Comprehensive dedup quality: type/project/source awareness

internal/agent/encoding/agent.go

@@ -72,8 +72,9 @@ type EncodingConfig struct {
 	BatchSizeEvent          int      // batch size for EncodeAllPending (default: 50)
 	BatchSizePoll           int      // batch size for polling loop (default: 10)
 	EmbedBatchSize          int      // max memories to batch-embed in one call (default 10)
-	DeduplicationThreshold  float32  // cosine sim above which new memory is a duplicate (default: 0.9)
-	SalienceFloor           float32  // min salience to encode; non-MCP sources below this are skipped (default: 0.0)
+	DeduplicationThreshold     float32  // cosine sim above which new memory is a duplicate (default: 0.95)
+	MCPDeduplicationThreshold  float32  // higher threshold for MCP-sourced memories (default: 0.98)
+	SalienceFloor              float32  // min salience to encode; non-MCP sources below this are skipped (default: 0.0)
 	DisablePolling          bool     // if true, skip the polling loop (MCP processes should not poll)
 }
 
@@ -124,8 +125,10 @@ func DefaultConfig() EncodingConfig {
 		BackoffThreshold:        3,
 		BackoffBaseSec:          30,
 		BackoffMaxSec:           300,
-		BatchSizeEvent:          50,
-		BatchSizePoll:           10,
+		BatchSizeEvent:             50,
+		BatchSizePoll:              10,
+		DeduplicationThreshold:     0.95,
+		MCPDeduplicationThreshold:  0.98,
 	}
 }
 
@@ -710,11 +713,8 @@ func (ea *EncodingAgent) finalizeEncodedMemory(ctx context.Context, raw store.Ra
 			ea.log.Warn("failed to search for similar memories", "raw_id", raw.ID, "error", err)
 		} else {
 			// Check for near-duplicate before creating a new memory
-			dedupThreshold := ea.config.DeduplicationThreshold
-			if dedupThreshold <= 0 {
-				dedupThreshold = 0.9
-			}
-			if dup := findDuplicate(similar, dedupThreshold); dup != nil {
+			dc := ea.buildDedupContext(raw)
+			if dup := findDuplicate(similar, dc); dup != nil {
 				ea.log.Info("dedup: boosting existing memory instead of creating duplicate",
 					"raw_id", raw.ID,
 					"existing_id", dup.Memory.ID,
@@ -990,11 +990,8 @@ func (ea *EncodingAgent) encodeMemory(ctx context.Context, rawID string) error {
 			ea.log.Debug("similarity search completed", "raw_id", raw.ID, "results", len(similar))
 
 			// Dedup check: if a near-duplicate already exists, boost it instead of creating a new memory
-			dedupThreshold := ea.config.DeduplicationThreshold
-			if dedupThreshold <= 0 {
-				dedupThreshold = 0.9
-			}
-			if dup := findDuplicate(similar, dedupThreshold); dup != nil {
+			dc := ea.buildDedupContext(raw)
+			if dup := findDuplicate(similar, dc); dup != nil {
 				ea.log.Info("dedup: boosting existing memory instead of creating duplicate",
 					"raw_id", raw.ID, "existing_id", dup.Memory.ID, "similarity", dup.Score)
 				newSalience := dup.Memory.Salience + 0.05
@@ -1939,12 +1936,64 @@ func truncateString(s string, maxLen int) string {
 	return string(runes[:maxLen]) + "..."
 }
 
-// findDuplicate returns the first result above the dedup threshold, or nil.
-func findDuplicate(results []store.RetrievalResult, threshold float32) *store.RetrievalResult {
+// buildDedupContext creates a dedup context from the agent config and raw memory.
+func (ea *EncodingAgent) buildDedupContext(raw store.RawMemory) dedupContext {
+	threshold := ea.config.DeduplicationThreshold
+	if threshold <= 0 {
+		threshold = 0.95
+	}
+	mcpThreshold := ea.config.MCPDeduplicationThreshold
+	if mcpThreshold <= 0 {
+		mcpThreshold = 0.98
+	}
+	return dedupContext{
+		Threshold:    threshold,
+		MCPThreshold: mcpThreshold,
+		RawSource:    raw.Source,
+		RawType:      raw.Type,
+		RawProject:   raw.Project,
+	}
+}
+
+// dedupContext holds the context needed for smart deduplication decisions.
+type dedupContext struct {
+	Threshold    float32 // base cosine similarity threshold
+	MCPThreshold float32 // higher threshold for MCP-sourced memories (explicit user input)
+	RawSource    string  // source of the incoming memory
+	RawType      string  // type of the incoming memory (decision, error, insight, etc.)
+	RawProject   string  // project of the incoming memory
+}
+
+// findDuplicate returns the best dedup candidate, applying type-aware,
+// project-aware, and source-aware filtering. Returns nil if no valid
+// duplicate is found.
+//
+// Rules:
+//   - Never dedup across different memory types (decision != error)
+//   - Never dedup across different projects
+//   - MCP-sourced memories use a higher threshold (default 0.98) since
+//     they represent explicit user/agent input worth preserving
+//   - All other sources use the base threshold (default 0.95)
+func findDuplicate(results []store.RetrievalResult, dc dedupContext) *store.RetrievalResult {
+	threshold := dc.Threshold
+	if dc.RawSource == "mcp" && dc.MCPThreshold > 0 {
+		threshold = dc.MCPThreshold
+	}
+
 	for i := range results {
-		if results[i].Score >= threshold {
-			return &results[i]
+		r := &results[i]
+		if r.Score < threshold {
+			continue
+		}
+		// Skip cross-type dedup: a decision and an error are never duplicates.
+		if dc.RawType != "" && r.Memory.Type != "" && dc.RawType != r.Memory.Type {
+			continue
+		}
+		// Skip cross-project dedup: same topic in different projects is distinct.
+		if dc.RawProject != "" && r.Memory.Project != "" && dc.RawProject != r.Memory.Project {
+			continue
 		}
+		return r
 	}
 	return nil
 }

internal/agent/encoding/agent_test.go

@@ -2100,12 +2100,14 @@ func TestCompressionResponseRoundTrip(t *testing.T) {
 // ---------------------------------------------------------------------------
 
 func TestFindDuplicate(t *testing.T) {
+	baseDC := dedupContext{Threshold: 0.9, MCPThreshold: 0.98}
+
 	t.Run("returns first result above threshold", func(t *testing.T) {
 		results := []store.RetrievalResult{
 			{Memory: store.Memory{ID: "a"}, Score: 0.95},
 			{Memory: store.Memory{ID: "b"}, Score: 0.85},
 		}
-		dup := findDuplicate(results, 0.9)
+		dup := findDuplicate(results, baseDC)
 		if dup == nil {
 			t.Fatal("expected duplicate to be found")
 		}
@@ -2119,14 +2121,14 @@ func TestFindDuplicate(t *testing.T) {
 			{Memory: store.Memory{ID: "a"}, Score: 0.85},
 			{Memory: store.Memory{ID: "b"}, Score: 0.70},
 		}
-		dup := findDuplicate(results, 0.9)
+		dup := findDuplicate(results, baseDC)
 		if dup != nil {
 			t.Errorf("expected nil, got %q", dup.Memory.ID)
 		}
 	})
 
 	t.Run("empty results returns nil", func(t *testing.T) {
-		dup := findDuplicate(nil, 0.9)
+		dup := findDuplicate(nil, baseDC)
 		if dup != nil {
 			t.Error("expected nil for empty results")
 		}
@@ -2136,9 +2138,75 @@ func TestFindDuplicate(t *testing.T) {
 		results := []store.RetrievalResult{
 			{Memory: store.Memory{ID: "a"}, Score: 0.9},
 		}
-		dup := findDuplicate(results, 0.9)
+		dup := findDuplicate(results, baseDC)
 		if dup == nil {
 			t.Fatal("expected duplicate at exact threshold")
 		}
 	})
+
+	t.Run("skips cross-type dedup", func(t *testing.T) {
+		results := []store.RetrievalResult{
+			{Memory: store.Memory{ID: "a", Type: "error"}, Score: 0.99},
+		}
+		dc := dedupContext{Threshold: 0.9, RawType: "decision"}
+		dup := findDuplicate(results, dc)
+		if dup != nil {
+			t.Error("should not dedup across different types")
+		}
+	})
+
+	t.Run("allows same-type dedup", func(t *testing.T) {
+		results := []store.RetrievalResult{
+			{Memory: store.Memory{ID: "a", Type: "decision"}, Score: 0.95},
+		}
+		dc := dedupContext{Threshold: 0.9, RawType: "decision"}
+		dup := findDuplicate(results, dc)
+		if dup == nil {
+			t.Fatal("should dedup same type above threshold")
+		}
+	})
+
+	t.Run("skips cross-project dedup", func(t *testing.T) {
+		results := []store.RetrievalResult{
+			{Memory: store.Memory{ID: "a", Project: "felix-lm"}, Score: 0.99},
+		}
+		dc := dedupContext{Threshold: 0.9, RawProject: "mnemonic"}
+		dup := findDuplicate(results, dc)
+		if dup != nil {
+			t.Error("should not dedup across different projects")
+		}
+	})
+
+	t.Run("MCP source uses higher threshold", func(t *testing.T) {
+		results := []store.RetrievalResult{
+			{Memory: store.Memory{ID: "a"}, Score: 0.96},
+		}
+		dc := dedupContext{Threshold: 0.95, MCPThreshold: 0.98, RawSource: "mcp"}
+		dup := findDuplicate(results, dc)
+		if dup != nil {
+			t.Error("MCP at 0.96 should NOT dedup when MCP threshold is 0.98")
+		}
+	})
+
+	t.Run("MCP source dedupes above MCP threshold", func(t *testing.T) {
+		results := []store.RetrievalResult{
+			{Memory: store.Memory{ID: "a"}, Score: 0.99},
+		}
+		dc := dedupContext{Threshold: 0.95, MCPThreshold: 0.98, RawSource: "mcp"}
+		dup := findDuplicate(results, dc)
+		if dup == nil {
+			t.Fatal("MCP at 0.99 should dedup when MCP threshold is 0.98")
+		}
+	})
+
+	t.Run("non-MCP source uses base threshold", func(t *testing.T) {
+		results := []store.RetrievalResult{
+			{Memory: store.Memory{ID: "a"}, Score: 0.96},
+		}
+		dc := dedupContext{Threshold: 0.95, MCPThreshold: 0.98, RawSource: "filesystem"}
+		dup := findDuplicate(results, dc)
+		if dup == nil {
+			t.Fatal("filesystem at 0.96 should dedup when base threshold is 0.95")
+		}
+	})
 }

internal/mcp/server.go

@@ -382,7 +382,7 @@ func (srv *MCPServer) handleRemember(ctx context.Context, args map[string]interf
 
 	srv.log.Info("memory stored", "id", raw.ID, "source", source, "type", memType, "project", project)
 
-	return toolResult(fmt.Sprintf("Stored memory %s (type: %s, project: %s)\n  Raw ID: %s\n  Initial salience: %.2f\n  Encoding: queued (async)\n\nTip: Use check_memory with raw_id \"%s\" to verify encoding status.",
+	return toolResult(fmt.Sprintf("Stored memory %s (type: %s, project: %s)\n  Raw ID: %s\n  Initial salience: %.2f\n  Encoding: queued (async)\n\nTip: Use check_memory with raw_id %q to verify encoding status. Dedup protections: same-type, same-project, source-aware thresholds.",
 		raw.ID, memType, project, raw.ID, raw.InitialSalience, raw.ID)), nil
 }
 
@@ -1757,9 +1757,9 @@ func (srv *MCPServer) handleCheckMemory(ctx context.Context, args map[string]int
 			if err != nil {
 				return toolResult(fmt.Sprintf("No memory found for raw_id %q or memory_id %q.", rawID, memoryID)), nil
 			}
-			status := "pending"
+			status := "pending encoding"
 			if raw.Processed {
-				status = "processed (encoding may have been deduplicated)"
+				status = "deduplicated — a similar memory already existed, so this one boosted its salience instead of creating a duplicate"
 			}
 			return toolResult(fmt.Sprintf("Raw memory %s found but not yet encoded.\n  Status: %s\n  Source: %s\n  Type: %s\n  Salience: %.2f\n  Created: %s",
 				raw.ID, status, raw.Source, raw.Type, raw.InitialSalience, raw.CreatedAt.Format(time.RFC3339))), nil