feat: Shared Memory Phase 4 — Advanced Cross-Peer Features

.envrc

@@ -0,0 +1 @@
+source_up

apps/application/src/baml_client/partial_types.ts

@@ -96,6 +96,11 @@ export namespace partial_types {
       peerCard: string[]
       recentMessages: string[]
       sessionSummaries: string[]
+      crossPeerPerspectives: CrossPeerPerspective[]
+    }
+    export interface CrossPeerPerspective {
+      label?: string | null
+      perspective?: string | null
     }
     export interface MemoryCore {
       userName?: string | null

apps/application/src/baml_client/type_builder.ts

@@ -45,7 +45,7 @@ export default class TypeBuilder {
 
     LocationLite: ClassViewer<'LocationLite', "lat" | "lon" | "timezone">;
 
-    MemoryContext: ClassViewer<'MemoryContext', "explicitFacts" | "deductiveFacts" | "peerCard" | "recentMessages" | "sessionSummaries">;
+    MemoryContext: ClassViewer<'MemoryContext', "explicitFacts" | "deductiveFacts" | "peerCard" | "recentMessages" | "sessionSummaries" | "crossPeerPerspectives">;
 
     MemoryCore: ClassViewer<'MemoryCore', "userName" | "userFacts" | "deductiveFacts">;
 
@@ -127,7 +127,7 @@ export default class TypeBuilder {
         ]);
 
         this.MemoryContext = this.tb.classViewer("MemoryContext", [
-          "explicitFacts","deductiveFacts","peerCard","recentMessages","sessionSummaries",
+          "explicitFacts","deductiveFacts","peerCard","recentMessages","sessionSummaries","crossPeerPerspectives",
         ]);
 
         this.MemoryCore = this.tb.classViewer("MemoryCore", [

apps/application/src/baml_client/types.ts

@@ -141,6 +141,13 @@ export interface MemoryContext {
   peerCard: string[]
   recentMessages: string[]
   sessionSummaries: string[]
+  crossPeerPerspectives: CrossPeerPerspective[]
+
+}
+
+export interface CrossPeerPerspective {
+  label: string
+  perspective: string
 
 }
 

apps/application/src/handlers/memory.ts

@@ -1,11 +1,13 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
 import type { Id } from "@convex/_generated/dataModel";
+import { Honcho } from "@honcho-ai/sdk";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
 import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
+import { env } from "../core/env";
 
 const memoryRunCallIds = new WeakMap<AppSession, number>();
 
@@ -198,6 +200,82 @@ export async function MemoryRecall(
 				peerRep = { explicit: [], deductive: [] };
 			}
 
+			// Cross-peer queries: fetch perspectives from connected users
+			let crossPeerPerspectives: Array<{ label: string; perspective: string }> = [];
+			try {
+				const connections = await convexClient.query(
+					api.connections.getActiveSharedMemoryConnectionsByMentraId,
+					{ mentraUserId },
+				);
+
+				const cappedConnections = connections.slice(0, 3);
+				if (cappedConnections.length > 0) {
+					session.logger.info(
+						`[startMemoryRecallFlow] Querying ${cappedConnections.length} cross-peer connections`,
+					);
+
+					const honchoClient = new Honcho({
+						apiKey: env.HONCHO_API_KEY,
+						environment: "production",
+						workspaceId: "with-context",
+					});
+
+					const rawPerspectives = await Promise.all(
+						cappedConnections.map(async (conn) => {
+							try {
+								const connectedPeer = await honchoClient.peer(
+									`${conn.connectedUserId}-diatribe`,
+								);
+								const rep = await connectedPeer.representation({
+									target: `${userId}-diatribe`,
+									searchQuery: textQuery,
+									searchTopK: 5,
+									maxConclusions: 10,
+								});
+								return {
+									label: conn.label ?? "Connected user",
+									perspective: typeof rep.representation === "string" ? rep.representation : "",
+								};
+							} catch (error) {
+								session.logger.warn(
+									`[startMemoryRecallFlow] Cross-peer query failed for ${conn.connectedUserId}: ${error}`,
+								);
+								return null;
+							}
+						}),
+					);
+
+					const validPerspectives = rawPerspectives.filter(
+						(p): p is { label: string; perspective: string } =>
+							p !== null && p.perspective.length > 0,
+					);
+
+					// Sensitivity gate
+					const sensitivityResults = await Promise.all(
+						validPerspectives.map(async (p) => {
+							try {
+								const category = await b.CheckSensitivity(p.perspective);
+								return { ...p, category };
+							} catch {
+								return { ...p, category: "SENSITIVE" as const };
+							}
+						}),
+					);
+
+					crossPeerPerspectives = sensitivityResults
+						.filter((p) => p.category === "SAFE")
+						.map(({ label, perspective }) => ({ label, perspective }));
+
+					session.logger.info(
+						`[startMemoryRecallFlow] Cross-peer: ${validPerspectives.length} valid, ${crossPeerPerspectives.length} safe`,
+					);
+				}
+			} catch (error) {
+				session.logger.warn(
+					`[startMemoryRecallFlow] Cross-peer lookup failed: ${error}`,
+				);
+			}
+
 			// Build memory context
 			const memoryContext = {
 				explicitFacts: peerRep.explicit.map((e) => e.content),
@@ -207,6 +285,7 @@ export async function MemoryRecall(
 				peerCard: contextData.peerCard,
 				recentMessages: contextData.messages.slice(-5).map((m) => m.content),
 				sessionSummaries,
+				crossPeerPerspectives,
 			};
 
 			// Synthesize response with BAML (replaces .chat() call)

baml_src/chat.baml

@@ -29,6 +29,7 @@ class ChatContext {
   userFacts string[] @description("Known facts about the user")
   deductiveFacts string[] @description("Inferred facts about the user")
   conversationHistory ChatConversationMessage[] @description("Previous messages in this chat")
+  crossPeerPerspectives CrossPeerPerspective[] @description("Perspectives from connected users with shared memory enabled")
 }
 
 function InterpretChatMessage(
@@ -50,6 +51,7 @@ Style:
 - Reference session context naturally
 - Keep responses brief - this is chat, not an essay
 - Don't be overly enthusiastic or use excessive exclamation marks
+- If connected user perspectives are available, weave them naturally with attribution (e.g. "Your wife mentioned...", "Alex has been looking into...")
 
 DATE: {{ context.date }}
 
@@ -75,6 +77,14 @@ USER PROFILE:
 - {{ fact }}
 {% endfor %}
 
+{% endif %}
+{% if context.crossPeerPerspectives | length > 0 %}
+CONNECTED USER PERSPECTIVES:
+{% for p in context.crossPeerPerspectives %}
+From "{{ p.label }}":
+{{ p.perspective }}
+{% endfor %}
+
 {% endif %}
 {% if context.conversationHistory | length > 0 %}
 CONVERSATION HISTORY:
@@ -114,6 +124,7 @@ test test_chat_simple_greeting {
         "User is interested in AI technologies"
       ]
       conversationHistory []
+      crossPeerPerspectives []
     }
   }
 }
@@ -142,6 +153,7 @@ test test_chat_with_history {
           createdAt "2024-12-24T16:00:00Z"
         }
       ]
+      crossPeerPerspectives []
     }
   }
 }
@@ -157,6 +169,7 @@ test test_chat_no_sessions {
       userFacts []
       deductiveFacts []
       conversationHistory []
+      crossPeerPerspectives []
     }
   }
 }

baml_src/core.baml

@@ -1,5 +1,10 @@
 // Common classes shared across BAML functions
 
+class CrossPeerPerspective {
+  label string @description("Display name of the connected user")
+  perspective string @description("Perspective or relevant facts from the connected user's memory")
+}
+
 class MemoryCore {
   userName string? @description("User's name if known")
   userFacts string[] @description("Relevant biographical facts about the user")

baml_src/email_reply.baml

@@ -20,6 +20,7 @@ class EmailContext {
   sessionTopics string[] @description("Topics from the linked session")
   peerCard string[] @description("User profile facts from Honcho")
   conversationHistory ConversationMessage[] @description("Previous messages in this email thread")
+  crossPeerPerspectives CrossPeerPerspective[] @description("Perspectives from connected users with shared memory enabled")
 }
 
 function InterpretEmailReply(
@@ -68,6 +69,14 @@ CONVERSATION HISTORY:
 ---
 {% endfor %}
 
+{% endif %}
+{% if context.crossPeerPerspectives | length > 0 %}
+CONNECTED USER PERSPECTIVES:
+{% for p in context.crossPeerPerspectives %}
+From "{{ p.label }}":
+{{ p.perspective }}
+{% endfor %}
+
 {% endif %}
 USER'S NEW REPLY:
 {{ userMessage }}
@@ -88,6 +97,7 @@ test test_email_reply_simple {
       conversationHistory [
         { direction "outbound", content "Here are your session notes from today..." }
       ]
+      crossPeerPerspectives []
     }
   }
 }
@@ -104,6 +114,7 @@ test test_email_reply_with_new_info {
       conversationHistory [
         { direction "outbound", content "Here are your session notes from today..." }
       ]
+      crossPeerPerspectives []
     }
   }
 }
@@ -118,6 +129,7 @@ test test_email_reply_no_session {
       sessionTopics []
       peerCard []
       conversationHistory []
+      crossPeerPerspectives []
     }
   }
 }

baml_src/followup.baml

@@ -16,6 +16,7 @@ class FollowupChatContext {
     conversationHistory FollowupConversationMessage[] @description("Previous chat messages about this follow-up")
     memory FollowupMemoryContext? @description("User memory context from Honcho")
     searchResults FollowupSearchResult[] @description("Web search results relevant to the topic")
+    crossPeerPerspectives CrossPeerPerspective[] @description("Perspectives from connected users with shared memory enabled")
 }
 
 class FollowupChatResponse {
@@ -136,6 +137,14 @@ CONVERSATION HISTORY:
 ---
 {% endfor %}
 
+{% endif %}
+{% if context.crossPeerPerspectives | length > 0 %}
+CONNECTED USER PERSPECTIVES:
+{% for p in context.crossPeerPerspectives %}
+From "{{ p.label }}":
+{{ p.perspective }}
+{% endfor %}
+
 {% endif %}
 USER'S MESSAGE:
 {{ userMessage }}
@@ -158,6 +167,7 @@ test test_followup_chat_simple {
             ]
             conversationHistory []
             searchResults []
+            crossPeerPerspectives []
         }
     }
 }
@@ -187,6 +197,7 @@ test test_followup_chat_with_history {
                 }
             ]
             searchResults []
+            crossPeerPerspectives []
         }
     }
 }

baml_src/sensitivity.baml

@@ -0,0 +1,37 @@
+// Sensitivity classification for cross-peer shared content
+
+enum SensitivityCategory {
+  SAFE @description("Content is safe to share across peers")
+  SENSITIVE @description("Content contains sensitive personal information that should not be shared")
+}
+
+function CheckSensitivity(crossPeerContext: string) -> SensitivityCategory {
+  client "Groq"
+
+  prompt #"
+Classify whether the following cross-peer context contains sensitive personal information that should NOT be shared between connected users.
+
+SENSITIVE includes: medical/health details, financial information, passwords/credentials, private relationship issues, legal matters, or anything the user would reasonably expect to remain private.
+
+SAFE includes: general interests, preferences, professional topics, hobbies, travel plans, food preferences, or other casual information.
+
+Content to classify:
+{{ crossPeerContext }}
+
+{{ ctx.output_format }}
+"#
+}
+
+test test_safe_content {
+  functions [CheckSensitivity]
+  args {
+    crossPeerContext "User enjoys hiking and is interested in AI technology. Works as a software engineer."
+  }
+}
+
+test test_sensitive_content {
+  functions [CheckSensitivity]
+  args {
+    crossPeerContext "User mentioned they are dealing with a medical diagnosis and has been seeing a therapist."
+  }
+}