Conversation Logging to Database

.github/workflows/staging.yml

@@ -35,5 +35,5 @@ jobs:
       run: bun install
 
     - name: Run tests
-      run: bun run test
+      run: bunx turbo run test --filter='!python-tools'
 

.github/workflows/test.yml

@@ -30,19 +30,9 @@ jobs:
       uses: oven-sh/setup-bun@v2
       with:
         bun-version: latest
-
-    - name: Setup uv
-      uses: astral-sh/setup-uv@v4
-      with:
-        version: "latest"
 
     - name: Install dependencies
       run: bun install
-
-    - name: Install Python dependencies
-      run: cd packages/python-tools && uv sync
 
     - name: Run tests
-      run: |
-        export PATH="$HOME/.local/bin:$PATH"
-        bun run test
+      run: bunx turbo run test --filter='!python-tools'

AGENTS.md

@@ -400,3 +400,56 @@ Environment Variables & Port Configuration
 - Add ngrok domains to `ALLOWED_ORIGINS` for external dev access
 
 Done. This guide covers the minimal, repeatable steps for adding new tools and flows with token‑efficient prompts and predictable UX.
+
+Conversation Logging (ML Training Data)
+
+The system captures all user transcripts and LLM responses to Convex for ML training and improvement.
+
+**Architecture:**
+- **Schema**: `packages/convex/schema.ts` → `conversationLogs` table
+- **Convex mutations**: `packages/convex/conversationLogs.ts` → `logConversation`, `updateResponse`
+- **Application utility**: `apps/application/src/core/conversationLogger.ts` → fire-and-forget logging functions
+
+**Data Flow:**
+1. `handleTranscription` receives utterance → routes via BAML `b.Route()`
+2. Immediately after routing: `logConversation(userId, sessionId, transcript, route)` creates initial record
+3. Handler processes request → displays response on glasses
+4. Handler calls `updateConversationResponse(userId, sessionId, transcript, response)` to capture formatted output
+
+**LogContext Pattern:**
+Handlers that capture responses receive an optional `logContext` parameter:
+```ts
+logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string }
+```
+
+After displaying lines, call:
+```ts
+if (logContext) {
+  const responseText = lines.map((l) => `W: ${l}`).join("\n");
+  updateConversationResponse(
+    logContext.convexUserId,
+    logContext.sessionId,
+    logContext.transcript,
+    responseText,
+  );
+}
+```
+
+**Coverage by Route:**
+| Route | Initial Log | Response Captured | Notes |
+|-------|-------------|-------------------|-------|
+| WEATHER | ✓ | ✓ | Weather summary lines |
+| MAPS | ✓ | ✓ | Place recommendations |
+| WEB_SEARCH | ✓ | ✓ | Search answer lines |
+| KNOWLEDGE | ✓ | ✓ | General knowledge answer |
+| MEMORY_RECALL | ✓ | ✓ | Synthesized memory lines |
+| MEMORY_CAPTURE | ✓ | ✗ | Silent operation (stores to Honcho) |
+| PASSTHROUGH | ✓ | ✓ (when hint shown) | Null case for ambient speech |
+| NOTE_THIS | ✓ | ✗ | Meta-action, not content response |
+| FOLLOW_UP | ✓ | ✗ | Meta-action, not content response |
+
+**Key Points:**
+- Use fire-and-forget pattern: `void convexClient.mutation(...)` with `.catch()` for error logging
+- Response field captures the formatted text shown on glasses (e.g., `"W: 72°F and sunny"`)
+- PASSTHROUGH with no hint = null response (valuable for training router to identify ambient speech)
+- Initial log happens in `transcriptionFlow.ts`; response update happens in individual handlers

apps/application/src/core/conversationLogger.ts

@@ -0,0 +1,53 @@
+import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
+import { convexClient } from "./convex";
+
+/**
+ * Logs a conversation to Convex for ML training data.
+ * Uses fire-and-forget pattern - errors are logged but don't block.
+ */
+export function logConversation(
+	userId: Id<"users">,
+	sessionId: string,
+	transcript: string,
+	route: string,
+	response?: string,
+): void {
+	void convexClient
+		.mutation(api.conversationLogs.logConversation, {
+			userId,
+			sessionId,
+			transcript,
+			route,
+			response,
+		})
+		.catch((error) => {
+			console.error("[ConversationLogger] Failed to log conversation:", error);
+		});
+}
+
+/**
+ * Updates the response field for a previously logged conversation.
+ * Used by handlers to capture the final formatted response shown on glasses.
+ * Uses fire-and-forget pattern - errors are logged but don't block.
+ */
+export function updateConversationResponse(
+	userId: Id<"users">,
+	sessionId: string,
+	transcript: string,
+	response: string,
+): void {
+	void convexClient
+		.mutation(api.conversationLogs.updateResponse, {
+			userId,
+			sessionId,
+			transcript,
+			response,
+		})
+		.catch((error) => {
+			console.error(
+				"[ConversationLogger] Failed to update conversation response:",
+				error,
+			);
+		});
+}

apps/application/src/handlers/hints.ts

@@ -1,7 +1,9 @@
 import { b, HintCategory } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 
@@ -14,6 +16,7 @@ export async function tryPassthroughHint(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ): Promise<void> {
 	const runId = Date.now();
 	hintRunIds.set(session, runId);
@@ -163,6 +166,16 @@ export async function tryPassthroughHint(
 			durationMs: 4000,
 			priority: 3,
 		});
+
+		// Log the hint response for ML training
+		if (logContext) {
+			updateConversationResponse(
+				logContext.convexUserId,
+				logContext.sessionId,
+				logContext.transcript,
+				`H: ${hintResult.hint}`,
+			);
+		}
 	} catch (error) {
 		session.logger.error(`[tryPassthroughHint] Error: ${String(error)}`);
 	}

apps/application/src/handlers/knowledge.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 import { showTextDuringOperation } from "../core/textWall";
@@ -17,6 +19,7 @@ export async function startKnowledgeFlow(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	knowledgeRunIds.set(session, runId);
@@ -228,6 +231,21 @@ export async function startKnowledgeFlow(
 					priority: 2,
 				});
 			}
+
+			if (logContext && answerLines.length > 0) {
+				const responseText = answerLines
+					.map((line, i) => {
+						const label = answerLines.length > 1 ? `A${i + 1}` : "A";
+						return `Q: ${questionLine}\n${label}: ${line}`;
+					})
+					.join("\n");
+				updateConversationResponse(
+					logContext.convexUserId,
+					logContext.sessionId,
+					logContext.transcript,
+					responseText,
+				);
+			}
 		} else {
 			displayQueue.enqueue({
 				text: "// Clairvoyant\nK: No question detected.",

apps/application/src/handlers/maps.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import {
 	checkUserIsPro,
 	convexClient,
@@ -30,6 +32,7 @@ async function processPlacesData(
 	places: PlaceSuggestion[],
 	runId: number,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	if (!places?.length) {
 		displayQueue.enqueue({
@@ -188,6 +191,16 @@ async function processPlacesData(
 			priority: 2,
 		});
 	}
+
+	if (logContext && lines.length > 0) {
+		const responseText = lines.map((l) => `M: ${l}`).join("\n");
+		updateConversationResponse(
+			logContext.convexUserId,
+			logContext.sessionId,
+			logContext.transcript,
+			responseText,
+		);
+	}
 }
 
 export async function startMapsFlow(
@@ -197,6 +210,7 @@ export async function startMapsFlow(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	mapsRunIds.set(session, runId);
@@ -287,6 +301,7 @@ export async function startMapsFlow(
 				places,
 				runId,
 				displayQueue,
+				logContext,
 			);
 		} catch (error) {
 			session.logger.error(`[startMapsFlow] Maps flow error: ${String(error)}`);
@@ -376,6 +391,7 @@ export async function startMapsFlow(
 					places,
 					runId,
 					displayQueue,
+					logContext,
 				);
 			} catch (error) {
 				session.logger.error(

apps/application/src/handlers/memory.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 
@@ -78,6 +80,7 @@ export async function MemoryRecall(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	memoryRunCallIds.set(session, runId);
@@ -251,6 +254,17 @@ export async function MemoryRecall(
 						priority: 2,
 					});
 				}
+
+				// Log the response for ML training
+				if (logContext) {
+					const responseText = lines.map((l) => `R: ${l}`).join("\n");
+					updateConversationResponse(
+						logContext.convexUserId,
+						logContext.sessionId,
+						logContext.transcript,
+						responseText,
+					);
+				}
 			} else {
 				session.logger.error(
 					`[startMemoryRecallFlow] No lines in synthesis results`,

apps/application/src/handlers/search.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 import { showTextDuringOperation } from "../core/textWall";
@@ -18,6 +20,7 @@ export async function startWebSearchFlow(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	webSearchRunIds.set(session, runId);
@@ -231,6 +234,16 @@ export async function startWebSearchFlow(
 					priority: 2,
 				});
 			}
+
+			if (logContext) {
+				const responseText = lines.map((l) => `S: ${l}`).join("\n");
+				updateConversationResponse(
+					logContext.convexUserId,
+					logContext.sessionId,
+					logContext.transcript,
+					responseText,
+				);
+			}
 		} else {
 			session.logger.error(`[startWebSearchFlow] No lines in answerLines`);
 		}