diff --git a/backend/package.json b/backend/package.json index 433c853..a34c40e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -13,6 +13,8 @@ "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "@mastra/core": "^1.36.0", + "@mastra/observability": "^1.13.0", + "@mastra/pg": "^1.11.1", "@openrouter/ai-sdk-provider": "^2.9.0", "ai": "^6.0.0", "convex": "^1.39.1", diff --git a/backend/src/index.ts b/backend/src/index.ts index 010df15..46868b1 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -13,6 +13,7 @@ import { sendTransactionalEmail } from "./email/send.js"; import { datasetReadyTemplate } from "./email/templates/dataset-ready.js"; import { capture, shutdown as shutdownAnalytics } from "./analytics/posthog.js"; import { EVENTS } from "./analytics/events.js"; +import { mastra } from "./mastra/index.js"; /** Domain part of an email, for analytics (we never log full addresses). */ function emailDomain(email: string): string { @@ -539,6 +540,7 @@ await fastify.register(async (instance) => { return reply.code(202).send({ success: true, runId: run.runId }); } catch (err) { + await mastra.observability.getDefaultInstance()?.flush(); const msg = err instanceof Error ? err.message : String(err); if (msg.includes("validator") || msg.includes("Invalid")) { return reply.code(400).send({ error: "Invalid datasetId" }); diff --git a/backend/src/mastra/index.ts b/backend/src/mastra/index.ts index 34d97e6..ffb3e62 100644 --- a/backend/src/mastra/index.ts +++ b/backend/src/mastra/index.ts @@ -2,7 +2,16 @@ import { Mastra } from "@mastra/core/mastra"; import { inferSchemaWorkflow } from "./workflows/infer-schema.js"; import { populateWorkflow } from "./workflows/populate.js"; import { updateWorkflow } from "./workflows/update.js"; - +import { PostgresStore } from '@mastra/pg' +import { + Observability, + MastraStorageExporter, + SensitiveDataFilter, +} from '@mastra/observability' +const storage = new PostgresStore({ + id: 'mastra-pg-storage', + connectionString: process.env.MASTRA_DATABASE_URL!, +}) /** * Mastra registry. * @@ -16,4 +25,18 @@ import { updateWorkflow } from "./workflows/update.js"; */ export const mastra = new Mastra({ workflows: { inferSchemaWorkflow, populateWorkflow, updateWorkflow }, -}); + + storage, + observability: new Observability({ + + configs: { + default: { + serviceName: 'bigset', + exporters: [new MastraStorageExporter({ strategy: 'auto' })], + spanOutputProcessors: [ + new SensitiveDataFilter(), + ], + }, + }, + }), +}); \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1afbf5b..07ead5e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -36,6 +36,7 @@ services: CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-} OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} TINYFISH_API_KEY: ${TINYFISH_API_KEY:-} + MASTRA_DATABASE_URL: postgresql://bigset:bigset@db:5432/bigset_internal # Transactional email — when unset, the email module no-ops. RESEND_API_KEY: ${RESEND_API_KEY:-} EMAIL_FROM: ${EMAIL_FROM:-BigSet } @@ -65,6 +66,7 @@ services: CONVEX_URL: http://convex:3210 CONVEX_SELF_HOSTED_ADMIN_KEY: ${CONVEX_SELF_HOSTED_ADMIN_KEY:-} TINYFISH_API_KEY: ${TINYFISH_API_KEY:-} + MASTRA_DATABASE_URL: postgresql://bigset:bigset@db:5432/bigset_internal depends_on: convex: condition: service_healthy