feat: add dataset sharing, public preview page, and import

backend/src/clerk-auth.ts

@@ -116,8 +116,8 @@ export async function requireAuth(
   const requestState = await req.server.clerk.authenticateRequest(
     clerkRequest,
     {
-      // Anyone consuming our backend is our own frontend; lock to its origin.
-      authorizedParties: [env.CLIENT_ORIGIN],
+      // Anyone consuming our backend is our own frontend; lock to allowed origins.
+      authorizedParties: env.CLIENT_ORIGINS,
     },
   );
 

backend/src/env.ts

@@ -18,11 +18,21 @@ function numberFromEnv(name: string, fallback: number): number {
   return Number.isFinite(parsed) ? parsed : fallback;
 }
 
+function stringListFromEnv(name: string, fallback: string[]): string[] {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  return raw
+    .split(",")
+    .map((value) => value.trim())
+    .filter(Boolean);
+}
+
 export const env = {
   PROD: process.env.PROD,
   IS_PROD: process.env.PROD === "1",
   IS_LOCAL_MODE: process.env.PROD !== "1",
   CLIENT_ORIGIN: process.env.CLIENT_ORIGIN || "http://localhost:3500",
+  CLIENT_ORIGINS: stringListFromEnv("CLIENT_ORIGIN", ["http://localhost:3500"]),
   CONVEX_URL: required("CONVEX_URL"),
   PORT: numberFromEnv("PORT", 3501),
 

backend/src/index.ts

@@ -8,7 +8,7 @@ import { inferSchema } from "./pipeline/schema-inference.js";
 import { datasetContextSchema, type DatasetContext } from "./pipeline/populate.js";
 import { populateWorkflow } from "./mastra/workflows/populate.js";
 import { updateWorkflow } from "./mastra/workflows/update.js";
-import { convex, internal } from "./convex.js";
+import { convex, api, internal } from "./convex.js";
 import { sendTransactionalEmail } from "./email/send.js";
 import { datasetReadyTemplate } from "./email/templates/dataset-ready.js";
 import { capture, shutdown as shutdownAnalytics } from "./analytics/posthog.js";
@@ -629,23 +629,25 @@ function startLocalRefreshScheduler(
 
 const fastify = Fastify({ logger: true });
 
-const allowedCorsOrigins = new Set([env.CLIENT_ORIGIN]);
+const allowedCorsOrigins = new Set(env.CLIENT_ORIGINS);
 if (env.IS_LOCAL_MODE) {
-  try {
-    const clientOrigin = new URL(env.CLIENT_ORIGIN);
-    if (
-      clientOrigin.hostname === "localhost" ||
-      clientOrigin.hostname === "127.0.0.1"
-    ) {
-      allowedCorsOrigins.add(
-        `${clientOrigin.protocol}//localhost${clientOrigin.port ? `:${clientOrigin.port}` : ""}`,
-      );
-      allowedCorsOrigins.add(
-        `${clientOrigin.protocol}//127.0.0.1${clientOrigin.port ? `:${clientOrigin.port}` : ""}`,
-      );
+  for (const origin of env.CLIENT_ORIGINS) {
+    try {
+      const clientOrigin = new URL(origin);
+      if (
+        clientOrigin.hostname === "localhost" ||
+        clientOrigin.hostname === "127.0.0.1"
+      ) {
+        allowedCorsOrigins.add(
+          `${clientOrigin.protocol}//localhost${clientOrigin.port ? `:${clientOrigin.port}` : ""}`,
+        );
+        allowedCorsOrigins.add(
+          `${clientOrigin.protocol}//127.0.0.1${clientOrigin.port ? `:${clientOrigin.port}` : ""}`,
+        );
+      }
+    } catch {
+      // Keep the configured origin only if the origin is not URL-shaped.
     }
-  } catch {
-    // Keep the configured origin only if CLIENT_ORIGIN is not URL-shaped.
   }
 }
 
@@ -682,6 +684,34 @@ fastify.addHook("onClose", async () => {
 
 fastify.get("/health", async () => ({ status: "ok" }));
 
+fastify.get("/share/:id", async (request, reply) => {
+  const { id } = request.params as { id: string };
+  reply.header("Access-Control-Allow-Origin", "*");
+  let dataset;
+  try {
+    dataset = await convex.query(api.datasets.get, { id });
+  } catch (err) {
+    request.log.error({ err, id }, "Failed to fetch dataset for share route");
+    return reply.code(502).send({ error: "Failed to fetch dataset" });
+  }
+  if (!dataset || dataset.visibility !== "public") {
+    return reply.code(404).send({ error: "Dataset not found" });
+  }
+  return {
+    name: dataset.name,
+    description: dataset.description,
+    rowCount: dataset.rowCount,
+    columns: dataset.columns,
+  };
+});
+
+fastify.options("/share/:id", async (_request, reply) => {
+  reply.header("Access-Control-Allow-Origin", "*");
+  reply.header("Access-Control-Allow-Methods", "GET, OPTIONS");
+  reply.header("Access-Control-Allow-Headers", "Content-Type");
+  return reply.code(204).send();
+});
+
 fastify.get("/local-setup/status", async (_req, reply) => {
   if (!env.IS_LOCAL_MODE) {
     return reply.code(404).send({ error: "Not found" });

backend/src/pipeline/schema-inference.ts

@@ -18,6 +18,7 @@ Your job is to:
    - \`hybrid\` — unclear; the pipeline will try search_fetch first and fall back to browser.
 5. Set \`source_hint\` to a specific URL whenever possible (e.g. \`https://www.ycombinator.com/companies?industry=Fintech\`). Avoid vague descriptions.
 6. Write a \`retrieval_hint\` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row.
+7. If the user's prompt mentions a specific number of items (e.g. "top 10", "list of 50", "25 companies"), set \`suggested_row_count\` to that number. Otherwise omit it.
 
 Rules:
 

backend/src/pipeline/types.ts

@@ -38,6 +38,7 @@ export const datasetSchemaSchema = z
     primary_key: z.union([z.string(), z.array(z.string())]),
     retrieval_strategy: retrievalStrategySchema,
     source_hint: z.string().min(1),
+    suggested_row_count: z.number().optional(),
   })
   .superRefine((data, ctx) => {
     const names = data.columns.map((c) => c.name);

docker-compose.dev.yml

@@ -28,7 +28,7 @@ services:
       - ./backend/src:/app/src
       - ./scripts:/scripts:ro
     environment:
-      CLIENT_ORIGIN: http://localhost:3500
+      CLIENT_ORIGIN: http://localhost:3500,http://localhost:3502
       CONVEX_URL: http://convex:3210
       PORT: 3501
       PROD: ${PROD:-}
@@ -109,6 +109,7 @@ services:
       - ./scripts:/scripts:ro
     environment:
       NEXT_PUBLIC_CONVEX_URL: http://localhost:3210
+      CONVEX_URL: http://convex:3210
       NEXT_PUBLIC_PROD: ${PROD:-}
       PROD: ${PROD:-}
       NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-}
@@ -122,6 +123,7 @@ services:
       NEXT_PUBLIC_CLERK_AFTER_SIGN_IN_URL: /dashboard
       NEXT_PUBLIC_CLERK_AFTER_SIGN_UP_URL: /dashboard
       NEXT_PUBLIC_BACKEND_URL: http://localhost:3501
+      BACKEND_URL: http://backend:3501
       # PostHog — analytics no-ops if unset
       NEXT_PUBLIC_POSTHOG_KEY: ${NEXT_PUBLIC_POSTHOG_KEY:-}
       NEXT_PUBLIC_POSTHOG_HOST: ${NEXT_PUBLIC_POSTHOG_HOST:-https://us.i.posthog.com}

frontend/app/api/share/[id]/route.ts

@@ -0,0 +1,40 @@
+import { NextResponse } from "next/server";
+
+const CORS = { "Access-Control-Allow-Origin": "*" };
+const BACKEND_URL =
+  process.env.BACKEND_URL ??
+  process.env.NEXT_PUBLIC_BACKEND_URL ??
+  "http://localhost:3501";
+
+export async function GET(
+  _req: Request,
+  { params }: { params: Promise<{ id: string }> },
+) {
+  const { id } = await params;
+  try {
+    const res = await fetch(`${BACKEND_URL}/share/${id}`, {
+      signal: AbortSignal.timeout(5000),
+    });
+    if (res.ok) {
+      const data = await res.json();
+      return NextResponse.json(data, { headers: CORS });
+    }
+    if (res.status === 404) {
+      return NextResponse.json({ error: "Dataset not found" }, { status: 404, headers: CORS });
+    }
+    return NextResponse.json({ error: "Upstream service error" }, { status: 502, headers: CORS });
+  } catch {
+    return NextResponse.json({ error: "Upstream service error" }, { status: 502, headers: CORS });
+  }
+}
+
+export async function OPTIONS() {
+  return new Response(null, {
+    status: 204,
+    headers: {
+      "Access-Control-Allow-Origin": "*",
+      "Access-Control-Allow-Methods": "GET, OPTIONS",
+      "Access-Control-Allow-Headers": "Content-Type",
+    },
+  });
+}