From bedc9d015982cd685ba43e95c073941301a9ed11 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Tue, 2 Jun 2026 14:40:05 -0700 Subject: [PATCH 01/21] Subscribe to theme changes --- frontend/components/ThemeToggle.tsx | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/components/ThemeToggle.tsx b/frontend/components/ThemeToggle.tsx index 346a509..c185991 100644 --- a/frontend/components/ThemeToggle.tsx +++ b/frontend/components/ThemeToggle.tsx @@ -32,11 +32,19 @@ function applyTheme(theme: Theme): void { function subscribeToThemeChange(onThemeChange: () => void): () => void { if (typeof window === "undefined") return () => {}; - window.addEventListener("storage", onThemeChange); - window.addEventListener(THEME_CHANGED_EVENT, onThemeChange); + const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); + function syncTheme() { + applyTheme(readEffectiveTheme()); + onThemeChange(); + } + + window.addEventListener("storage", syncTheme); + window.addEventListener(THEME_CHANGED_EVENT, syncTheme); + mediaQuery.addEventListener("change", syncTheme); return () => { - window.removeEventListener("storage", onThemeChange); - window.removeEventListener(THEME_CHANGED_EVENT, onThemeChange); + window.removeEventListener("storage", syncTheme); + window.removeEventListener(THEME_CHANGED_EVENT, syncTheme); + mediaQuery.removeEventListener("change", syncTheme); }; } From 764a1fa273322a45ea41cbd66b3aaed51b39e840 Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Tue, 2 Jun 2026 17:18:56 -0700 Subject: [PATCH 02/21] OR Base URL --- .env.example | 3 +++ backend/src/config/models.ts | 26 +++++++++++++----------- backend/src/mastra/agents/investigate.ts | 1 + backend/src/mastra/agents/populate.ts | 1 + backend/src/mastra/agents/refresh.ts | 1 + backend/src/mastra/workflows/populate.ts | 1 + backend/src/pipeline/schema-inference.ts | 5 ++++- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index f522353..4daed18 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,9 @@ TINYFISH_API_KEY= # Generate at https://openrouter.ai/settings/keys OPENROUTER_API_KEY=sk-or-... +# Optional — route OpenRouter-compatible calls through another base URL. +OPENROUTER_BASE_URL= + # OpenRouter model slugs for each AI task. # Defaults (used when no user preference is saved): # SCHEMA_INFERENCE_MODEL: anthropic/claude-sonnet-4.6 (powerful for schema inference) diff --git a/backend/src/config/models.ts b/backend/src/config/models.ts index f63b419..746d7de 100644 --- a/backend/src/config/models.ts +++ b/backend/src/config/models.ts @@ -135,15 +135,17 @@ export async function fetchModelsFromOpenRouter(): Promise { throw new Error("OPENROUTER_API_KEY is not set"); } + const baseUrl = (process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1").replace(/\/+$/, ""); + const url = new URL(`${baseUrl}/models`); + url.searchParams.set("output_modalities", "text"); + url.searchParams.set("supported_parameters", "tools"); + // Only text-based models that support tools - const response = await fetch( - "https://openrouter.ai/api/v1/models?output_modalities=text&supported_parameters=tools", - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ); + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); if (!response.ok) { throw new Error(`OpenRouter API failed: ${response.status} ${response.statusText}`); @@ -152,8 +154,8 @@ export async function fetchModelsFromOpenRouter(): Promise { const json = (await response.json()) as { data: Array<{ id: string; - name: string; - context_length: number; + name?: string; + context_length?: number; pricing?: { completion?: string; prompt?: string }; }>; }; @@ -163,7 +165,7 @@ export async function fetchModelsFromOpenRouter(): Promise { const models = json.data .filter((m) => !EXCLUDED_MODEL_SLUGS.includes(m.id)) .map((model) => ({ - modelName: model.name, + modelName: model.name ?? model.id, canonicalSlug: model.id, contextLength: model.context_length ?? 0, promptCost: parseFloat(model.pricing?.prompt ?? "0") * 1_000_000, @@ -171,4 +173,4 @@ export async function fetchModelsFromOpenRouter(): Promise { })); return models; -} \ No newline at end of file +} diff --git a/backend/src/mastra/agents/investigate.ts b/backend/src/mastra/agents/investigate.ts index 4cdc32e..a747bde 100644 --- a/backend/src/mastra/agents/investigate.ts +++ b/backend/src/mastra/agents/investigate.ts @@ -7,6 +7,7 @@ import type { PopulateColumn } from "../../pipeline/populate.js"; const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY!, + baseURL: process.env.OPENROUTER_BASE_URL, }); function buildInvestigateInstructions(columns: PopulateColumn[]): string { diff --git a/backend/src/mastra/agents/populate.ts b/backend/src/mastra/agents/populate.ts index 85edf53..3942fce 100644 --- a/backend/src/mastra/agents/populate.ts +++ b/backend/src/mastra/agents/populate.ts @@ -8,6 +8,7 @@ import type { RunMetrics } from "../run-metrics.js"; const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY!, + baseURL: process.env.OPENROUTER_BASE_URL, }); const INSTRUCTIONS = `You are an expert dataset builder. You conduct research using your web tools. diff --git a/backend/src/mastra/agents/refresh.ts b/backend/src/mastra/agents/refresh.ts index 2215686..0ee31bd 100644 --- a/backend/src/mastra/agents/refresh.ts +++ b/backend/src/mastra/agents/refresh.ts @@ -7,6 +7,7 @@ import type { PopulateColumn } from "../../pipeline/populate.js"; const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY!, + baseURL: process.env.OPENROUTER_BASE_URL, }); function buildRefreshInstructions(columns: PopulateColumn[]): string { diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index ea06e0e..b477dc6 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -109,6 +109,7 @@ Respond with EXACTLY one word: scraper or search`; try { const openrouter = createOpenRouter({ apiKey: process.env.OPENROUTER_API_KEY!, + baseURL: process.env.OPENROUTER_BASE_URL, }); const modelSlug = inputData.authContext?.modelConfig?.schemaInference ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 1f1ea2a..14ba1e9 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -31,7 +31,10 @@ function getModel(modelSlug?: string) { if (!apiKey) { throw new Error("Missing required environment variable: OPENROUTER_API_KEY"); } - const openrouter = createOpenRouter({ apiKey }); + const openrouter = createOpenRouter({ + apiKey, + baseURL: process.env.OPENROUTER_BASE_URL, + }); const resolvedSlug = modelSlug ?? DEFAULT_MODEL_IDS.SCHEMA_INFERENCE; return openrouter(resolvedSlug); } From 228550c492d7b4279ac6a5a47d0a5e855572a93a Mon Sep 17 00:00:00 2001 From: Adam Xu Date: Wed, 3 Jun 2026 12:16:00 -0700 Subject: [PATCH 03/21] add local mode --- .env.example | 53 --- AGENTS.md | 1 - backend/src/clerk-auth.ts | 9 +- backend/src/config/models.ts | 6 +- backend/src/env.ts | 3 + backend/src/index.ts | 107 +++++- backend/src/local-credentials.ts | 244 ++++++++++++ backend/src/mastra/agents/investigate.ts | 10 +- backend/src/mastra/agents/populate.ts | 11 +- backend/src/mastra/agents/refresh.ts | 13 +- backend/src/mastra/tools/investigate-tool.ts | 2 + backend/src/mastra/tools/web-tools.ts | 10 +- backend/src/mastra/workflows/populate.ts | 5 +- backend/src/mastra/workflows/update.ts | 9 +- backend/src/pipeline/schema-inference.ts | 10 +- docker-compose.dev.yml | 8 +- frontend/app/convex-provider.tsx | 10 +- frontend/app/dashboard/page.tsx | 120 +++--- frontend/app/dashboard/settings/layout.tsx | 102 ++--- .../app/dashboard/settings/models/page.tsx | 45 ++- frontend/app/dataset/[id]/page.tsx | 12 +- frontend/app/dataset/new/page.tsx | 8 +- frontend/app/layout.tsx | 14 +- frontend/app/local-setup-gate.tsx | 63 ++++ frontend/app/page.tsx | 20 +- .../app/setup/openrouter/callback/page.tsx | 64 ++++ frontend/app/setup/page.tsx | 344 +++++++++++++++++ frontend/app/sign-in/[[...sign-in]]/page.tsx | 6 + frontend/app/sign-up/[[...sign-up]]/page.tsx | 6 + frontend/components/LocalUtilityMenu.tsx | 84 +++++ frontend/components/QuotaBadge.tsx | 10 +- .../settings/LocalCredentialsPanel.tsx | 357 ++++++++++++++++++ .../table/use-row-change-detection.ts | 35 +- frontend/convex/auth.config.ts | 21 +- frontend/convex/lib/authz.ts | 18 +- frontend/convex/lib/quota.ts | 28 +- frontend/convex/localCredentials.ts | 54 +++ frontend/convex/schema.ts | 8 + frontend/lib/analytics-provider.tsx | 4 +- frontend/lib/app-auth.tsx | 69 ++++ frontend/lib/app-mode.ts | 2 + frontend/lib/backend.ts | 83 ++++ frontend/lib/openrouter-oauth.ts | 55 +++ frontend/next.config.ts | 6 +- frontend/proxy.ts | 6 +- makefiles/Makefile | 31 +- 46 files changed, 1907 insertions(+), 279 deletions(-) delete mode 100644 .env.example create mode 100644 backend/src/local-credentials.ts create mode 100644 frontend/app/local-setup-gate.tsx create mode 100644 frontend/app/setup/openrouter/callback/page.tsx create mode 100644 frontend/app/setup/page.tsx create mode 100644 frontend/components/LocalUtilityMenu.tsx create mode 100644 frontend/components/settings/LocalCredentialsPanel.tsx create mode 100644 frontend/convex/localCredentials.ts create mode 100644 frontend/lib/app-auth.tsx create mode 100644 frontend/lib/app-mode.ts create mode 100644 frontend/lib/openrouter-oauth.ts diff --git a/.env.example b/.env.example deleted file mode 100644 index 4daed18..0000000 --- a/.env.example +++ /dev/null @@ -1,53 +0,0 @@ -# This is the only local env file BigSet expects. -# Copy this file to .env and fill in your values. - -# TinyFish (required) — web search + dataset population. -# Generate at https://agent.tinyfish.ai/api-keys?utm_source=github&utm_medium=organic&utm_campaign=bigset-developer-2026q2 -TINYFISH_API_KEY= - -# OpenRouter (required) — schema inference + AI agents. -# Generate at https://openrouter.ai/settings/keys -OPENROUTER_API_KEY=sk-or-... - -# Optional — route OpenRouter-compatible calls through another base URL. -OPENROUTER_BASE_URL= - -# OpenRouter model slugs for each AI task. -# Defaults (used when no user preference is saved): -# SCHEMA_INFERENCE_MODEL: anthropic/claude-sonnet-4.6 (powerful for schema inference) -# POPULATE_ORCHESTRATOR_MODEL: qwen/qwen3.7-max (cost-effective orchestrator) -# INVESTIGATE_SUBAGENT_MODEL: qwen/qwen3.7-max (cost-effective subagent) -# Find model IDs at https://openrouter.ai/models — any OpenRouter model slug is valid. -SCHEMA_INFERENCE_MODEL=anthropic/claude-sonnet-4.6 -POPULATE_ORCHESTRATOR_MODEL=qwen/qwen3.7-max -INVESTIGATE_SUBAGENT_MODEL=qwen/qwen3.7-max - -# Clerk (required) — user authentication. -# Create a free app at https://dashboard.clerk.com -# Enable the Clerk JWT Templates -> Convex template, then set your issuer URL. -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_... -CLERK_SECRET_KEY=sk_test_... -CLERK_JWT_ISSUER_DOMAIN=https://your-app.clerk.accounts.dev - -# Auto-generated by `make dev` on first run. Do not fill in manually. -CONVEX_SELF_HOSTED_ADMIN_KEY= - -# Local service URLs -CLIENT_ORIGIN=http://localhost:3500 -CONVEX_URL=http://localhost:3210 -NEXT_PUBLIC_CONVEX_URL=http://127.0.0.1:3210 -CONVEX_SELF_HOSTED_URL=http://127.0.0.1:3210 -NEXT_PUBLIC_BACKEND_URL=http://localhost:3501 -PORT=3501 - -# Optional — the following keys are not required to run BigSet. - -# Resend (optional — transactional emails when a populate workflow finishes). -# Unset → email module logs and no-ops. Generate at https://resend.com/api-keys -RESEND_API_KEY= -EMAIL_FROM="BigSet " - -# PostHog (optional — leave blank to disable analytics entirely in local dev). -# Get from https://us.posthog.com/project/settings/general. -NEXT_PUBLIC_POSTHOG_KEY= -NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com diff --git a/AGENTS.md b/AGENTS.md index fae66c0..d821a46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,6 @@ ## What not to do -- Do not add Clerk, Auth0, or any third-party auth service. We use Better Auth (self-hosted). - Do not add API routes to the frontend. All API logic belongs in the backend. - Do not hardcode ports. Read from env vars (`PORT`, `CLIENT_ORIGIN`, `BETTER_AUTH_URL`). - Do not commit `.env` files or secrets. diff --git a/backend/src/clerk-auth.ts b/backend/src/clerk-auth.ts index 0a046ae..de28dca 100644 --- a/backend/src/clerk-auth.ts +++ b/backend/src/clerk-auth.ts @@ -8,6 +8,7 @@ import fp from "fastify-plugin"; import { createClerkClient, type ClerkClient } from "@clerk/backend"; import { env } from "./env.js"; +import { LOCAL_USER_ID } from "./local-credentials.js"; /** * Clerk JWT verification for the Fastify backend. @@ -41,7 +42,7 @@ declare module "fastify" { } const clerkPlugin: FastifyPluginAsync = async (fastify: FastifyInstance) => { - if (!env.CLERK_SECRET_KEY) { + if (env.IS_PROD && !env.CLERK_SECRET_KEY) { fastify.log.warn( "CLERK_SECRET_KEY not set — protected routes will reject all requests. " + "Set it before adding routes that require auth.", @@ -66,6 +67,7 @@ export async function getUserEmail( clerk: ClerkClient, userId: string, ): Promise { + if (env.IS_LOCAL_MODE) return null; try { const user = await clerk.users.getUser(userId); return user.primaryEmailAddress?.emailAddress ?? null; @@ -87,6 +89,11 @@ export async function requireAuth( req: FastifyRequest, reply: FastifyReply, ): Promise { + if (env.IS_LOCAL_MODE) { + req.auth = { userId: LOCAL_USER_ID }; + return; + } + if (!env.CLERK_SECRET_KEY) { req.log.error("CLERK_SECRET_KEY is not set; cannot verify request"); await reply.code(500).send({ error: "Auth not configured" }); diff --git a/backend/src/config/models.ts b/backend/src/config/models.ts index 746d7de..1d28cbf 100644 --- a/backend/src/config/models.ts +++ b/backend/src/config/models.ts @@ -6,6 +6,7 @@ import { api, internal, convex } from "../convex.js"; import { env } from "../env.js"; +import { requireOpenRouterApiKey } from "../local-credentials.js"; export interface OpenRouterModel { modelName: string; @@ -130,10 +131,7 @@ export async function getModelConfig( * for Convex storage. */ export async function fetchModelsFromOpenRouter(): Promise { - const apiKey = env.OPENROUTER_API_KEY; - if (!apiKey) { - throw new Error("OPENROUTER_API_KEY is not set"); - } + const apiKey = await requireOpenRouterApiKey(); const baseUrl = (process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1").replace(/\/+$/, ""); const url = new URL(`${baseUrl}/models`); diff --git a/backend/src/env.ts b/backend/src/env.ts index f1fbf17..798b03c 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -19,6 +19,9 @@ function numberFromEnv(name: string, fallback: number): number { } export const env = { + PROD: process.env.PROD, + IS_PROD: process.env.PROD === "1", + IS_LOCAL_MODE: process.env.PROD !== "1", CLIENT_ORIGIN: process.env.CLIENT_ORIGIN || "http://localhost:3500", CONVEX_URL: required("CONVEX_URL"), PORT: numberFromEnv("PORT", 3501), diff --git a/backend/src/index.ts b/backend/src/index.ts index 61c7314..442f5d3 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,4 +1,4 @@ -import Fastify, { type FastifyBaseLogger } from "fastify"; +import Fastify, { type FastifyBaseLogger, type FastifyReply } from "fastify"; import fastifyCors from "@fastify/cors"; import type { ClerkClient } from "@clerk/backend"; @@ -14,6 +14,14 @@ import { datasetReadyTemplate } from "./email/templates/dataset-ready.js"; import { capture, shutdown as shutdownAnalytics } from "./analytics/posthog.js"; import { EVENTS } from "./analytics/events.js"; import { registerDataset, deregisterDataset, abortDataset } from "./abort-registry.js"; +import { + exchangeOpenRouterOAuthCode, + getLocalSetupStatus, + requireLocalSetupComplete, + saveLocalCredential, + verifyOpenRouterApiKey, + verifyTinyFishApiKey, +} from "./local-credentials.js"; /** Domain part of an email, for analytics (we never log full addresses). */ function emailDomain(email: string): string { @@ -84,6 +92,8 @@ async function sendDatasetReadyNotification({ rowCount: number; workflowType?: "populate" | "update"; }): Promise { + if (env.IS_LOCAL_MODE) return; + const baseProps = { datasetId, datasetName, @@ -142,6 +152,18 @@ async function sendDatasetReadyNotification({ } } +async function ensureLocalSetupReady(reply: FastifyReply): Promise { + try { + await requireLocalSetupComplete(); + return true; + } catch { + await reply.code(428).send({ + error: "Local setup is incomplete. Connect TinyFish and OpenRouter first.", + }); + return false; + } +} + /** * Shared stop-success path: set the dataset live, send the ready email. * @@ -523,6 +545,11 @@ function startLocalRefreshScheduler( ticking = true; try { + if (env.IS_LOCAL_MODE) { + const setup = await getLocalSetupStatus(); + if (!setup.complete) return; + } + const now = Date.now(); const dueDatasets = await convex.query( internal.datasets.listDueForRefreshInternal, @@ -629,6 +656,81 @@ fastify.addHook("onClose", async () => { fastify.get("/health", async () => ({ status: "ok" })); +fastify.get("/local-setup/status", async (_req, reply) => { + if (!env.IS_LOCAL_MODE) { + return reply.code(404).send({ error: "Not found" }); + } + return await getLocalSetupStatus(); +}); + +fastify.post("/local-setup/tinyfish", async (req, reply) => { + if (!env.IS_LOCAL_MODE) { + return reply.code(404).send({ error: "Not found" }); + } + + const body = req.body as { apiKey?: string }; + const apiKey = body?.apiKey?.trim(); + if (!apiKey) { + return reply.code(400).send({ error: "TinyFish API key is required" }); + } + + try { + await verifyTinyFishApiKey(apiKey); + await saveLocalCredential("tinyfish", apiKey, "api_key"); + return await getLocalSetupStatus(); + } catch (err) { + const message = err instanceof Error ? err.message : "TinyFish verification failed"; + req.log.warn({ err }, "TinyFish local setup verification failed"); + return reply.code(400).send({ error: message }); + } +}); + +fastify.post("/local-setup/openrouter-key", async (req, reply) => { + if (!env.IS_LOCAL_MODE) { + return reply.code(404).send({ error: "Not found" }); + } + + const body = req.body as { apiKey?: string }; + const apiKey = body?.apiKey?.trim(); + if (!apiKey) { + return reply.code(400).send({ error: "OpenRouter API key is required" }); + } + + try { + await verifyOpenRouterApiKey(apiKey); + await saveLocalCredential("openrouter", apiKey, "api_key"); + return await getLocalSetupStatus(); + } catch (err) { + const message = err instanceof Error ? err.message : "OpenRouter verification failed"; + req.log.warn({ err }, "OpenRouter local setup verification failed"); + return reply.code(400).send({ error: message }); + } +}); + +fastify.post("/local-setup/openrouter-oauth", async (req, reply) => { + if (!env.IS_LOCAL_MODE) { + return reply.code(404).send({ error: "Not found" }); + } + + const body = req.body as { code?: string; codeVerifier?: string }; + const code = body?.code?.trim(); + const codeVerifier = body?.codeVerifier?.trim(); + if (!code || !codeVerifier) { + return reply.code(400).send({ error: "OpenRouter OAuth code is required" }); + } + + try { + const apiKey = await exchangeOpenRouterOAuthCode({ code, codeVerifier }); + await verifyOpenRouterApiKey(apiKey); + await saveLocalCredential("openrouter", apiKey, "oauth"); + return await getLocalSetupStatus(); + } catch (err) { + const message = err instanceof Error ? err.message : "OpenRouter OAuth failed"; + req.log.warn({ err }, "OpenRouter OAuth setup failed"); + return reply.code(400).send({ error: message }); + } +}); + fastify.post("/openrouter/refresh", { preHandler: requireAuth }, async (req, reply) => { const { fetchModelsFromOpenRouter, upsertModelBatch } = await import("./config/models.js"); @@ -714,6 +816,7 @@ await fastify.register(async (instance) => { if (!body?.prompt || typeof body.prompt !== "string" || !body.prompt.trim()) { return reply.code(400).send({ error: "prompt is required" }); } + if (!(await ensureLocalSetupReady(reply))) return; try { const auth = req.auth; @@ -743,6 +846,7 @@ await fastify.register(async (instance) => { details: parsed.error.flatten().fieldErrors, }); } + if (!(await ensureLocalSetupReady(reply))) return; try { const auth = req.auth; @@ -815,6 +919,7 @@ await fastify.register(async (instance) => { details: parsed.error.flatten().fieldErrors, }); } + if (!(await ensureLocalSetupReady(reply))) return; try { const auth = req.auth; diff --git a/backend/src/local-credentials.ts b/backend/src/local-credentials.ts new file mode 100644 index 0000000..eda6266 --- /dev/null +++ b/backend/src/local-credentials.ts @@ -0,0 +1,244 @@ +import { convex, internal } from "./convex.js"; +import { env } from "./env.js"; + +export const LOCAL_USER_ID = "local_user_default"; + +export type LocalCredentialService = "tinyfish" | "openrouter"; +export type ConnectionMethod = "api_key" | "oauth"; + +export interface ServiceSetupStatus { + configured: boolean; + source: "local" | "env" | null; + connectionMethod: ConnectionMethod | null; + verifiedAt: number | null; +} + +export interface LocalSetupStatus { + mode: "local" | "production"; + required: boolean; + complete: boolean; + services: Record; +} + +function isPlaceholder(value: string, service: LocalCredentialService): boolean { + if (!value.trim()) return true; + if (value.includes("...")) return true; + if (service === "openrouter" && value === "sk-or-...") return true; + return false; +} + +function envCredential(service: LocalCredentialService): string | undefined { + const value = + service === "tinyfish" ? process.env.TINYFISH_API_KEY : env.OPENROUTER_API_KEY; + if (!value || isPlaceholder(value, service)) return undefined; + return value; +} + +async function localCredential(service: LocalCredentialService): Promise<{ + apiKey: string; + connectionMethod: ConnectionMethod; + verifiedAt: number; +} | null> { + if (!env.IS_LOCAL_MODE) return null; + const row = await convex.query(internal.localCredentials.getInternal, { + service, + }); + if (!row?.apiKey) return null; + return { + apiKey: row.apiKey, + connectionMethod: row.connectionMethod, + verifiedAt: row.verifiedAt, + }; +} + +export async function resolveCredential( + service: LocalCredentialService, +): Promise<{ apiKey: string; source: "local" | "env" } | null> { + const local = await localCredential(service); + if (local) return { apiKey: local.apiKey, source: "local" }; + + const fromEnv = envCredential(service); + if (fromEnv) return { apiKey: fromEnv, source: "env" }; + + return null; +} + +export async function getOpenRouterApiKey(): Promise { + return (await resolveCredential("openrouter"))?.apiKey; +} + +export async function requireOpenRouterApiKey(): Promise { + const apiKey = await getOpenRouterApiKey(); + if (!apiKey) { + throw new Error("OpenRouter is not configured. Complete local setup first."); + } + return apiKey; +} + +export async function getTinyFishApiKey(): Promise { + return (await resolveCredential("tinyfish"))?.apiKey; +} + +export function tinyFishHeaders(apiKey: string): Record { + return { + "X-API-Key": apiKey, + "X-TF-ORIGIN": "BigSet", + "X-TF-Request-Origin": "BigSet", + }; +} + +export async function requireLocalSetupComplete(): Promise { + if (!env.IS_LOCAL_MODE) return; + const status = await getLocalSetupStatus(); + if (!status.complete) { + throw new Error("Local setup is incomplete."); + } +} + +export async function getLocalSetupStatus(): Promise { + if (!env.IS_LOCAL_MODE) { + const tinyfish = envCredential("tinyfish"); + const openrouter = envCredential("openrouter"); + return { + mode: "production", + required: false, + complete: true, + services: { + tinyfish: { + configured: !!tinyfish, + source: tinyfish ? "env" : null, + connectionMethod: tinyfish ? "api_key" : null, + verifiedAt: null, + }, + openrouter: { + configured: !!openrouter, + source: openrouter ? "env" : null, + connectionMethod: openrouter ? "api_key" : null, + verifiedAt: null, + }, + }, + }; + } + + const tinyfishLocal = await localCredential("tinyfish"); + const openrouterLocal = await localCredential("openrouter"); + const tinyfishEnv = envCredential("tinyfish"); + const openrouterEnv = envCredential("openrouter"); + + const tinyfish: ServiceSetupStatus = tinyfishLocal + ? { + configured: true, + source: "local", + connectionMethod: tinyfishLocal.connectionMethod, + verifiedAt: tinyfishLocal.verifiedAt, + } + : { + configured: !!tinyfishEnv, + source: tinyfishEnv ? "env" : null, + connectionMethod: tinyfishEnv ? "api_key" : null, + verifiedAt: null, + }; + + const openrouter: ServiceSetupStatus = openrouterLocal + ? { + configured: true, + source: "local", + connectionMethod: openrouterLocal.connectionMethod, + verifiedAt: openrouterLocal.verifiedAt, + } + : { + configured: !!openrouterEnv, + source: openrouterEnv ? "env" : null, + connectionMethod: openrouterEnv ? "api_key" : null, + verifiedAt: null, + }; + + return { + mode: "local", + required: true, + complete: tinyfish.configured && openrouter.configured, + services: { tinyfish, openrouter }, + }; +} + +export async function saveLocalCredential( + service: LocalCredentialService, + apiKey: string, + connectionMethod: ConnectionMethod, +): Promise { + if (!env.IS_LOCAL_MODE) { + throw new Error("Local credential storage is disabled when PROD=1."); + } + await convex.mutation(internal.localCredentials.upsertInternal, { + service, + apiKey, + connectionMethod, + verifiedAt: Date.now(), + }); +} + +export async function verifyTinyFishApiKey(apiKey: string): Promise { + const url = new URL("https://api.search.tinyfish.ai"); + url.searchParams.set("query", "BigSet"); + + const response = await fetch(url, { + headers: tinyFishHeaders(apiKey), + }); + + if (!response.ok) { + if (response.status === 401) { + throw new Error("TinyFish rejected that API key."); + } + throw new Error( + `TinyFish verification failed with HTTP ${response.status}.`, + ); + } +} + +export async function verifyOpenRouterApiKey(apiKey: string): Promise { + const baseUrl = ( + process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1" + ).replace(/\/+$/, ""); + const response = await fetch(`${baseUrl}/models`, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new Error("OpenRouter rejected that API key."); + } + throw new Error( + `OpenRouter verification failed with HTTP ${response.status}.`, + ); + } +} + +export async function exchangeOpenRouterOAuthCode({ + code, + codeVerifier, +}: { + code: string; + codeVerifier: string; +}): Promise { + const response = await fetch("https://openrouter.ai/api/v1/auth/keys", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + code, + code_verifier: codeVerifier, + code_challenge_method: "S256", + }), + }); + + if (!response.ok) { + throw new Error( + `OpenRouter OAuth exchange failed with HTTP ${response.status}.`, + ); + } + + const body = (await response.json()) as { key?: string }; + if (!body.key) { + throw new Error("OpenRouter OAuth exchange did not return an API key."); + } + return body.key; +} diff --git a/backend/src/mastra/agents/investigate.ts b/backend/src/mastra/agents/investigate.ts index a747bde..63a7b8c 100644 --- a/backend/src/mastra/agents/investigate.ts +++ b/backend/src/mastra/agents/investigate.ts @@ -5,11 +5,6 @@ import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; -const openrouter = createOpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY!, - baseURL: process.env.OPENROUTER_BASE_URL, -}); - function buildInvestigateInstructions(columns: PopulateColumn[]): string { const columnNames = columns.map((c) => c.name); const columnsDesc = columns @@ -60,8 +55,13 @@ export function buildInvestigateAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], + openRouterApiKey: string, ): Agent { const modelSlug = authContext.modelConfig!.investigateSubagent; + const openrouter = createOpenRouter({ + apiKey: openRouterApiKey, + baseURL: process.env.OPENROUTER_BASE_URL, + }); const { insert_row } = buildPopulateTools( authorizedDatasetId, diff --git a/backend/src/mastra/agents/populate.ts b/backend/src/mastra/agents/populate.ts index 3942fce..9fb8f41 100644 --- a/backend/src/mastra/agents/populate.ts +++ b/backend/src/mastra/agents/populate.ts @@ -6,11 +6,6 @@ import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; import type { RunMetrics } from "../run-metrics.js"; -const openrouter = createOpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY!, - baseURL: process.env.OPENROUTER_BASE_URL, -}); - const INSTRUCTIONS = `You are an expert dataset builder. You conduct research using your web tools. You do broad research to see which rows to add, and then you spin up sub-agents that can do the deep research and fill in each row for you. Your job is to make sure you dispatch and manage your army of sub agents to build up a dataset with 100 rows in it. Stop as soon as the dataset reaches 100 rows. @@ -43,9 +38,14 @@ export function buildPopulateAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], + openRouterApiKey: string, metrics?: RunMetrics, ): Agent { const modelSlug = authContext.modelConfig!.populateOrchestrator; + const openrouter = createOpenRouter({ + apiKey: openRouterApiKey, + baseURL: process.env.OPENROUTER_BASE_URL, + }); return new Agent({ id: "populate-agent", @@ -59,6 +59,7 @@ export function buildPopulateAgent( authorizedDatasetId, authContext, columns, + openRouterApiKey, metrics, ), }, diff --git a/backend/src/mastra/agents/refresh.ts b/backend/src/mastra/agents/refresh.ts index 0ee31bd..144065a 100644 --- a/backend/src/mastra/agents/refresh.ts +++ b/backend/src/mastra/agents/refresh.ts @@ -5,11 +5,6 @@ import { searchWebTool, fetchPageTool } from "../tools/web-tools.js"; import type { AuthContext } from "../workflows/populate.js"; import type { PopulateColumn } from "../../pipeline/populate.js"; -const openrouter = createOpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY!, - baseURL: process.env.OPENROUTER_BASE_URL, -}); - function buildRefreshInstructions(columns: PopulateColumn[]): string { const columnNames = columns.map((c) => c.name); const columnsDesc = columns @@ -56,7 +51,13 @@ export function buildRefreshAgent( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], + openRouterApiKey: string, ): Agent { + const modelSlug = authContext.modelConfig!.investigateSubagent; + const openrouter = createOpenRouter({ + apiKey: openRouterApiKey, + baseURL: process.env.OPENROUTER_BASE_URL, + }); const { update_row } = buildPopulateTools( authorizedDatasetId, authContext, @@ -65,7 +66,7 @@ export function buildRefreshAgent( id: "refresh-agent", name: "Dataset Refresh Agent", instructions: buildRefreshInstructions(columns), - model: openrouter("qwen/qwen3.7-max"), + model: openrouter(modelSlug), tools: { update_row, search_web: searchWebTool, diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index 534ed7f..c66b048 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -77,6 +77,7 @@ export function buildSubagentTool( authorizedDatasetId: string, authContext: AuthContext, columns: PopulateColumn[], + openRouterApiKey: string, metrics?: RunMetrics, ) { return createTool({ @@ -108,6 +109,7 @@ export function buildSubagentTool( authorizedDatasetId, authContext, columns, + openRouterApiKey, ); const pkBlock = Object.entries(primary_keys) diff --git a/backend/src/mastra/tools/web-tools.ts b/backend/src/mastra/tools/web-tools.ts index 961897e..97cac93 100644 --- a/backend/src/mastra/tools/web-tools.ts +++ b/backend/src/mastra/tools/web-tools.ts @@ -1,5 +1,6 @@ import { createTool } from "@mastra/core/tools"; import { z } from "zod"; +import { getTinyFishApiKey, tinyFishHeaders } from "../../local-credentials.js"; const FETCH_TIMEOUT_MS = 30_000; @@ -24,7 +25,7 @@ export const searchWebTool = createTool({ if (!query?.trim()) return { error: "query is required and cannot be empty." }; - const apiKey = process.env.TINYFISH_API_KEY; + const apiKey = await getTinyFishApiKey(); if (!apiKey) return { error: "TINYFISH_API_KEY is not configured. Web search is unavailable — use synthetic data instead." }; @@ -35,7 +36,7 @@ export const searchWebTool = createTool({ const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); try { const res = await fetch(url, { - headers: { "X-API-Key": apiKey, "X-TF-Request-Origin": "bigset" }, + headers: tinyFishHeaders(apiKey), signal: controller.signal, }); clearTimeout(timeout); @@ -90,7 +91,7 @@ export const fetchPageTool = createTool({ if (!targetUrl.startsWith("http://") && !targetUrl.startsWith("https://")) return { error: `Invalid URL "${targetUrl}". Must start with http:// or https://.` }; - const apiKey = process.env.TINYFISH_API_KEY; + const apiKey = await getTinyFishApiKey(); if (!apiKey) return { error: "TINYFISH_API_KEY is not configured. Page fetch is unavailable — use data from search snippets instead." }; @@ -103,8 +104,7 @@ export const fetchPageTool = createTool({ method: "POST", headers: { "Content-Type": "application/json", - "X-API-Key": apiKey, - "X-TF-Request-Origin": "bigset", + ...tinyFishHeaders(apiKey), }, body: JSON.stringify({ urls: [targetUrl], format: "markdown" }), signal: controller.signal, diff --git a/backend/src/mastra/workflows/populate.ts b/backend/src/mastra/workflows/populate.ts index ef7c58b..a606d8c 100644 --- a/backend/src/mastra/workflows/populate.ts +++ b/backend/src/mastra/workflows/populate.ts @@ -5,6 +5,7 @@ import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { datasetContextSchema, populateColumnSchema } from "../../pipeline/populate.js"; import { convex, internal } from "../../convex.js"; import { DEFAULT_MODEL_IDS } from "../../config/models.js"; +import { requireOpenRouterApiKey } from "../../local-credentials.js"; import { buildPopulateAgent } from "../agents/populate.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; @@ -108,8 +109,9 @@ Respond with EXACTLY one word: scraper or search`; let classification: "scraper" | "search" = "search"; try { + const apiKey = await requireOpenRouterApiKey(); const openrouter = createOpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY!, + apiKey, baseURL: process.env.OPENROUTER_BASE_URL, }); const modelSlug = @@ -247,6 +249,7 @@ const agentStep = createStep({ inputData.authorizedDatasetId, inputData.authContext, inputData.columns, + await requireOpenRouterApiKey(), metrics, ); const abortSignal = getSignal(inputData.authorizedDatasetId); diff --git a/backend/src/mastra/workflows/update.ts b/backend/src/mastra/workflows/update.ts index 45e3421..28aadee 100644 --- a/backend/src/mastra/workflows/update.ts +++ b/backend/src/mastra/workflows/update.ts @@ -4,6 +4,7 @@ import { datasetContextSchema, populateColumnSchema } from "../../pipeline/popul import { convex, internal } from "../../convex.js"; import { buildRefreshAgent } from "../agents/refresh.js"; import { authContextSchema } from "./populate.js"; +import { requireOpenRouterApiKey } from "../../local-credentials.js"; import { RunMetrics } from "../run-metrics.js"; import { saveRunMetrics } from "../save-run-metrics.js"; import { getSignal } from "../../abort-registry.js"; @@ -99,12 +100,18 @@ const refreshRowsStep = createStep({ const metrics = new RunMetrics(); const startedAt = Date.now(); + const openRouterApiKey = await requireOpenRouterApiKey(); const pkColumns = columns.filter((c) => c.isPrimaryKey); async function processRow(row: z.infer) { try { - const agent = buildRefreshAgent(datasetId, authContext, columns); + const agent = buildRefreshAgent( + datasetId, + authContext, + columns, + openRouterApiKey, + ); const pkBlock = pkColumns.length > 0 diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 14ba1e9..467a393 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -2,6 +2,7 @@ import { generateText, Output, NoObjectGeneratedError } from "ai"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { DEFAULT_MODEL_IDS } from "../config/models.js"; +import { requireOpenRouterApiKey } from "../local-credentials.js"; import { datasetSchemaSchema, type DatasetSchema } from "./types.js"; const SYSTEM_PROMPT = `You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you produce a precise schema definition. @@ -26,11 +27,8 @@ Rules: - Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. - When a column is a scalar numeric rating (e.g. average score like 4.3/5 for restaurants, cafes, hotels, products, apps): name it generically (e.g. "rating" not "yelp_rating") and write a retrieval_hint explaining that review sites (Yelp, TripAdvisor, Google Maps) block direct page fetches, so the agent must extract ratings from **search result snippets**. The hint should say: "Search for \\" rating reviews\\" and include location terms only when location is part of the entity identity. Look for ratings in snippets from TripAdvisor (\\"rated X.X of 5\\"), Yelp search listings (\\"X.X (N reviews)\\"), or aggregator sites (Birdeye, joe.coffee, giftly, Uber Eats, menufyy). Do NOT try to fetch yelp.com or tripadvisor.com directly — they block automated access. Accept ratings from any reputable source." If including a rating column, also add a "rating_source" text column so the agent records where the rating came from. Do not rename review-count or review-text fields to "rating" — keep those as distinct columns (e.g. "review_count") when the user explicitly asks for them.`; -function getModel(modelSlug?: string) { - const apiKey = process.env.OPENROUTER_API_KEY; - if (!apiKey) { - throw new Error("Missing required environment variable: OPENROUTER_API_KEY"); - } +async function getModel(modelSlug?: string) { + const apiKey = await requireOpenRouterApiKey(); const openrouter = createOpenRouter({ apiKey, baseURL: process.env.OPENROUTER_BASE_URL, @@ -40,7 +38,7 @@ function getModel(modelSlug?: string) { } export async function inferSchema(prompt: string, modelSlug?: string): Promise { - const model = getModel(modelSlug); + const model = await getModel(modelSlug); try { return await callOnce(model, prompt); } catch (error) { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1a12432..f2b9298 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -31,6 +31,7 @@ services: CLIENT_ORIGIN: http://localhost:3500 CONVEX_URL: http://convex:3210 PORT: 3501 + PROD: ${PROD:-} CONVEX_SELF_HOSTED_ADMIN_KEY: ${CONVEX_SELF_HOSTED_ADMIN_KEY:-} CLERK_SECRET_KEY: ${CLERK_SECRET_KEY:-} CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-} @@ -65,6 +66,7 @@ services: environment: HOST: 0.0.0.0 PORT: 4111 + PROD: ${PROD:-} OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} CONVEX_URL: http://convex:3210 CONVEX_SELF_HOSTED_ADMIN_KEY: ${CONVEX_SELF_HOSTED_ADMIN_KEY:-} @@ -95,8 +97,10 @@ services: - ./scripts:/scripts:ro environment: NEXT_PUBLIC_CONVEX_URL: http://localhost:3210 - NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY} - CLERK_SECRET_KEY: ${CLERK_SECRET_KEY} + NEXT_PUBLIC_PROD: ${PROD:-} + PROD: ${PROD:-} + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-} + CLERK_SECRET_KEY: ${CLERK_SECRET_KEY:-} NEXT_PUBLIC_CLERK_SIGN_IN_URL: /sign-in NEXT_PUBLIC_CLERK_SIGN_UP_URL: /sign-up NEXT_PUBLIC_CLERK_SIGN_IN_FALLBACK_REDIRECT_URL: /dashboard diff --git a/frontend/app/convex-provider.tsx b/frontend/app/convex-provider.tsx index 7d876e0..be33d3e 100644 --- a/frontend/app/convex-provider.tsx +++ b/frontend/app/convex-provider.tsx @@ -1,15 +1,21 @@ "use client"; import { ConvexReactClient } from "convex/react"; +import { ConvexProvider } from "convex/react"; import { ConvexProviderWithClerk } from "convex/react-clerk"; -import { useAuth } from "@clerk/nextjs"; import { type ReactNode } from "react"; +import { useAuth } from "@clerk/nextjs"; +import { isLocalMode } from "@/lib/app-mode"; const convex = new ConvexReactClient( - process.env.NEXT_PUBLIC_CONVEX_URL as string + process.env.NEXT_PUBLIC_CONVEX_URL || "http://127.0.0.1:3210" ); export function ConvexClientProvider({ children }: { children: ReactNode }) { + if (isLocalMode) { + return {children}; + } + return ( {children} diff --git a/frontend/app/dashboard/page.tsx b/frontend/app/dashboard/page.tsx index 747e5c0..7c3ca80 100644 --- a/frontend/app/dashboard/page.tsx +++ b/frontend/app/dashboard/page.tsx @@ -3,8 +3,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import Link from "next/link"; import { useRouter } from "next/navigation"; -import { useQuery, useConvexAuth } from "convex/react"; -import { useUser, useClerk } from "@clerk/nextjs"; +import { useQuery } from "convex/react"; import { api } from "@/convex/_generated/api"; import { DatasetCard, @@ -12,29 +11,34 @@ import { } from "@/components/dataset/DatasetCard"; import { useTheme } from "@/components/ThemeToggle"; import { QuotaBadge } from "@/components/QuotaBadge"; +import { LocalUtilityMenu } from "@/components/LocalUtilityMenu"; import { EVENTS, track } from "@/lib/analytics"; import type { ProfileUser } from "@/lib/profile-user"; +import { useAppClerk, useAppConvexAuth, useAppUser } from "@/lib/app-auth"; +import { isLocalMode } from "@/lib/app-mode"; export default function DashboardPage() { - const { isAuthenticated, isLoading } = useConvexAuth(); - const { user } = useUser(); - const { signOut } = useClerk(); + const { isAuthenticated, isLoading } = useAppConvexAuth(); + const { user } = useAppUser(); + const { signOut } = useAppClerk(); const [search, setSearch] = useState(""); const mine = useQuery( api.datasets.listMine, isAuthenticated ? {} : "skip", ); - // Public datasets are open to anonymous users too, so no `skip` gate. - const curated = useQuery(api.datasets.listPublic, {}); + const showCurated = !isLocalMode; + const curated = useQuery( + api.datasets.listPublic, + showCurated ? {} : "skip", + ); - // Quota state drives the "+ New Dataset" button — disabled when the - // user is at their free-tier limit. `undefined` while loading. + // Quota limits are cloud-only. Local mode can create datasets without this gate. const usage = useQuery( api.quota.getMy, - isAuthenticated ? {} : "skip", + !isLocalMode && isAuthenticated ? {} : "skip", ); - const atLimit = usage !== undefined && usage.remaining === 0; + const atLimit = !isLocalMode && usage !== undefined && usage.remaining === 0; // Fire dashboard_viewed once per mount when both queries have resolved, // so we attach accurate counts. `dashboardFired` prevents the effect @@ -45,15 +49,15 @@ export default function DashboardPage() { !dashboardFired.current && isAuthenticated && mine !== undefined && - curated !== undefined + (!showCurated || curated !== undefined) ) { dashboardFired.current = true; track(EVENTS.DASHBOARD_VIEWED, { owned_count: mine.length, - curated_count: curated.length, + curated_count: showCurated ? (curated?.length ?? 0) : 0, }); } - }, [isAuthenticated, mine, curated]); + }, [isAuthenticated, mine, curated, showCurated]); const { filteredMine, filteredCurated } = useMemo(() => { const q = search.trim().toLowerCase(); @@ -85,9 +89,15 @@ export default function DashboardPage() { BigSet BigSet
- -
- signOut()} /> + {isLocalMode ? ( + + ) : ( + <> + +
+ signOut()} /> + + )}
@@ -187,19 +197,23 @@ export default function DashboardPage() { } /> -
+ {!isLocalMode && ( + <> +
-
+
+ + )}
); @@ -300,14 +314,16 @@ function ProfileMenu({ {open && ( )} diff --git a/frontend/app/dashboard/settings/layout.tsx b/frontend/app/dashboard/settings/layout.tsx index 5073466..b254f05 100644 --- a/frontend/app/dashboard/settings/layout.tsx +++ b/frontend/app/dashboard/settings/layout.tsx @@ -1,17 +1,19 @@ "use client"; import Link from "next/link"; -import { useUser, useClerk } from "@clerk/nextjs"; import { useTheme } from "@/components/ThemeToggle"; +import { LocalUtilityMenu } from "@/components/LocalUtilityMenu"; import { useEffect, useRef, useState } from "react"; +import { useAppClerk, useAppUser } from "@/lib/app-auth"; +import { isLocalMode } from "@/lib/app-mode"; export default function SettingsLayout({ children, }: { children: React.ReactNode; }) { - const { user } = useUser(); - const { signOut } = useClerk(); + const { user } = useAppUser(); + const { signOut } = useAppClerk(); const [profileOpen, setProfileOpen] = useState(false); const profileRef = useRef(null); const { theme, toggle: toggleTheme } = useTheme(); @@ -46,52 +48,56 @@ export default function SettingsLayout({ ← Back to Dashboard
-
- + {isLocalMode ? ( + + ) : ( +
+ - {profileOpen && ( -
-
-

{name}

- {email && ( -

- {email} -

- )} + {profileOpen && ( +
+
+

{name}

+ {email && ( +

+ {email} +

+ )} +
+
+ + +
-
- - -
-
- )} -
+ )} +
+ )}
@@ -100,4 +106,4 @@ export default function SettingsLayout({
); -} \ No newline at end of file +} diff --git a/frontend/app/dashboard/settings/models/page.tsx b/frontend/app/dashboard/settings/models/page.tsx index 5531f9e..6a3acb4 100644 --- a/frontend/app/dashboard/settings/models/page.tsx +++ b/frontend/app/dashboard/settings/models/page.tsx @@ -2,18 +2,19 @@ import { useState, useEffect } from "react"; import { useQuery } from "convex/react"; -import { useAuth } from "@clerk/nextjs"; import { api } from "@/convex/_generated/api"; import { getModelConfig, saveModelConfig, getOpenRouterModels, refreshOpenRouterModels, type EffectiveModelConfig, type OpenRouterModel } from "@/lib/backend"; import { SettingsPageLayout } from "@/components/settings/SettingsPageLayout"; import { SettingsHeader } from "@/components/settings/SettingsHeader"; import { SettingsTile } from "@/components/settings/SettingsTile"; +import { LocalCredentialsPanel } from "@/components/settings/LocalCredentialsPanel"; import { ModelSideSheet } from "@/components/settings/ModelSideSheet"; import { MODEL_ROLES, type ModelRole } from "@/components/settings/types"; import { SkeletonList } from "@/components/settings/Skeleton"; +import { useAppAuth } from "@/lib/app-auth"; export default function ModelSettingsPage() { - const { getToken } = useAuth(); + const { getToken } = useAppAuth(); const convexModels = useQuery(api.openRouterModels.list, {}); const [effectiveConfig, setEffectiveConfig] = useState(null); @@ -115,25 +116,29 @@ export default function ModelSettingsPage() { return ( - +
+ -
- {isLoading ? ( - - ) : ( - MODEL_ROLES.map((role) => ( - openSideSheet(role)} - /> - )) - )} + + +
+ {isLoading ? ( + + ) : ( + MODEL_ROLES.map((role) => ( + openSideSheet(role)} + /> + )) + )} +
{activeSheet && ( diff --git a/frontend/app/dataset/[id]/page.tsx b/frontend/app/dataset/[id]/page.tsx index d9ee1b6..28af2b0 100644 --- a/frontend/app/dataset/[id]/page.tsx +++ b/frontend/app/dataset/[id]/page.tsx @@ -3,8 +3,7 @@ import { useParams } from "next/navigation"; import Link from "next/link"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { useMutation, useQuery, useConvexAuth } from "convex/react"; -import { useAuth, useUser, useClerk } from "@clerk/nextjs"; +import { useMutation, useQuery } from "convex/react"; import { api } from "@/convex/_generated/api"; import type { Id } from "@/convex/_generated/dataModel"; import { DatasetTable } from "@/components/table"; @@ -22,13 +21,14 @@ import { type RefreshCadence, } from "@/lib/refresh-cadence"; import type { ProfileUser } from "@/lib/profile-user"; +import { useAppAuth, useAppClerk, useAppConvexAuth, useAppUser } from "@/lib/app-auth"; export default function DatasetPage() { const params = useParams(); - const { isLoading: authLoading } = useConvexAuth(); - const { userId, getToken } = useAuth(); - const { user } = useUser(); - const { signOut } = useClerk(); + const { isLoading: authLoading } = useAppConvexAuth(); + const { userId, getToken } = useAppAuth(); + const { user } = useAppUser(); + const { signOut } = useAppClerk(); const [exporting, setExporting] = useState<"csv" | "xlsx" | null>(null); const [populating, setPopulating] = useState(false); const [updating, setUpdating] = useState(false); diff --git a/frontend/app/dataset/new/page.tsx b/frontend/app/dataset/new/page.tsx index 285cbde..868f437 100644 --- a/frontend/app/dataset/new/page.tsx +++ b/frontend/app/dataset/new/page.tsx @@ -3,11 +3,11 @@ import { useEffect, useState, useRef } from "react"; import { useRouter } from "next/navigation"; import Link from "next/link"; -import { useAuth } from "@clerk/nextjs"; -import { useMutation, useConvexAuth } from "convex/react"; +import { useMutation } from "convex/react"; import { api } from "@/convex/_generated/api"; import { EVENTS, track } from "@/lib/analytics"; import { inferSchema, type InferredColumn } from "@/lib/backend"; +import { useAppAuth, useAppConvexAuth } from "@/lib/app-auth"; import { REFRESH_CADENCE_OPTIONS, type RefreshCadence, @@ -76,7 +76,7 @@ function TypeSelector({ value, onChange }: { value: ColumnType; onChange: (v: Co export default function NewDatasetPage() { const router = useRouter(); - const { isAuthenticated, isLoading } = useConvexAuth(); + const { isAuthenticated, isLoading } = useAppConvexAuth(); const [step, setStep] = useState("describe"); const [prompt, setPrompt] = useState(""); @@ -89,7 +89,7 @@ export default function NewDatasetPage() { "search_fetch" | "browser" | "hybrid" | null >(null); const [sourceHint, setSourceHint] = useState(""); - const { getToken } = useAuth(); + const { getToken } = useAppAuth(); const createDataset = useMutation(api.datasets.create); diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index c4955a3..6fb7515 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -1,8 +1,9 @@ import type { Metadata } from "next"; import { Geist, Geist_Mono } from "next/font/google"; -import { ClerkProvider } from "@clerk/nextjs"; import { ConvexClientProvider } from "./convex-provider"; +import { AppAuthProvider } from "@/lib/app-auth"; import { AnalyticsProvider } from "@/lib/analytics-provider"; +import { LocalSetupGate } from "./local-setup-gate"; import "./globals.css"; const geistSans = Geist({ @@ -46,14 +47,13 @@ export default function RootLayout({