From b5bdda9481da9519d13e719ea5c71740b141ceff Mon Sep 17 00:00:00 2001 From: Edward Tran Date: Mon, 25 May 2026 11:03:31 +0700 Subject: [PATCH 1/2] Unify local env loading --- .env.example | 20 +++++-- .gitignore | 3 +- CLAUDE.md | 8 +-- README.md | 27 ++++----- backend/.env.example | 31 ---------- backend/README.md | 14 +++-- backend/package.json | 6 +- backend/src/convex.ts | 6 +- backend/src/env.ts | 9 ++- backend/src/index.ts | 18 ++++-- backend/src/mastra/tools/investigate-tool.ts | 2 +- backend/src/pipeline/schema-inference.ts | 2 +- docker-compose.dev.yml | 6 ++ frontend/.env.example | 19 ------- frontend/.gitignore | 3 +- frontend/README.md | 10 ++-- frontend/components/ThemeToggle.tsx | 35 +++++++----- frontend/components/table/ColumnHeader.tsx | 7 +-- frontend/components/table/DatasetTable.tsx | 2 +- frontend/components/table/TableHeader.tsx | 8 +-- frontend/lib/analytics.ts | 9 ++- frontend/package.json | 8 +-- makefiles/Makefile | 59 ++++++++++++++++---- scripts/with-root-env.mjs | 47 ++++++++++++++++ 24 files changed, 220 insertions(+), 139 deletions(-) delete mode 100644 backend/.env.example delete mode 100644 frontend/.env.example create mode 100644 scripts/with-root-env.mjs diff --git a/.env.example b/.env.example index 042fae4..7edaa64 100644 --- a/.env.example +++ b/.env.example @@ -1,23 +1,33 @@ -# These are read by docker-compose.dev.yml. +# This is the only local env file BigSet expects. # Copy this file to .env and fill in your values. +# Local service URLs +CLIENT_ORIGIN=http://localhost:3500 +CONVEX_URL=http://localhost:3210 +NEXT_PUBLIC_CONVEX_URL=http://127.0.0.1:3210 +CONVEX_SELF_HOSTED_URL=http://127.0.0.1:3210 +NEXT_PUBLIC_BACKEND_URL=http://localhost:3501 +PORT=3501 + # Clerk — create a free app at https://dashboard.clerk.com +# Enable the Clerk JWT Templates -> Convex template, then set your issuer URL. NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_... CLERK_SECRET_KEY=sk_test_... +CLERK_JWT_ISSUER_DOMAIN=https://your-app.clerk.accounts.dev # OpenRouter — required by backend + Mastra for AI model calls. # Generate at https://openrouter.ai/settings/keys OPENROUTER_API_KEY=sk-or-... +# TinyFish — used by the backend's populate agent for web search and fetch. +# Generate at https://agent.tinyfish.ai/api-keys +TINYFISH_API_KEY= + # Generate once after the first `make dev` with: # docker compose exec convex ./generate_admin_key.sh # Used by the backend container to call internal Convex functions. CONVEX_SELF_HOSTED_ADMIN_KEY= -# TinyFish — used by the backend's populate agent for web search and fetch. -# Generate at https://agent.tinyfish.ai/api-keys -TINYFISH_API_KEY= - # Resend (optional — transactional emails when a populate workflow finishes). # Unset → email module logs and no-ops. Generate at https://resend.com/api-keys RESEND_API_KEY= diff --git a/.gitignore b/.gitignore index 7632c39..60e55f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store node_modules/ +.npm-cache/ .env .env.local Project_BigSet_brief.md @@ -26,4 +27,4 @@ temp/ *.tgz # Internal docs -BigSet Technical Specs & Goals.md \ No newline at end of file +BigSet Technical Specs & Goals.md diff --git a/CLAUDE.md b/CLAUDE.md index 4df3522..4cb032b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,14 +8,14 @@ Frontend on :3500, backend on :3501, Mastra Studio on :4111, Convex dashboard on 1. Create a free Clerk account at https://clerk.com and create an application. 2. In the Clerk dashboard, go to **JWT Templates** and enable the **Convex** template. -3. Copy `frontend/.env.example` to `frontend/.env.local` and fill in your Clerk keys: +3. Copy `.env.example` to `.env` and fill in your Clerk keys: - `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` — from Clerk API Keys - `CLERK_SECRET_KEY` — from Clerk API Keys - `CLERK_JWT_ISSUER_DOMAIN` — your Frontend API URL (e.g. `https://your-app.clerk.accounts.dev`) -4. Add an OpenRouter API key to the root `.env` file: `OPENROUTER_API_KEY=sk-or-...` (get one at https://openrouter.ai/settings/keys). Docker Compose reads the root `.env` and passes it to the backend and Mastra containers. +4. Add an OpenRouter API key to the root `.env` file: `OPENROUTER_API_KEY=sk-or-...` (get one at https://openrouter.ai/settings/keys). 4b. Add a TinyFish API key to the root `.env` file: `TINYFISH_API_KEY=...` (get one at https://agent.tinyfish.ai/api-keys). This enables the populate agent to search the web and fetch page content. 5. Run `make dev` — this starts all Docker services AND pushes Convex functions automatically. -6. Generate a Convex admin key (first run only): `docker compose exec convex ./generate_admin_key.sh` and add it as `CONVEX_SELF_HOSTED_ADMIN_KEY` in `frontend/.env.local`, then re-run `make dev`. +6. Generate a Convex admin key (first run only): `docker compose exec convex ./generate_admin_key.sh` and add it as `CONVEX_SELF_HOSTED_ADMIN_KEY` in root `.env`, then re-run `make dev`. ## Architecture @@ -35,7 +35,7 @@ Convex functions use `ctx.auth.getUserIdentity()` to get the authenticated user. ## Environment Variables -Docker Compose interpolates variables from the root `.env` file. Key variables: +Root `.env` is the only local env file. Docker Compose, package scripts, and Convex CLI helper targets all read it. Key variables: - `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY`, `CLERK_SECRET_KEY` — shared by frontend and backend - `OPENROUTER_API_KEY` — used by backend and Mastra for AI model calls - `CONVEX_SELF_HOSTED_ADMIN_KEY` — used by backend for system-level Convex writes diff --git a/README.md b/README.md index 7067053..679f5ce 100644 --- a/README.md +++ b/README.md @@ -44,16 +44,12 @@ cd bigset Create a Clerk application at [dashboard.clerk.com](https://dashboard.clerk.com), then go to **JWT Templates** and enable the **Convex** template. -### 2. Configure env files +### 2. Configure env ```bash -# Root .env — used by Docker for the frontend container cp .env.example .env -# Fill in NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY and CLERK_SECRET_KEY - -# Frontend .env.local — used by Next.js and Convex CLI -cp frontend/.env.example frontend/.env.local -# Fill in all three Clerk keys (publishable, secret, and JWT issuer domain) +# Fill in NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, CLERK_SECRET_KEY, +# CLERK_JWT_ISSUER_DOMAIN, OPENROUTER_API_KEY, and optional service keys. ``` > **Required for the create-dataset wizard:** set `OPENROUTER_API_KEY` (used by the schema-inference pipeline). Get one at [openrouter.ai](https://openrouter.ai). Without it the wizard's "Generate Schema" step will fail. @@ -66,7 +62,9 @@ cp frontend/.env.example frontend/.env.local make dev ``` -This starts all Docker services, waits for Convex to be healthy, and deploys Convex functions automatically. Once it's up: +This starts all Docker services, waits for Convex to be healthy, and deploys Convex functions automatically. +`make dev` checks that root `.env` contains real Clerk and OpenRouter values before it starts Docker. +Once it's up: - App: http://localhost:3500 - Convex dashboard: http://localhost:6791 @@ -78,26 +76,26 @@ This starts all Docker services, waits for Convex to be healthy, and deploys Con docker compose exec convex ./generate_admin_key.sh ``` -Paste the output into `frontend/.env.local` as `CONVEX_SELF_HOSTED_ADMIN_KEY`, then re-run `make dev`. +Paste the output into `.env` as `CONVEX_SELF_HOSTED_ADMIN_KEY`, then re-run `make dev`. ### 5. Load curated public datasets The landing page and the dashboard's "Curated" section read from a set of 9 system-owned datasets. Load them with: ```bash -cd frontend -npx convex run publicSeed:seedPublicDatasets +make seed-public-datasets ``` The script is **idempotent** — rerunning it skips datasets that already exist (matched by a stable `seedKey`, so renaming a curated dataset never creates a duplicate). To add a 10th curated dataset, append it to `PUBLIC_DATASETS` in [frontend/convex/publicSeed.ts](frontend/convex/publicSeed.ts) with a fresh `seedKey` and rerun the command. To replace existing curated content in place, pass `force: true`: ```bash -npx convex run publicSeed:seedPublicDatasets '{"force":true}' +cd frontend +node ../scripts/with-root-env.mjs npx convex run publicSeed:seedPublicDatasets '{"force":true}' ``` Open [localhost:3500](http://localhost:3500) and click **Get started** to sign in. -> **Note:** Backend env needs no setup — `backend/.env.example` has correct defaults. If you edit Convex functions in `frontend/convex/`, run `make convex-push` to deploy the changes. +> **Note:** root `.env` is the only local env file. If you edit Convex functions in `frontend/convex/`, run `make convex-push` to deploy the changes. > **Free tier:** each signed-in account gets **2,500 row operations per calendar month** (resets on the 1st, UTC). The header shows a live usage badge; system-owned curated datasets bypass the quota. @@ -123,14 +121,13 @@ Open [localhost:3500](http://localhost:3500) and click **Get started** to sign i bigset/ ├── frontend/ Next.js 16 — UI + Convex schema & functions │ ├── convex/ Convex functions, schema, authz + quota helpers -│ └── .env.local Clerk + Convex keys (not committed) ├── backend/ Fastify + Mastra — schema inference + populate agent │ ├── src/pipeline/ Pure pipelines: schema inference + populate context │ ├── src/mastra/ Mastra workflows, agents, and tools (Studio at :4111 in dev) │ ├── src/email/ Transactional email (Resend) — sends "dataset ready" notifications │ └── src/analytics/ Server-side PostHog wrapper for backend-only events ├── scripts/ One-off scripts (e.g. verify-authz.sh) -├── .env Clerk keys for docker-compose (not committed) +├── .env Local env for frontend, backend, Convex CLI, and Docker (not committed) ├── docker-compose.dev.yml └── Makefile ``` diff --git a/backend/.env.example b/backend/.env.example deleted file mode 100644 index 1c0dee9..0000000 --- a/backend/.env.example +++ /dev/null @@ -1,31 +0,0 @@ -CLIENT_ORIGIN=http://localhost:3500 -CONVEX_URL=http://localhost:3210 -PORT=3501 - -# Required once the backend starts writing rows via internal Convex mutations. -# Generate with: docker compose exec convex ./generate_admin_key.sh -CONVEX_SELF_HOSTED_ADMIN_KEY= - -# Required for user-facing protected routes (JWT verification). -# Same values as the frontend's Clerk keys. -CLERK_SECRET_KEY= -CLERK_PUBLISHABLE_KEY= - -# OpenRouter API key — required by schema inference. -# Generate at https://openrouter.ai/settings/keys -OPENROUTER_API_KEY=sk-or-... - -# TinyFish API key — used by the populate agent for web search and fetch. -# Generate at https://agent.tinyfish.ai/api-keys -TINYFISH_API_KEY= - -# Resend (transactional email) — optional. When unset, the email module -# logs and no-ops. Generate at https://resend.com/api-keys -RESEND_API_KEY= -# Sender address. The domain must be verified in the Resend dashboard. -EMAIL_FROM="BigSet " - -# PostHog server-side analytics — optional. Same project key as the -# frontend (phc_...). Used to track email-lifecycle events server-side. -POSTHOG_KEY= -POSTHOG_HOST=https://us.i.posthog.com diff --git a/backend/README.md b/backend/README.md index 107b90c..2507f84 100644 --- a/backend/README.md +++ b/backend/README.md @@ -5,10 +5,11 @@ Fastify server that handles auth, database, and talks to TinyFish APIs. ## Running ```bash +# From the repo root: cp .env.example .env -# Set BETTER_AUTH_SECRET (openssl rand -base64 32) +# Fill in the root .env file. +cd backend npm install -npx drizzle-kit push npm run dev ``` @@ -17,9 +18,9 @@ Starts on [localhost:3501](http://localhost:3501). ## Key Paths - `src/index.ts` — Fastify server + route setup -- `src/auth.ts` — Better Auth config -- `src/schema.ts` — Drizzle table definitions -- `src/db.ts` — Database connection +- `src/clerk-auth.ts` — Clerk JWT verification +- `src/convex.ts` — Convex HTTP client +- `src/env.ts` — root env loader ## Scripts @@ -27,4 +28,5 @@ Starts on [localhost:3501](http://localhost:3501). |---------|-------------| | `npm run dev` | Start with hot reload | | `npm run build` | Compile TypeScript | -| `npm run db:push` | Push schema changes to Postgres | + +Local backend scripts load the repo-root `.env` through `../scripts/with-root-env.mjs`. diff --git a/backend/package.json b/backend/package.json index daf0ad5..433c853 100644 --- a/backend/package.json +++ b/backend/package.json @@ -4,10 +4,10 @@ "type": "module", "private": true, "scripts": { - "dev": "tsx watch src/index.ts", + "dev": "node ../scripts/with-root-env.mjs tsx watch src/index.ts", "build": "tsc", - "start": "node dist/index.js", - "mastra:dev": "mastra dev" + "start": "node ../scripts/with-root-env.mjs node dist/index.js", + "mastra:dev": "node ../scripts/with-root-env.mjs mastra dev" }, "dependencies": { "@clerk/backend": "^3.4.11", diff --git a/backend/src/convex.ts b/backend/src/convex.ts index 2b7e267..770640d 100644 --- a/backend/src/convex.ts +++ b/backend/src/convex.ts @@ -3,6 +3,10 @@ import { anyApi } from "convex/server"; import { env } from "./env.js"; +type ConvexHttpClientWithAdminAuth = ConvexHttpClient & { + setAdminAuth(token: string): void; +}; + /** * Convex client for SYSTEM-LEVEL operations from the backend. * @@ -27,5 +31,5 @@ export const internal = anyApi; export const convex = new ConvexHttpClient(env.CONVEX_URL); if (env.CONVEX_ADMIN_KEY) { - convex.setAdminAuth(env.CONVEX_ADMIN_KEY); + (convex as ConvexHttpClientWithAdminAuth).setAdminAuth(env.CONVEX_ADMIN_KEY); } diff --git a/backend/src/env.ts b/backend/src/env.ts index cd4079a..9ae3c09 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -1,4 +1,7 @@ -import "dotenv/config"; +import { config as loadDotenv } from "dotenv"; +import { fileURLToPath } from "node:url"; + +loadDotenv({ path: fileURLToPath(new URL("../../.env", import.meta.url)) }); function required(name: string): string { const value = process.env[name]; @@ -21,7 +24,9 @@ export const env = { // Used by ./clerk-auth.ts to verify JWTs on protected routes (e.g. // /infer-schema). Required for the backend to function. CLERK_SECRET_KEY: process.env.CLERK_SECRET_KEY, - CLERK_PUBLISHABLE_KEY: process.env.CLERK_PUBLISHABLE_KEY, + CLERK_PUBLISHABLE_KEY: + process.env.CLERK_PUBLISHABLE_KEY ?? + process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, diff --git a/backend/src/index.ts b/backend/src/index.ts index 8c9f195..a606b45 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -78,6 +78,11 @@ await fastify.register(async (instance) => { } try { + const auth = req.auth; + if (!auth) { + return reply.code(401).send({ error: "Authentication required" }); + } + // Ownership check uses the INTERNAL (admin-callable, no-authz) getter. // We can't use `api.datasets.get` here because that runs through // `loadReadableDataset`, which requires either a Clerk-identified @@ -94,7 +99,7 @@ await fastify.register(async (instance) => { if (!dataset) { return reply.code(404).send({ error: "Dataset not found" }); } - if (dataset.ownerId !== req.auth.userId) { + if (dataset.ownerId !== auth.userId) { return reply.code(403).send({ error: "Not authorized to populate this dataset" }); } @@ -108,7 +113,7 @@ await fastify.register(async (instance) => { inputData: { ...parsed.data, authContext: { - authorizedUserId: req.auth!.userId, + authorizedUserId: auth.userId, workflowRunId: run.runId, }, }, @@ -257,13 +262,18 @@ await fastify.register(async (instance) => { } try { + const auth = req.auth; + if (!auth) { + return reply.code(401).send({ error: "Authentication required" }); + } + const dataset = await convex.query(internal.datasets.getInternal, { id: parsed.data.datasetId, }); if (!dataset) { return reply.code(404).send({ error: "Dataset not found" }); } - if (dataset.ownerId !== req.auth.userId) { + if (dataset.ownerId !== auth.userId) { return reply.code(403).send({ error: "Not authorized to update this dataset" }); } @@ -272,7 +282,7 @@ await fastify.register(async (instance) => { inputData: { ...parsed.data, authContext: { - authorizedUserId: req.auth!.userId, + authorizedUserId: auth.userId, workflowRunId: run.runId, }, }, diff --git a/backend/src/mastra/tools/investigate-tool.ts b/backend/src/mastra/tools/investigate-tool.ts index f9000b6..3324079 100644 --- a/backend/src/mastra/tools/investigate-tool.ts +++ b/backend/src/mastra/tools/investigate-tool.ts @@ -43,7 +43,7 @@ function parseInvestigateResult( const reasonMatch = text.match(/REASON:\s*(.+?)$/is); return { - inserted: insertedMatch?.[1]?.toLowerCase() === "true" ?? false, + inserted: insertedMatch?.[1]?.toLowerCase() === "true", row_summary: summaryMatch?.[1]?.trim() || undefined, clues: cluesMatch?.[1]?.trim() || undefined, reason: reasonMatch?.[1]?.trim() || text.slice(0, 300), diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 0b12015..36d8561 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -54,7 +54,7 @@ async function callOnce( model, output: Output.object({ schema: datasetSchemaSchema }), system: SYSTEM_PROMPT, - maxTokens: 4096, + maxOutputTokens: 4096, prompt, }); if (!output) throw new Error("Model did not generate a valid schema object"); diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d3c0f35..365b402 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -20,6 +20,8 @@ services: build: context: ./backend dockerfile: Dockerfile.dev + env_file: + - .env ports: - "3501:3501" volumes: @@ -48,6 +50,8 @@ services: build: context: ./backend dockerfile: Dockerfile.mastra + env_file: + - .env ports: - "4111:4111" volumes: @@ -67,6 +71,8 @@ services: build: context: ./frontend dockerfile: Dockerfile.dev + env_file: + - .env ports: - "3500:3500" volumes: diff --git a/frontend/.env.example b/frontend/.env.example deleted file mode 100644 index 912ced3..0000000 --- a/frontend/.env.example +++ /dev/null @@ -1,19 +0,0 @@ -# Convex (self-hosted) -NEXT_PUBLIC_CONVEX_URL=http://127.0.0.1:3210 -CONVEX_SELF_HOSTED_URL=http://127.0.0.1:3210 -CONVEX_SELF_HOSTED_ADMIN_KEY= - -# Clerk — create a free app at https://dashboard.clerk.com -# 1. Create a Clerk application -# 2. Go to JWT Templates → enable the "Convex" template -# 3. Copy your keys below -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_... -CLERK_SECRET_KEY=sk_test_... -CLERK_JWT_ISSUER_DOMAIN=https://your-app.clerk.accounts.dev - -# Backend API (Fastify) -NEXT_PUBLIC_BACKEND_URL=http://localhost:3501 - -# PostHog (optional — leave blank to disable analytics entirely in local dev) -NEXT_PUBLIC_POSTHOG_KEY= -NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com diff --git a/frontend/.gitignore b/frontend/.gitignore index c712c9b..598418e 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -30,9 +30,8 @@ yarn-debug.log* yarn-error.log* .pnpm-debug.log* -# env files (can opt-in for committing if needed) +# env files (root .env is canonical) .env* -!.env.example # package manager: this project uses bun (bun.lock is the source of truth). # Reject npm/yarn lockfiles so they don't drift from bun's resolution. diff --git a/frontend/README.md b/frontend/README.md index 883bcf2..56b2be7 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -9,15 +9,17 @@ bun install bun dev --port 3500 ``` -Opens on [localhost:3500](http://localhost:3500). Expects the backend running on 3501 (auth requests are proxied via Next.js rewrites). +Opens on [localhost:3500](http://localhost:3500). Package scripts load root +`.env` before starting Next.js. The supported full-stack dev path is still +`make dev` from the repo root. ## Key Paths - `app/page.tsx` — Landing page -- `app/auth/` — Sign in + sign up +- `app/sign-in/` and `app/sign-up/` — Clerk sign in + sign up - `app/dashboard/` — Main dashboard (protected) -- `lib/auth-client.ts` — Better Auth React client -- `next.config.ts` — Rewrites `/api/auth/*` to the backend +- `lib/backend.ts` — Backend API client +- `proxy.ts` — Clerk route protection ## Scripts diff --git a/frontend/components/ThemeToggle.tsx b/frontend/components/ThemeToggle.tsx index 89eebe5..c0b54d8 100644 --- a/frontend/components/ThemeToggle.tsx +++ b/frontend/components/ThemeToggle.tsx @@ -1,11 +1,12 @@ "use client"; -import { useEffect, useState } from "react"; +import { useSyncExternalStore } from "react"; import { EVENTS, track } from "@/lib/analytics"; type Theme = "light" | "dark"; const STORAGE_KEY = "bigset:theme"; +const THEME_CHANGED_EVENT = "bigset:theme-changed"; /** * The same selection logic that runs in the inline `` script @@ -29,21 +30,29 @@ function applyTheme(theme: Theme): void { document.documentElement.setAttribute("data-theme", theme); } -export function ThemeToggle({ className = "" }: { className?: string }) { - // We don't know the theme until we've mounted (server can't read - // localStorage). Render the toggle invisible-but-laid-out until then - // so it doesn't pop in and shift layout. - const [mounted, setMounted] = useState(false); - const [theme, setTheme] = useState("light"); +function subscribeToThemeChange(onThemeChange: () => void): () => void { + if (typeof window === "undefined") return () => {}; + window.addEventListener("storage", onThemeChange); + window.addEventListener(THEME_CHANGED_EVENT, onThemeChange); + return () => { + window.removeEventListener("storage", onThemeChange); + window.removeEventListener(THEME_CHANGED_EVENT, onThemeChange); + }; +} + +function readServerTheme(): Theme { + return "light"; +} - useEffect(() => { - setTheme(readEffectiveTheme()); - setMounted(true); - }, []); +export function ThemeToggle({ className = "" }: { className?: string }) { + const theme = useSyncExternalStore( + subscribeToThemeChange, + readEffectiveTheme, + readServerTheme, + ); function toggle() { const next: Theme = theme === "dark" ? "light" : "dark"; - setTheme(next); applyTheme(next); try { window.localStorage.setItem(STORAGE_KEY, next); @@ -51,6 +60,7 @@ export function ThemeToggle({ className = "" }: { className?: string }) { // localStorage may be blocked (Safari private mode etc.) — toggle // still works for the session, just doesn't persist. } + window.dispatchEvent(new Event(THEME_CHANGED_EVENT)); track(EVENTS.THEME_CHANGED, { theme: next }); } @@ -63,7 +73,6 @@ export function ThemeToggle({ className = "" }: { className?: string }) { aria-label={label} title={label} className={`inline-flex items-center justify-center h-7 w-7 text-muted hover:text-foreground transition-colors ${className}`} - style={{ opacity: mounted ? 1 : 0 }} > {/* Both icons rendered, one shown based on theme. Avoids a flash when switching since neither has to mount/unmount. */} diff --git a/frontend/components/table/ColumnHeader.tsx b/frontend/components/table/ColumnHeader.tsx index 27adfee..ca5b8c0 100644 --- a/frontend/components/table/ColumnHeader.tsx +++ b/frontend/components/table/ColumnHeader.tsx @@ -1,6 +1,5 @@ "use client"; -import type { RefObject } from "react"; import type { Header } from "@tanstack/react-table"; import type { DatasetRow, DatasetColumn } from "./types"; import { ColumnIcon } from "./ColumnIcon"; @@ -10,12 +9,12 @@ export function ColumnHeader({ header, column, isResizing, - tableContainerRef, + containerHeight, }: { header: Header; column?: DatasetColumn; isResizing: boolean; - tableContainerRef: RefObject; + containerHeight: number; }) { return (
)} diff --git a/frontend/components/table/DatasetTable.tsx b/frontend/components/table/DatasetTable.tsx index 9a06a7a..5644cff 100644 --- a/frontend/components/table/DatasetTable.tsx +++ b/frontend/components/table/DatasetTable.tsx @@ -150,7 +150,7 @@ export function DatasetTable({ allState={selection.allState} toggleAll={selection.toggleAll} resizingColumnId={resizingColumnId} - tableContainerRef={tableContainerRef} + containerHeight={containerHeight} /> []; columns: DatasetColumn[]; allState: boolean | "indeterminate"; toggleAll: () => void; resizingColumnId: string | false; - tableContainerRef: RefObject; + containerHeight: number; }) { const checkboxRef = useRef(null); @@ -58,7 +58,7 @@ export function TableHeader({ header={header} column={columns[i]} isResizing={resizingColumnId === header.id} - tableContainerRef={tableContainerRef} + containerHeight={containerHeight} /> ))}
diff --git a/frontend/lib/analytics.ts b/frontend/lib/analytics.ts index 7b60702..6b620f7 100644 --- a/frontend/lib/analytics.ts +++ b/frontend/lib/analytics.ts @@ -84,8 +84,8 @@ export function initAnalytics(): boolean { // - maskInputOptions: every form input/textarea value is masked // unconditionally. Catches the search box, the wizard prompt, // Clerk's email + password fields. - // - recordConsole: console.error/warn shows up alongside the - // replay timeline — invaluable for "user says it broke". + // - logs.captureConsoleLogs: console.error/warn is captured for + // "user says it broke" debugging. // - recordCrossOriginIframes: false → Clerk's hosted iframes // (if any) are not pierced into. session_recording: { @@ -97,7 +97,10 @@ export function initAnalytics(): boolean { email: true, }, recordCrossOriginIframes: false, - recordConsole: true, + }, + + logs: { + captureConsoleLogs: true, }, loaded: () => { diff --git a/frontend/package.json b/frontend/package.json index d7d7a0c..5dc165c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,10 +3,10 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "next dev", - "build": "next build", - "start": "next start", - "lint": "eslint" + "dev": "node ../scripts/with-root-env.mjs next dev", + "build": "node ../scripts/with-root-env.mjs next build", + "start": "node ../scripts/with-root-env.mjs next start", + "lint": "node ../scripts/with-root-env.mjs eslint" }, "dependencies": { "@clerk/nextjs": "^7.3.7", diff --git a/makefiles/Makefile b/makefiles/Makefile index 497efef..fb1778e 100644 --- a/makefiles/Makefile +++ b/makefiles/Makefile @@ -1,8 +1,8 @@ -.PHONY: all dev down clean convex-push convex-env +.PHONY: all dev validate-dev-env down clean convex-push convex-env seed-public-datasets all: dev -dev: +dev: validate-dev-env docker compose -f docker-compose.dev.yml up --build -d @echo "Waiting for Convex to be healthy..." @for i in $$(seq 1 120); do \ @@ -18,20 +18,57 @@ dev: @echo " Mastra Studio: http://localhost:4111" docker compose -f docker-compose.dev.yml logs -f +validate-dev-env: + @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } + @check_env() { \ + key="$$1"; placeholder="$$2"; \ + value="$$(grep "^$$key=" .env | cut -d= -f2-)"; \ + if [[ -z "$$value" || "$$value" == "$$placeholder" || "$$value" == *"..."* ]]; then \ + echo "Error: $$key must be set in root .env"; \ + exit 1; \ + fi; \ + }; \ + check_env NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY pk_test_...; \ + check_env CLERK_SECRET_KEY sk_test_...; \ + check_env CLERK_JWT_ISSUER_DOMAIN https://your-app.clerk.accounts.dev; \ + check_env OPENROUTER_API_KEY sk-or-... + convex-env: - @test -f frontend/.env.local || { echo "Error: frontend/.env.local not found"; exit 1; } - @grep -q CLERK_JWT_ISSUER_DOMAIN frontend/.env.local || { echo "Error: CLERK_JWT_ISSUER_DOMAIN not set in frontend/.env.local"; exit 1; } - @grep -q CONVEX_SELF_HOSTED_ADMIN_KEY frontend/.env.local || { echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY not set in frontend/.env.local"; exit 1; } - @cd frontend && npx convex env set CLERK_JWT_ISSUER_DOMAIN "$$(grep CLERK_JWT_ISSUER_DOMAIN .env.local | cut -d= -f2-)" \ + @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } + @issuer="$$(grep '^CLERK_JWT_ISSUER_DOMAIN=' .env | cut -d= -f2-)"; \ + admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2-)"; \ + if [[ -z "$$issuer" || "$$issuer" == "https://your-app.clerk.accounts.dev" ]]; then \ + echo "Error: CLERK_JWT_ISSUER_DOMAIN must be your Clerk issuer URL in root .env"; \ + exit 1; \ + fi; \ + if [[ -z "$$admin_key" ]]; then \ + echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY is missing in root .env"; \ + echo "Generate it after Convex starts with:"; \ + echo " docker compose -f docker-compose.dev.yml exec convex ./generate_admin_key.sh"; \ + echo "Then paste it into root .env and rerun make dev."; \ + exit 1; \ + fi; \ + cd frontend && node ../scripts/with-root-env.mjs npx convex env set CLERK_JWT_ISSUER_DOMAIN "$$issuer" \ --url http://127.0.0.1:3210 \ - --admin-key "$$(grep CONVEX_SELF_HOSTED_ADMIN_KEY .env.local | cut -d= -f2-)" + --admin-key "$$admin_key" convex-push: - @test -f frontend/.env.local || { echo "Error: frontend/.env.local not found"; exit 1; } - @grep -q CONVEX_SELF_HOSTED_ADMIN_KEY frontend/.env.local || { echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY not set in frontend/.env.local"; exit 1; } - @cd frontend && npx convex deploy \ + @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } + @admin_key="$$(grep '^CONVEX_SELF_HOSTED_ADMIN_KEY=' .env | cut -d= -f2-)"; \ + if [[ -z "$$admin_key" ]]; then \ + echo "Error: CONVEX_SELF_HOSTED_ADMIN_KEY is missing in root .env"; \ + echo "Generate it after Convex starts with:"; \ + echo " docker compose -f docker-compose.dev.yml exec convex ./generate_admin_key.sh"; \ + echo "Then paste it into root .env and rerun make dev."; \ + exit 1; \ + fi; \ + cd frontend && node ../scripts/with-root-env.mjs npx convex deploy \ --url http://127.0.0.1:3210 \ - --admin-key "$$(grep CONVEX_SELF_HOSTED_ADMIN_KEY .env.local | cut -d= -f2-)" + --admin-key "$$admin_key" + +seed-public-datasets: + @test -f .env || { echo "Error: .env not found. Run: cp .env.example .env"; exit 1; } + @cd frontend && node ../scripts/with-root-env.mjs npx convex run publicSeed:seedPublicDatasets down: docker compose -f docker-compose.dev.yml down diff --git a/scripts/with-root-env.mjs b/scripts/with-root-env.mjs new file mode 100644 index 0000000..238000d --- /dev/null +++ b/scripts/with-root-env.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const rootEnvPath = resolve(repoRoot, ".env"); + +if (existsSync(rootEnvPath)) { + for (const line of readFileSync(rootEnvPath, "utf8").split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const separatorIndex = trimmed.indexOf("="); + if (separatorIndex <= 0) continue; + + const key = trimmed.slice(0, separatorIndex).trim(); + let value = trimmed.slice(separatorIndex + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + process.env[key] ??= value; + } +} + +const [command, ...args] = process.argv.slice(2); +if (!command) { + console.error("Usage: node scripts/with-root-env.mjs [...args]"); + process.exit(2); +} + +const child = spawn(command, args, { + env: process.env, + shell: process.platform === "win32", + stdio: "inherit", +}); + +child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 1); +}); From 5605f9c5be41246f03edf485c0ee21ea6b44b264 Mon Sep 17 00:00:00 2001 From: Edward Tran Date: Mon, 25 May 2026 11:45:08 +0700 Subject: [PATCH 2/2] Mount root env loader in dev containers --- docker-compose.dev.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 365b402..1afbf5b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,6 +26,7 @@ services: - "3501:3501" volumes: - ./backend/src:/app/src + - ./scripts:/scripts:ro environment: CLIENT_ORIGIN: http://localhost:3500 CONVEX_URL: http://convex:3210 @@ -56,6 +57,7 @@ services: - "4111:4111" volumes: - ./backend/src:/app/src + - ./scripts:/scripts:ro environment: HOST: 0.0.0.0 PORT: 4111 @@ -86,6 +88,7 @@ services: # time and silently ignores local edits. - ./frontend/proxy.ts:/app/proxy.ts - ./frontend/next.config.ts:/app/next.config.ts + - ./scripts:/scripts:ro environment: NEXT_PUBLIC_CONVEX_URL: http://localhost:3210 NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY}