From 70451ca2adf1335dc44e88a32f844e14ed161d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20Emir=20Y=C4=B1ld=C4=B1r=C4=B1m?= Date: Fri, 3 Jul 2026 11:37:23 +0300 Subject: [PATCH] Add Lottie Studio: web UI + agent job-bridge + MP4 export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A small web UI (public/studio.html at /studio) that lets you drop a photo/SVG, write a prompt, and pick options; a Vite dev-server plugin exposes a job queue (/api/jobs) writing requests to .studio-jobs//. A watching coding agent authors the scene under public/projects/ and the player shows it live — no API key in the app. Also adds MP4 export: GET /api/export renders a scene frame-by-frame through Skottie (CanvasKit) and encodes H.264 via a bundled ffmpeg-static binary (no system ffmpeg required), streamed back as a download. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 + STUDIO.md | 83 +++++++ package.json | 1 + public/studio.html | 477 +++++++++++++++++++++++++++++++++++++++++ vite-plugins/studio.ts | 297 +++++++++++++++++++++++++ vite.config.ts | 2 + 6 files changed, 863 insertions(+) create mode 100644 STUDIO.md create mode 100644 public/studio.html create mode 100644 vite-plugins/studio.ts diff --git a/.gitignore b/.gitignore index 1db670e..d28764b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ app.config.timestamp_*.js # Temp gitignore +# Studio job queue (agent bridge) +.studio-jobs + # System Files .DS_Store Thumbs.db diff --git a/STUDIO.md b/STUDIO.md new file mode 100644 index 0000000..36155eb --- /dev/null +++ b/STUDIO.md @@ -0,0 +1,83 @@ +# Lottie Studio + +A small web UI + agent bridge on top of [diffusion-studio/lottie](https://github.com/diffusionstudio/lottie) +(the official Skia/Skottie Lottie player). You drop a photo/SVG, write a prompt, +pick options — a coding agent (Claude Code) picks the job up, authors the Lottie +scene, and the player shows it live. + +This is an early proof of concept, not a finished product. See **Roadmap** below. + +## Run + +```bash +npm install +npm run dev # player on http://localhost:3030 +``` + +Open the studio at **http://localhost:3030/studio.html** (or `/studio`). + +## How it works (the bridge) + +No API key lives in the app — a watching agent is the generation engine. + +``` +Studio page ──POST /api/jobs──▶ .studio-jobs//job.json (+ image) [status: pending] + (browser) │ + ▼ + Claude Code watches for pending jobs, + authors public/projects///lottie.json, + writes status: done + scene back into job.json + │ +Studio polls /api/jobs/ ◀────────────┘ → embeds the player once the scene exists +``` + +- **`public/studio.html`** — the studio UI: image dropzone, prompt, options + (project, size, fps, frames, background transparent/color, loop, editable + controls, notes), live player iframe, recent-jobs feed. +- **`vite-plugins/studio.ts`** — the job queue API (`POST/GET /api/jobs`, + `GET /api/jobs/:id`, `/studio` redirect). Jobs live under `.studio-jobs/` + (gitignored). +- **Scenes** are authored under `public/projects///` as + `lottie.json` (+ optional `controls.json`, image/font assets) — the same + contract the base player already watches. Raster grounding images are embedded + as Lottie image assets so the original art is preserved pixel-for-pixel; overlay + layers (blink/wink/wave/camera) are parented to the image so they track it. + +## MP4 export + +Each finished scene has an **MP4 indir** button (background colour selectable — +MP4 has no alpha). `GET /api/export?project=&scene=&bg=&fps=&loops=` renders the +scene frame-by-frame through Skottie and encodes H.264 via a bundled +`ffmpeg-static` binary (no system ffmpeg needed), then streams the file back. + +## Roadmap (what would make it good) + +**Quality engine** +1. Agentic generate → render → **vision-check** → self-correct loop (align + eyes, catch edge gaps, verify the loop seam) instead of one-shot authoring. +2. Layer/part separation & rigging (SVG input or auto-segmentation + background + removal) for real limb motion — walk, arm-wave, lip-sync. +3. Automatic face/landmark detection so blink/wink/lip-sync auto-align to any + character (no hand-tuned coordinates). +4. A parameterized motion-preset library (blink, wink, wave, bounce, float, + pop-in, shake, breathe, parallax, camera push/pull, confetti…). + +**Studio UX** +5. Timeline: play/pause/scrub, frame pinning, before/after, chat-style "refine" + on an existing scene, regenerate. +6. Export to GIF / MP4 / WebM + copy-ready snippets (web, RN Skottie, iOS, + Android, Flutter). +7. SVG-first grounding, multiple references, region capture; background + transparent/color/gradient/keep-source with auto source-color extraction. + +**Infra** +8. Headless mode (run with an API key, no live agent session) + queue + robustness (retry/cancel/multi-job) + a persistent gallery. +9. Quality guardrails on every generation: JSON validate, render-check, + loop-seam check, blank-canvas rejection. + +## Credit + +Built on **diffusion-studio/lottie** (Text-to-Lottie). Base player, skill, and +player contract are theirs; the Studio UI, job-bridge, and generated scenes are +the additions here. diff --git a/package.json b/package.json index 34c1608..3fae622 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "fflate": "^0.8.3", + "ffmpeg-static": "^5.3.0", "lucide-solid": "^1.18.0", "shadcn": "^4.11.0", "solid-js": "^1.9.9", diff --git a/public/studio.html b/public/studio.html new file mode 100644 index 0000000..4bfe08d --- /dev/null +++ b/public/studio.html @@ -0,0 +1,477 @@ + + + + + + Text-to-Lottie Studio + + + +
+
+ + + +

Text-to-Lottie Studio

+
+

Foto (opsiyonel) + prompt ver, ayarları seç → arkada Claude Code sahneyi üretir, oynatıcı burada canlı gösterir. Player: localhost:3030

+ +
+ +
+
+
+ +
+ +
Sürükle-bırak ya da tıkla — SVG, PNG, ekran görüntüsü, logo
+ +
+
+ +
+
+ +
+
+
+ +
+ + +
Motion tabiri kullan (ease-in / ease-out), kamera hareketi iste, FPS/süre belirt.
+
+
+ +
+
+
+ + +
+
+ + +
+
+ + + +
+
+
+
+ + +
+
+ +
+ +
+ + +
+ +
+ +
+ +
+ + +
+
+ +
+ + +
+
+ + +
+ + +
+
+
+ +
Ürettiğin animasyon burada canlı görünecek.
+
+ + +
+ +
+

Son işler

+
+
+
+
+
+ + + + diff --git a/vite-plugins/studio.ts b/vite-plugins/studio.ts new file mode 100644 index 0000000..b16405e --- /dev/null +++ b/vite-plugins/studio.ts @@ -0,0 +1,297 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import crypto from "node:crypto"; +import { spawn } from "node:child_process"; +import { createRequire } from "node:module"; +import type { Plugin } from "vite"; +import type { IncomingMessage, ServerResponse } from "node:http"; + +const require = createRequire(import.meta.url); + +/** + * Studio: a tiny job queue that bridges the browser and a coding agent. + * + * The Studio page (public/studio.html) POSTs a generation request — prompt, + * options, and an optional grounding image — to `/api/jobs`. Each request is + * written to `.studio-jobs//job.json` (plus `input.` for the image) + * with `status: "pending"`. A watching agent (Claude Code) picks up pending + * jobs, authors the Lottie scene under `public/projects/...`, and writes the + * result back into `job.json` (`status: "done"`, `scene`, `sceneUrl`). The page + * polls `/api/jobs/` and embeds the player once the scene exists. + * + * No API key lives here — the agent is the generation engine. + */ + +const MIME_EXT: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/webp": "webp", + "image/gif": "gif", + "image/svg+xml": "svg", +}; + +const SAFE_ID = /^[A-Za-z0-9._-]+$/; + +// CanvasKit (full/skottie build) is heavy to instantiate — load once, reuse +// across exports. +let canvasKitPromise: Promise | null = null; +function getCanvasKit(binDir: string): Promise { + if (!canvasKitPromise) { + const init = require(path.join(binDir, "canvaskit.js")); + canvasKitPromise = init({ locateFile: (f: string) => path.join(binDir, f) }); + } + return canvasKitPromise; +} + +/** Render a scene frame-by-frame through Skottie and pipe raw RGBA to ffmpeg → MP4. */ +async function renderMp4(opts: { + ck: any; sceneDir: string; doc: any; fps: number; + bg: [number, number, number] | null; loops: number; ffmpegPath: string; +}): Promise { + const { ck, sceneDir, doc, fps, bg, loops, ffmpegPath } = opts; + + const assets: Record = {}; + for (const a of doc.assets || []) { + if (a?.p && typeof a.p === "string" && !a.p.startsWith("data:")) { + const f = path.join(sceneDir, a.p); + if (fs.existsSync(f)) assets[a.p] = fs.readFileSync(f); + } + } + + const W = doc.w || 512, H = doc.h || 512; + const anim = ck.MakeManagedAnimation(JSON.stringify(doc), assets); + const total = Math.max(1, Math.round(anim.duration() * anim.fps())); + const surface = ck.MakeSurface(W, H); + const canvas = surface.getCanvas(); + // MP4/H.264 has no alpha — composite transparent scenes on a solid colour. + const [r, g, b] = bg ?? [255, 255, 255]; + const outFile = path.join(os.tmpdir(), `lottie-${crypto.randomUUID()}.mp4`); + + const ff = spawn(ffmpegPath, [ + "-y", "-f", "rawvideo", "-pixel_format", "rgba", + "-video_size", `${W}x${H}`, "-framerate", String(fps), "-i", "pipe:0", + "-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2,format=yuv420p", + "-c:v", "libx264", "-preset", "medium", "-crf", "18", + "-movflags", "+faststart", outFile, + ]); + let ffErr = ""; + ff.stderr.on("data", (d) => { ffErr += d.toString(); if (ffErr.length > 4000) ffErr = ffErr.slice(-2000); }); + const closed = new Promise((resolve, reject) => { + ff.on("error", reject); + ff.on("close", (code) => code === 0 ? resolve() : reject(new Error(`ffmpeg exit ${code}: ${ffErr.slice(-500)}`))); + }); + + try { + for (let l = 0; l < loops; l++) { + for (let f = 0; f < total; f++) { + anim.seekFrame(f); + canvas.clear(ck.Color(r, g, b, 255)); + anim.render(canvas, ck.LTRBRect(0, 0, W, H)); + surface.flush(); + const img = surface.makeImageSnapshot(); + const px: Uint8Array = img.readPixels(0, 0, { + width: W, height: H, + colorType: ck.ColorType.RGBA_8888, + alphaType: ck.AlphaType.Unpremul, + colorSpace: ck.ColorSpace.SRGB, + }); + img.delete(); + const buf = Buffer.from(px.buffer, px.byteOffset, px.byteLength); + if (!ff.stdin.write(buf)) await new Promise((res) => ff.stdin.once("drain", res)); + } + } + ff.stdin.end(); + await closed; + } finally { + surface.delete(); + anim.delete(); + } + + const out = fs.readFileSync(outFile); + fs.unlink(outFile, () => {}); + return out; +} + +interface Job { + id: string; + createdAt: string; + updatedAt: string; + status: "pending" | "processing" | "done" | "error"; + prompt: string; + options: Record; + imageFile: string | null; + scene: string | null; // "/" + sceneUrl: string | null; // "//" + note: string | null; // agent-facing progress / summary + error: string | null; +} + +function readBody(req: IncomingMessage): Promise { + return new Promise((resolve) => { + let data = ""; + req.on("data", (chunk) => (data += chunk)); + req.on("end", () => resolve(data)); + req.on("error", () => resolve("")); + }); +} + +function readJob(jobsDir: string, id: string): Job | null { + if (!SAFE_ID.test(id)) return null; + const file = path.join(jobsDir, id, "job.json"); + if (!file.startsWith(jobsDir + path.sep) || !fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, "utf8")) as Job; + } catch { + return null; + } +} + +function listJobs(jobsDir: string): Job[] { + if (!fs.existsSync(jobsDir)) return []; + return fs + .readdirSync(jobsDir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => readJob(jobsDir, e.name)) + .filter((j): j is Job => j !== null) + .sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +} + +export function studioPlugin(): Plugin { + let jobsDir = ""; + let projectsDir = ""; + let ckBinDir = ""; + + return { + name: "studio-jobs", + + configResolved(config) { + jobsDir = path.resolve(config.root, ".studio-jobs"); + projectsDir = path.resolve(config.root, "public/projects"); + ckBinDir = path.resolve(config.root, "node_modules/canvaskit-wasm/bin/full"); + }, + + configureServer(server) { + fs.mkdirSync(jobsDir, { recursive: true }); + + const json = (res: ServerResponse, status: number, body: unknown) => { + res.statusCode = status; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify(body)); + }; + + // connect strips the "/api/jobs" mount prefix, so req.url is "/" (collection) + // or "/" (single job). + server.middlewares.use("/api/jobs", async (req, res) => { + const rest = (req.url || "/").split("?")[0]; + const id = rest.replace(/^\/+/, "").split("/")[0]; + + // GET /api/jobs — list + if (req.method === "GET" && !id) { + return json(res, 200, { jobs: listJobs(jobsDir) }); + } + + // GET /api/jobs/ — single + if (req.method === "GET" && id) { + const job = readJob(jobsDir, id); + return job ? json(res, 200, job) : json(res, 404, { error: "not found" }); + } + + // POST /api/jobs — enqueue a new generation job + if (req.method === "POST" && !id) { + let body: Record = {}; + try { + body = JSON.parse((await readBody(req)) || "{}"); + } catch { + return json(res, 400, { error: "invalid JSON body" }); + } + const prompt = String(body.prompt ?? "").trim(); + if (!prompt) return json(res, 400, { error: "prompt is required" }); + + const now = new Date().toISOString(); + const jobId = `${Date.now().toString(36)}-${crypto.randomUUID().slice(0, 8)}`; + const dir = path.join(jobsDir, jobId); + fs.mkdirSync(dir, { recursive: true }); + + let imageFile: string | null = null; + const image = typeof body.image === "string" ? body.image : ""; + const match = image.match(/^data:([^;]+);base64,(.*)$/s); + if (match) { + const ext = MIME_EXT[match[1].toLowerCase()] ?? "bin"; + imageFile = `input.${ext}`; + fs.writeFileSync(path.join(dir, imageFile), Buffer.from(match[2], "base64")); + } + + const job: Job = { + id: jobId, + createdAt: now, + updatedAt: now, + status: "pending", + prompt, + options: (body.options as Record) ?? {}, + imageFile, + scene: null, + sceneUrl: null, + note: null, + error: null, + }; + fs.writeFileSync(path.join(dir, "job.json"), JSON.stringify(job, null, 2)); + return json(res, 201, job); + } + + json(res, 405, { error: "method not allowed" }); + }); + + // GET /api/export?project=&scene=&bg=&fps=&loops= -> download an MP4 + server.middlewares.use("/api/export", async (req, res) => { + if (req.method !== "GET") return json(res, 405, { error: "method not allowed" }); + try { + const q = new URL(req.url || "/", "http://x").searchParams; + const project = q.get("project") || ""; + const scene = q.get("scene") || ""; + if (!SAFE_ID.test(project) || !SAFE_ID.test(scene)) { + return json(res, 400, { error: "bad project/scene" }); + } + const sceneDir = path.join(projectsDir, project, scene); + const lottiePath = path.join(sceneDir, "lottie.json"); + if (!lottiePath.startsWith(projectsDir + path.sep) || !fs.existsSync(lottiePath)) { + return json(res, 404, { error: "scene not found" }); + } + const doc = JSON.parse(fs.readFileSync(lottiePath, "utf8")); + + const bgParam = (q.get("bg") || "ffffff").toLowerCase(); + let bg: [number, number, number] | null = [255, 255, 255]; + if (bgParam === "transparent") bg = [255, 255, 255]; // no alpha in mp4 + else if (/^[0-9a-f]{6}$/.test(bgParam)) { + bg = [parseInt(bgParam.slice(0, 2), 16), parseInt(bgParam.slice(2, 4), 16), parseInt(bgParam.slice(4, 6), 16)]; + } + const fps = Math.min(60, Math.max(1, Number(q.get("fps")) || doc.fr || 30)); + const loops = Math.min(8, Math.max(1, Number(q.get("loops")) || 1)); + + const ffmpegPath: string = require("ffmpeg-static"); + const ck = await getCanvasKit(ckBinDir); + const buf = await renderMp4({ ck, sceneDir, doc, fps, bg, loops, ffmpegPath }); + + res.statusCode = 200; + res.setHeader("Content-Type", "video/mp4"); + res.setHeader("Content-Length", String(buf.length)); + res.setHeader("Content-Disposition", `attachment; filename="${project}-${scene}.mp4"`); + res.end(buf); + } catch (e) { + const msg = String((e as Error)?.message ?? e); + const hint = msg.includes("ENOENT") ? "ffmpeg binary not found (ffmpeg-static)" : msg; + json(res, 500, { error: hint }); + } + }); + + // Convenience: /studio -> the Studio page. + server.middlewares.use("/studio", (req, res, next) => { + if (req.url && req.url !== "/" && req.url !== "") return next(); + res.statusCode = 302; + res.setHeader("Location", "/studio.html"); + res.end(); + }); + }, + }; +} diff --git a/vite.config.ts b/vite.config.ts index 9db8f1f..86fe603 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -5,6 +5,7 @@ import solidSvg from 'vite-plugin-solid-svg'; import solidPlugin from 'vite-plugin-solid'; import devtools from 'solid-devtools/vite'; import { scenesPlugin } from './vite-plugins/scenes'; +import { studioPlugin } from './vite-plugins/studio'; export default defineConfig({ plugins: [ @@ -13,6 +14,7 @@ export default defineConfig({ tailwindcss(), solidSvg({ defaultAsComponent: true }), scenesPlugin(), + studioPlugin(), ], server: { port: 3030,