Skip to content

HTML5 WebP capture + Google Font auto-registration on export #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dazzatronus wants to merge 3 commits into
base: main
Choose a base branch
from
Open

HTML5 WebP capture + Google Font auto-registration on export #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f8db74
fix: update HTML5 player to use WebP format for frame capture and imp…
dazzatronus May 25, 2026
35b1e59
refactor: remove Html5Player duration detection logic
dazzatronus May 25, 2026
c2e4dfa
fix: auto-register catalogued Google Fonts on export
dazzatronus May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 50 additions & 79 deletions src/components/canvas/players/html5-player.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,7 @@ import type { Edit } from "@core/edit-session";
import { EditEvent } from "@core/events/edit-events";
import { type Size } from "@layouts/geometry";
import type { ResolvedClip } from "@schemas";
import {
Html5AssetSchema,
composeHtml5IframeSrcdoc,
computeHtml5FrameCount,
detectHtml5DurationWithRetry,
type Html5Asset
} from "@shotstack/shotstack-canvas";
import { Html5AssetSchema, composeHtml5IframeSrcdoc, computeHtml5FrameCount, type Html5Asset } from "@shotstack/shotstack-canvas";
import * as pixi from "pixi.js";

import { computeHtml5CacheKey, html5CacheGet, html5CachePut } from "./html5-cache";
Expand All @@ -21,10 +15,9 @@ const DECODED_FRAME_LIMIT = 30;

type HarnessWindow = Window & {
["__shotstackSeek"]?: (ms: number) => void;
["__shotstackDetectDurationMs"]?: () => number;
};
const SEEK_KEY = "__shotstackSeek" as const;
const DETECT_KEY = "__shotstackDetectDurationMs" as const;
const CAPTURE_CONCURRENCY = 4;

function yieldFrame(): Promise<void> {
return new Promise<void>(resolve => {
Expand Down Expand Up @@ -56,7 +49,7 @@ function waitForIframeLoad(iframe: HTMLIFrameElement, timeoutMs: number = IFRAME
});
}

async function foreignObjectSvgToPng(svg: string, width: number, height: number): Promise<Uint8Array> {
async function foreignObjectSvgToWebp(svg: string, width: number, height: number): Promise<Uint8Array> {
const url = `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`;
const img = new Image();
img.src = url;
Expand All @@ -68,7 +61,7 @@ async function foreignObjectSvgToPng(svg: string, width: number, height: number)
if (!ctx) throw new Error("2D context unavailable for foreignObject rasterise");
ctx.drawImage(img, 0, 0, width, height);
const blob = await new Promise<Blob | null>(resolve => {
canvas.toBlob(resolve, "image/png");
canvas.toBlob(resolve, "image/webp", 0.85);
});
if (!blob) throw new Error(`canvas.toBlob returned null — taint? (svg bytes=${svg.length})`);
return new Uint8Array(await blob.arrayBuffer());
Expand Down Expand Up @@ -158,21 +151,6 @@ export class Html5Player extends Player {
}
}

/**
* Returns the harness-reported duration in ms, or null if unavailable.
*/
private probeDurationMs(): number | null {
const detect = this.harnessWindow?.[DETECT_KEY];
if (typeof detect !== "function") return null;
try {
const ms = detect();
if (typeof ms === "number" && Number.isFinite(ms) && ms > 0) return ms;
} catch (err) {
console.warn("[Html5Player] __shotstackDetectDurationMs threw:", err);
}
return null;
}

private emitCaptureFailed(error: unknown, fallback: string): void {
const message = error instanceof Error ? error.message : String(error);
try {
Expand Down Expand Up @@ -220,7 +198,7 @@ export class Html5Player extends Player {
}
await this.mountIframe(validation.data);
this.configureKeyframes();
this.prewarmCapture();
this.beginCapture();
} catch (error) {
console.error("Failed to render html5 asset:", error instanceof Error ? `${error.message}\n${error.stack}` : error);
this.createFallbackGraphic();
Expand Down Expand Up @@ -292,8 +270,12 @@ export class Html5Player extends Player {
// ─── capture pipeline ──────────────────────────────────────────────────────

private async captureFrames(): Promise<Blob[] | null> {
if (this.capturedFrames) return this.capturedFrames;
if (this.captureInFlight) return this.captureInFlight;
if (this.capturedFrames && this.capturedHash === this.contentHash) {
return this.capturedFrames;
}
if (this.captureInFlight) {
return this.captureInFlight;
}
const previous = Html5Player.captureChain;
this.captureInFlight = previous
.then(() => {
Expand Down Expand Up @@ -334,11 +316,7 @@ export class Html5Player extends Player {
await yieldFrame();
if (stale()) return null;

const detectedDurationMs = await detectHtml5DurationWithRetry(() => this.probeDurationMs(), stale);
if (stale()) return null;

const { frameCount } = computeHtml5FrameCount({
detectedDurationMs,
clipLengthSeconds: this.getLength(),
jsContent: this.asset.js,
cssContent: this.asset.css,
Expand All @@ -350,25 +328,24 @@ export class Html5Player extends Player {
this.captureFramesTotal = frameCount;
this.captureFramesDone = 0;

const rasterisePromises: Promise<Uint8Array>[] = [];
const onFrameDone = (png: Uint8Array): Uint8Array => {
this.captureFramesDone += 1;
return png;
};

for (let i = 0; i < frameCount; i += 1) {
const blobs: Blob[] = [];
for (let i = 0; i < frameCount; i += CAPTURE_CONCURRENCY) {
await yieldFrame();
if (stale()) return null;
this.seekHarness(i / fps);
forceLayout(this.iframe.contentDocument.body);
const svg = this.captureIframeAsForeignObjectSvg(W, H);
rasterisePromises.push(foreignObjectSvgToPng(svg, W, H).then(onFrameDone));
const batch: Promise<Uint8Array>[] = [];
for (let j = i; j < Math.min(i + CAPTURE_CONCURRENCY, frameCount); j += 1) {
this.seekHarness(j / fps);
forceLayout(this.iframe.contentDocument.body);
batch.push(foreignObjectSvgToWebp(this.captureIframeAsForeignObjectSvg(W, H), W, H));
}
const frames = await Promise.all(batch);
if (stale()) return null;
for (const frame of frames) {
this.captureFramesDone += 1;
blobs.push(new Blob([frame as BlobPart], { type: "image/webp" }));
}
}

const pngs = await Promise.all(rasterisePromises);
if (stale()) return null;
const blobs: Blob[] = pngs.map(png => new Blob([png as BlobPart], { type: "image/png" }));

this.capturedFrames = blobs;
this.capturedHash = cacheKey;
this.emitCaptureCompleted(frameCount);
Expand Down Expand Up @@ -484,13 +461,14 @@ export class Html5Player extends Player {
if (newHash === this.contentHash) return;
this.contentHash = newHash;
this.transitionToStale();
this.disposeCapturedFrames();
this.captureInFlight = null;
this.hasTriggeredCapture = false;
this.seekErrorReported = false;
this.iframe.srcdoc = composeHtml5IframeSrcdoc(this.asset);
try {
await waitForIframeLoad(this.iframe, undefined, true);
this.prewarmCapture();
this.beginCapture();
} catch (err) {
console.warn("[Html5Player] reload iframe load failed:", err);
this.emitCaptureFailed(err, "static-placeholder");
Expand Down Expand Up @@ -531,54 +509,47 @@ export class Html5Player extends Player {
}
}

/**
* Run capture in the background without changing UI mode.
*/
private prewarmCapture(): void {
private beginCapture(): void {
if (this.disposed || !this.iframe) return;
if (this.capturedFrames || this.captureInFlight) return;
this.captureFrames().catch(err => {
console.warn("[Html5Player] prewarm capture failed:", err);
});
}

private triggerCaptureIfNeeded(): void {
if (this.hasTriggeredCapture) return;
if (!this.iframe || !this.edit.isPlaying) return;
if (this.mode !== "editing" && this.mode !== "stale") return;
if (this.mode === "capturing" || this.mode === "playback") return;
if (this.captureInFlight) return;

if (this.capturedFrames && this.capturedHash === this.contentHash) {
const hashAtTrigger = this.contentHash;
this.hasTriggeredCapture = true;
this.transitionToPlayback()
.catch(err => {
console.warn("[Html5Player] transitionToPlayback failed:", err);
this.transitionToEditing();
})
.finally(() => {
if (this.contentHash !== hashAtTrigger) this.hasTriggeredCapture = false;
});
this.transitionToPlayback().catch(err => {
console.warn("[Html5Player] transitionToPlayback failed:", err);
this.transitionToEditing();
});
return;
}

this.hasTriggeredCapture = true;
this.transitionToCapturing();
const hashAtTrigger = this.contentHash;

const hashAtStart = this.contentHash;
this.captureFrames()
.then(async frames => {
if (!frames || frames.length === 0) return;
if (this.capturedHash !== hashAtTrigger || this.contentHash !== hashAtTrigger) return;
if (this.disposed) return;
const fresh = !!frames && frames.length > 0 && this.capturedHash === hashAtStart && this.contentHash === hashAtStart;
if (!fresh) {
// Stale/empty result — fall back to the live iframe instead of stranding the loader.
if (this.mode === "capturing") this.transitionToEditing();
return;
}
await this.transitionToPlayback();
})
.catch(err => {
console.warn("[Html5Player] capture failed:", err);
this.emitCaptureFailed(err, "live-iframe");
this.transitionToEditing();
if (this.mode === "capturing") this.transitionToEditing();
});
}

private triggerCaptureIfNeeded(): void {
if (this.hasTriggeredCapture) return;
if (!this.iframe || !this.edit.isPlaying) return;
if (this.mode !== "editing" && this.mode !== "stale") return;
this.hasTriggeredCapture = true;
this.beginCapture();
}

private mountLoadingGraphic(): void {
if (this.loadingGraphic) return;
const width = this.renderedWidth || this.edit.size.width;
Expand Down
59 changes: 29 additions & 30 deletions src/components/canvas/players/placeholder-graphic.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,40 +22,38 @@ export function createPlaceholderGraphic(width: number, height: number): pixi.Gr
export function createCaptureLoadingGraphic(width: number, height: number): { container: pixi.Container; setProgress: (fraction: number) => void } {
const container = new pixi.Container();

// Background
const bg = new pixi.Graphics();
bg.fillStyle = { color: "#0f172a", alpha: 0.92 };
bg.rect(0, 0, width, height);
bg.fill();
container.addChild(bg);
const sideMargin = Math.max(12, Math.round(width * 0.06));
const barWidth = width - sideMargin * 2;
const barHeight = Math.max(4, Math.round(height / 220));
const radius = barHeight / 2;
const barX = sideMargin;
const barY = height - Math.max(18, Math.round(height / 36)) - barHeight;

// Border
const border = new pixi.Graphics();
border.strokeStyle = { color: "#334155", width: 2 };
border.rect(1, 1, width - 2, height - 2);
border.stroke();
container.addChild(border);

const labelStyle = new pixi.TextStyle({
fontFamily: "system-ui, sans-serif",
fontSize: Math.min(48, Math.max(18, height / 16)),
fill: 0xe2e8f0,
fontWeight: "600"
const fontSize = Math.min(30, Math.max(13, Math.round(height / 42)));
const label = new pixi.Text({
text: "Loading video…",
style: new pixi.TextStyle({ fontFamily: "system-ui, sans-serif", fontSize, fill: 0xe2e8f0, fontWeight: "600" })
});
const label = new pixi.Text({ text: "Loading clip...", style: labelStyle });
label.anchor.set(0.5, 0.5);
label.x = width / 2;
label.y = height / 2 - 10;

// Small rounded backing behind the label so it stays legible over any clip content.
const padX = Math.round(fontSize * 0.6);
const padY = Math.round(fontSize * 0.3);
const labelW = label.width + padX * 2;
const labelH = label.height + padY * 2;
const labelY = barY - labelH - Math.round(barHeight * 2) - 4;
const labelBg = new pixi.Graphics();
labelBg.fillStyle = { color: "#0f172a", alpha: 0.7 };
labelBg.roundRect(barX, labelY, labelW, labelH, Math.round(labelH / 2));
labelBg.fill();
container.addChild(labelBg);
label.x = barX + padX;
label.y = labelY + padY;
container.addChild(label);

// Progress bar
const trackWidth = Math.min(360, width * 0.5);
const trackHeight = 6;
const trackX = (width - trackWidth) / 2;
const trackY = height / 2 + 60;
// Progress track, pinned to the bottom edge.
const track = new pixi.Graphics();
track.fillStyle = { color: "#1e293b", alpha: 1 };
track.rect(trackX, trackY, trackWidth, trackHeight);
track.fillStyle = { color: "#0f172a", alpha: 0.6 };
track.roundRect(barX, barY, barWidth, barHeight, radius);
track.fill();
container.addChild(track);

Expand All @@ -64,8 +62,9 @@ export function createCaptureLoadingGraphic(width: number, height: number): { co
const setProgress = (fraction: number) => {
const f = Math.max(0, Math.min(1, fraction));
fill.clear();
if (f <= 0) return;
fill.fillStyle = { color: "#34d399", alpha: 1 };
fill.rect(trackX, trackY, trackWidth * f, trackHeight);
fill.roundRect(barX, barY, Math.max(barHeight, barWidth * f), barHeight, radius);
fill.fill();
};
setProgress(0);
Expand Down
15 changes: 15 additions & 0 deletions src/core/edit-document.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import type { Size } from "@layouts/geometry";

import { resolveGoogleFontUrl } from "./fonts/font-config";
import type { Clip, Track, Edit, Soundtrack } from "./schemas";
import { setNestedValue } from "./shared/utils";

Expand Down Expand Up @@ -621,6 +622,9 @@ export class EditDocument {
const result = structuredClone(this.data);
const includeIds = options?.includeIds ?? false;

const fonts = result.timeline.fonts ?? [];
const registeredSrcs = new Set(fonts.map(f => f.src));

// Restore placeholders from document bindings before optionally stripping IDs
for (const track of result.timeline.tracks) {
for (const clip of track.clips) {
Expand All @@ -637,9 +641,20 @@ export class EditDocument {
// Strip internal ID — render API ignores it; default keeps payloads clean.
delete (clip as InternalClip).id;
}

const family = (clip.asset as { font?: { family?: string } } | undefined)?.font?.family;
const fontSrc = family ? resolveGoogleFontUrl(family) : undefined;
if (fontSrc && !registeredSrcs.has(fontSrc)) {
fonts.push({ src: fontSrc });
registeredSrcs.add(fontSrc);
}
}
}

if (fonts.length > 0) {
result.timeline.fonts = fonts;
}

if (result.merge?.length === 0) {
delete result.merge;
}
Expand Down
4 changes: 4 additions & 0 deletions src/core/fonts/font-config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ export function isGoogleFont(fontFamily: string): boolean {
return GOOGLE_FONTS_BY_FILENAME.has(fontFamily) || GOOGLE_FONTS_BY_NAME.has(fontFamily);
}

export function resolveGoogleFontUrl(fontFamily: string): string | undefined {
return (GOOGLE_FONTS_BY_FILENAME.get(fontFamily) ?? GOOGLE_FONTS_BY_NAME.get(fontFamily))?.url;
}

/**
* Get the display name for a font (resolves Google Font filename hashes to readable names)
*/
Expand Down
Loading
Loading