From 3c4def15c0cf23ba65f6d3a43abf8066d250830a Mon Sep 17 00:00:00 2001 From: Verion1 Date: Tue, 26 May 2026 15:23:22 +0300 Subject: [PATCH 1/3] feat(realtime): add preferredVideoCodec connect option Exposes a preferredVideoCodec ("h264" | "vp9") option on realtime.connect() to control the local track publish codec. VP9 publishes use a 2.6 Mbps cap; Safari remains pinned to vp8. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/sdk/index.html | 15 +++++++++++++++ packages/sdk/src/realtime/client.ts | 11 +++++++++-- packages/sdk/src/realtime/config-realtime.ts | 1 + packages/sdk/src/realtime/media-channel.ts | 18 +++++++++++------- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 02dcd652..857db353 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -343,6 +343,14 @@

Configuration

+
+ + +
@@ -562,6 +570,7 @@

Console Logs

modelSelect: document.getElementById('model-select'), realtimeBaseUrl: document.getElementById('realtime-base-url'), resolutionSelect: document.getElementById('resolution-select'), + codecSelect: document.getElementById('codec-select'), initialPrompt: document.getElementById('initial-prompt'), cameraFps: document.getElementById('camera-fps'), streamAudioToggle: document.getElementById('stream-audio-toggle'), @@ -891,9 +900,15 @@

Console Logs

const resolution = elements.resolutionSelect.value; addLog(`Resolution: ${resolution}`, 'info'); + const preferredVideoCodec = elements.codecSelect.value || undefined; + if (preferredVideoCodec) { + addLog(`Preferred video codec: ${preferredVideoCodec}`, 'info'); + } + decartRealtime = await decartClient.realtime.connect(localStream, { model, resolution, + ...(preferredVideoCodec && { preferredVideoCodec }), onRemoteStream: (stream) => { addLog('Received remote stream from Decart', 'success'); elements.remoteVideo.srcObject = stream; diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index a19396dc..ae70e6f4 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -13,6 +13,7 @@ import { imageToBase64 } from "../utils/media"; import { isDesktopSafari } from "../utils/platform"; import { createEventBuffer } from "./event-buffer"; import { realtimeMethods, type SetInput } from "./methods"; +import type { VideoCodec } from "./media-channel"; import { createMirroredStream, type MirroredStream, shouldMirrorTrack } from "./mirror-stream"; import type { DiagnosticEvent } from "./observability/diagnostics"; import { RealtimeObservability } from "./observability/realtime-observability"; @@ -53,6 +54,7 @@ const realTimeClientConnectOptionsSchema = z.object({ queryParams: z.record(z.string(), z.string()).optional(), mirror: z.union([z.literal("auto"), z.boolean()]).optional(), resolution: z.enum(["720p", "1080p"]).optional(), + preferredVideoCodec: z.enum(["h264", "vp9"]).optional(), }); export type RealTimeClientConnectOptions = Omit, "model"> & { model: ModelDefinition | CustomModelDefinition; @@ -99,7 +101,8 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options); if (!parsedOptions.success) throw parsedOptions.error; - const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data; + const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution, preferredVideoCodec } = + parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; let inputStream: MediaStream = stream ?? new MediaStream(); @@ -145,6 +148,10 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { }); const safariCodec = isDesktopSafari() ? "vp8" : undefined; + if (safariCodec && preferredVideoCodec) { + logger.warn("preferredVideoCodec ignored on Desktop Safari; using vp8"); + } + const publishCodec: VideoCodec | undefined = safariCodec ?? preferredVideoCodec; const queryParams = new URLSearchParams({ ...(safariCodec ? { livekit_server_codec: safariCodec } : {}), @@ -163,7 +170,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialImageRef, initialPrompt, logger, - videoCodec: safariCodec, + videoCodec: publishCodec, }); let sessionId: string | null = null; diff --git a/packages/sdk/src/realtime/config-realtime.ts b/packages/sdk/src/realtime/config-realtime.ts index 1b3d9bee..7ebf9dca 100644 --- a/packages/sdk/src/realtime/config-realtime.ts +++ b/packages/sdk/src/realtime/config-realtime.ts @@ -33,6 +33,7 @@ export const REALTIME_CONFIG = { }, defaultVideoCodec: "h264", defaultMaxVideoBitrateBps: 3_500_000, + vp9MaxVideoBitrateBps: 2_600_000, defaultPublishFps: 30, }, observability: { diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 29cbea65..b611183b 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -17,16 +17,20 @@ import type { RealtimeObservability } from "./observability/realtime-observabili export type VideoCodec = "h264" | "vp8" | "vp9" | "av1"; export function getDefaultVideoPublishOptions(videoCodec?: VideoCodec): TrackPublishOptions { - const videoEncoding = { - maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps, - maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps, - }; + const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec; + const maxBitrate = + resolvedCodec === "vp9" + ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps + : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps; return { source: Track.Source.Camera, - videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec, - simulcast: true, - videoEncoding, + videoCodec: resolvedCodec, + simulcast: false, + videoEncoding: { + maxBitrate, + maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps, + }, }; } From 1b67e4a23763238dcef652a6843e90551e5c7ca8 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Tue, 26 May 2026 15:24:52 +0300 Subject: [PATCH 2/3] chore: organize imports per biome Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/sdk/src/realtime/client.ts | 2 +- packages/sdk/src/realtime/media-channel.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index ae70e6f4..8ec77495 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -12,8 +12,8 @@ import { createConsoleLogger, type Logger } from "../utils/logger"; import { imageToBase64 } from "../utils/media"; import { isDesktopSafari } from "../utils/platform"; import { createEventBuffer } from "./event-buffer"; -import { realtimeMethods, type SetInput } from "./methods"; import type { VideoCodec } from "./media-channel"; +import { realtimeMethods, type SetInput } from "./methods"; import { createMirroredStream, type MirroredStream, shouldMirrorTrack } from "./mirror-stream"; import type { DiagnosticEvent } from "./observability/diagnostics"; import { RealtimeObservability } from "./observability/realtime-observability"; diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index b611183b..c5e8b636 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -26,7 +26,7 @@ export function getDefaultVideoPublishOptions(videoCodec?: VideoCodec): TrackPub return { source: Track.Source.Camera, videoCodec: resolvedCodec, - simulcast: false, + simulcast: true, videoEncoding: { maxBitrate, maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps, From 0878f7c7a6bfc140bd2db634a153747164990d76 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Tue, 26 May 2026 15:47:52 +0300 Subject: [PATCH 3/3] address PR feedback - Set VP9 max bitrate to 3 Mbps. - Disable simulcast only for VP9 (SVC); keep simulcast on for h264/vp8/av1. - Drop the Safari override warn; document the fallback on the schema field instead. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/sdk/src/realtime/client.ts | 4 +--- packages/sdk/src/realtime/config-realtime.ts | 2 +- packages/sdk/src/realtime/media-channel.ts | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 8ec77495..b35987d6 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -54,6 +54,7 @@ const realTimeClientConnectOptionsSchema = z.object({ queryParams: z.record(z.string(), z.string()).optional(), mirror: z.union([z.literal("auto"), z.boolean()]).optional(), resolution: z.enum(["720p", "1080p"]).optional(), + /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */ preferredVideoCodec: z.enum(["h264", "vp9"]).optional(), }); export type RealTimeClientConnectOptions = Omit, "model"> & { @@ -148,9 +149,6 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { }); const safariCodec = isDesktopSafari() ? "vp8" : undefined; - if (safariCodec && preferredVideoCodec) { - logger.warn("preferredVideoCodec ignored on Desktop Safari; using vp8"); - } const publishCodec: VideoCodec | undefined = safariCodec ?? preferredVideoCodec; const queryParams = new URLSearchParams({ diff --git a/packages/sdk/src/realtime/config-realtime.ts b/packages/sdk/src/realtime/config-realtime.ts index 7ebf9dca..a359aa6f 100644 --- a/packages/sdk/src/realtime/config-realtime.ts +++ b/packages/sdk/src/realtime/config-realtime.ts @@ -33,7 +33,7 @@ export const REALTIME_CONFIG = { }, defaultVideoCodec: "h264", defaultMaxVideoBitrateBps: 3_500_000, - vp9MaxVideoBitrateBps: 2_600_000, + vp9MaxVideoBitrateBps: 3_000_000, defaultPublishFps: 30, }, observability: { diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index c5e8b636..725943df 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -26,7 +26,7 @@ export function getDefaultVideoPublishOptions(videoCodec?: VideoCodec): TrackPub return { source: Track.Source.Camera, videoCodec: resolvedCodec, - simulcast: true, + simulcast: resolvedCodec !== "vp9", videoEncoding: { maxBitrate, maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps,