feat(realtime): add preferredVideoCodec connect option

packages/sdk/index.html

@@ -343,6 +343,14 @@ <h3>Configuration</h3>
                 <option value="1080p">1080p</option>
             </select>
         </div>
+        <div class="control-group">
+            <label for="codec-select">Preferred Video Codec:</label>
+            <select id="codec-select">
+                <option value="" selected>Default (h264)</option>
+                <option value="h264">h264</option>
+                <option value="vp9">vp9</option>
+            </select>
+        </div>
         <div class="control-group">
             <label for="initial-prompt">Initial Prompt (empty = passthrough):</label>
             <input type="text" id="initial-prompt" placeholder="e.g. Lego World">
@@ -562,6 +570,7 @@ <h3>Console Logs</h3>
             modelSelect: document.getElementById('model-select'),
             realtimeBaseUrl: document.getElementById('realtime-base-url'),
             resolutionSelect: document.getElementById('resolution-select'),
+            codecSelect: document.getElementById('codec-select'),
             initialPrompt: document.getElementById('initial-prompt'),
             cameraFps: document.getElementById('camera-fps'),
             streamAudioToggle: document.getElementById('stream-audio-toggle'),
@@ -891,9 +900,15 @@ <h3>Console Logs</h3>
                 const resolution = elements.resolutionSelect.value;
                 addLog(`Resolution: ${resolution}`, 'info');
 
+                const preferredVideoCodec = elements.codecSelect.value || undefined;
+                if (preferredVideoCodec) {
+                    addLog(`Preferred video codec: ${preferredVideoCodec}`, 'info');
+                }
+
                 decartRealtime = await decartClient.realtime.connect(localStream, {
                     model,
                     resolution,
+                    ...(preferredVideoCodec && { preferredVideoCodec }),
                     onRemoteStream: (stream) => {
                         addLog('Received remote stream from Decart', 'success');
                         elements.remoteVideo.srcObject = stream;

packages/sdk/src/realtime/client.ts

@@ -12,6 +12,7 @@ import { createConsoleLogger, type Logger } from "../utils/logger";
 import { imageToBase64 } from "../utils/media";
 import { isDesktopSafari } from "../utils/platform";
 import { createEventBuffer } from "./event-buffer";
+import type { VideoCodec } from "./media-channel";
 import { realtimeMethods, type SetInput } from "./methods";
 import { createMirroredStream, type MirroredStream, shouldMirrorTrack } from "./mirror-stream";
 import type { DiagnosticEvent } from "./observability/diagnostics";
@@ -53,6 +54,8 @@ const realTimeClientConnectOptionsSchema = z.object({
   queryParams: z.record(z.string(), z.string()).optional(),
   mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
   resolution: z.enum(["720p", "1080p"]).optional(),
+  /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */
+  preferredVideoCodec: z.enum(["h264", "vp9"]).optional(),
 });
 export type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
   model: ModelDefinition | CustomModelDefinition;
@@ -99,7 +102,8 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
     const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
     if (!parsedOptions.success) throw parsedOptions.error;
 
-    const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data;
+    const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution, preferredVideoCodec } =
+      parsedOptions.data;
     const mirror = parsedOptions.data.mirror ?? false;
     let inputStream: MediaStream = stream ?? new MediaStream();
 
@@ -145,6 +149,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
       });
 
       const safariCodec = isDesktopSafari() ? "vp8" : undefined;
+      const publishCodec: VideoCodec | undefined = safariCodec ?? preferredVideoCodec;
 
       const queryParams = new URLSearchParams({
         ...(safariCodec ? { livekit_server_codec: safariCodec } : {}),
@@ -163,7 +168,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         initialImageRef,
         initialPrompt,
         logger,
-        videoCodec: safariCodec,
+        videoCodec: publishCodec,
       });
 
       let sessionId: string | null = null;

packages/sdk/src/realtime/config-realtime.ts

@@ -33,6 +33,7 @@ export const REALTIME_CONFIG = {
     },
     defaultVideoCodec: "h264",
     defaultMaxVideoBitrateBps: 3_500_000,
+    vp9MaxVideoBitrateBps: 3_000_000,
     defaultPublishFps: 30,
   },
   observability: {

packages/sdk/src/realtime/media-channel.ts

@@ -17,16 +17,20 @@ import type { RealtimeObservability } from "./observability/realtime-observabili
 export type VideoCodec = "h264" | "vp8" | "vp9" | "av1";
 
 export function getDefaultVideoPublishOptions(videoCodec?: VideoCodec): TrackPublishOptions {
-  const videoEncoding = {
-    maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps,
-    maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps,
-  };
+  const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec;
+  const maxBitrate =
+    resolvedCodec === "vp9"
+      ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps
+      : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps;
 
   return {
     source: Track.Source.Camera,
-    videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec,
-    simulcast: true,
-    videoEncoding,
+    videoCodec: resolvedCodec,
+    simulcast: resolvedCodec !== "vp9",
+    videoEncoding: {
+      maxBitrate,
+      maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps,
+    },
   };
 }