diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 02dcd65..857db35 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -343,6 +343,14 @@

Configuration

+
+ + +
@@ -562,6 +570,7 @@

Console Logs

modelSelect: document.getElementById('model-select'), realtimeBaseUrl: document.getElementById('realtime-base-url'), resolutionSelect: document.getElementById('resolution-select'), + codecSelect: document.getElementById('codec-select'), initialPrompt: document.getElementById('initial-prompt'), cameraFps: document.getElementById('camera-fps'), streamAudioToggle: document.getElementById('stream-audio-toggle'), @@ -891,9 +900,15 @@

Console Logs

const resolution = elements.resolutionSelect.value; addLog(`Resolution: ${resolution}`, 'info'); + const preferredVideoCodec = elements.codecSelect.value || undefined; + if (preferredVideoCodec) { + addLog(`Preferred video codec: ${preferredVideoCodec}`, 'info'); + } + decartRealtime = await decartClient.realtime.connect(localStream, { model, resolution, + ...(preferredVideoCodec && { preferredVideoCodec }), onRemoteStream: (stream) => { addLog('Received remote stream from Decart', 'success'); elements.remoteVideo.srcObject = stream; diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index a19396d..b35987d 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -12,6 +12,7 @@ import { createConsoleLogger, type Logger } from "../utils/logger"; import { imageToBase64 } from "../utils/media"; import { isDesktopSafari } from "../utils/platform"; import { createEventBuffer } from "./event-buffer"; +import type { VideoCodec } from "./media-channel"; import { realtimeMethods, type SetInput } from "./methods"; import { createMirroredStream, type MirroredStream, shouldMirrorTrack } from "./mirror-stream"; import type { DiagnosticEvent } from "./observability/diagnostics"; @@ -53,6 +54,8 @@ const realTimeClientConnectOptionsSchema = z.object({ queryParams: z.record(z.string(), z.string()).optional(), mirror: z.union([z.literal("auto"), z.boolean()]).optional(), resolution: z.enum(["720p", "1080p"]).optional(), + /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */ + preferredVideoCodec: z.enum(["h264", "vp9"]).optional(), }); export type RealTimeClientConnectOptions = Omit, "model"> & { model: ModelDefinition | CustomModelDefinition; @@ -99,7 +102,8 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options); if (!parsedOptions.success) throw parsedOptions.error; - const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data; + const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution, preferredVideoCodec } = + parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; let inputStream: MediaStream = stream ?? new MediaStream(); @@ -145,6 +149,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { }); const safariCodec = isDesktopSafari() ? "vp8" : undefined; + const publishCodec: VideoCodec | undefined = safariCodec ?? preferredVideoCodec; const queryParams = new URLSearchParams({ ...(safariCodec ? { livekit_server_codec: safariCodec } : {}), @@ -163,7 +168,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialImageRef, initialPrompt, logger, - videoCodec: safariCodec, + videoCodec: publishCodec, }); let sessionId: string | null = null; diff --git a/packages/sdk/src/realtime/config-realtime.ts b/packages/sdk/src/realtime/config-realtime.ts index 1b3d9be..a359aa6 100644 --- a/packages/sdk/src/realtime/config-realtime.ts +++ b/packages/sdk/src/realtime/config-realtime.ts @@ -33,6 +33,7 @@ export const REALTIME_CONFIG = { }, defaultVideoCodec: "h264", defaultMaxVideoBitrateBps: 3_500_000, + vp9MaxVideoBitrateBps: 3_000_000, defaultPublishFps: 30, }, observability: { diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 29cbea6..725943d 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -17,16 +17,20 @@ import type { RealtimeObservability } from "./observability/realtime-observabili export type VideoCodec = "h264" | "vp8" | "vp9" | "av1"; export function getDefaultVideoPublishOptions(videoCodec?: VideoCodec): TrackPublishOptions { - const videoEncoding = { - maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps, - maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps, - }; + const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec; + const maxBitrate = + resolvedCodec === "vp9" + ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps + : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps; return { source: Track.Source.Camera, - videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec, - simulcast: true, - videoEncoding, + videoCodec: resolvedCodec, + simulcast: resolvedCodec !== "vp9", + videoEncoding: { + maxBitrate, + maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps, + }, }; }