From aca3c826491e7c447c040418aa470d1d50fe2cfc Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 20 May 2026 15:52:13 +0300 Subject: [PATCH 1/7] feat(realtime): expose granular server error types and payloads Previously, real-time server errors were generic. This change introduces `RealtimeWebSocketErrorType` and attaches it, along with the full server payload, to `ServerError` objects. New `REALTIME_` error codes are added to `DecartSDKError` to enable more specific error handling and debugging for different real-time issues like invalid API keys, insufficient credits, or moderation violations. --- packages/sdk/src/index.ts | 11 ++- .../sdk/src/realtime/signaling-channel.ts | 6 +- packages/sdk/src/realtime/types.ts | 27 +++++- packages/sdk/src/utils/errors.ts | 60 +++++++++++++- packages/sdk/tests/realtime.unit.test.ts | 82 ++++++++++++++++++- 5 files changed, 175 insertions(+), 11 deletions(-) diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index b2444264..6c911927 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -41,7 +41,14 @@ export type { SubscribeEvents, SubscribeOptions, } from "./realtime/subscribe-client"; -export type { ConnectionState, GenerationEndedMessage, QueuePosition, QueuePositionMessage } from "./realtime/types"; +export type { + ConnectionState, + GenerationEndedMessage, + QueuePosition, + QueuePositionMessage, + RealtimeWebSocketErrorMessage, + RealtimeWebSocketErrorType, +} from "./realtime/types"; export { type CanonicalModel, type CustomModelDefinition, @@ -67,7 +74,7 @@ export { } from "./shared/model"; export type { ModelState } from "./shared/types"; export type { CreateTokenOptions, CreateTokenResponse, TokensClient } from "./tokens/client"; -export { type DecartSDKError, ERROR_CODES } from "./utils/errors"; +export { type DecartSDKError, ERROR_CODES, type RealtimeServerErrorData } from "./utils/errors"; export { createConsoleLogger, type Logger, type LogLevel, noopLogger } from "./utils/logger"; // Schema with validation to ensure proxy and apiKey are mutually exclusive diff --git a/packages/sdk/src/realtime/signaling-channel.ts b/packages/sdk/src/realtime/signaling-channel.ts index 82216fe0..f90e0e00 100644 --- a/packages/sdk/src/realtime/signaling-channel.ts +++ b/packages/sdk/src/realtime/signaling-channel.ts @@ -30,7 +30,7 @@ export type SignalingChannelEvents = { queuePosition: QueuePosition; generationTick: GenerationTick; generationEnded: GenerationEnded; - serverError: Error; + serverError: ServerError; closed: ConnectionClosed; }; @@ -364,7 +364,9 @@ export class SignalingChannel { case "error": { const error = new Error(msg.error) as ServerError; error.source = "server"; - this.logger.error("signaling: server error received", { error: msg.error }); + error.errorType = msg.error_type; + error.serverPayload = msg; + this.logger.error("signaling: server error received", { error: msg.error, errorType: msg.error_type }); this.events.emit("serverError", error); this.rejectPendingRoomInfo(error); this.rejectAllPending(error); diff --git a/packages/sdk/src/realtime/types.ts b/packages/sdk/src/realtime/types.ts index d5f7776e..dcb9b2a8 100644 --- a/packages/sdk/src/realtime/types.ts +++ b/packages/sdk/src/realtime/types.ts @@ -11,10 +11,31 @@ export type PromptAckMessage = { error: null | string; }; -export type ErrorMessage = { +export type RealtimeWebSocketErrorType = + | "invalid_api_key" + | "origin_not_allowed" + | "invalid_model" + | "removed_model" + | "model_not_available_for_trial" + | "insufficient_credits" + | "upstream_capacity" + | "upstream_rejected" + | "upstream_timeout" + | "model_server_disconnected" + | "model_setup_timeout" + | "session_duration_limit" + | "session_not_found" + | "server_shutdown" + | "moderation_violation" + | "internal_error"; + +export type RealtimeWebSocketErrorMessage = { type: "error"; error: string; -}; + error_type?: RealtimeWebSocketErrorType; +} & Record; + +export type ErrorMessage = RealtimeWebSocketErrorMessage; export type SetImageMessage = { type: "set_image"; @@ -99,6 +120,8 @@ export type InitialPrompt = { export type ServerError = Error & { source?: string; + errorType?: RealtimeWebSocketErrorType; + serverPayload?: RealtimeWebSocketErrorMessage; }; export type PromptSendOptions = { diff --git a/packages/sdk/src/utils/errors.ts b/packages/sdk/src/utils/errors.ts index a8cecd05..c2a3c93f 100644 --- a/packages/sdk/src/utils/errors.ts +++ b/packages/sdk/src/utils/errors.ts @@ -1,3 +1,5 @@ +import type { RealtimeWebSocketErrorMessage, RealtimeWebSocketErrorType, ServerError } from "../realtime/types"; + export type DecartSDKError = { code: string; message: string; @@ -5,6 +7,11 @@ export type DecartSDKError = { cause?: Error; }; +export type RealtimeServerErrorData = { + errorType?: RealtimeWebSocketErrorType; + serverPayload?: RealtimeWebSocketErrorMessage; +}; + export const ERROR_CODES = { INVALID_API_KEY: "INVALID_API_KEY", INVALID_BASE_URL: "INVALID_BASE_URL", @@ -23,8 +30,44 @@ export const ERROR_CODES = { WEBRTC_TIMEOUT_ERROR: "WEBRTC_TIMEOUT_ERROR", WEBRTC_SERVER_ERROR: "WEBRTC_SERVER_ERROR", WEBRTC_SIGNALING_ERROR: "WEBRTC_SIGNALING_ERROR", + // Realtime server error codes + REALTIME_INVALID_API_KEY: "REALTIME_INVALID_API_KEY", + REALTIME_ORIGIN_NOT_ALLOWED: "REALTIME_ORIGIN_NOT_ALLOWED", + REALTIME_INVALID_MODEL: "REALTIME_INVALID_MODEL", + REALTIME_REMOVED_MODEL: "REALTIME_REMOVED_MODEL", + REALTIME_MODEL_NOT_AVAILABLE_FOR_TRIAL: "REALTIME_MODEL_NOT_AVAILABLE_FOR_TRIAL", + REALTIME_INSUFFICIENT_CREDITS: "REALTIME_INSUFFICIENT_CREDITS", + REALTIME_UPSTREAM_CAPACITY: "REALTIME_UPSTREAM_CAPACITY", + REALTIME_UPSTREAM_REJECTED: "REALTIME_UPSTREAM_REJECTED", + REALTIME_UPSTREAM_TIMEOUT: "REALTIME_UPSTREAM_TIMEOUT", + REALTIME_MODEL_SERVER_DISCONNECTED: "REALTIME_MODEL_SERVER_DISCONNECTED", + REALTIME_MODEL_SETUP_TIMEOUT: "REALTIME_MODEL_SETUP_TIMEOUT", + REALTIME_SESSION_DURATION_LIMIT: "REALTIME_SESSION_DURATION_LIMIT", + REALTIME_SESSION_NOT_FOUND: "REALTIME_SESSION_NOT_FOUND", + REALTIME_SERVER_SHUTDOWN: "REALTIME_SERVER_SHUTDOWN", + REALTIME_MODERATION_VIOLATION: "REALTIME_MODERATION_VIOLATION", + REALTIME_INTERNAL_ERROR: "REALTIME_INTERNAL_ERROR", } as const; +const REALTIME_SERVER_ERROR_CODES = { + invalid_api_key: ERROR_CODES.REALTIME_INVALID_API_KEY, + origin_not_allowed: ERROR_CODES.REALTIME_ORIGIN_NOT_ALLOWED, + invalid_model: ERROR_CODES.REALTIME_INVALID_MODEL, + removed_model: ERROR_CODES.REALTIME_REMOVED_MODEL, + model_not_available_for_trial: ERROR_CODES.REALTIME_MODEL_NOT_AVAILABLE_FOR_TRIAL, + insufficient_credits: ERROR_CODES.REALTIME_INSUFFICIENT_CREDITS, + upstream_capacity: ERROR_CODES.REALTIME_UPSTREAM_CAPACITY, + upstream_rejected: ERROR_CODES.REALTIME_UPSTREAM_REJECTED, + upstream_timeout: ERROR_CODES.REALTIME_UPSTREAM_TIMEOUT, + model_server_disconnected: ERROR_CODES.REALTIME_MODEL_SERVER_DISCONNECTED, + model_setup_timeout: ERROR_CODES.REALTIME_MODEL_SETUP_TIMEOUT, + session_duration_limit: ERROR_CODES.REALTIME_SESSION_DURATION_LIMIT, + session_not_found: ERROR_CODES.REALTIME_SESSION_NOT_FOUND, + server_shutdown: ERROR_CODES.REALTIME_SERVER_SHUTDOWN, + moderation_violation: ERROR_CODES.REALTIME_MODERATION_VIOLATION, + internal_error: ERROR_CODES.REALTIME_INTERNAL_ERROR, +} as const satisfies Record; + export function createSDKError( code: string, message: string, @@ -63,8 +106,13 @@ export function createWebrtcTimeoutError(phase: string, timeoutMs?: number, caus ); } -export function createWebrtcServerError(message: string): DecartSDKError { - return createSDKError(ERROR_CODES.WEBRTC_SERVER_ERROR, message); +export function createWebrtcServerError( + message: string, + data?: RealtimeServerErrorData, + cause?: Error, +): DecartSDKError { + const code = data?.errorType ? REALTIME_SERVER_ERROR_CODES[data.errorType] : ERROR_CODES.WEBRTC_SERVER_ERROR; + return createSDKError(code, message, data, cause); } export function createWebrtcSignalingError(error: Error): DecartSDKError { @@ -76,10 +124,14 @@ export function createWebrtcSignalingError(error: Error): DecartSDKError { */ export function classifyWebrtcError(error: Error): DecartSDKError { const msg = error.message.toLowerCase(); - const source = (error as Error & { source?: string }).source; + const serverError = error as ServerError; + const source = serverError.source; if (source === "server") { - return createWebrtcServerError(error.message); + const data: RealtimeServerErrorData = {}; + if (serverError.errorType) data.errorType = serverError.errorType; + if (serverError.serverPayload) data.serverPayload = serverError.serverPayload; + return createWebrtcServerError(error.message, Object.keys(data).length > 0 ? data : undefined, error); } if (msg.includes("websocket")) { diff --git a/packages/sdk/tests/realtime.unit.test.ts b/packages/sdk/tests/realtime.unit.test.ts index deee25c1..c138822e 100644 --- a/packages/sdk/tests/realtime.unit.test.ts +++ b/packages/sdk/tests/realtime.unit.test.ts @@ -519,6 +519,44 @@ describe("SignalingChannel initial handshake", () => { await expect(initialStateAck).rejects.toThrow("initial state failed"); }); + it("preserves typed server error payloads", async () => { + const { SignalingChannel } = await import("../src/realtime/signaling-channel.js"); + const channel = new SignalingChannel({ url: "wss://example.test/realtime" }); + const serverErrors: ServerError[] = []; + channel.on("serverError", (error) => serverErrors.push(error)); + + const openPromise = channel.openAndJoin({ + initialState: { image: "base64-image" }, + }); + + const ws = FakeWebSocket.instances[0]; + ws.onopen?.(); + await flushMicrotasks(); + + ws.receive({ + type: "livekit_room_info", + livekit_url: "wss://livekit.example.test", + token: "token", + room_name: "room", + session_id: "session", + }); + + const { initialStateAck } = await openPromise; + const payload = { + type: "error", + error: "moderation_violation", + error_type: "moderation_violation", + provider_response: { category: "safety" }, + } as const; + ws.receive(payload); + + await expect(initialStateAck).rejects.toThrow("moderation_violation"); + expect(serverErrors).toHaveLength(1); + expect(serverErrors[0].source).toBe("server"); + expect(serverErrors[0].errorType).toBe("moderation_violation"); + expect(serverErrors[0].serverPayload).toEqual(payload); + }); + it("rejects pending initial-state ack on close", async () => { const { SignalingChannel } = await import("../src/realtime/signaling-channel.js"); const channel = new SignalingChannel({ url: "wss://example.test/realtime" }); @@ -963,9 +1001,51 @@ describe("WebRTC Error Classification", () => { const { classifyWebrtcError, ERROR_CODES } = await import("../src/utils/errors.js"); const error = new Error("Insufficient credits") as ServerError; error.source = "server"; + error.errorType = "insufficient_credits"; + error.serverPayload = { + type: "error", + error: "Insufficient credits", + error_type: "insufficient_credits", + }; const result = classifyWebrtcError(error); - expect(result.code).toBe(ERROR_CODES.WEBRTC_SERVER_ERROR); + expect(result.code).toBe(ERROR_CODES.REALTIME_INSUFFICIENT_CREDITS); expect(result.message).toBe("Insufficient credits"); + expect(result.data).toEqual({ + errorType: "insufficient_credits", + serverPayload: { + type: "error", + error: "Insufficient credits", + error_type: "insufficient_credits", + }, + }); + }); + + it("classifies moderation violations with a specific realtime error code", async () => { + const { classifyWebrtcError, ERROR_CODES } = await import("../src/utils/errors.js"); + const error = new Error("moderation_violation") as ServerError; + error.source = "server"; + error.errorType = "moderation_violation"; + error.serverPayload = { + type: "error", + error: "moderation_violation", + error_type: "moderation_violation", + }; + + const result = classifyWebrtcError(error); + + expect(result.code).toBe(ERROR_CODES.REALTIME_MODERATION_VIOLATION); + expect(result.data?.errorType).toBe("moderation_violation"); + }); + + it("keeps legacy server errors on the generic server error code", async () => { + const { classifyWebrtcError, ERROR_CODES } = await import("../src/utils/errors.js"); + const error = new Error("Server overloaded") as ServerError; + error.source = "server"; + + const result = classifyWebrtcError(error); + + expect(result.code).toBe(ERROR_CODES.WEBRTC_SERVER_ERROR); + expect(result.data).toBeUndefined(); }); it("classifies unknown errors as signaling errors", async () => { From 42ea6a2b9adbb7b9221c41ef56f99122144a675d Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 20 May 2026 20:23:20 +0300 Subject: [PATCH 2/7] testing logic --- packages/sdk/index.html | 579 +++++++++++++++++- packages/sdk/src/realtime/client.ts | 19 +- packages/sdk/src/realtime/media-channel.ts | 76 ++- packages/sdk/src/realtime/stream-session.ts | 9 + packages/sdk/src/realtime/subscribe-client.ts | 21 + 5 files changed, 692 insertions(+), 12 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index c9d5c3a2..7d38f7fb 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -343,6 +343,21 @@

Configuration

+
+ + +
+
@@ -362,13 +377,21 @@

Configuration

+
+ + +
+ + + 0s
+

Subscribe / Viewer Mode

@@ -511,6 +534,187 @@

Console Logs

let currentSessionId = null; let currentSubscribeToken = null; let model = models.realtime("lucy-2.1"); + + // ----------------------------------------------------------------------- + // LiveKit publish profiles for 1:1 streaming to the AI inference server. + // Profile A-B and F-H candidates keep H.264 and maxFramerate: 30. + // The default entry intentionally preserves the SDK's simulcast baseline. + // For a single backend consumer, single-layer H.264 is the expected best path; + // simulcast remains here only as a control to measure its overhead. + // Profiles C-E keep the VP9/AV1 SVC candidates for comparison. + // ----------------------------------------------------------------------- + const PUBLISH_PROFILES = { + default: { + label: 'H.264 + simulcast (current SDK default)', + description: 'Same as Profile A but with simulcast enabled — represents what the SDK ships today before any profile tuning.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: true, + videoEncoding: { + maxBitrate: 2_500_000, + maxFramerate: 30, + }, + degradationPreference: 'balanced', + }, + }, + A: { + label: 'H.264 baseline, no simulcast (balanced)', + description: 'H.264 single layer, default degradation strategy. The control profile.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { + maxBitrate: 2_500_000, + maxFramerate: 30, + }, + degradationPreference: 'balanced', + }, + }, + B: { + label: 'H.264 + maintain-framerate, priority=high', + description: 'H.264 single layer, drops resolution before FPS under pressure, DSCP-prioritized.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { + maxBitrate: 3_000_000, + maxFramerate: 30, + priority: 'high', + }, + degradationPreference: 'maintain-framerate', + }, + }, + C: { + label: 'VP9 SVC L1T3 (temporal only)', + description: 'VP9 with 3 temporal layers (single resolution). Loss recovery without spatial encode cost.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'vp9', + scalabilityMode: 'L1T3', + simulcast: false, + videoEncoding: { + maxBitrate: 1_500_000, + maxFramerate: 30, + }, + backupCodec: { codec: 'h264' }, + degradationPreference: 'maintain-framerate', + }, + }, + D: { + label: 'VP9 SVC L3T3_KEY (spatial+temporal)', + description: 'VP9 full SVC: 3 spatial x 3 temporal layers. Best resilience on bad networks.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'vp9', + scalabilityMode: 'L3T3_KEY', + simulcast: false, + videoEncoding: { + maxBitrate: 1_800_000, + maxFramerate: 30, + }, + backupCodec: { codec: 'h264' }, + degradationPreference: 'maintain-framerate', + }, + }, + E: { + label: 'AV1 SVC L3T3_KEY (compression-first)', + description: 'AV1 full SVC. Best compression, but watch client CPU. Falls back to H.264 if unsupported.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'av1', + scalabilityMode: 'L3T3_KEY', + simulcast: false, + videoEncoding: { + maxBitrate: 1_000_000, + maxFramerate: 30, + }, + backupCodec: { codec: 'h264' }, + degradationPreference: 'maintain-framerate', + }, + }, + F: { + label: 'H.264 constrained, maintain-framerate', + description: 'H.264 single layer with a lower bitrate ceiling for weaker networks; drops resolution before FPS.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { + maxBitrate: 1_800_000, + maxFramerate: 30, + priority: 'high', + }, + degradationPreference: 'maintain-framerate', + }, + }, + G: { + label: 'H.264 high bitrate, maintain-framerate', + description: 'H.264 single layer with more bitrate headroom for clean networks while still preserving FPS under pressure.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { + maxBitrate: 2_000_000, + maxFramerate: 30, + priority: 'high', + }, + degradationPreference: 'maintain-resolution', + }, + }, + H: { + label: 'H.264 adaptive receive, maintain-framerate', + description: 'H.264 single layer plus adaptiveStream for the processed-video subscription; upstream to the server remains single-layer H.264.', + roomOptions: { + adaptiveStream: true, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { + maxBitrate: 2_500_000, + maxFramerate: 30, + priority: 'high', + }, + degradationPreference: 'maintain-framerate', + }, + }, + }; + + function getSelectedPublishProfile() { + const key = elements.publishProfileSelect.value; + const profile = PUBLISH_PROFILES[key]; + if (!profile) return { key: 'default', ...PUBLISH_PROFILES.default }; + return { key, ...profile }; + } // DOM elements const elements = { @@ -518,12 +722,18 @@

Console Logs

modelSelect: document.getElementById('model-select'), realtimeBaseUrl: document.getElementById('realtime-base-url'), resolutionSelect: document.getElementById('resolution-select'), + publishProfileSelect: document.getElementById('publish-profile-select'), + publishProfileDescription: document.getElementById('publish-profile-description'), initialPrompt: document.getElementById('initial-prompt'), cameraFps: document.getElementById('camera-fps'), streamAudioToggle: document.getElementById('stream-audio-toggle'), + sourceFile: document.getElementById('source-file'), + startFileSource: document.getElementById('start-file-source'), startCamera: document.getElementById('start-camera'), connectBtn: document.getElementById('connect-btn'), disconnectBtn: document.getElementById('disconnect-btn'), + printStatsBtn: document.getElementById('print-stats-btn'), + sessionTimer: document.getElementById('session-timer'), localVideo: document.getElementById('local-video'), remoteVideo: document.getElementById('remote-video'), promptInput: document.getElementById('prompt-input'), @@ -562,7 +772,7 @@

Console Logs

processStatusText: document.getElementById('process-status-text'), processedContainer: document.getElementById('processed-container'), originalVideo: document.getElementById('original-video'), - processedVideo: document.getElementById('processed-video') + processedVideo: document.getElementById('processed-video'), }; // Pre-populate API key from environment variable if available @@ -638,6 +848,231 @@

Console Logs

}; } + // --- Stats aggregator: tracks running sum + count per numeric field, per group key. --- + const STATS_POLL_INTERVAL_MS = 5000; + let statsPollTimer = null; + let statsRunCount = 0; + // Map> + const statsAggregates = new Map(); + + function aggregateStatObject(groupKey, statObj) { + if (!statObj) return; + let groupAgg = statsAggregates.get(groupKey); + if (!groupAgg) { + groupAgg = new Map(); + statsAggregates.set(groupKey, groupAgg); + } + for (const [field, value] of Object.entries(statObj)) { + if (typeof value !== 'number' || !Number.isFinite(value)) continue; + const slot = groupAgg.get(field) ?? { sum: 0, count: 0 }; + slot.sum += value; + slot.count += 1; + groupAgg.set(field, slot); + } + } + + function formatStatsForLog(statObj) { + const slim = {}; + for (const [k, v] of Object.entries(statObj)) { + if (v === null || v === undefined) continue; + if (typeof v === 'object') continue; + slim[k] = v; + } + return JSON.stringify(slim); + } + + async function pollAndAggregateStats() { + if (!decartRealtime?.getVideoStats) return; + let result; + try { + result = await decartRealtime.getVideoStats(); + } catch (error) { + addLog(`Stats poll failed: ${error.message}`, 'error'); + return; + } + statsRunCount += 1; + const { sender = [], receiver = [] } = result; + if (sender.length === 0 && receiver.length === 0) { + addLog(`[stats #${statsRunCount}] no tracks yet`, 'info'); + return; + } + sender.forEach((stats, idx) => { + const key = `sender[${stats.rid || idx}]`; + aggregateStatObject(key, stats); + addLog(`[stats #${statsRunCount}] ${key} ${formatStatsForLog(stats)}`, 'info'); + }); + receiver.forEach((stats, idx) => { + const key = `receiver[${idx}]`; + aggregateStatObject(key, stats); + addLog(`[stats #${statsRunCount}] ${key} ${formatStatsForLog(stats)}`, 'info'); + }); + } + + // --- Session timer --- + let sessionTimerInterval = null; + let sessionStartedAt = 0; + + function startSessionTimer() { + stopSessionTimer(); + sessionStartedAt = Date.now(); + elements.sessionTimer.textContent = '0s'; + sessionTimerInterval = setInterval(() => { + const seconds = Math.floor((Date.now() - sessionStartedAt) / 1000); + elements.sessionTimer.textContent = `${seconds}s`; + }, 1000); + } + + function stopSessionTimer() { + if (sessionTimerInterval) { + clearInterval(sessionTimerInterval); + sessionTimerInterval = null; + } + } + + // --- Prompt cycler --- + const CYCLE_PROMPTS = [ + 'add a red baseball cap', + 'put a black leather jacket on the person', + 'replace the shirt with a white tank top', + 'add round sunglasses', + 'add a knitted winter scarf', + 'put a yellow raincoat on the person', + 'add a gold chain necklace', + ]; + const PROMPT_CYCLE_INTERVAL_MS = 10000; + let promptCycleTimer = null; + let promptCycleIndex = 0; + + function sendCyclePrompt() { + if (!decartRealtime || !isConnected || activeConnectionMode !== 'publisher') return; + const prompt = CYCLE_PROMPTS[promptCycleIndex % CYCLE_PROMPTS.length]; + promptCycleIndex += 1; + try { + decartRealtime.setPrompt(prompt, { enhance: true }); + addLog(`[cycle ${promptCycleIndex}/${CYCLE_PROMPTS.length}] prompt → "${prompt}"`, 'info'); + } catch (error) { + addLog(`Cycle prompt failed: ${error.message}`, 'error'); + } + } + + function startPromptCycler() { + stopPromptCycler(); + promptCycleIndex = 0; + sendCyclePrompt(); + promptCycleTimer = setInterval(sendCyclePrompt, PROMPT_CYCLE_INTERVAL_MS); + addLog(`Started prompt cycler (${CYCLE_PROMPTS.length} prompts × ${PROMPT_CYCLE_INTERVAL_MS / 1000}s)`, 'info'); + } + + function stopPromptCycler() { + if (promptCycleTimer) { + clearInterval(promptCycleTimer); + promptCycleTimer = null; + } + } + + function startStatsPolling() { + stopStatsPolling(); + statsPollTimer = setInterval(pollAndAggregateStats, STATS_POLL_INTERVAL_MS); + elements.printStatsBtn.disabled = false; + startSessionTimer(); + addLog(`Started stats polling every ${STATS_POLL_INTERVAL_MS / 1000}s`, 'info'); + } + + function stopStatsPolling() { + if (statsPollTimer) { + clearInterval(statsPollTimer); + statsPollTimer = null; + } + } + + function resetStatsAggregates() { + statsAggregates.clear(); + statsRunCount = 0; + addLog('Cleared stats aggregates', 'info'); + } + + function buildStatsSummary() { + const summary = { samples: statsRunCount }; + for (const [groupKey, fields] of statsAggregates) { + const group = {}; + for (const [field, { sum, count }] of fields) { + group[field] = sum; + } + summary[groupKey] = group; + } + return summary; + } + + function printStatsSummary() { + const summary = buildStatsSummary(); + console.log('[Decart SDK] Stats averages:', summary); + addLog('Stats averages printed to console (see DevTools for full object)', 'success'); + } + + // --- MediaRecorder for the Decart remote stream --- + let mediaRecorder = null; + let recordedChunks = []; + + function pickSupportedMimeType() { + const candidates = [ + 'video/webm;codecs=vp9,opus', + 'video/webm;codecs=vp8,opus', + 'video/webm;codecs=h264,opus', + 'video/webm', + 'video/mp4', + ]; + return candidates.find((m) => window.MediaRecorder?.isTypeSupported?.(m)) || ''; + } + + function startRecording() { + if (mediaRecorder) return; + const remoteStream = elements.remoteVideo.srcObject; + if (!(remoteStream instanceof MediaStream) || remoteStream.getVideoTracks().length === 0) { + addLog('No remote stream available to record', 'error'); + return; + } + if (!window.MediaRecorder) { + addLog('MediaRecorder is not supported in this browser', 'error'); + return; + } + const mimeType = pickSupportedMimeType(); + recordedChunks = []; + try { + mediaRecorder = new MediaRecorder(remoteStream, mimeType ? { mimeType } : undefined); + } catch (error) { + addLog(`Failed to create MediaRecorder: ${error.message}`, 'error'); + mediaRecorder = null; + return; + } + mediaRecorder.ondataavailable = (event) => { + if (event.data && event.data.size > 0) recordedChunks.push(event.data); + }; + mediaRecorder.onstop = () => { + const finalType = mediaRecorder?.mimeType || 'video/webm'; + const blob = new Blob(recordedChunks, { type: finalType }); + const ext = finalType.includes('mp4') ? 'mp4' : 'webm'; + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `decart_remote_${Date.now()}.${ext}`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setTimeout(() => URL.revokeObjectURL(url), 1000); + addLog(`Recording stopped. Downloaded ${(blob.size / 1024 / 1024).toFixed(2)} MB (${ext})`, 'success'); + recordedChunks = []; + mediaRecorder = null; + }; + mediaRecorder.start(1000); + addLog(`Auto-recording remote stream (${mediaRecorder.mimeType || 'default'})`, 'success'); + } + + function stopRecording() { + if (mediaRecorder && mediaRecorder.state !== 'inactive') { + mediaRecorder.stop(); + } + } + // Update connection status function updateStatus(status) { const statusClass = { @@ -685,6 +1120,21 @@

Console Logs

setTimeout(() => waitForPublisherSubscribeToken(attemptsRemaining - 1), 250); } + // Publish profile selection: update the inline description and log on change. + function formatProfileTitle({ key, label }) { + return key === 'default' ? `Default baseline — ${label}` : `Profile ${key} — ${label}`; + } + function updatePublishProfileDescription() { + const profile = getSelectedPublishProfile(); + elements.publishProfileDescription.textContent = `${formatProfileTitle(profile)}: ${profile.description}`; + } + elements.publishProfileSelect.addEventListener('change', () => { + const profile = getSelectedPublishProfile(); + updatePublishProfileDescription(); + addLog(`Selected publish profile: ${formatProfileTitle(profile)}`, 'info'); + }); + updatePublishProfileDescription(); + // Model selection handler elements.modelSelect.addEventListener('change', (e) => { const selectedModel = e.target.value; @@ -696,6 +1146,7 @@

Console Logs

if (localStream) { stopCameraResources(); elements.startCamera.disabled = false; + elements.startFileSource.disabled = !elements.sourceFile.files[0]; elements.connectBtn.disabled = true; addLog('Camera stopped. Please restart camera with new model settings.', 'info'); } @@ -730,6 +1181,71 @@

Console Logs

}; } + let fileSource = null; + + function stopFileSource() { + if (!fileSource) return; + if (fileSource.rafId) cancelAnimationFrame(fileSource.rafId); + try { fileSource.video.pause(); } catch (_) {} + fileSource.video.removeAttribute('src'); + fileSource.video.load(); + try { URL.revokeObjectURL(fileSource.objectUrl); } catch (_) {} + fileSource = null; + } + + async function beginFileSource(file) { + stopCameraResources(); + stopFileSource(); + + const camera = getSelectedCameraSettings(); + const video = document.createElement('video'); + video.muted = true; + video.loop = true; + video.playsInline = true; + video.autoplay = true; + const objectUrl = URL.createObjectURL(file); + video.src = objectUrl; + + await new Promise((resolve, reject) => { + video.addEventListener('loadedmetadata', resolve, { once: true }); + video.addEventListener('error', () => reject(new Error('Failed to load video file')), { once: true }); + }); + await video.play(); + + const canvas = document.createElement('canvas'); + canvas.width = camera.width; + canvas.height = camera.height; + const ctx = canvas.getContext('2d'); + + const drawCoverFrame = () => { + if (!fileSource) return; + const vw = video.videoWidth; + const vh = video.videoHeight; + if (vw && vh) { + const scale = Math.max(canvas.width / vw, canvas.height / vh); + const dw = vw * scale; + const dh = vh * scale; + const dx = (canvas.width - dw) / 2; + const dy = (canvas.height - dh) / 2; + ctx.drawImage(video, dx, dy, dw, dh); + } + fileSource.rafId = requestAnimationFrame(drawCoverFrame); + }; + + const stream = canvas.captureStream(camera.fps); + fileSource = { video, canvas, objectUrl, rafId: 0, stream }; + drawCoverFrame(); + + localStream = stream; + elements.localVideo.srcObject = stream; + await elements.localVideo.play(); + + addLog(`File source streaming: ${file.name} → ${camera.width}×${camera.height} @ ${camera.fps}fps (looped)`, 'success'); + elements.startCamera.disabled = true; + elements.startFileSource.disabled = true; + elements.connectBtn.disabled = false; + } + function stopCameraResources() { if (localStream) { localStream.getTracks().forEach((track) => { @@ -737,6 +1253,7 @@

Console Logs

}); localStream = null; } + stopFileSource(); elements.localVideo.srcObject = null; } @@ -772,7 +1289,27 @@

Console Logs

stopCameraResources(); } }); - + + elements.sourceFile.addEventListener('change', () => { + elements.startFileSource.disabled = !elements.sourceFile.files[0]; + }); + + elements.startFileSource.addEventListener('click', async () => { + const file = elements.sourceFile.files[0]; + if (!file) { + addLog('Please choose a video file to stream', 'error'); + return; + } + try { + await beginFileSource(file); + } catch (error) { + addLog(`Failed to start file source: ${error.message}`, 'error'); + stopCameraResources(); + } + }); + + elements.printStatsBtn.addEventListener('click', printStatsSummary); + // Connect to Decart elements.connectBtn.addEventListener('click', async () => { const apiKey = elements.apiKey.value.trim(); @@ -828,9 +1365,16 @@

Console Logs

const resolution = elements.resolutionSelect.value; addLog(`Resolution: ${resolution}`, 'info'); + const profile = getSelectedPublishProfile(); + addLog(`Publish profile: ${formatProfileTitle(profile)}`, 'info'); + addLog(`Publish options: ${JSON.stringify(profile.publishOptions)}`, 'info'); + addLog(`Room options: ${JSON.stringify(profile.roomOptions)}`, 'info'); + decartRealtime = await decartClient.realtime.connect(localStream, { model, resolution, + publishOptions: profile.publishOptions, + roomOptions: profile.roomOptions, onRemoteStream: (stream) => { addLog('Received remote stream from Decart', 'success'); elements.remoteVideo.srcObject = stream; @@ -840,6 +1384,10 @@

Console Logs

if (state === 'connected' || state === 'generating') { syncPublisherSubscribeToken(); } + if (state === 'generating') { + startRecording(); + startPromptCycler(); + } }, onQueuePosition: (queuePosition) => { addLog(`Queue position: ${queuePosition.position}/${queuePosition.queueSize}`, 'info'); @@ -866,7 +1414,10 @@

Console Logs

syncPublisherSubscribeToken(); waitForPublisherSubscribeToken(); elements.sessionInfo.style.display = 'block'; - + + resetStatsAggregates(); + startStatsPolling(); + addLog('Successfully connected to Decart!', 'success'); addLog(`Session ID: ${currentSessionId || '(pending)'}`, 'success'); } else { @@ -885,6 +1436,9 @@

Console Logs

elements.sessionInfo.style.display = 'none'; elements.publisherTokenInfo.style.display = 'none'; elements.subscribeBtn.disabled = false; + stopStatsPolling(); + stopPromptCycler(); + stopSessionTimer(); } }); @@ -945,6 +1499,7 @@

Console Logs

onRemoteStream: (stream) => { addLog('Received subscribed stream from Decart', 'success'); elements.remoteVideo.srcObject = stream; + startRecording(); }, onConnectionChange: (state) => { updateStatus(state); @@ -966,6 +1521,8 @@

Console Logs

elements.setImage.disabled = true; elements.sessionInfo.style.display = 'none'; elements.publisherTokenInfo.style.display = 'none'; + resetStatsAggregates(); + startStatsPolling(); addLog('Successfully subscribed to stream. No local media was requested or published.', 'success'); } else { throw new Error('Subscribe established but not in connected state'); @@ -979,22 +1536,32 @@

Console Logs

activeConnectionMode = null; elements.subscribeBtn.disabled = false; elements.startCamera.disabled = false; + stopStatsPolling(); + stopSessionTimer(); } }); // Disconnect elements.disconnectBtn.addEventListener('click', () => { + stopRecording(); + stopStatsPolling(); + stopPromptCycler(); + stopSessionTimer(); + if (statsAggregates.size > 0) { + printStatsSummary(); + } + if (decartRealtime) { addLog('Disconnecting from Decart...', 'info'); decartRealtime.disconnect(); decartRealtime = null; decartClient = null; } - + stopCameraResources(); elements.remoteVideo.srcObject = null; - + isConnected = false; updateStatus('disconnected'); currentSessionId = null; @@ -1003,9 +1570,11 @@

Console Logs

// Reset UI elements.startCamera.disabled = false; + elements.startFileSource.disabled = !elements.sourceFile.files[0]; elements.connectBtn.disabled = true; elements.subscribeBtn.disabled = false; elements.disconnectBtn.disabled = true; + elements.printStatsBtn.disabled = true; elements.promptInput.disabled = true; elements.sendPrompt.disabled = true; elements.promptInput.value = ''; diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 73be6b3c..178868e5 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -18,6 +18,7 @@ import { RealtimeObservability } from "./observability/realtime-observability"; import type { WebRTCStats } from "./observability/webrtc-stats"; import { StreamSession } from "./stream-session"; import type { ConnectionState, GenerationEnded, GenerationTick, ImageSetOptions, QueuePosition } from "./types"; +import type { RoomOptions, TrackPublishOptions, VideoReceiverStats, VideoSenderStats } from "livekit-client"; export type RealTimeClientOptions = { baseUrl: string; @@ -52,6 +53,10 @@ const realTimeClientConnectOptionsSchema = z.object({ queryParams: z.record(z.string(), z.string()).optional(), mirror: z.union([z.literal("auto"), z.boolean()]).optional(), resolution: z.enum(["720p", "1080p"]).optional(), + publishOptions: z + .custom>((val) => val === undefined || typeof val === "object") + .optional(), + roomOptions: z.custom>((val) => val === undefined || typeof val === "object").optional(), }); export type RealTimeClientConnectOptions = Omit, "model"> & { model: ModelDefinition | CustomModelDefinition; @@ -79,6 +84,7 @@ export type RealTimeClient = { subscribeToken: string | null; getSubscribeToken: () => string | null; setImage: (image: Blob | File | string | null, options?: ImageSetOptions) => Promise; + getVideoStats: () => Promise<{ sender: VideoSenderStats[]; receiver: VideoReceiverStats[] }>; }; export const createRealTimeClient = (opts: RealTimeClientOptions) => { @@ -92,7 +98,15 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options); if (!parsedOptions.success) throw parsedOptions.error; - const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data; + const { + onRemoteStream, + onConnectionChange, + onQueuePosition, + initialState, + resolution, + publishOptions, + roomOptions, + } = parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; let inputStream: MediaStream = stream ?? new MediaStream(); @@ -154,6 +168,8 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialPrompt, logger, videoCodec: safariCodec, + publishOptions, + roomOptions, }); let sessionId: string | null = null; @@ -209,6 +225,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { return subscribeToken; }, getSubscribeToken: () => subscribeToken, + getVideoStats: () => activeSession.getVideoStats(), setImage: async (image: Blob | File | string | null, imgOptions?: ImageSetOptions) => { if (image === null) return activeSession.setImage(null, imgOptions); const base64 = await imageToBase64(image); diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 29cbea65..2691a89a 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -1,12 +1,17 @@ import { type DisconnectReason, + LocalVideoTrack, type RemoteParticipant, type RemoteTrack, + RemoteVideoTrack, Room, RoomEvent, + type RoomOptions, Track, TrackEvent, type TrackPublishOptions, + type VideoReceiverStats, + type VideoSenderStats, } from "livekit-client"; import mitt, { type Emitter } from "mitt"; @@ -16,17 +21,26 @@ import type { RealtimeObservability } from "./observability/realtime-observabili export type VideoCodec = "h264" | "vp8" | "vp9" | "av1"; -export function getDefaultVideoPublishOptions(videoCodec?: VideoCodec): TrackPublishOptions { - const videoEncoding = { +export function getDefaultVideoPublishOptions( + videoCodec?: VideoCodec, + overrides?: Partial, +): TrackPublishOptions { + const defaultEncoding = { maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps, maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps, }; return { source: Track.Source.Camera, - videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec, simulcast: true, - videoEncoding, + videoCodec: REALTIME_CONFIG.livekit.defaultVideoCodec, + ...overrides, + // Caller-provided codec (e.g. Safari forced vp8) wins over profile overrides. + ...(videoCodec ? { videoCodec } : {}), + videoEncoding: { + ...defaultEncoding, + ...overrides?.videoEncoding, + }, }; } @@ -41,6 +55,8 @@ export interface MediaChannelConfig { localStream: MediaStream | null; logger?: Logger; videoCodec?: VideoCodec; + publishOptions?: Partial; + roomOptions?: Partial; } export type MediaConnectOptions = { @@ -71,7 +87,10 @@ export class MediaChannel { } async connect(opts: MediaConnectOptions): Promise { - this.room ??= new Room(REALTIME_CONFIG.livekit.roomOptions); + this.room ??= new Room({ + ...REALTIME_CONFIG.livekit.roomOptions, + ...this.config.roomOptions, + }); const room = this.room; room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => { @@ -103,6 +122,42 @@ export class MediaChannel { this.config.observability?.setLiveKitRoom(room); } + async getVideoStats(): Promise<{ sender: VideoSenderStats[]; receiver: VideoReceiverStats[] }> { + const room = this.room; + if (!room) return { sender: [], receiver: [] }; + + const sender: VideoSenderStats[] = []; + for (const pub of room.localParticipant.videoTrackPublications.values()) { + const track = pub.track; + if (track instanceof LocalVideoTrack) { + try { + const layers = await track.getSenderStats(); + sender.push(...layers); + } catch (error) { + this.logger.debug("getSenderStats failed", { error: (error as Error).message }); + } + } + } + + const receiver: VideoReceiverStats[] = []; + for (const participant of room.remoteParticipants.values()) { + if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) continue; + for (const pub of participant.videoTrackPublications.values()) { + const track = pub.track; + if (track instanceof RemoteVideoTrack) { + try { + const stats = await track.getReceiverStats(); + if (stats) receiver.push(stats); + } catch (error) { + this.logger.debug("getReceiverStats failed", { error: (error as Error).message }); + } + } + } + } + + return { sender, receiver }; + } + async publishLocalTracks(): Promise { if (!this.config.localStream) return; this.config.observability?.startPhase("publish-local-track"); @@ -124,7 +179,16 @@ export class MediaChannel { if (!this.room) return; for (const track of stream.getTracks()) { if (track.kind === "video") { - await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.config.videoCodec)); + const publishOptions = getDefaultVideoPublishOptions(this.config.videoCodec, this.config.publishOptions); + this.logger.info("livekit: publishing video track", { + videoCodec: publishOptions.videoCodec, + simulcast: publishOptions.simulcast, + scalabilityMode: publishOptions.scalabilityMode, + degradationPreference: publishOptions.degradationPreference, + maxBitrate: publishOptions.videoEncoding?.maxBitrate, + maxFramerate: publishOptions.videoEncoding?.maxFramerate, + }); + await this.room.localParticipant.publishTrack(track, publishOptions); } else { await this.room.localParticipant.publishTrack(track); } diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts index b9249c99..34840849 100644 --- a/packages/sdk/src/realtime/stream-session.ts +++ b/packages/sdk/src/realtime/stream-session.ts @@ -5,6 +5,7 @@ import { createConsoleLogger, type Logger } from "../utils/logger"; import { REALTIME_CONFIG } from "./config-realtime"; import { InitialStateGate } from "./initial-state-gate"; import { MediaChannel, type VideoCodec } from "./media-channel"; +import type { RoomOptions, TrackPublishOptions, VideoReceiverStats, VideoSenderStats } from "livekit-client"; import type { RealtimeObservability } from "./observability/realtime-observability"; import { SignalingChannel } from "./signaling-channel"; import type { @@ -59,6 +60,8 @@ interface StreamSessionConfig { initialPrompt?: InitialPrompt; logger?: Logger; videoCodec?: VideoCodec; + publishOptions?: Partial; + roomOptions?: Partial; } export class StreamSession { @@ -123,6 +126,10 @@ export class StreamSession { return this.signaling.sendPrompt(text, opts); } + async getVideoStats(): Promise<{ sender: VideoSenderStats[]; receiver: VideoReceiverStats[] }> { + return this.media.getVideoStats(); + } + async setImage(image: string | null, opts?: ImageSetOptions): Promise { this.assertConnected(); return this.signaling.setImage(image, opts); @@ -307,6 +314,8 @@ export class StreamSession { localStream: this.config.localStream, logger: this.logger, videoCodec: this.config.videoCodec, + publishOptions: this.config.publishOptions, + roomOptions: this.config.roomOptions, }); this.wireSignalingEvents(); this.wireMediaEvents(); diff --git a/packages/sdk/src/realtime/subscribe-client.ts b/packages/sdk/src/realtime/subscribe-client.ts index 3782fd66..f7a2b7da 100644 --- a/packages/sdk/src/realtime/subscribe-client.ts +++ b/packages/sdk/src/realtime/subscribe-client.ts @@ -2,9 +2,11 @@ import { ConnectionState as LiveKitConnectionState, type RemoteParticipant, type RemoteTrack, + RemoteVideoTrack, Room, RoomEvent, Track, + type VideoReceiverStats, } from "livekit-client"; import { classifyWebrtcError, type DecartSDKError } from "../utils/errors"; @@ -55,6 +57,7 @@ export type RealTimeSubscribeClient = { disconnect: () => void; on: (event: K, listener: (data: SubscribeEvents[K]) => void) => void; off: (event: K, listener: (data: SubscribeEvents[K]) => void) => void; + getVideoStats: () => Promise<{ sender: never[]; receiver: VideoReceiverStats[] }>; }; export type SubscribeOptions = { @@ -181,6 +184,24 @@ export const createRealTimeSubscribeClient = (opts: RealTimeSubscribeClientOptio }, on: emitter.on, off: emitter.off, + getVideoStats: async () => { + const receiver: VideoReceiverStats[] = []; + for (const participant of activeRoom.remoteParticipants.values()) { + if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) continue; + for (const pub of participant.videoTrackPublications.values()) { + const track = pub.track; + if (track instanceof RemoteVideoTrack) { + try { + const stats = await track.getReceiverStats(); + if (stats) receiver.push(stats); + } catch (error) { + logger.debug("getReceiverStats failed", { error: (error as Error).message }); + } + } + } + } + return { sender: [], receiver }; + }, }; flush(); From 874fedf65b1ea186c055529704a74861573c1b45 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Thu, 21 May 2026 12:43:14 +0300 Subject: [PATCH 3/7] increase max bitrate --- packages/sdk/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 7d38f7fb..2138d8b4 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -664,7 +664,7 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 1_800_000, + maxBitrate: 2_500_000, maxFramerate: 30, priority: 'high', }, @@ -673,7 +673,7 @@

Console Logs

}, G: { label: 'H.264 high bitrate, maintain-framerate', - description: 'H.264 single layer with more bitrate headroom for clean networks while still preserving FPS under pressure.', + description: 'H.264 single layer with more bitrate headroom for clean networks while still preserving resolution under pressure.', roomOptions: { adaptiveStream: false, dynacast: false, @@ -682,7 +682,7 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_000_000, + maxBitrate: 2_500_000, maxFramerate: 30, priority: 'high', }, From e71ec423643618f7c75ccaba5a42a661248363e3 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Fri, 22 May 2026 11:35:05 +0300 Subject: [PATCH 4/7] added new instrumentation --- packages/sdk/index.html | 1062 ++++++++++++++--- packages/sdk/src/realtime/client.ts | 14 +- packages/sdk/src/realtime/media-channel.ts | 74 +- packages/sdk/src/realtime/mirror-stream.ts | 13 +- packages/sdk/src/realtime/stream-session.ts | 9 +- packages/sdk/src/realtime/subscribe-client.ts | 9 +- 6 files changed, 972 insertions(+), 209 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 2138d8b4..6dce9193 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -281,6 +281,7 @@ } } +

🎭 Decart SDK Test Page

@@ -344,18 +345,8 @@

Configuration

- - + +
@@ -373,10 +364,6 @@

Configuration

-
- - -
@@ -392,6 +379,42 @@

Configuration

+ +
+

Benchmark Dashboard

+
+ + +
+
+ + + + + + + +
+
+ Status: idle +   Elapsed: 0s +   Phase: (none) +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

Subscribe / Viewer Mode

@@ -545,8 +568,8 @@

Console Logs

// ----------------------------------------------------------------------- const PUBLISH_PROFILES = { default: { - label: 'H.264 + simulcast (current SDK default)', - description: 'Same as Profile A but with simulcast enabled — represents what the SDK ships today before any profile tuning.', + label: 'H.264 + simulcast (current SDK default, capped at 2 Mbps)', + description: 'Simulcast control at the same bitrate cap as F/G/H, so the only knob varying vs them is simulcast on/off.', roomOptions: { adaptiveStream: false, dynacast: false, @@ -555,15 +578,16 @@

Console Logs

videoCodec: 'h264', simulcast: true, videoEncoding: { - maxBitrate: 2_500_000, + maxBitrate: 2_000_000, maxFramerate: 30, + priority: 'high', }, degradationPreference: 'balanced', }, }, A: { - label: 'H.264 baseline, no simulcast (balanced)', - description: 'H.264 single layer, default degradation strategy. The control profile.', + label: 'H.264 economy — 1.2 Mbps, maintain-resolution', + description: 'Weak-network champion. ~60% of F/G\'s budget at the same resolution and cadence target, biased to preserve pixel fidelity (drop fps before res). Tests how much bandwidth we can shave before the model\'s input quality breaks.', roomOptions: { adaptiveStream: false, dynacast: false, @@ -572,15 +596,16 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_500_000, + maxBitrate: 1_200_000, maxFramerate: 30, + priority: 'high', }, - degradationPreference: 'balanced', + degradationPreference: 'maintain-resolution', }, }, B: { - label: 'H.264 + maintain-framerate, priority=high', - description: 'H.264 single layer, drops resolution before FPS under pressure, DSCP-prioritized.', + label: 'H.264 sharper-frames — 2 Mbps @ 20 fps, maintain-resolution', + description: 'Bits-per-frame champion. Same budget as F/G but only 20 fps target, so each frame gets ~50% more bits and visibly sharper detail. Bets the inference model values per-frame fidelity over cadence — and that 20 fps is enough for its temporal needs.', roomOptions: { adaptiveStream: false, dynacast: false, @@ -589,11 +614,11 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 3_000_000, - maxFramerate: 30, + maxBitrate: 2_000_000, + maxFramerate: 20, priority: 'high', }, - degradationPreference: 'maintain-framerate', + degradationPreference: 'maintain-resolution', }, }, C: { @@ -646,7 +671,7 @@

Console Logs

scalabilityMode: 'L3T3_KEY', simulcast: false, videoEncoding: { - maxBitrate: 1_000_000, + maxBitrate: 1_500_000, maxFramerate: 30, }, backupCodec: { codec: 'h264' }, @@ -664,7 +689,7 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_500_000, + maxBitrate: 2_000_000, maxFramerate: 30, priority: 'high', }, @@ -682,7 +707,7 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_500_000, + maxBitrate: 2_000_000, maxFramerate: 30, priority: 'high', }, @@ -690,31 +715,56 @@

Console Logs

}, }, H: { - label: 'H.264 adaptive receive, maintain-framerate', - description: 'H.264 single layer plus adaptiveStream for the processed-video subscription; upstream to the server remains single-layer H.264.', + label: 'H.264 balanced — like F/G but no degradation bias', + description: 'Identical to F/G except degradationPreference=balanced — lets WebRTC pick what to drop. Control profile: tells us whether explicit maintain-framerate or maintain-resolution actually moves the needle vs the engine\'s default behaviour.', roomOptions: { - adaptiveStream: true, + adaptiveStream: false, dynacast: false, }, publishOptions: { videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_500_000, + maxBitrate: 2_000_000, maxFramerate: 30, priority: 'high', }, - degradationPreference: 'maintain-framerate', + degradationPreference: 'balanced', }, }, }; + // Chain state: array of profile keys to run sequentially, plus the index currently active. + const CHAIN_INTERVAL_MS = 142_000; + const chainedProfileKeys = ['default']; + let chainIndex = 0; + let chainActive = false; + let chainAdvanceTimer = null; + function getSelectedPublishProfile() { - const key = elements.publishProfileSelect.value; + const key = chainedProfileKeys[chainIndex] ?? chainedProfileKeys[0] ?? 'default'; const profile = PUBLISH_PROFILES[key]; if (!profile) return { key: 'default', ...PUBLISH_PROFILES.default }; return { key, ...profile }; } + + const PROFILE_SHORT_LABELS = { + default: 'Default — H.264 + simulcast @ 2 Mbps', + A: 'A — H.264 economy 1.2 Mbps', + B: 'B — H.264 sharper 2 Mbps @ 20 fps', + C: 'C — VP9 SVC L1T3', + D: 'D — VP9 SVC L3T3_KEY', + E: 'E — AV1 SVC L3T3_KEY', + F: 'F — H.264 2 Mbps, maintain-framerate', + G: 'G — H.264 2 Mbps, maintain-resolution', + H: 'H — H.264 2 Mbps, balanced', + }; + + function buildProfileOptionsHtml() { + return Object.keys(PUBLISH_PROFILES) + .map((key) => ``) + .join(''); + } // DOM elements const elements = { @@ -722,11 +772,10 @@

Console Logs

modelSelect: document.getElementById('model-select'), realtimeBaseUrl: document.getElementById('realtime-base-url'), resolutionSelect: document.getElementById('resolution-select'), - publishProfileSelect: document.getElementById('publish-profile-select'), + chainedProfilesContainer: document.getElementById('chained-profiles-container'), publishProfileDescription: document.getElementById('publish-profile-description'), initialPrompt: document.getElementById('initial-prompt'), cameraFps: document.getElementById('camera-fps'), - streamAudioToggle: document.getElementById('stream-audio-toggle'), sourceFile: document.getElementById('source-file'), startFileSource: document.getElementById('start-file-source'), startCamera: document.getElementById('start-camera'), @@ -734,6 +783,27 @@

Console Logs

disconnectBtn: document.getElementById('disconnect-btn'), printStatsBtn: document.getElementById('print-stats-btn'), sessionTimer: document.getElementById('session-timer'), + // Benchmark dashboard + benchDuration: document.getElementById('bench-duration'), + benchStart: document.getElementById('bench-start'), + benchStop: document.getElementById('bench-stop'), + benchPhaseName: document.getElementById('bench-phase-name'), + benchMarkPhase: document.getElementById('bench-mark-phase'), + benchExportJson: document.getElementById('bench-export-json'), + benchExportCsv: document.getElementById('bench-export-csv'), + benchClear: document.getElementById('bench-clear'), + benchStatusText: document.getElementById('bench-status-text'), + benchElapsedText: document.getElementById('bench-elapsed-text'), + benchPhaseText: document.getElementById('bench-phase-text'), + benchLive: document.getElementById('bench-live'), + benchChartFps: document.getElementById('bench-chart-fps'), + benchChartBitrate: document.getElementById('bench-chart-bitrate'), + benchChartRtt: document.getElementById('bench-chart-rtt'), + benchChartLoss: document.getElementById('bench-chart-loss'), + benchChartResolution: document.getElementById('bench-chart-resolution'), + benchChartNack: document.getElementById('bench-chart-nack'), + benchChartKeyframes: document.getElementById('bench-chart-keyframes'), + benchChartSummary: document.getElementById('bench-chart-summary'), localVideo: document.getElementById('local-video'), remoteVideo: document.getElementById('remote-video'), promptInput: document.getElementById('prompt-input'), @@ -1009,15 +1079,463 @@

Console Logs

addLog('Stats averages printed to console (see DevTools for full object)', 'success'); } + // --- Benchmark engine --- + const BENCH_POLL_INTERVAL_MS = 1000; + const BENCH_WARMUP_FPS_THRESHOLD = 28; + const BENCH_WARMUP_REQUIRED_SAMPLES = 3; + const BENCH_PALETTE = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']; + + const benchmarkRuns = []; + let currentBenchRun = null; + let benchPollTimer = null; + let benchRunTimer = null; + let benchElapsedTimer = null; + let benchPrevRaw = null; + let benchCharts = null; + + function nowMs() { return Date.now(); } + + function ensureBenchCharts() { + if (benchCharts || typeof Chart === 'undefined') return benchCharts; + const commonOpts = { + animation: false, + responsive: true, + maintainAspectRatio: true, + interaction: { mode: 'nearest', intersect: false }, + scales: { + x: { type: 'linear', title: { display: true, text: 't (s)' } }, + y: { beginAtZero: true }, + }, + }; + benchCharts = { + fps: new Chart(elements.benchChartFps, { + type: 'line', + data: { datasets: [] }, + options: { ...commonOpts, plugins: { title: { display: true, text: 'Receiver FPS (per second)' } } }, + }), + bitrate: new Chart(elements.benchChartBitrate, { + type: 'line', + data: { datasets: [] }, + options: { ...commonOpts, plugins: { title: { display: true, text: 'Bitrate (kbps): rx solid, tx dashed' } } }, + }), + rtt: new Chart(elements.benchChartRtt, { + type: 'line', + data: { datasets: [] }, + options: { ...commonOpts, plugins: { title: { display: true, text: 'RTT (ms)' } } }, + }), + loss: new Chart(elements.benchChartLoss, { + type: 'line', + data: { datasets: [] }, + options: { ...commonOpts, plugins: { title: { display: true, text: 'Packet loss % (cumulative)' } } }, + }), + resolution: new Chart(elements.benchChartResolution, { + type: 'line', + data: { datasets: [] }, + options: { + ...commonOpts, + plugins: { title: { display: true, text: 'Receiver frame width (px)' } }, + elements: { line: { stepped: 'before' } }, + }, + }), + nack: new Chart(elements.benchChartNack, { + type: 'line', + data: { datasets: [] }, + options: { ...commonOpts, plugins: { title: { display: true, text: 'NACKs sent by receiver (per second)' } } }, + }), + keyframes: new Chart(elements.benchChartKeyframes, { + type: 'bar', + data: { datasets: [] }, + options: { ...commonOpts, plugins: { title: { display: true, text: 'Keyframes encoded (sender) per second — solid; decoded (receiver) — hatched' } } }, + }), + summary: new Chart(elements.benchChartSummary, { + type: 'bar', + data: { labels: [], datasets: [] }, + options: { + animation: false, + responsive: true, + plugins: { title: { display: true, text: 'Per-run summary' } }, + scales: { y: { beginAtZero: true } }, + }, + }), + }; + return benchCharts; + } + + function runColor(idx) { return BENCH_PALETTE[idx % BENCH_PALETTE.length]; } + + function runLabel(run, idx) { return `#${idx + 1} ${run.profile}`; } + + function refreshTimeSeriesCharts() { + const charts = ensureBenchCharts(); + if (!charts) return; + const runs = [...benchmarkRuns, ...(currentBenchRun ? [currentBenchRun] : [])]; + const datasets = { fps: [], bitrate: [], rtt: [], loss: [], resolution: [], nack: [], keyframes: [] }; + runs.forEach((run, idx) => { + const color = runColor(idx); + const label = runLabel(run, idx); + const points = run.samples.map((s) => ({ x: +(s.tMs / 1000).toFixed(2), d: s.derived })); + datasets.fps.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.receiverFps })), spanGaps: true, pointRadius: 0 }); + datasets.bitrate.push({ label: `${label} rx`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.receiverBitrateBps != null ? p.d.receiverBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); + datasets.bitrate.push({ label: `${label} tx`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.senderBitrateBps != null ? p.d.senderBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); + datasets.rtt.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.rtt })), spanGaps: true, pointRadius: 0 }); + datasets.loss.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.cumulativeLossPct })), spanGaps: true, pointRadius: 0 }); + datasets.resolution.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.frameWidth })), spanGaps: true, pointRadius: 0, stepped: 'before' }); + datasets.nack.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.nackCountDelta })), spanGaps: true, pointRadius: 0 }); + datasets.keyframes.push({ label: `${label} encoded`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.keyFramesEncodedDelta })) }); + datasets.keyframes.push({ label: `${label} decoded`, borderColor: color, backgroundColor: 'transparent', borderDash: [3, 3], data: points.map((p) => ({ x: p.x, y: p.d.keyFramesDecodedDelta })) }); + }); + charts.fps.data.datasets = datasets.fps; + charts.bitrate.data.datasets = datasets.bitrate; + charts.rtt.data.datasets = datasets.rtt; + charts.loss.data.datasets = datasets.loss; + charts.resolution.data.datasets = datasets.resolution; + charts.nack.data.datasets = datasets.nack; + charts.keyframes.data.datasets = datasets.keyframes; + charts.fps.update(); + charts.bitrate.update(); + charts.rtt.update(); + charts.loss.update(); + charts.resolution.update(); + charts.nack.update(); + charts.keyframes.update(); + } + + function refreshSummaryChart() { + const charts = ensureBenchCharts(); + if (!charts) return; + const labels = benchmarkRuns.map((r, i) => runLabel(r, i)); + const pick = (key) => benchmarkRuns.map((r) => key(r.summary)); + charts.summary.data.labels = labels; + charts.summary.data.datasets = [ + { label: 'Warmup (s)', data: pick((s) => s.warmupMs != null ? s.warmupMs / 1000 : null), backgroundColor: '#1f77b4' }, + { label: 'Avg FPS', data: pick((s) => s.fps.avg), backgroundColor: '#2ca02c' }, + { label: 'p5 FPS', data: pick((s) => s.fps.p5), backgroundColor: '#ff7f0e' }, + { label: 'FPS σ', data: pick((s) => s.fps.stddev), backgroundColor: '#9467bd' }, + { label: 'Avg rx kbps / 100', data: pick((s) => s.receiverBitrateBps / 100000), backgroundColor: '#17becf' }, + { label: 'Freezes', data: pick((s) => s.freezeCount), backgroundColor: '#d62728' }, + { label: 'Loss %', data: pick((s) => s.packetLoss * 100), backgroundColor: '#8c564b' }, + ]; + charts.summary.update(); + } + + function deriveSampleMetrics(curr, prev) { + const d = { + receiverFps: null, + receiverBitrateBps: null, + senderBitrateBps: null, + senderFps: null, + packetsLostDelta: 0, + packetsReceivedDelta: 0, + cumulativeLossPct: null, + rtt: null, + jitter: null, + frameWidth: null, + frameHeight: null, + nackCountTotal: null, + nackCountDelta: 0, + keyFramesEncodedTotal: curr.keyFramesEncoded ?? 0, + keyFramesDecodedTotal: curr.keyFramesDecoded ?? 0, + keyFramesEncodedDelta: prev ? Math.max(0, (curr.keyFramesEncoded ?? 0) - (prev.keyFramesEncoded ?? 0)) : 0, + keyFramesDecodedDelta: prev ? Math.max(0, (curr.keyFramesDecoded ?? 0) - (prev.keyFramesDecoded ?? 0)) : 0, + qualityLimitationReason: null, + }; + const recv = curr.receiver?.[0]; + if (recv) { + d.frameWidth = recv.frameWidth ?? null; + d.frameHeight = recv.frameHeight ?? null; + d.jitter = recv.jitter ?? null; + d.nackCountTotal = recv.nackCount ?? null; + const prevRecv = prev?.receiver?.[0]; + if (prevRecv) { + const dtMs = recv.timestamp - prevRecv.timestamp; + if (dtMs > 0) { + d.receiverFps = ((recv.framesDecoded - prevRecv.framesDecoded) * 1000) / dtMs; + d.receiverBitrateBps = ((recv.bytesReceived - prevRecv.bytesReceived) * 8 * 1000) / dtMs; + } + d.packetsLostDelta = Math.max(0, (recv.packetsLost ?? 0) - (prevRecv.packetsLost ?? 0)); + d.packetsReceivedDelta = Math.max(0, (recv.packetsReceived ?? 0) - (prevRecv.packetsReceived ?? 0)); + d.nackCountDelta = Math.max(0, (recv.nackCount ?? 0) - (prevRecv.nackCount ?? 0)); + } + const totalLost = recv.packetsLost ?? 0; + const totalRecv = recv.packetsReceived ?? 0; + const denom = totalLost + totalRecv; + d.cumulativeLossPct = denom > 0 ? (totalLost / denom) * 100 : null; + } + if (curr.sender && curr.sender.length > 0) { + d.senderFps = curr.sender.reduce((m, s) => Math.max(m, s.framesPerSecond ?? 0), 0); + d.qualityLimitationReason = curr.sender[0].qualityLimitationReason ?? null; + let bytesNow = 0; + let bytesPrev = 0; + let dtMs = 0; + for (let i = 0; i < curr.sender.length; i++) { + const s = curr.sender[i]; + bytesNow += s.bytesSent ?? 0; + const p = prev?.sender?.[i]; + if (p) { + bytesPrev += p.bytesSent ?? 0; + if (!dtMs) dtMs = s.timestamp - p.timestamp; + } + } + if (dtMs > 0) d.senderBitrateBps = ((bytesNow - bytesPrev) * 8 * 1000) / dtMs; + d.rtt = curr.sender.reduce((m, s) => Math.max(m, (s.roundTripTime ?? 0) * 1000), 0) || null; + } + return d; + } + + function currentPhase() { + if (!currentBenchRun || currentBenchRun.phases.length === 0) return '(none)'; + return currentBenchRun.phases[currentBenchRun.phases.length - 1].name; + } + + async function collectBenchSample() { + if (!currentBenchRun || !decartRealtime?.getVideoStats) return; + let raw; + try { raw = await decartRealtime.getVideoStats(); } catch (err) { addLog(`Benchmark sample failed: ${err.message}`, 'error'); return; } + const tMs = nowMs() - currentBenchRun.startedAtMs; + const derived = deriveSampleMetrics(raw, benchPrevRaw); + const sample = { tMs, phase: currentPhase(), sender: raw.sender, receiver: raw.receiver, derived }; + currentBenchRun.samples.push(sample); + benchPrevRaw = raw; + elements.benchLive.textContent = + `t=${(tMs / 1000).toFixed(0)}s phase=${sample.phase} ` + + `rxFps=${(derived.receiverFps ?? 0).toFixed(1)} rxKbps=${((derived.receiverBitrateBps ?? 0) / 1000).toFixed(0)} ` + + `txKbps=${((derived.senderBitrateBps ?? 0) / 1000).toFixed(0)} rtt=${(derived.rtt ?? 0).toFixed(0)}ms ` + + `res=${derived.frameWidth ?? '?'}×${derived.frameHeight ?? '?'} qLim=${derived.qualityLimitationReason ?? 'none'}`; + if (currentBenchRun.warmupMs === null && derived.receiverFps != null) { + currentBenchRun.warmupRun.push(derived.receiverFps); + if (currentBenchRun.warmupRun.length > BENCH_WARMUP_REQUIRED_SAMPLES) currentBenchRun.warmupRun.shift(); + if (currentBenchRun.warmupRun.length === BENCH_WARMUP_REQUIRED_SAMPLES && + currentBenchRun.warmupRun.every((f) => f >= BENCH_WARMUP_FPS_THRESHOLD)) { + currentBenchRun.warmupMs = tMs; + addLog(`Warmup reached: ${(tMs / 1000).toFixed(1)}s`, 'success'); + } + } + if (currentBenchRun.samples.length % 2 === 0) refreshTimeSeriesCharts(); + } + + function avg(arr) { return arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : 0; } + function stddev(arr) { + if (arr.length < 2) return 0; + const m = avg(arr); + return Math.sqrt(arr.reduce((acc, v) => acc + (v - m) ** 2, 0) / arr.length); + } + function percentile(arr, p) { + if (!arr.length) return null; + const s = [...arr].sort((a, b) => a - b); + const i = Math.min(s.length - 1, Math.max(0, Math.floor((p / 100) * s.length))); + return s[i]; + } + + function computeBenchSummary(run) { + const fpsArr = run.samples.map((s) => s.derived.receiverFps).filter((v) => v != null && Number.isFinite(v)); + const rxBr = run.samples.map((s) => s.derived.receiverBitrateBps).filter((v) => v != null && Number.isFinite(v)); + const txBr = run.samples.map((s) => s.derived.senderBitrateBps).filter((v) => v != null && Number.isFinite(v)); + const rtts = run.samples.map((s) => s.derived.rtt).filter((v) => v != null && Number.isFinite(v) && v > 0); + const jitters = run.samples.map((s) => s.derived.jitter).filter((v) => v != null && Number.isFinite(v)); + const totalLost = run.samples.reduce((a, s) => a + (s.derived.packetsLostDelta || 0), 0); + const totalReceived = run.samples.reduce((a, s) => a + (s.derived.packetsReceivedDelta || 0), 0); + const loss = (totalLost + totalReceived) > 0 ? totalLost / (totalLost + totalReceived) : 0; + let freezeCount = 0; + let freezeDurMs = 0; + let inFreeze = false; + for (let i = 1; i < run.samples.length; i++) { + const f = run.samples[i].derived.receiverFps; + const isFreeze = f != null && f < 1; + if (isFreeze) { + freezeDurMs += run.samples[i].tMs - run.samples[i - 1].tMs; + if (!inFreeze) { freezeCount++; inFreeze = true; } + } else { inFreeze = false; } + } + const last = run.samples[run.samples.length - 1]; + const resolution = last?.derived.frameWidth && last.derived.frameHeight + ? `${last.derived.frameWidth}×${last.derived.frameHeight}` : 'n/a'; + const avgFps = avg(fpsArr); + const avgRx = avg(rxBr); + const bitratePerFrame = avgFps > 0 ? avgRx / avgFps : 0; + return { + durationMs: run.endedAtMs - run.startedAtMs, + samples: run.samples.length, + warmupMs: run.warmupMs, + fps: { + avg: avgFps, + min: fpsArr.length ? Math.min(...fpsArr) : null, + p5: percentile(fpsArr, 5), + p95: percentile(fpsArr, 95), + stddev: stddev(fpsArr), + largestDip: fpsArr.length ? avgFps - Math.min(...fpsArr) : null, + }, + senderBitrateBps: avg(txBr), + receiverBitrateBps: avgRx, + bytesPerFrame: bitratePerFrame / 8, + rttMs: avg(rtts), + jitter: avg(jitters), + packetLoss: loss, + freezeCount, + totalFreezesDurationMs: freezeDurMs, + resolution, + }; + } + + function startBenchmark() { + if (currentBenchRun) { addLog('Benchmark already running', 'error'); return; } + if (!decartRealtime?.getVideoStats) { addLog('Not connected — cannot start benchmark', 'error'); return; } + const profileKey = getSelectedPublishProfile().key; + const durationSeconds = Math.max(10, Math.min(600, Number(elements.benchDuration.value) || 120)); + currentBenchRun = { + profile: profileKey, + sessionId: currentSessionId, + durationMs: durationSeconds * 1000, + startedAtMs: nowMs(), + endedAtMs: null, + phases: [{ name: 'start', tMs: 0 }], + samples: [], + warmupMs: null, + warmupRun: [], + }; + benchPrevRaw = null; + elements.benchStart.disabled = true; + elements.benchStop.disabled = false; + elements.benchMarkPhase.disabled = false; + elements.benchExportJson.disabled = false; + elements.benchExportCsv.disabled = false; + elements.benchClear.disabled = false; + elements.benchStatusText.textContent = `running (profile=${profileKey})`; + elements.benchPhaseText.textContent = 'start'; + elements.benchElapsedText.textContent = '0s'; + benchPollTimer = setInterval(collectBenchSample, BENCH_POLL_INTERVAL_MS); + benchRunTimer = setTimeout(stopBenchmark, durationSeconds * 1000); + benchElapsedTimer = setInterval(() => { + const s = Math.floor((nowMs() - currentBenchRun.startedAtMs) / 1000); + elements.benchElapsedText.textContent = `${s}s / ${durationSeconds}s`; + }, 1000); + addLog(`Benchmark started: profile=${profileKey} duration=${durationSeconds}s pollInterval=${BENCH_POLL_INTERVAL_MS}ms`, 'success'); + } + + function stopBenchmark() { + if (!currentBenchRun) return; + if (benchPollTimer) { clearInterval(benchPollTimer); benchPollTimer = null; } + if (benchRunTimer) { clearTimeout(benchRunTimer); benchRunTimer = null; } + if (benchElapsedTimer) { clearInterval(benchElapsedTimer); benchElapsedTimer = null; } + currentBenchRun.endedAtMs = nowMs(); + currentBenchRun.summary = computeBenchSummary(currentBenchRun); + benchmarkRuns.push(currentBenchRun); + console.log('[Benchmark] Run complete:', currentBenchRun); + addLog(`Benchmark stopped. profile=${currentBenchRun.profile} warmup=${currentBenchRun.summary.warmupMs != null ? (currentBenchRun.summary.warmupMs / 1000).toFixed(1) + 's' : 'n/a'} avgFps=${currentBenchRun.summary.fps.avg.toFixed(1)} p5Fps=${(currentBenchRun.summary.fps.p5 ?? 0).toFixed(1)}`, 'success'); + const finished = currentBenchRun; + currentBenchRun = null; + elements.benchStart.disabled = !decartRealtime?.getVideoStats; + elements.benchStop.disabled = true; + elements.benchMarkPhase.disabled = true; + elements.benchStatusText.textContent = `idle (last: ${finished.profile})`; + refreshTimeSeriesCharts(); + refreshSummaryChart(); + } + + function markBenchPhase() { + if (!currentBenchRun) return; + const raw = elements.benchPhaseName.value.trim(); + const name = raw || `phase-${currentBenchRun.phases.length}`; + const tMs = nowMs() - currentBenchRun.startedAtMs; + currentBenchRun.phases.push({ name, tMs }); + elements.benchPhaseName.value = ''; + elements.benchPhaseText.textContent = name; + addLog(`Phase "${name}" marked at ${(tMs / 1000).toFixed(1)}s`, 'info'); + } + + function triggerDownload(blob, filename) { + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setTimeout(() => URL.revokeObjectURL(url), 1000); + } + + function exportBenchJson() { + const payload = { exportedAt: new Date().toISOString(), runs: benchmarkRuns }; + const blob = new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }); + triggerDownload(blob, `benchmark_${Date.now()}.json`); + addLog(`Exported ${benchmarkRuns.length} run(s) as JSON`, 'success'); + } + + function exportBenchCsv() { + const header = [ + 'run', 'profile', 'sessionId', 'tMs', 'phase', + 'receiverFps', 'receiverKbps', 'senderKbps', 'senderFps', + 'rttMs', 'jitter', 'packetsLostDelta', 'packetsReceivedDelta', + 'cumulativeLossPct', 'frameWidth', 'frameHeight', + 'nackCountTotal', 'nackCountDelta', + 'keyFramesEncodedTotal', 'keyFramesEncodedDelta', + 'keyFramesDecodedTotal', 'keyFramesDecodedDelta', + 'qualityLimitationReason', + ]; + const rows = [header.join(',')]; + benchmarkRuns.forEach((run, idx) => { + run.samples.forEach((s) => { + const d = s.derived; + rows.push([ + idx + 1, + run.profile, + run.sessionId ?? '', + s.tMs, + JSON.stringify(s.phase), + d.receiverFps?.toFixed(3) ?? '', + d.receiverBitrateBps != null ? (d.receiverBitrateBps / 1000).toFixed(2) : '', + d.senderBitrateBps != null ? (d.senderBitrateBps / 1000).toFixed(2) : '', + d.senderFps?.toFixed(3) ?? '', + d.rtt?.toFixed(2) ?? '', + d.jitter ?? '', + d.packetsLostDelta ?? '', + d.packetsReceivedDelta ?? '', + d.cumulativeLossPct?.toFixed(4) ?? '', + d.frameWidth ?? '', + d.frameHeight ?? '', + d.nackCountTotal ?? '', + d.nackCountDelta ?? '', + d.keyFramesEncodedTotal ?? '', + d.keyFramesEncodedDelta ?? '', + d.keyFramesDecodedTotal ?? '', + d.keyFramesDecodedDelta ?? '', + d.qualityLimitationReason ?? '', + ].join(',')); + }); + }); + const blob = new Blob([rows.join('\n')], { type: 'text/csv' }); + triggerDownload(blob, `benchmark_${Date.now()}.csv`); + addLog(`Exported ${benchmarkRuns.length} run(s) as CSV`, 'success'); + } + + function clearBenchRuns() { + if (currentBenchRun) { addLog('Stop the current benchmark first', 'error'); return; } + benchmarkRuns.length = 0; + elements.benchExportJson.disabled = true; + elements.benchExportCsv.disabled = true; + elements.benchClear.disabled = true; + elements.benchLive.textContent = ''; + elements.benchStatusText.textContent = 'idle'; + refreshTimeSeriesCharts(); + refreshSummaryChart(); + addLog('Benchmark runs cleared', 'info'); + } + + elements.benchStart.addEventListener('click', startBenchmark); + elements.benchStop.addEventListener('click', stopBenchmark); + elements.benchMarkPhase.addEventListener('click', markBenchPhase); + elements.benchExportJson.addEventListener('click', exportBenchJson); + elements.benchExportCsv.addEventListener('click', exportBenchCsv); + elements.benchClear.addEventListener('click', clearBenchRuns); + // --- MediaRecorder for the Decart remote stream --- let mediaRecorder = null; let recordedChunks = []; function pickSupportedMimeType() { const candidates = [ - 'video/webm;codecs=vp9,opus', - 'video/webm;codecs=vp8,opus', - 'video/webm;codecs=h264,opus', + 'video/webm;codecs=vp9', + 'video/webm;codecs=vp8', + 'video/webm;codecs=h264', 'video/webm', 'video/mp4', ]; @@ -1051,6 +1569,14 @@

Console Logs

const finalType = mediaRecorder?.mimeType || 'video/webm'; const blob = new Blob(recordedChunks, { type: finalType }); const ext = finalType.includes('mp4') ? 'mp4' : 'webm'; + const sizeMb = (blob.size / 1024 / 1024).toFixed(2); + recordedChunks = []; + mediaRecorder = null; + const shouldDownload = window.confirm(`Download recorded remote stream? (${sizeMb} MB ${ext})`); + if (!shouldDownload) { + addLog(`Recording stopped. Discarded ${sizeMb} MB (user declined download)`, 'info'); + return; + } const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; @@ -1059,9 +1585,7 @@

Console Logs

a.click(); document.body.removeChild(a); setTimeout(() => URL.revokeObjectURL(url), 1000); - addLog(`Recording stopped. Downloaded ${(blob.size / 1024 / 1024).toFixed(2)} MB (${ext})`, 'success'); - recordedChunks = []; - mediaRecorder = null; + addLog(`Recording stopped. Downloaded ${sizeMb} MB (${ext})`, 'success'); }; mediaRecorder.start(1000); addLog(`Auto-recording remote stream (${mediaRecorder.mimeType || 'default'})`, 'success'); @@ -1120,19 +1644,90 @@

Console Logs

setTimeout(() => waitForPublisherSubscribeToken(attemptsRemaining - 1), 250); } - // Publish profile selection: update the inline description and log on change. function formatProfileTitle({ key, label }) { return key === 'default' ? `Default baseline — ${label}` : `Profile ${key} — ${label}`; } + function updatePublishProfileDescription() { - const profile = getSelectedPublishProfile(); - elements.publishProfileDescription.textContent = `${formatProfileTitle(profile)}: ${profile.description}`; + const lines = chainedProfileKeys.map((key, idx) => { + const profile = PUBLISH_PROFILES[key]; + const title = profile ? formatProfileTitle({ key, label: profile.label }) : `${key} (unknown)`; + const desc = profile?.description ?? ''; + const marker = chainActive && idx === chainIndex ? '▶ ' : `${idx + 1}. `; + return `${marker}${title}: ${desc}`; + }); + elements.publishProfileDescription.innerHTML = lines.join('
'); + } + + function renderChainedProfiles() { + const container = elements.chainedProfilesContainer; + container.innerHTML = ''; + const optionsHtml = buildProfileOptionsHtml(); + + chainedProfileKeys.forEach((key, idx) => { + const wrapper = document.createElement('span'); + wrapper.style.display = 'inline-flex'; + wrapper.style.alignItems = 'center'; + wrapper.style.gap = '4px'; + + if (chainedProfileKeys.length > 1) { + const badge = document.createElement('span'); + badge.textContent = `${idx + 1}`; + badge.style.fontFamily = "'Courier New', monospace"; + badge.style.fontSize = '12px'; + badge.style.color = chainActive && idx === chainIndex ? '#4CAF50' : '#888'; + badge.style.fontWeight = 'bold'; + wrapper.appendChild(badge); + } + + const select = document.createElement('select'); + select.innerHTML = optionsHtml; + select.value = key; + select.style.width = 'auto'; + select.style.minWidth = '280px'; + select.style.fontSize = '13px'; + select.style.padding = '6px 8px'; + select.disabled = chainActive; + select.addEventListener('change', () => { + chainedProfileKeys[idx] = select.value; + updatePublishProfileDescription(); + addLog(`Chain slot ${idx + 1}: ${select.value}`, 'info'); + }); + wrapper.appendChild(select); + container.appendChild(wrapper); + }); + + const addBtn = document.createElement('button'); + addBtn.type = 'button'; + addBtn.textContent = '+'; + addBtn.title = 'Add another profile to the chain'; + addBtn.style.width = '32px'; + addBtn.style.minWidth = '32px'; + addBtn.style.padding = '6px 8px'; + addBtn.style.fontSize = '16px'; + addBtn.style.marginRight = '0'; + addBtn.style.marginBottom = '0'; + addBtn.disabled = chainActive; + addBtn.addEventListener('click', () => { + const last = chainedProfileKeys[chainedProfileKeys.length - 1] ?? 'default'; + chainedProfileKeys.push(last); + renderChainedProfiles(); + updatePublishProfileDescription(); + addLog(`Chain extended to ${chainedProfileKeys.length} profiles`, 'info'); + }); + container.appendChild(addBtn); } - elements.publishProfileSelect.addEventListener('change', () => { - const profile = getSelectedPublishProfile(); + + function setChainUiLocked(locked) { + chainActive = locked; + elements.apiKey.disabled = locked; + elements.realtimeBaseUrl.disabled = locked; + elements.modelSelect.disabled = locked; + renderChainedProfiles(); updatePublishProfileDescription(); - addLog(`Selected publish profile: ${formatProfileTitle(profile)}`, 'info'); - }); + } + + renderChainedProfiles(); updatePublishProfileDescription(); // Model selection handler @@ -1182,6 +1777,8 @@

Console Logs

} let fileSource = null; + let lastSourceMode = null; + let lastSourceFile = null; function stopFileSource() { if (!fileSource) return; @@ -1240,12 +1837,44 @@

Console Logs

elements.localVideo.srcObject = stream; await elements.localVideo.play(); + lastSourceMode = 'file'; + lastSourceFile = file; addLog(`File source streaming: ${file.name} → ${camera.width}×${camera.height} @ ${camera.fps}fps (looped)`, 'success'); elements.startCamera.disabled = true; elements.startFileSource.disabled = true; elements.connectBtn.disabled = false; } + async function acquireCamera() { + if (!navigator.mediaDevices?.getUserMedia) { + throw new Error('Camera API is not available in this browser/context'); + } + const camera = getSelectedCameraSettings(); + const constraints = { + video: getCameraVideoConstraints(camera), + audio: false, + }; + addLog(`Camera target: ${camera.orientation}, ${camera.width}×${camera.height} @ ${camera.fps}fps`, 'info'); + stopCameraResources(); + localStream = await navigator.mediaDevices.getUserMedia(constraints); + elements.localVideo.srcObject = localStream; + await elements.localVideo.play(); + lastSourceMode = 'camera'; + lastSourceFile = null; + } + + async function refreshSourceForChain() { + if (lastSourceMode === 'file' && lastSourceFile) { + addLog('Restarting file source for next chain step', 'info'); + await beginFileSource(lastSourceFile); + return; + } + if (lastSourceMode === 'camera') { + addLog('Re-acquiring camera for next chain step', 'info'); + await acquireCamera(); + } + } + function stopCameraResources() { if (localStream) { localStream.getTracks().forEach((track) => { @@ -1257,32 +1886,12 @@

Console Logs

elements.localVideo.srcObject = null; } - // Start camera elements.startCamera.addEventListener('click', async () => { try { - if (!navigator.mediaDevices?.getUserMedia) { - throw new Error('Camera API is not available in this browser/context'); - } - addLog('Requesting camera access...', 'info'); - const streamAudio = elements.streamAudioToggle.checked; - const camera = getSelectedCameraSettings(); - const constraints = { - video: getCameraVideoConstraints(camera), - audio: streamAudio - ? { echoCancellation: true, noiseSuppression: true, autoGainControl: true } - : false, - }; - - addLog(`Camera target: ${camera.orientation}, ${camera.width}×${camera.height} @ ${camera.fps}fps`, 'info'); - addLog(`Microphone audio: ${streamAudio ? 'enabled' : 'disabled'}`, 'info'); - - localStream = await navigator.mediaDevices.getUserMedia(constraints); - elements.localVideo.srcObject = localStream; - await elements.localVideo.play(); + await acquireCamera(); elements.startCamera.disabled = true; elements.connectBtn.disabled = false; - addLog('Camera access granted successfully', 'success'); } catch (error) { addLog(`Failed to access camera: ${error.message}`, 'error'); @@ -1310,120 +1919,203 @@

Console Logs

elements.printStatsBtn.addEventListener('click', printStatsSummary); - // Connect to Decart - elements.connectBtn.addEventListener('click', async () => { + function scheduleAutoStartBenchmark() { + setTimeout(() => { + if (currentBenchRun) return; + if (!decartRealtime?.getVideoStats) return; + if (!isConnected) return; + addLog('Auto-starting benchmark (3s after remote stream)', 'info'); + startBenchmark(); + }, 3000); + } + + async function connectPublisher() { const apiKey = elements.apiKey.value.trim(); - if (!apiKey) { - addLog('Please enter an API key', 'error'); - return; + if (!apiKey) throw new Error('API key required'); + if (!localStream) throw new Error('Camera not started'); + + updateStatus('connecting'); + elements.connectBtn.disabled = true; + elements.subscribeBtn.disabled = true; + activeConnectionMode = 'publisher'; + + addLog(`Initializing Decart client (chain slot ${chainIndex + 1}/${chainedProfileKeys.length})...`, 'info'); + + const realtimeBaseUrl = elements.realtimeBaseUrl.value.trim(); + decartClient = createDecartClient({ + apiKey, + ...(realtimeBaseUrl && { realtimeBaseUrl }), + logger: createDemoLogger("debug"), + }); + + addLog('Connecting to Decart server via LiveKit...', 'info'); + + const initialPromptText = elements.initialPrompt.value.trim(); + const promptValue = initialPromptText ? { text: initialPromptText } : undefined; + if (promptValue) { + addLog(`Initial prompt: "${initialPromptText}"`, 'info'); + } else { + addLog('Initial prompt: passthrough (no prompt)', 'info'); } - - if (!localStream) { - addLog('Please start camera first', 'error'); - return; + + let initialImage; + if (promptValue && (model.name === 'lucy-2.1' || model.name === 'mirage_v2')) { + const initialImageResponse = await fetch('./tests/fixtures/image.png'); + initialImage = await initialImageResponse.blob(); } - if (isConnected) { - addLog('Already connected. Disconnect before starting another mode.', 'error'); - return; + const resolution = elements.resolutionSelect.value; + const profile = getSelectedPublishProfile(); + addLog(`Resolution: ${resolution}`, 'info'); + addLog(`Publish profile: ${formatProfileTitle(profile)}`, 'info'); + addLog(`Publish options: ${JSON.stringify(profile.publishOptions)}`, 'info'); + addLog(`Room options: ${JSON.stringify(profile.roomOptions)}`, 'info'); + + decartRealtime = await decartClient.realtime.connect(localStream, { + model, + resolution, + publishOptions: profile.publishOptions, + roomOptions: profile.roomOptions, + remoteVideoElement: elements.remoteVideo, + onRemoteStream: (stream) => { + addLog('Received remote stream from Decart', 'success'); + elements.remoteVideo.srcObject = stream; + if (!chainActive) startRecording(); + startPromptCycler(); + elements.benchStart.disabled = false; + scheduleAutoStartBenchmark(); + if (chainActive) { + if (chainAdvanceTimer) clearTimeout(chainAdvanceTimer); + chainAdvanceTimer = setTimeout(advanceChain, CHAIN_INTERVAL_MS); + addLog(`Chain advance armed in ${CHAIN_INTERVAL_MS / 1000}s`, 'info'); + } + }, + onConnectionChange: (state) => { + updateStatus(state); + if (state === 'connected' || state === 'generating') { + syncPublisherSubscribeToken(); + } + }, + onQueuePosition: (queuePosition) => { + addLog(`Queue position: ${queuePosition.position}/${queuePosition.queueSize}`, 'info'); + }, + initialState: { + ...(promptValue && { prompt: promptValue }), + ...(initialImage && { image: initialImage }), + }, + }); + + if (!decartRealtime.isConnected()) { + throw new Error('Connection established but not in connected state'); } - - try { - updateStatus('connecting'); - elements.connectBtn.disabled = true; - elements.subscribeBtn.disabled = true; - activeConnectionMode = 'publisher'; - - addLog('Initializing Decart client...', 'info'); - - const realtimeBaseUrl = elements.realtimeBaseUrl.value.trim(); - decartClient = createDecartClient({ - apiKey, - ...(realtimeBaseUrl && { realtimeBaseUrl }), - logger: createDemoLogger("debug"), - }); - - addLog('Connecting to Decart server via LiveKit...', 'info'); - - // Build initial prompt from text field (empty / null toggle = passthrough) - const initialPromptText = elements.initialPrompt.value.trim(); - const promptValue = initialPromptText ? { text: initialPromptText } : undefined; - if (promptValue) { - addLog(`Initial prompt: "${initialPromptText}"`, 'info'); - } else { - addLog('Initial prompt: passthrough (no prompt)', 'info'); - } + isConnected = true; + updateStatus('connected'); + elements.disconnectBtn.disabled = false; + elements.promptInput.disabled = false; + elements.sendPrompt.disabled = false; + elements.promisePromptInput.disabled = false; + elements.sendPromisePrompt.disabled = false; + elements.referenceImage.disabled = false; + elements.subscribeBtn.disabled = true; - // Load initial reference image only when there's an actual prompt - let initialImage; - if (promptValue && (model.name === 'lucy-2.1' || model.name === 'mirage_v2')) { - const initialImageResponse = await fetch('./tests/fixtures/image.png'); - initialImage = await initialImageResponse.blob(); - } + syncPublisherSubscribeToken(); + waitForPublisherSubscribeToken(); + elements.sessionInfo.style.display = 'block'; - const resolution = elements.resolutionSelect.value; - addLog(`Resolution: ${resolution}`, 'info'); + resetStatsAggregates(); + startStatsPolling(); - const profile = getSelectedPublishProfile(); - addLog(`Publish profile: ${formatProfileTitle(profile)}`, 'info'); - addLog(`Publish options: ${JSON.stringify(profile.publishOptions)}`, 'info'); - addLog(`Room options: ${JSON.stringify(profile.roomOptions)}`, 'info'); + addLog('Successfully connected to Decart!', 'success'); + addLog(`Session ID: ${currentSessionId || '(pending)'}`, 'success'); + } - decartRealtime = await decartClient.realtime.connect(localStream, { - model, - resolution, - publishOptions: profile.publishOptions, - roomOptions: profile.roomOptions, - onRemoteStream: (stream) => { - addLog('Received remote stream from Decart', 'success'); - elements.remoteVideo.srcObject = stream; - }, - onConnectionChange: (state) => { - updateStatus(state); - if (state === 'connected' || state === 'generating') { - syncPublisherSubscribeToken(); - } - if (state === 'generating') { - startRecording(); - startPromptCycler(); - } - }, - onQueuePosition: (queuePosition) => { - addLog(`Queue position: ${queuePosition.position}/${queuePosition.queueSize}`, 'info'); - }, - initialState: { - ...(promptValue && { prompt: promptValue }), - ...(initialImage && { image: initialImage }), - }, - }); - - // Check connection status - if (decartRealtime.isConnected()) { - isConnected = true; - updateStatus('connected'); - elements.disconnectBtn.disabled = false; - elements.promptInput.disabled = false; - elements.sendPrompt.disabled = false; - elements.promisePromptInput.disabled = false; - elements.sendPromisePrompt.disabled = false; - elements.referenceImage.disabled = false; - elements.subscribeBtn.disabled = true; - - // Show session ID - syncPublisherSubscribeToken(); - waitForPublisherSubscribeToken(); - elements.sessionInfo.style.display = 'block'; + function teardownRealtimeKeepingCamera() { + stopRecording(); + stopStatsPolling(); + stopPromptCycler(); + stopSessionTimer(); + if (currentBenchRun) stopBenchmark(); + if (decartRealtime) { + try { decartRealtime.disconnect(); } catch (_) {} + } + decartRealtime = null; + decartClient = null; + isConnected = false; + elements.remoteVideo.srcObject = null; + updateStatus('disconnected'); + } - resetStatsAggregates(); - startStatsPolling(); + async function advanceChain() { + chainAdvanceTimer = null; + addLog(`Chain interval elapsed for profile ${chainedProfileKeys[chainIndex]}`, 'info'); + teardownRealtimeKeepingCamera(); + chainIndex += 1; + if (chainIndex >= chainedProfileKeys.length) { + addLog(`Chain complete — ran ${chainedProfileKeys.length} profile(s)`, 'success'); + finishChain(); + return; + } + addLog(`Reconnecting with profile ${chainedProfileKeys[chainIndex]} (slot ${chainIndex + 1}/${chainedProfileKeys.length})`, 'info'); + updatePublishProfileDescription(); + renderChainedProfiles(); + try { + await refreshSourceForChain(); + await connectPublisher(); + } catch (err) { + addLog(`Chain advance failed: ${err.message}`, 'error'); + finishChain(); + } + } - addLog('Successfully connected to Decart!', 'success'); - addLog(`Session ID: ${currentSessionId || '(pending)'}`, 'success'); - } else { - throw new Error('Connection established but not in connected state'); - } - + function finishChain() { + if (chainAdvanceTimer) { + clearTimeout(chainAdvanceTimer); + chainAdvanceTimer = null; + } + chainIndex = 0; + setChainUiLocked(false); + activeConnectionMode = null; + currentSessionId = null; + currentSubscribeToken = null; + elements.startCamera.disabled = !!localStream; + elements.connectBtn.disabled = !localStream; + elements.subscribeBtn.disabled = false; + elements.disconnectBtn.disabled = true; + elements.printStatsBtn.disabled = true; + elements.promptInput.disabled = true; + elements.sendPrompt.disabled = true; + elements.promisePromptInput.disabled = true; + elements.sendPromisePrompt.disabled = true; + elements.sessionInfo.style.display = 'none'; + elements.publisherTokenInfo.style.display = 'none'; + elements.publisherSubscribeToken.value = ''; + elements.copySubscribeToken.disabled = true; + elements.referenceImage.disabled = true; + elements.setImage.disabled = true; + elements.benchStart.disabled = true; + elements.benchMarkPhase.disabled = true; + if (benchmarkRuns.length > 0) { + addLog(`Aggregated ${benchmarkRuns.length} benchmark run(s) — export JSON/CSV to review`, 'success'); + } + } + + elements.connectBtn.addEventListener('click', async () => { + const apiKey = elements.apiKey.value.trim(); + if (!apiKey) { addLog('Please enter an API key', 'error'); return; } + if (!localStream) { addLog('Please start camera first', 'error'); return; } + if (isConnected) { addLog('Already connected. Disconnect before starting another mode.', 'error'); return; } + + chainIndex = 0; + if (chainedProfileKeys.length > 1) { + setChainUiLocked(true); + addLog(`Starting chain of ${chainedProfileKeys.length} profiles × ${CHAIN_INTERVAL_MS / 1000}s = ${(chainedProfileKeys.length * CHAIN_INTERVAL_MS / 1000)}s total`, 'success'); + } else { + setChainUiLocked(false); + } + + try { + await connectPublisher(); } catch (error) { addLog(`Failed to connect: ${error.message}`, 'error'); updateStatus('disconnected'); @@ -1439,6 +2131,9 @@

Console Logs

stopStatsPolling(); stopPromptCycler(); stopSessionTimer(); + elements.benchStart.disabled = true; + elements.benchMarkPhase.disabled = true; + if (chainActive) finishChain(); } }); @@ -1496,10 +2191,13 @@

Console Logs

decartRealtime = await decartClient.realtime.subscribe({ token, + remoteVideoElement: elements.remoteVideo, onRemoteStream: (stream) => { addLog('Received subscribed stream from Decart', 'success'); elements.remoteVideo.srcObject = stream; startRecording(); + elements.benchStart.disabled = false; + scheduleAutoStartBenchmark(); }, onConnectionChange: (state) => { updateStatus(state); @@ -1538,15 +2236,29 @@

Console Logs

elements.startCamera.disabled = false; stopStatsPolling(); stopSessionTimer(); + elements.benchStart.disabled = true; + elements.benchMarkPhase.disabled = true; } }); // Disconnect elements.disconnectBtn.addEventListener('click', () => { + if (chainAdvanceTimer) { + clearTimeout(chainAdvanceTimer); + chainAdvanceTimer = null; + } + if (chainActive) { + addLog('Chain cancelled by user', 'info'); + setChainUiLocked(false); + chainIndex = 0; + } stopRecording(); stopStatsPolling(); stopPromptCycler(); stopSessionTimer(); + if (currentBenchRun) stopBenchmark(); + elements.benchStart.disabled = true; + elements.benchMarkPhase.disabled = true; if (statsAggregates.size > 0) { printStatsSummary(); } diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 178868e5..23704e02 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -57,6 +57,11 @@ const realTimeClientConnectOptionsSchema = z.object({ .custom>((val) => val === undefined || typeof val === "object") .optional(), roomOptions: z.custom>((val) => val === undefined || typeof val === "object").optional(), + remoteVideoElement: z + .custom( + (val) => val === undefined || (typeof val === "object" && val !== null && "srcObject" in val), + ) + .optional(), }); export type RealTimeClientConnectOptions = Omit, "model"> & { model: ModelDefinition | CustomModelDefinition; @@ -84,7 +89,12 @@ export type RealTimeClient = { subscribeToken: string | null; getSubscribeToken: () => string | null; setImage: (image: Blob | File | string | null, options?: ImageSetOptions) => Promise; - getVideoStats: () => Promise<{ sender: VideoSenderStats[]; receiver: VideoReceiverStats[] }>; + getVideoStats: () => Promise<{ + sender: VideoSenderStats[]; + receiver: VideoReceiverStats[]; + keyFramesEncoded: number; + keyFramesDecoded: number; + }>; }; export const createRealTimeClient = (opts: RealTimeClientOptions) => { @@ -106,6 +116,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { resolution, publishOptions, roomOptions, + remoteVideoElement, } = parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; let inputStream: MediaStream = stream ?? new MediaStream(); @@ -170,6 +181,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { videoCodec: safariCodec, publishOptions, roomOptions, + remoteVideoElement, }); let sessionId: string | null = null; diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 2691a89a..ec0fc00c 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -33,10 +33,8 @@ export function getDefaultVideoPublishOptions( return { source: Track.Source.Camera, simulcast: true, - videoCodec: REALTIME_CONFIG.livekit.defaultVideoCodec, + videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec, ...overrides, - // Caller-provided codec (e.g. Safari forced vp8) wins over profile overrides. - ...(videoCodec ? { videoCodec } : {}), videoEncoding: { ...defaultEncoding, ...overrides?.videoEncoding, @@ -57,6 +55,7 @@ export interface MediaChannelConfig { videoCodec?: VideoCodec; publishOptions?: Partial; roomOptions?: Partial; + remoteVideoElement?: HTMLVideoElement; } export type MediaConnectOptions = { @@ -95,9 +94,13 @@ export class MediaChannel { room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => { if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return; - if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return; + if (track.kind !== Track.Kind.Video) return; - track.attach(); + if (this.config.remoteVideoElement) { + track.attach(this.config.remoteVideoElement); + } else { + track.attach(); + } const mediaStreamTrack = track.mediaStreamTrack; if (mediaStreamTrack) { this.remoteStream ??= new MediaStream(); @@ -122,11 +125,17 @@ export class MediaChannel { this.config.observability?.setLiveKitRoom(room); } - async getVideoStats(): Promise<{ sender: VideoSenderStats[]; receiver: VideoReceiverStats[] }> { + async getVideoStats(): Promise<{ + sender: VideoSenderStats[]; + receiver: VideoReceiverStats[]; + keyFramesEncoded: number; + keyFramesDecoded: number; + }> { const room = this.room; - if (!room) return { sender: [], receiver: [] }; + if (!room) return { sender: [], receiver: [], keyFramesEncoded: 0, keyFramesDecoded: 0 }; const sender: VideoSenderStats[] = []; + let keyFramesEncoded = 0; for (const pub of room.localParticipant.videoTrackPublications.values()) { const track = pub.track; if (track instanceof LocalVideoTrack) { @@ -136,10 +145,22 @@ export class MediaChannel { } catch (error) { this.logger.debug("getSenderStats failed", { error: (error as Error).message }); } + try { + const report = await track.getRTCStatsReport(); + report?.forEach((stat: unknown) => { + const s = stat as { type?: string; kind?: string; keyFramesEncoded?: number }; + if (s.type === "outbound-rtp" && s.kind === "video" && typeof s.keyFramesEncoded === "number") { + keyFramesEncoded += s.keyFramesEncoded; + } + }); + } catch (error) { + this.logger.debug("getRTCStatsReport (sender) failed", { error: (error as Error).message }); + } } } const receiver: VideoReceiverStats[] = []; + let keyFramesDecoded = 0; for (const participant of room.remoteParticipants.values()) { if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) continue; for (const pub of participant.videoTrackPublications.values()) { @@ -151,11 +172,22 @@ export class MediaChannel { } catch (error) { this.logger.debug("getReceiverStats failed", { error: (error as Error).message }); } + try { + const report = await track.getRTCStatsReport(); + report?.forEach((stat: unknown) => { + const s = stat as { type?: string; kind?: string; keyFramesDecoded?: number }; + if (s.type === "inbound-rtp" && s.kind === "video" && typeof s.keyFramesDecoded === "number") { + keyFramesDecoded += s.keyFramesDecoded; + } + }); + } catch (error) { + this.logger.debug("getRTCStatsReport (receiver) failed", { error: (error as Error).message }); + } } } } - return { sender, receiver }; + return { sender, receiver, keyFramesEncoded, keyFramesDecoded }; } async publishLocalTracks(): Promise { @@ -177,21 +209,17 @@ export class MediaChannel { private async publishTracks(stream: MediaStream): Promise { if (!this.room) return; - for (const track of stream.getTracks()) { - if (track.kind === "video") { - const publishOptions = getDefaultVideoPublishOptions(this.config.videoCodec, this.config.publishOptions); - this.logger.info("livekit: publishing video track", { - videoCodec: publishOptions.videoCodec, - simulcast: publishOptions.simulcast, - scalabilityMode: publishOptions.scalabilityMode, - degradationPreference: publishOptions.degradationPreference, - maxBitrate: publishOptions.videoEncoding?.maxBitrate, - maxFramerate: publishOptions.videoEncoding?.maxFramerate, - }); - await this.room.localParticipant.publishTrack(track, publishOptions); - } else { - await this.room.localParticipant.publishTrack(track); - } + for (const track of stream.getVideoTracks()) { + const publishOptions = getDefaultVideoPublishOptions(this.config.videoCodec, this.config.publishOptions); + this.logger.info("livekit: publishing video track", { + videoCodec: publishOptions.videoCodec, + simulcast: publishOptions.simulcast, + scalabilityMode: publishOptions.scalabilityMode, + degradationPreference: publishOptions.degradationPreference, + maxBitrate: publishOptions.videoEncoding?.maxBitrate, + maxFramerate: publishOptions.videoEncoding?.maxFramerate, + }); + await this.room.localParticipant.publishTrack(track, publishOptions); } } } diff --git a/packages/sdk/src/realtime/mirror-stream.ts b/packages/sdk/src/realtime/mirror-stream.ts index 78ccdfca..5fc5d220 100644 --- a/packages/sdk/src/realtime/mirror-stream.ts +++ b/packages/sdk/src/realtime/mirror-stream.ts @@ -39,19 +39,18 @@ export function shouldMirrorTrack(track: MediaStreamTrack): boolean { export function createMirroredStream(input: MediaStream, opts: MirroredStreamOptions): MirroredStream { const [sourceVideo] = input.getVideoTracks(); - const audioTracks = input.getAudioTracks(); if (!sourceVideo) { return { stream: input, dispose: () => {}, impl: "noop" }; } if (isMediaStreamTrackProcessorSupported()) { - return createWithTrackProcessor(sourceVideo, audioTracks); + return createWithTrackProcessor(sourceVideo); } - return createWithCanvas(sourceVideo, audioTracks, opts.fps); + return createWithCanvas(sourceVideo, opts.fps); } -function createWithTrackProcessor(sourceVideo: MediaStreamTrack, audioTracks: MediaStreamTrack[]): MirroredStream { +function createWithTrackProcessor(sourceVideo: MediaStreamTrack): MirroredStream { const Processor = (globalThis as unknown as { MediaStreamTrackProcessor: MediaStreamTrackProcessorCtor }) .MediaStreamTrackProcessor; const Generator = (globalThis as unknown as { MediaStreamTrackGenerator: MediaStreamTrackGeneratorCtor }) @@ -101,7 +100,7 @@ function createWithTrackProcessor(sourceVideo: MediaStreamTrack, audioTracks: Me .pipeTo(generator.writable) .catch(() => {}); - const stream = new MediaStream([generator, ...audioTracks]); + const stream = new MediaStream([generator]); let disposed = false; return { @@ -115,7 +114,7 @@ function createWithTrackProcessor(sourceVideo: MediaStreamTrack, audioTracks: Me }; } -function createWithCanvas(sourceVideo: MediaStreamTrack, audioTracks: MediaStreamTrack[], fps: number): MirroredStream { +function createWithCanvas(sourceVideo: MediaStreamTrack, fps: number): MirroredStream { if (typeof document === "undefined") { throw new Error("createMirroredStream requires a DOM environment (document is undefined)"); } @@ -164,7 +163,7 @@ function createWithCanvas(sourceVideo: MediaStreamTrack, audioTracks: MediaStrea rafHandle = requestAnimationFrame(draw); return { - stream: new MediaStream([flippedTrack, ...audioTracks]), + stream: new MediaStream([flippedTrack]), impl: "canvas", dispose: () => { if (disposed) return; diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts index 34840849..54a47126 100644 --- a/packages/sdk/src/realtime/stream-session.ts +++ b/packages/sdk/src/realtime/stream-session.ts @@ -62,6 +62,7 @@ interface StreamSessionConfig { videoCodec?: VideoCodec; publishOptions?: Partial; roomOptions?: Partial; + remoteVideoElement?: HTMLVideoElement; } export class StreamSession { @@ -126,7 +127,12 @@ export class StreamSession { return this.signaling.sendPrompt(text, opts); } - async getVideoStats(): Promise<{ sender: VideoSenderStats[]; receiver: VideoReceiverStats[] }> { + async getVideoStats(): Promise<{ + sender: VideoSenderStats[]; + receiver: VideoReceiverStats[]; + keyFramesEncoded: number; + keyFramesDecoded: number; + }> { return this.media.getVideoStats(); } @@ -316,6 +322,7 @@ export class StreamSession { videoCodec: this.config.videoCodec, publishOptions: this.config.publishOptions, roomOptions: this.config.roomOptions, + remoteVideoElement: this.config.remoteVideoElement, }); this.wireSignalingEvents(); this.wireMediaEvents(); diff --git a/packages/sdk/src/realtime/subscribe-client.ts b/packages/sdk/src/realtime/subscribe-client.ts index f7a2b7da..1b555e70 100644 --- a/packages/sdk/src/realtime/subscribe-client.ts +++ b/packages/sdk/src/realtime/subscribe-client.ts @@ -64,6 +64,7 @@ export type SubscribeOptions = { token: string; onRemoteStream: (stream: MediaStream) => void; onConnectionChange?: (state: ConnectionState) => void; + remoteVideoElement?: HTMLVideoElement; }; export type RealTimeSubscribeClientOptions = { @@ -150,9 +151,13 @@ export const createRealTimeSubscribeClient = (opts: RealTimeSubscribeClientOptio activeRoom.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => { if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return; - if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return; + if (track.kind !== Track.Kind.Video) return; - track.attach(); + if (options.remoteVideoElement) { + track.attach(options.remoteVideoElement); + } else { + track.attach(); + } const mediaStreamTrack = track.mediaStreamTrack; if (!mediaStreamTrack) return; remoteStream ??= new MediaStream(); From 8f269646bdf874344e3510c650616f1fff75a0cb Mon Sep 17 00:00:00 2001 From: Verion1 Date: Fri, 22 May 2026 12:08:45 +0300 Subject: [PATCH 5/7] fixed bugs --- packages/sdk/index.html | 72 +++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 6dce9193..2e53c44d 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -407,7 +407,6 @@

Benchmark Dashboard

-
@@ -416,7 +415,7 @@

Benchmark Dashboard

-
+

Subscribe / Viewer Mode

@@ -735,7 +734,7 @@

Console Logs

}; // Chain state: array of profile keys to run sequentially, plus the index currently active. - const CHAIN_INTERVAL_MS = 142_000; + const CHAIN_INTERVAL_MS = 132_000; const chainedProfileKeys = ['default']; let chainIndex = 0; let chainActive = false; @@ -800,7 +799,6 @@

Console Logs

benchChartBitrate: document.getElementById('bench-chart-bitrate'), benchChartRtt: document.getElementById('bench-chart-rtt'), benchChartLoss: document.getElementById('bench-chart-loss'), - benchChartResolution: document.getElementById('bench-chart-resolution'), benchChartNack: document.getElementById('bench-chart-nack'), benchChartKeyframes: document.getElementById('bench-chart-keyframes'), benchChartSummary: document.getElementById('bench-chart-summary'), @@ -823,6 +821,7 @@

Console Logs

publisherSubscribeToken: document.getElementById('publisher-subscribe-token'), copySubscribeToken: document.getElementById('copy-subscribe-token'), subscribeTokenInput: document.getElementById('subscribe-token-input'), + subscribeSection: document.getElementById('subscribe-section'), subscribeBtn: document.getElementById('subscribe-btn'), logsContainer: document.getElementById('logs-container'), // Image reference elements @@ -1097,11 +1096,29 @@

Console Logs

function ensureBenchCharts() { if (benchCharts || typeof Chart === 'undefined') return benchCharts; + const sharedTooltip = { + mode: 'index', + intersect: false, + position: 'nearest', + itemSort: (a, b) => (b.parsed.y ?? -Infinity) - (a.parsed.y ?? -Infinity), + callbacks: { + label: (ctx) => { + const v = ctx.parsed.y; + const value = v == null || Number.isNaN(v) ? '—' : Number(v).toLocaleString(undefined, { maximumFractionDigits: 2 }); + return `${ctx.dataset.label}: ${value}`; + }, + }, + }; + const timeSeriesPlugins = (titleText) => ({ + title: { display: true, text: titleText }, + tooltip: sharedTooltip, + legend: { labels: { boxWidth: 10 } }, + }); const commonOpts = { animation: false, responsive: true, maintainAspectRatio: true, - interaction: { mode: 'nearest', intersect: false }, + interaction: { mode: 'index', intersect: false, axis: 'x' }, scales: { x: { type: 'linear', title: { display: true, text: 't (s)' } }, y: { beginAtZero: true }, @@ -1111,41 +1128,32 @@

Console Logs

fps: new Chart(elements.benchChartFps, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: { title: { display: true, text: 'Receiver FPS (per second)' } } }, + options: { ...commonOpts, plugins: timeSeriesPlugins('Receiver FPS (per second)') }, }), bitrate: new Chart(elements.benchChartBitrate, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: { title: { display: true, text: 'Bitrate (kbps): rx solid, tx dashed' } } }, + options: { ...commonOpts, plugins: timeSeriesPlugins('Bitrate (kbps): rx solid, tx dashed') }, }), rtt: new Chart(elements.benchChartRtt, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: { title: { display: true, text: 'RTT (ms)' } } }, + options: { ...commonOpts, plugins: timeSeriesPlugins('RTT (ms)') }, }), loss: new Chart(elements.benchChartLoss, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: { title: { display: true, text: 'Packet loss % (cumulative)' } } }, - }), - resolution: new Chart(elements.benchChartResolution, { - type: 'line', - data: { datasets: [] }, - options: { - ...commonOpts, - plugins: { title: { display: true, text: 'Receiver frame width (px)' } }, - elements: { line: { stepped: 'before' } }, - }, + options: { ...commonOpts, plugins: timeSeriesPlugins('Packet loss % (cumulative)') }, }), nack: new Chart(elements.benchChartNack, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: { title: { display: true, text: 'NACKs sent by receiver (per second)' } } }, + options: { ...commonOpts, plugins: timeSeriesPlugins('NACKs sent by receiver (per second)') }, }), keyframes: new Chart(elements.benchChartKeyframes, { - type: 'bar', + type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: { title: { display: true, text: 'Keyframes encoded (sender) per second — solid; decoded (receiver) — hatched' } } }, + options: { ...commonOpts, plugins: timeSeriesPlugins('Keyframes total (cumulative): encoded solid, decoded dashed') }, }), summary: new Chart(elements.benchChartSummary, { type: 'bar', @@ -1153,7 +1161,11 @@

Console Logs

options: { animation: false, responsive: true, - plugins: { title: { display: true, text: 'Per-run summary' } }, + interaction: { mode: 'index', intersect: false, axis: 'x' }, + plugins: { + title: { display: true, text: 'Per-run summary' }, + tooltip: sharedTooltip, + }, scales: { y: { beginAtZero: true } }, }, }), @@ -1169,7 +1181,7 @@

Console Logs

const charts = ensureBenchCharts(); if (!charts) return; const runs = [...benchmarkRuns, ...(currentBenchRun ? [currentBenchRun] : [])]; - const datasets = { fps: [], bitrate: [], rtt: [], loss: [], resolution: [], nack: [], keyframes: [] }; + const datasets = { fps: [], bitrate: [], rtt: [], loss: [], nack: [], keyframes: [] }; runs.forEach((run, idx) => { const color = runColor(idx); const label = runLabel(run, idx); @@ -1179,23 +1191,20 @@

Console Logs

datasets.bitrate.push({ label: `${label} tx`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.senderBitrateBps != null ? p.d.senderBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); datasets.rtt.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.rtt })), spanGaps: true, pointRadius: 0 }); datasets.loss.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.cumulativeLossPct })), spanGaps: true, pointRadius: 0 }); - datasets.resolution.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.frameWidth })), spanGaps: true, pointRadius: 0, stepped: 'before' }); datasets.nack.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.nackCountDelta })), spanGaps: true, pointRadius: 0 }); - datasets.keyframes.push({ label: `${label} encoded`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.keyFramesEncodedDelta })) }); - datasets.keyframes.push({ label: `${label} decoded`, borderColor: color, backgroundColor: 'transparent', borderDash: [3, 3], data: points.map((p) => ({ x: p.x, y: p.d.keyFramesDecodedDelta })) }); + datasets.keyframes.push({ label: `${label} encoded`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.keyFramesEncodedTotal })), spanGaps: true, pointRadius: 0 }); + datasets.keyframes.push({ label: `${label} decoded`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.keyFramesDecodedTotal })), spanGaps: true, pointRadius: 0 }); }); charts.fps.data.datasets = datasets.fps; charts.bitrate.data.datasets = datasets.bitrate; charts.rtt.data.datasets = datasets.rtt; charts.loss.data.datasets = datasets.loss; - charts.resolution.data.datasets = datasets.resolution; charts.nack.data.datasets = datasets.nack; charts.keyframes.data.datasets = datasets.keyframes; charts.fps.update(); charts.bitrate.update(); charts.rtt.update(); charts.loss.update(); - charts.resolution.update(); charts.nack.update(); charts.keyframes.update(); } @@ -1212,6 +1221,7 @@

Console Logs

{ label: 'p5 FPS', data: pick((s) => s.fps.p5), backgroundColor: '#ff7f0e' }, { label: 'FPS σ', data: pick((s) => s.fps.stddev), backgroundColor: '#9467bd' }, { label: 'Avg rx kbps / 100', data: pick((s) => s.receiverBitrateBps / 100000), backgroundColor: '#17becf' }, + { label: 'Avg tx kbps / 100', data: pick((s) => s.senderBitrateBps / 100000), backgroundColor: '#0b5d6f' }, { label: 'Freezes', data: pick((s) => s.freezeCount), backgroundColor: '#d62728' }, { label: 'Loss %', data: pick((s) => s.packetLoss * 100), backgroundColor: '#8c564b' }, ]; @@ -2018,6 +2028,7 @@

Console Logs

elements.sendPromisePrompt.disabled = false; elements.referenceImage.disabled = false; elements.subscribeBtn.disabled = true; + elements.subscribeSection.style.display = 'none'; syncPublisherSubscribeToken(); waitForPublisherSubscribeToken(); @@ -2089,6 +2100,7 @@

Console Logs

elements.sendPromisePrompt.disabled = true; elements.sessionInfo.style.display = 'none'; elements.publisherTokenInfo.style.display = 'none'; + elements.subscribeSection.style.display = ''; elements.publisherSubscribeToken.value = ''; elements.copySubscribeToken.disabled = true; elements.referenceImage.disabled = true; @@ -2127,6 +2139,7 @@

Console Logs

currentSubscribeToken = null; elements.sessionInfo.style.display = 'none'; elements.publisherTokenInfo.style.display = 'none'; + elements.subscribeSection.style.display = ''; elements.subscribeBtn.disabled = false; stopStatsPolling(); stopPromptCycler(); @@ -2219,6 +2232,7 @@

Console Logs

elements.setImage.disabled = true; elements.sessionInfo.style.display = 'none'; elements.publisherTokenInfo.style.display = 'none'; + elements.subscribeSection.style.display = 'none'; resetStatsAggregates(); startStatsPolling(); addLog('Successfully subscribed to stream. No local media was requested or published.', 'success'); @@ -2234,6 +2248,7 @@

Console Logs

activeConnectionMode = null; elements.subscribeBtn.disabled = false; elements.startCamera.disabled = false; + elements.subscribeSection.style.display = ''; stopStatsPolling(); stopSessionTimer(); elements.benchStart.disabled = true; @@ -2296,6 +2311,7 @@

Console Logs

elements.promiseStatus.style.display = 'none'; elements.sessionInfo.style.display = 'none'; elements.publisherTokenInfo.style.display = 'none'; + elements.subscribeSection.style.display = ''; elements.publisherSubscribeToken.value = ''; elements.copySubscribeToken.disabled = true; elements.referenceImage.disabled = true; From 48a31d1542ad43fe75b51565966f6377cefa893a Mon Sep 17 00:00:00 2001 From: Verion1 Date: Sat, 23 May 2026 14:54:07 +0300 Subject: [PATCH 6/7] updated playground metrics to fit better profiles judgment --- packages/sdk/index.html | 616 ++++++++++++++---- packages/sdk/src/realtime/client.ts | 11 +- packages/sdk/src/realtime/media-channel.ts | 208 ++++-- packages/sdk/src/realtime/stream-session.ts | 11 +- packages/sdk/src/realtime/subscribe-client.ts | 64 +- 5 files changed, 725 insertions(+), 185 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 2e53c44d..4755cf9f 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -282,6 +282,7 @@ } +

🎭 Decart SDK Test Page

@@ -405,6 +406,8 @@

Benchmark Dashboard

+
+
@@ -566,36 +569,18 @@

Console Logs

// Profiles C-E keep the VP9/AV1 SVC candidates for comparison. // ----------------------------------------------------------------------- const PUBLISH_PROFILES = { - default: { - label: 'H.264 + simulcast (current SDK default, capped at 2 Mbps)', - description: 'Simulcast control at the same bitrate cap as F/G/H, so the only knob varying vs them is simulcast on/off.', - roomOptions: { - adaptiveStream: false, - dynacast: false, - }, - publishOptions: { - videoCodec: 'h264', - simulcast: true, - videoEncoding: { - maxBitrate: 2_000_000, - maxFramerate: 30, - priority: 'high', - }, - degradationPreference: 'balanced', - }, - }, A: { - label: 'H.264 economy — 1.2 Mbps, maintain-resolution', + label: 'H.264 economy — 1.6 Mbps, maintain-resolution', description: 'Weak-network champion. ~60% of F/G\'s budget at the same resolution and cadence target, biased to preserve pixel fidelity (drop fps before res). Tests how much bandwidth we can shave before the model\'s input quality breaks.', roomOptions: { adaptiveStream: false, - dynacast: false, + dynacast: true, }, publishOptions: { videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 1_200_000, + maxBitrate: 1_600_000, maxFramerate: 30, priority: 'high', }, @@ -603,18 +588,18 @@

Console Logs

}, }, B: { - label: 'H.264 sharper-frames — 2 Mbps @ 20 fps, maintain-resolution', + label: 'H.264 sharper-frames — 2 Mbps @ 24 fps, maintain-resolution', description: 'Bits-per-frame champion. Same budget as F/G but only 20 fps target, so each frame gets ~50% more bits and visibly sharper detail. Bets the inference model values per-frame fidelity over cadence — and that 20 fps is enough for its temporal needs.', roomOptions: { adaptiveStream: false, - dynacast: false, + dynacast: true, }, publishOptions: { videoCodec: 'h264', simulcast: false, videoEncoding: { maxBitrate: 2_000_000, - maxFramerate: 20, + maxFramerate: 24, priority: 'high', }, degradationPreference: 'maintain-resolution', @@ -733,22 +718,29 @@

Console Logs

}, }; + const DEFAULT_PROFILE_KEY = Object.keys(PUBLISH_PROFILES)[0]; + // Chain state: array of profile keys to run sequentially, plus the index currently active. - const CHAIN_INTERVAL_MS = 132_000; - const chainedProfileKeys = ['default']; + // Read lazily so it tracks the current bench-duration input value at chain time. + function getChainIntervalMs() { + const durationSeconds = Number(elements?.benchDuration?.value); + return Number.isFinite(durationSeconds) && durationSeconds > 0 + ? (durationSeconds * 1000) + 6000 + : 127_000; + } + const chainedProfileKeys = [DEFAULT_PROFILE_KEY]; let chainIndex = 0; let chainActive = false; let chainAdvanceTimer = null; function getSelectedPublishProfile() { - const key = chainedProfileKeys[chainIndex] ?? chainedProfileKeys[0] ?? 'default'; + const key = chainedProfileKeys[chainIndex] ?? chainedProfileKeys[0] ?? DEFAULT_PROFILE_KEY; const profile = PUBLISH_PROFILES[key]; - if (!profile) return { key: 'default', ...PUBLISH_PROFILES.default }; + if (!profile) return { key: DEFAULT_PROFILE_KEY, ...PUBLISH_PROFILES[DEFAULT_PROFILE_KEY] }; return { key, ...profile }; } const PROFILE_SHORT_LABELS = { - default: 'Default — H.264 + simulcast @ 2 Mbps', A: 'A — H.264 economy 1.2 Mbps', B: 'B — H.264 sharper 2 Mbps @ 20 fps', C: 'C — VP9 SVC L1T3', @@ -797,6 +789,8 @@

Console Logs

benchLive: document.getElementById('bench-live'), benchChartFps: document.getElementById('bench-chart-fps'), benchChartBitrate: document.getElementById('bench-chart-bitrate'), + benchChartQp: document.getElementById('bench-chart-qp'), + benchChartBpp: document.getElementById('bench-chart-bpp'), benchChartRtt: document.getElementById('bench-chart-rtt'), benchChartLoss: document.getElementById('bench-chart-loss'), benchChartNack: document.getElementById('bench-chart-nack'), @@ -1081,7 +1075,7 @@

Console Logs

// --- Benchmark engine --- const BENCH_POLL_INTERVAL_MS = 1000; const BENCH_WARMUP_FPS_THRESHOLD = 28; - const BENCH_WARMUP_REQUIRED_SAMPLES = 3; + const BENCH_WARMUP_REQUIRED_SAMPLES = 1; const BENCH_PALETTE = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']; const benchmarkRuns = []; @@ -1089,13 +1083,50 @@

Console Logs

let benchPollTimer = null; let benchRunTimer = null; let benchElapsedTimer = null; - let benchPrevRaw = null; let benchCharts = null; function nowMs() { return Date.now(); } + let annotationPluginRegistered = false; + function ensureAnnotationPlugin() { + if (annotationPluginRegistered || typeof Chart === 'undefined') return; + const plugin = window['chartjs-plugin-annotation']; + if (plugin) { + Chart.register(plugin); + annotationPluginRegistered = true; + } + } + + const QP_MAX = { H264: 51, VP8: 127, VP9: 255, AV1: 255 }; + function normalizeQp(qp, codec) { + if (qp == null || !Number.isFinite(qp)) return null; + const key = (codec || '').toUpperCase().replace(/[^A-Z0-9]/g, ''); + const max = QP_MAX[key] ?? 51; + return Math.max(0, Math.min(100, (qp / max) * 100)); + } + + const stripePatternCache = new Map(); + function makeStripePattern(color) { + if (stripePatternCache.has(color)) return stripePatternCache.get(color); + const c = document.createElement('canvas'); + c.width = 8; c.height = 8; + const ctx = c.getContext('2d'); + ctx.fillStyle = color; + ctx.fillRect(0, 0, 8, 8); + ctx.strokeStyle = 'rgba(255,255,255,0.65)'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(-2, 6); ctx.lineTo(6, -2); + ctx.moveTo(2, 10); ctx.lineTo(10, 2); + ctx.stroke(); + const pattern = ctx.createPattern(c, 'repeat'); + stripePatternCache.set(color, pattern); + return pattern; + } + function ensureBenchCharts() { if (benchCharts || typeof Chart === 'undefined') return benchCharts; + ensureAnnotationPlugin(); const sharedTooltip = { mode: 'index', intersect: false, @@ -1133,7 +1164,7 @@

Console Logs

bitrate: new Chart(elements.benchChartBitrate, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: timeSeriesPlugins('Bitrate (kbps): rx solid, tx dashed') }, + options: { ...commonOpts, plugins: timeSeriesPlugins('Bitrate (kbps): tx solid (out), rx dashed (in)') }, }), rtt: new Chart(elements.benchChartRtt, { type: 'line', @@ -1148,12 +1179,90 @@

Console Logs

nack: new Chart(elements.benchChartNack, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: timeSeriesPlugins('NACKs sent by receiver (per second)') }, + options: { ...commonOpts, plugins: timeSeriesPlugins('NACKs per sample: out solid (NACKs received by sender), in dashed (NACKs sent by receiver)') }, }), keyframes: new Chart(elements.benchChartKeyframes, { type: 'line', data: { datasets: [] }, - options: { ...commonOpts, plugins: timeSeriesPlugins('Keyframes total (cumulative): encoded solid, decoded dashed') }, + options: { ...commonOpts, plugins: timeSeriesPlugins('Keyframes per sample: out solid (sender), in dashed (receiver / PLI storms)') }, + }), + qp: new Chart(elements.benchChartQp, { + type: 'line', + data: { datasets: [] }, + options: { + animation: false, + responsive: true, + maintainAspectRatio: true, + interaction: { mode: 'index', intersect: false, axis: 'x' }, + scales: { + x: { type: 'linear', title: { display: true, text: 't (s)' } }, + y: { + type: 'linear', + position: 'left', + min: 0, + max: 100, + title: { display: true, text: 'Avg QP % (0–100, lower = better)' }, + }, + }, + plugins: { + title: { display: true, text: 'Avg QP % per frame — out solid, in dashed (normalized across codecs)' }, + tooltip: { + ...sharedTooltip, + callbacks: { + label(ctx) { + const ds = ctx.dataset; + const v = ctx.parsed.y; + const value = v == null || Number.isNaN(v) ? '—' : Number(v).toFixed(1); + if (ds._qpRaw) { + const raw = ds._qpRaw[ctx.dataIndex]; + const codec = ds._qpCodec; + const rawStr = raw != null && Number.isFinite(raw) ? raw.toFixed(1) : '—'; + return `${ds.label}: ${value}% (raw ${rawStr} ${codec ?? '?'})`; + } + return `${ds.label}: ${value}`; + }, + }, + }, + legend: { labels: { boxWidth: 10 } }, + annotation: { + annotations: { + qpGreen: { type: 'box', yScaleID: 'y', yMin: 0, yMax: 40, backgroundColor: 'rgba(76, 175, 80, 0.06)', borderWidth: 0 }, + qpYellow: { type: 'box', yScaleID: 'y', yMin: 40, yMax: 65, backgroundColor: 'rgba(255, 193, 7, 0.08)', borderWidth: 0 }, + qpRed: { type: 'box', yScaleID: 'y', yMin: 65, yMax: 100, backgroundColor: 'rgba(244, 67, 54, 0.08)', borderWidth: 0 }, + }, + }, + }, + }, + }), + bpp: new Chart(elements.benchChartBpp, { + type: 'line', + data: { datasets: [] }, + options: { + animation: false, + responsive: true, + maintainAspectRatio: true, + interaction: { mode: 'index', intersect: false, axis: 'x' }, + scales: { + x: { type: 'linear', title: { display: true, text: 't (s)' } }, + y: { + type: 'linear', + position: 'left', + beginAtZero: true, + title: { display: true, text: 'BPP (bits per pixel)' }, + }, + }, + plugins: { + title: { display: true, text: 'BPP — out solid (sender), in dashed (receiver), 0.07–0.15 reference' }, + tooltip: sharedTooltip, + legend: { labels: { boxWidth: 10 } }, + annotation: { + annotations: { + bppLow: { type: 'line', yScaleID: 'y', yMin: 0.07, yMax: 0.07, borderColor: 'rgba(0,0,0,0.4)', borderWidth: 1, borderDash: [4, 4], label: { display: true, content: '0.07', position: 'end', backgroundColor: 'rgba(0,0,0,0.0)', color: 'rgba(0,0,0,0.6)' } }, + bppHigh: { type: 'line', yScaleID: 'y', yMin: 0.15, yMax: 0.15, borderColor: 'rgba(0,0,0,0.4)', borderWidth: 1, borderDash: [4, 4], label: { display: true, content: '0.15', position: 'end', backgroundColor: 'rgba(0,0,0,0.0)', color: 'rgba(0,0,0,0.6)' } }, + }, + }, + }, + }, }), summary: new Chart(elements.benchChartSummary, { type: 'bar', @@ -1164,7 +1273,25 @@

Console Logs

interaction: { mode: 'index', intersect: false, axis: 'x' }, plugins: { title: { display: true, text: 'Per-run summary' }, - tooltip: sharedTooltip, + tooltip: { + ...sharedTooltip, + callbacks: { + label(ctx) { + const ds = ctx.dataset; + const v = ctx.parsed.y; + const value = v == null || Number.isNaN(v) + ? '—' + : Number(v).toLocaleString(undefined, { maximumFractionDigits: 2 }); + if (ds._qpRaw) { + const raw = ds._qpRaw[ctx.dataIndex]; + const codec = ds._qpCodec[ctx.dataIndex]; + const rawStr = raw != null && Number.isFinite(raw) ? raw.toFixed(1) : '—'; + return `${ds.label}: ${value}% (raw ${rawStr} ${codec ?? '?'})`; + } + return `${ds.label}: ${value}`; + }, + }, + }, }, scales: { y: { beginAtZero: true } }, }, @@ -1181,19 +1308,49 @@

Console Logs

const charts = ensureBenchCharts(); if (!charts) return; const runs = [...benchmarkRuns, ...(currentBenchRun ? [currentBenchRun] : [])]; - const datasets = { fps: [], bitrate: [], rtt: [], loss: [], nack: [], keyframes: [] }; + const datasets = { fps: [], bitrate: [], rtt: [], loss: [], nack: [], keyframes: [], qp: [], bpp: [] }; runs.forEach((run, idx) => { const color = runColor(idx); const label = runLabel(run, idx); const points = run.samples.map((s) => ({ x: +(s.tMs / 1000).toFixed(2), d: s.derived })); + const outCodec = points.find((p) => p.d.outboundCodec)?.d.outboundCodec ?? '?'; + const inCodec = points.find((p) => p.d.inboundCodec)?.d.inboundCodec ?? '?'; datasets.fps.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.receiverFps })), spanGaps: true, pointRadius: 0 }); - datasets.bitrate.push({ label: `${label} rx`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.receiverBitrateBps != null ? p.d.receiverBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); - datasets.bitrate.push({ label: `${label} tx`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.senderBitrateBps != null ? p.d.senderBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); + datasets.bitrate.push({ label: `${label} tx`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.senderBitrateBps != null ? p.d.senderBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); + datasets.bitrate.push({ label: `${label} rx`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.receiverBitrateBps != null ? p.d.receiverBitrateBps / 1000 : null })), spanGaps: true, pointRadius: 0 }); datasets.rtt.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.rtt })), spanGaps: true, pointRadius: 0 }); datasets.loss.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.cumulativeLossPct })), spanGaps: true, pointRadius: 0 }); - datasets.nack.push({ label, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.nackCountDelta })), spanGaps: true, pointRadius: 0 }); - datasets.keyframes.push({ label: `${label} encoded`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.keyFramesEncodedTotal })), spanGaps: true, pointRadius: 0 }); - datasets.keyframes.push({ label: `${label} decoded`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.keyFramesDecodedTotal })), spanGaps: true, pointRadius: 0 }); + datasets.nack.push({ label: `${label} out`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.nackOutDelta })), spanGaps: true, pointRadius: 0 }); + datasets.nack.push({ label: `${label} in`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.nackInDelta })), spanGaps: true, pointRadius: 0 }); + datasets.keyframes.push({ label: `${label} out`, borderColor: color, backgroundColor: color, data: points.map((p) => ({ x: p.x, y: p.d.keyFramesOutDelta })), spanGaps: true, pointRadius: 0 }); + datasets.keyframes.push({ label: `${label} in`, borderColor: color, backgroundColor: color, borderDash: [4, 4], data: points.map((p) => ({ x: p.x, y: p.d.keyFramesInDelta })), spanGaps: true, pointRadius: 0 }); + const qpOutRawSeries = points.map((p) => p.d.avgQpOut); + const qpInRawSeries = points.map((p) => p.d.avgQpIn); + datasets.qp.push({ + label: `${label} out (${outCodec})`, + borderColor: color, + backgroundColor: color, + yAxisID: 'y', + data: points.map((p) => ({ x: p.x, y: normalizeQp(p.d.avgQpOut, outCodec) })), + spanGaps: true, + pointRadius: 0, + _qpRaw: qpOutRawSeries, + _qpCodec: outCodec, + }); + datasets.qp.push({ + label: `${label} in (${inCodec})`, + borderColor: color, + backgroundColor: color, + borderDash: [4, 4], + yAxisID: 'y', + data: points.map((p) => ({ x: p.x, y: normalizeQp(p.d.avgQpIn, inCodec) })), + spanGaps: true, + pointRadius: 0, + _qpRaw: qpInRawSeries, + _qpCodec: inCodec, + }); + datasets.bpp.push({ label: `${label} out`, borderColor: color, backgroundColor: color, yAxisID: 'y', data: points.map((p) => ({ x: p.x, y: p.d.bppOut })), spanGaps: true, pointRadius: 0 }); + datasets.bpp.push({ label: `${label} in`, borderColor: color, backgroundColor: color, borderDash: [4, 4], yAxisID: 'y', data: points.map((p) => ({ x: p.x, y: p.d.bppIn })), spanGaps: true, pointRadius: 0 }); }); charts.fps.data.datasets = datasets.fps; charts.bitrate.data.datasets = datasets.bitrate; @@ -1201,12 +1358,16 @@

Console Logs

charts.loss.data.datasets = datasets.loss; charts.nack.data.datasets = datasets.nack; charts.keyframes.data.datasets = datasets.keyframes; + charts.qp.data.datasets = datasets.qp; + charts.bpp.data.datasets = datasets.bpp; charts.fps.update(); charts.bitrate.update(); charts.rtt.update(); charts.loss.update(); charts.nack.update(); charts.keyframes.update(); + charts.qp.update(); + charts.bpp.update(); } function refreshSummaryChart() { @@ -1214,47 +1375,154 @@

Console Logs

if (!charts) return; const labels = benchmarkRuns.map((r, i) => runLabel(r, i)); const pick = (key) => benchmarkRuns.map((r) => key(r.summary)); + + const BITRATE_COLOR = '#17becf'; + const BPP_COLOR = '#b15928'; + const QP_COLOR = '#6a3d9a'; + + const qpInRaw = pick((s) => s.avgQpIn); + const qpOutRaw = pick((s) => s.avgQpOut); + const qpInCodec = pick((s) => s.inboundCodec); + const qpOutCodec = pick((s) => s.outboundCodec); + charts.summary.data.labels = labels; charts.summary.data.datasets = [ - { label: 'Warmup (s)', data: pick((s) => s.warmupMs != null ? s.warmupMs / 1000 : null), backgroundColor: '#1f77b4' }, { label: 'Avg FPS', data: pick((s) => s.fps.avg), backgroundColor: '#2ca02c' }, { label: 'p5 FPS', data: pick((s) => s.fps.p5), backgroundColor: '#ff7f0e' }, { label: 'FPS σ', data: pick((s) => s.fps.stddev), backgroundColor: '#9467bd' }, - { label: 'Avg rx kbps / 100', data: pick((s) => s.receiverBitrateBps / 100000), backgroundColor: '#17becf' }, - { label: 'Avg tx kbps / 100', data: pick((s) => s.senderBitrateBps / 100000), backgroundColor: '#0b5d6f' }, - { label: 'Freezes', data: pick((s) => s.freezeCount), backgroundColor: '#d62728' }, - { label: 'Loss %', data: pick((s) => s.packetLoss * 100), backgroundColor: '#8c564b' }, + + { + label: 'Avg tx kbps / 100', + data: pick((s) => s.senderBitrateBps / 100000), + backgroundColor: BITRATE_COLOR, + borderColor: BITRATE_COLOR, + }, + { + label: 'Avg rx kbps / 100', + data: pick((s) => s.receiverBitrateBps / 100000), + backgroundColor: makeStripePattern(BITRATE_COLOR), + borderColor: BITRATE_COLOR, + borderWidth: 1, + }, + + { + label: 'Avg QP % out', + data: qpOutRaw.map((q, i) => normalizeQp(q, qpOutCodec[i])), + backgroundColor: QP_COLOR, + borderColor: QP_COLOR, + _qpRaw: qpOutRaw, + _qpCodec: qpOutCodec, + }, + { + label: 'Avg QP % in', + data: qpInRaw.map((q, i) => normalizeQp(q, qpInCodec[i])), + backgroundColor: makeStripePattern(QP_COLOR), + borderColor: QP_COLOR, + borderWidth: 1, + _qpRaw: qpInRaw, + _qpCodec: qpInCodec, + }, + + { + label: 'Avg BPP out × 100', + data: pick((s) => s.avgBppOut != null ? s.avgBppOut * 100 : null), + backgroundColor: BPP_COLOR, + borderColor: BPP_COLOR, + }, + { + label: 'Avg BPP in × 100', + data: pick((s) => s.avgBppIn != null ? s.avgBppIn * 100 : null), + backgroundColor: makeStripePattern(BPP_COLOR), + borderColor: BPP_COLOR, + borderWidth: 1, + }, + + { label: 'Freezes total', data: pick((s) => s.freezeCount), backgroundColor: '#d62728' }, ]; charts.summary.update(); } + function sumLayerField(layers, field) { + let total = 0; + let seen = false; + for (const layer of layers ?? []) { + const value = layer?.[field]; + if (typeof value === 'number' && Number.isFinite(value)) { + total += value; + seen = true; + } + } + return seen ? total : null; + } + + function safeDelta(currTotal, prevTotal) { + if (currTotal == null || prevTotal == null) return null; + const delta = currTotal - prevTotal; + return Number.isFinite(delta) && delta >= 0 ? delta : null; + } + + function firstCodec(items) { + for (const item of items ?? []) { + if (item?.codecMimeType) return item.codecMimeType; + } + return null; + } + function deriveSampleMetrics(curr, prev) { const d = { + outboundCodec: null, + inboundCodec: null, receiverFps: null, receiverBitrateBps: null, senderBitrateBps: null, senderFps: null, packetsLostDelta: 0, - packetsReceivedDelta: 0, cumulativeLossPct: null, rtt: null, - jitter: null, - frameWidth: null, - frameHeight: null, - nackCountTotal: null, - nackCountDelta: 0, - keyFramesEncodedTotal: curr.keyFramesEncoded ?? 0, - keyFramesDecodedTotal: curr.keyFramesDecoded ?? 0, - keyFramesEncodedDelta: prev ? Math.max(0, (curr.keyFramesEncoded ?? 0) - (prev.keyFramesEncoded ?? 0)) : 0, - keyFramesDecodedDelta: prev ? Math.max(0, (curr.keyFramesDecoded ?? 0) - (prev.keyFramesDecoded ?? 0)) : 0, - qualityLimitationReason: null, + outFrameWidth: null, + outFrameHeight: null, + inFrameWidth: null, + inFrameHeight: null, + inFps: null, + nackOutDelta: 0, + nackInDelta: 0, + keyFramesOutTotal: null, + keyFramesOutDelta: 0, + keyFramesInTotal: null, + keyFramesInDelta: 0, + qlReasonOut: null, + bppOut: null, + bppIn: null, + qpSumOutTotal: null, + framesEncodedTotal: null, + avgQpOut: null, + totalEncodeTimeSec: null, + avgEncodeMs: null, + qpSumInTotal: null, + framesDecodedTotal: null, + avgQpIn: null, + freezeCountTotal: null, + freezeCountDelta: 0, + totalFreezesDurationSec: null, + pauseCountTotal: null, + pauseCountDelta: 0, + framesDroppedInTotal: null, + framesDroppedInDelta: 0, }; + const recv = curr.receiver?.[0]; if (recv) { - d.frameWidth = recv.frameWidth ?? null; - d.frameHeight = recv.frameHeight ?? null; - d.jitter = recv.jitter ?? null; - d.nackCountTotal = recv.nackCount ?? null; + d.inFrameWidth = recv.frameWidth ?? null; + d.inFrameHeight = recv.frameHeight ?? null; + d.inFps = typeof recv.framesPerSecond === 'number' ? recv.framesPerSecond : null; + d.qpSumInTotal = recv.qpSum ?? null; + d.framesDecodedTotal = recv.framesDecoded ?? null; + d.keyFramesInTotal = recv.keyFramesDecoded ?? null; + d.freezeCountTotal = recv.freezeCount ?? null; + d.pauseCountTotal = recv.pauseCount ?? null; + d.framesDroppedInTotal = recv.framesDropped ?? null; + d.totalFreezesDurationSec = recv.totalFreezesDuration ?? null; + const prevRecv = prev?.receiver?.[0]; if (prevRecv) { const dtMs = recv.timestamp - prevRecv.timestamp; @@ -1263,17 +1531,35 @@

Console Logs

d.receiverBitrateBps = ((recv.bytesReceived - prevRecv.bytesReceived) * 8 * 1000) / dtMs; } d.packetsLostDelta = Math.max(0, (recv.packetsLost ?? 0) - (prevRecv.packetsLost ?? 0)); - d.packetsReceivedDelta = Math.max(0, (recv.packetsReceived ?? 0) - (prevRecv.packetsReceived ?? 0)); - d.nackCountDelta = Math.max(0, (recv.nackCount ?? 0) - (prevRecv.nackCount ?? 0)); + d.nackInDelta = Math.max(0, (recv.nackCount ?? 0) - (prevRecv.nackCount ?? 0)); + d.freezeCountDelta = Math.max(0, (recv.freezeCount ?? 0) - (prevRecv.freezeCount ?? 0)); + d.pauseCountDelta = Math.max(0, (recv.pauseCount ?? 0) - (prevRecv.pauseCount ?? 0)); + d.framesDroppedInDelta = Math.max(0, (recv.framesDropped ?? 0) - (prevRecv.framesDropped ?? 0)); + d.keyFramesInDelta = Math.max(0, (recv.keyFramesDecoded ?? 0) - (prevRecv.keyFramesDecoded ?? 0)); + + const qpDelta = safeDelta(recv.qpSum, prevRecv.qpSum); + const decodedDelta = safeDelta(recv.framesDecoded, prevRecv.framesDecoded); + if (qpDelta != null && decodedDelta && decodedDelta > 0) { + d.avgQpIn = qpDelta / decodedDelta; + } } const totalLost = recv.packetsLost ?? 0; const totalRecv = recv.packetsReceived ?? 0; const denom = totalLost + totalRecv; d.cumulativeLossPct = denom > 0 ? (totalLost / denom) * 100 : null; } + if (curr.sender && curr.sender.length > 0) { d.senderFps = curr.sender.reduce((m, s) => Math.max(m, s.framesPerSecond ?? 0), 0); - d.qualityLimitationReason = curr.sender[0].qualityLimitationReason ?? null; + d.qlReasonOut = curr.sender[0].qualityLimitationReason ?? null; + d.outFrameWidth = curr.sender[0].frameWidth ?? null; + d.outFrameHeight = curr.sender[0].frameHeight ?? null; + d.qpSumOutTotal = sumLayerField(curr.sender, 'qpSum'); + d.framesEncodedTotal = sumLayerField(curr.sender, 'framesEncoded'); + d.totalEncodeTimeSec = sumLayerField(curr.sender, 'totalEncodeTime'); + d.keyFramesOutTotal = sumLayerField(curr.sender, 'keyFramesEncoded'); + const nackOutTotal = sumLayerField(curr.sender, 'nackCount'); + let bytesNow = 0; let bytesPrev = 0; let dtMs = 0; @@ -1288,7 +1574,51 @@

Console Logs

} if (dtMs > 0) d.senderBitrateBps = ((bytesNow - bytesPrev) * 8 * 1000) / dtMs; d.rtt = curr.sender.reduce((m, s) => Math.max(m, (s.roundTripTime ?? 0) * 1000), 0) || null; + + if (prev?.sender) { + const prevQpOut = sumLayerField(prev.sender, 'qpSum'); + const prevFramesEncoded = sumLayerField(prev.sender, 'framesEncoded'); + const prevTotalEncodeTime = sumLayerField(prev.sender, 'totalEncodeTime'); + const prevKeyFramesEncoded = sumLayerField(prev.sender, 'keyFramesEncoded'); + const prevNackOut = sumLayerField(prev.sender, 'nackCount'); + + const qpDelta = safeDelta(d.qpSumOutTotal, prevQpOut); + const framesEncodedDelta = safeDelta(d.framesEncodedTotal, prevFramesEncoded); + const totalEncodeTimeDelta = safeDelta(d.totalEncodeTimeSec, prevTotalEncodeTime); + + if (qpDelta != null && framesEncodedDelta && framesEncodedDelta > 0) { + d.avgQpOut = qpDelta / framesEncodedDelta; + } + if (totalEncodeTimeDelta != null && framesEncodedDelta && framesEncodedDelta > 0) { + d.avgEncodeMs = (totalEncodeTimeDelta * 1000) / framesEncodedDelta; + } + d.keyFramesOutDelta = safeDelta(d.keyFramesOutTotal, prevKeyFramesEncoded) ?? 0; + d.nackOutDelta = safeDelta(nackOutTotal, prevNackOut) ?? 0; + } + } + + d.outboundCodec = firstCodec(curr.sender); + d.inboundCodec = firstCodec(curr.receiver); + + let senderPixelsPerSec = 0; + for (const s of curr.sender ?? []) { + const w = s.frameWidth; + const h = s.frameHeight; + const fps = s.framesPerSecond; + if (w && h && fps) senderPixelsPerSec += w * h * fps; + } + if (senderPixelsPerSec > 0 && d.senderBitrateBps != null && d.senderBitrateBps > 0) { + d.bppOut = d.senderBitrateBps / senderPixelsPerSec; } + + const inFpsForBpp = d.inFps && d.inFps > 0 ? d.inFps : d.receiverFps; + if ( + d.inFrameWidth && d.inFrameHeight && inFpsForBpp && inFpsForBpp > 0 && + d.receiverBitrateBps != null && d.receiverBitrateBps > 0 + ) { + d.bppIn = d.receiverBitrateBps / (d.inFrameWidth * d.inFrameHeight * inFpsForBpp); + } + return d; } @@ -1302,15 +1632,18 @@

Console Logs

let raw; try { raw = await decartRealtime.getVideoStats(); } catch (err) { addLog(`Benchmark sample failed: ${err.message}`, 'error'); return; } const tMs = nowMs() - currentBenchRun.startedAtMs; - const derived = deriveSampleMetrics(raw, benchPrevRaw); - const sample = { tMs, phase: currentPhase(), sender: raw.sender, receiver: raw.receiver, derived }; + const derived = deriveSampleMetrics(raw, currentBenchRun.prevRaw); + const sample = { tMs, sender: raw.sender, receiver: raw.receiver, derived }; currentBenchRun.samples.push(sample); - benchPrevRaw = raw; + currentBenchRun.prevRaw = raw; elements.benchLive.textContent = - `t=${(tMs / 1000).toFixed(0)}s phase=${sample.phase} ` + + `t=${(tMs / 1000).toFixed(0)}s ` + `rxFps=${(derived.receiverFps ?? 0).toFixed(1)} rxKbps=${((derived.receiverBitrateBps ?? 0) / 1000).toFixed(0)} ` + `txKbps=${((derived.senderBitrateBps ?? 0) / 1000).toFixed(0)} rtt=${(derived.rtt ?? 0).toFixed(0)}ms ` + - `res=${derived.frameWidth ?? '?'}×${derived.frameHeight ?? '?'} qLim=${derived.qualityLimitationReason ?? 'none'}`; + `out=${derived.outFrameWidth ?? '?'}×${derived.outFrameHeight ?? '?'} (${derived.outboundCodec ?? '?'}) ` + + `in=${derived.inFrameWidth ?? '?'}×${derived.inFrameHeight ?? '?'} (${derived.inboundCodec ?? '?'}) ` + + `qpOut=${derived.avgQpOut?.toFixed?.(1) ?? '?'} qpIn=${derived.avgQpIn?.toFixed?.(1) ?? '?'} ` + + `bppOut=${derived.bppOut?.toFixed?.(3) ?? '?'} bppIn=${derived.bppIn?.toFixed?.(3) ?? '?'}`; if (currentBenchRun.warmupMs === null && derived.receiverFps != null) { currentBenchRun.warmupRun.push(derived.receiverFps); if (currentBenchRun.warmupRun.length > BENCH_WARMUP_REQUIRED_SAMPLES) currentBenchRun.warmupRun.shift(); @@ -1336,32 +1669,46 @@

Console Logs

return s[i]; } + function meanIgnoringNulls(arr) { + const filtered = arr.filter((v) => v != null && Number.isFinite(v)); + return filtered.length ? filtered.reduce((a, b) => a + b, 0) / filtered.length : null; + } + function computeBenchSummary(run) { const fpsArr = run.samples.map((s) => s.derived.receiverFps).filter((v) => v != null && Number.isFinite(v)); const rxBr = run.samples.map((s) => s.derived.receiverBitrateBps).filter((v) => v != null && Number.isFinite(v)); const txBr = run.samples.map((s) => s.derived.senderBitrateBps).filter((v) => v != null && Number.isFinite(v)); const rtts = run.samples.map((s) => s.derived.rtt).filter((v) => v != null && Number.isFinite(v) && v > 0); - const jitters = run.samples.map((s) => s.derived.jitter).filter((v) => v != null && Number.isFinite(v)); const totalLost = run.samples.reduce((a, s) => a + (s.derived.packetsLostDelta || 0), 0); - const totalReceived = run.samples.reduce((a, s) => a + (s.derived.packetsReceivedDelta || 0), 0); - const loss = (totalLost + totalReceived) > 0 ? totalLost / (totalLost + totalReceived) : 0; - let freezeCount = 0; - let freezeDurMs = 0; - let inFreeze = false; - for (let i = 1; i < run.samples.length; i++) { - const f = run.samples[i].derived.receiverFps; - const isFreeze = f != null && f < 1; - if (isFreeze) { - freezeDurMs += run.samples[i].tMs - run.samples[i - 1].tMs; - if (!inFreeze) { freezeCount++; inFreeze = true; } - } else { inFreeze = false; } - } - const last = run.samples[run.samples.length - 1]; - const resolution = last?.derived.frameWidth && last.derived.frameHeight - ? `${last.derived.frameWidth}×${last.derived.frameHeight}` : 'n/a'; + const totalReceived = run.samples.reduce( + (a, s) => a + ((s.receiver?.[0]?.packetsReceived ?? 0) - (a > 0 ? 0 : 0)), + 0, + ); + const lastRecv = run.samples[run.samples.length - 1]?.receiver?.[0]; + const firstRecv = run.samples.find((s) => s.receiver?.[0])?.receiver?.[0]; + const denom = (lastRecv?.packetsReceived ?? 0) - (firstRecv?.packetsReceived ?? 0) + + (lastRecv?.packetsLost ?? 0) - (firstRecv?.packetsLost ?? 0); + const loss = denom > 0 ? totalLost / denom : 0; + + const lastSample = run.samples[run.samples.length - 1]; + const firstSampleWithFreeze = run.samples.find((s) => s.derived.freezeCountTotal != null); + const freezeCount = lastSample?.derived.freezeCountTotal != null + ? lastSample.derived.freezeCountTotal - (firstSampleWithFreeze?.derived.freezeCountTotal ?? 0) + : 0; + const freezeDurSec = lastSample?.derived.totalFreezesDurationSec != null + ? lastSample.derived.totalFreezesDurationSec - (firstSampleWithFreeze?.derived.totalFreezesDurationSec ?? 0) + : 0; + + const outResolution = lastSample?.derived.outFrameWidth && lastSample.derived.outFrameHeight + ? `${lastSample.derived.outFrameWidth}×${lastSample.derived.outFrameHeight}` : 'n/a'; + const inResolution = lastSample?.derived.inFrameWidth && lastSample.derived.inFrameHeight + ? `${lastSample.derived.inFrameWidth}×${lastSample.derived.inFrameHeight}` : 'n/a'; const avgFps = avg(fpsArr); const avgRx = avg(rxBr); const bitratePerFrame = avgFps > 0 ? avgRx / avgFps : 0; + const outboundCodec = run.samples.find((s) => s.derived.outboundCodec)?.derived.outboundCodec ?? null; + const inboundCodec = run.samples.find((s) => s.derived.inboundCodec)?.derived.inboundCodec ?? null; + const sumDeltas = (key) => run.samples.reduce((a, s) => a + (s.derived[key] || 0), 0); return { durationMs: run.endedAtMs - run.startedAtMs, samples: run.samples.length, @@ -1378,11 +1725,22 @@

Console Logs

receiverBitrateBps: avgRx, bytesPerFrame: bitratePerFrame / 8, rttMs: avg(rtts), - jitter: avg(jitters), packetLoss: loss, freezeCount, - totalFreezesDurationMs: freezeDurMs, - resolution, + totalFreezesDurationSec: freezeDurSec, + outResolution, + inResolution, + outboundCodec, + inboundCodec, + avgQpOut: meanIgnoringNulls(run.samples.map((s) => s.derived.avgQpOut)), + avgQpIn: meanIgnoringNulls(run.samples.map((s) => s.derived.avgQpIn)), + avgBppOut: meanIgnoringNulls(run.samples.map((s) => s.derived.bppOut)), + avgBppIn: meanIgnoringNulls(run.samples.map((s) => s.derived.bppIn)), + avgEncodeMs: meanIgnoringNulls(run.samples.map((s) => s.derived.avgEncodeMs)), + keyFramesOutDeltaTotal: sumDeltas('keyFramesOutDelta'), + keyFramesInDeltaTotal: sumDeltas('keyFramesInDelta'), + nackOutDeltaTotal: sumDeltas('nackOutDelta'), + nackInDeltaTotal: sumDeltas('nackInDelta'), }; } @@ -1401,8 +1759,8 @@

Console Logs

samples: [], warmupMs: null, warmupRun: [], + prevRaw: null, }; - benchPrevRaw = null; elements.benchStart.disabled = true; elements.benchStop.disabled = false; elements.benchMarkPhase.disabled = false; @@ -1472,14 +1830,21 @@

Console Logs

function exportBenchCsv() { const header = [ - 'run', 'profile', 'sessionId', 'tMs', 'phase', + 'run', 'profile', 'sessionId', 'tMs', + 'outboundCodec', 'inboundCodec', 'receiverFps', 'receiverKbps', 'senderKbps', 'senderFps', - 'rttMs', 'jitter', 'packetsLostDelta', 'packetsReceivedDelta', - 'cumulativeLossPct', 'frameWidth', 'frameHeight', - 'nackCountTotal', 'nackCountDelta', - 'keyFramesEncodedTotal', 'keyFramesEncodedDelta', - 'keyFramesDecodedTotal', 'keyFramesDecodedDelta', - 'qualityLimitationReason', + 'outFrameWidth', 'outFrameHeight', 'inFrameWidth', 'inFrameHeight', 'inFps', + 'bppOut', 'bppIn', + 'qpSumOutTotal', 'framesEncodedTotal', 'avgQpOut', + 'qpSumInTotal', 'framesDecodedTotal', 'avgQpIn', + 'totalEncodeTimeSec', 'avgEncodeMs', + 'rttMs', 'packetsLostDelta', 'cumulativeLossPct', + 'nackOutDelta', 'nackInDelta', + 'keyFramesOutTotal', 'keyFramesOutDelta', 'keyFramesInTotal', 'keyFramesInDelta', + 'framesDroppedInTotal', 'framesDroppedInDelta', + 'freezeCountTotal', 'freezeCountDelta', 'totalFreezesDurationSec', + 'pauseCountTotal', 'pauseCountDelta', + 'qlReasonOut', ]; const rows = [header.join(',')]; benchmarkRuns.forEach((run, idx) => { @@ -1490,25 +1855,44 @@

Console Logs

run.profile, run.sessionId ?? '', s.tMs, - JSON.stringify(s.phase), + d.outboundCodec ?? '', + d.inboundCodec ?? '', d.receiverFps?.toFixed(3) ?? '', d.receiverBitrateBps != null ? (d.receiverBitrateBps / 1000).toFixed(2) : '', d.senderBitrateBps != null ? (d.senderBitrateBps / 1000).toFixed(2) : '', d.senderFps?.toFixed(3) ?? '', + d.outFrameWidth ?? '', + d.outFrameHeight ?? '', + d.inFrameWidth ?? '', + d.inFrameHeight ?? '', + d.inFps != null ? d.inFps.toFixed(3) : '', + d.bppOut != null ? d.bppOut.toFixed(6) : '', + d.bppIn != null ? d.bppIn.toFixed(6) : '', + d.qpSumOutTotal ?? '', + d.framesEncodedTotal ?? '', + d.avgQpOut != null ? d.avgQpOut.toFixed(3) : '', + d.qpSumInTotal ?? '', + d.framesDecodedTotal ?? '', + d.avgQpIn != null ? d.avgQpIn.toFixed(3) : '', + d.totalEncodeTimeSec != null ? d.totalEncodeTimeSec.toFixed(4) : '', + d.avgEncodeMs != null ? d.avgEncodeMs.toFixed(3) : '', d.rtt?.toFixed(2) ?? '', - d.jitter ?? '', d.packetsLostDelta ?? '', - d.packetsReceivedDelta ?? '', d.cumulativeLossPct?.toFixed(4) ?? '', - d.frameWidth ?? '', - d.frameHeight ?? '', - d.nackCountTotal ?? '', - d.nackCountDelta ?? '', - d.keyFramesEncodedTotal ?? '', - d.keyFramesEncodedDelta ?? '', - d.keyFramesDecodedTotal ?? '', - d.keyFramesDecodedDelta ?? '', - d.qualityLimitationReason ?? '', + d.nackOutDelta ?? '', + d.nackInDelta ?? '', + d.keyFramesOutTotal ?? '', + d.keyFramesOutDelta ?? '', + d.keyFramesInTotal ?? '', + d.keyFramesInDelta ?? '', + d.framesDroppedInTotal ?? '', + d.framesDroppedInDelta ?? '', + d.freezeCountTotal ?? '', + d.freezeCountDelta ?? '', + d.totalFreezesDurationSec != null ? d.totalFreezesDurationSec.toFixed(3) : '', + d.pauseCountTotal ?? '', + d.pauseCountDelta ?? '', + d.qlReasonOut ?? '', ].join(',')); }); }); @@ -1655,7 +2039,7 @@

Console Logs

} function formatProfileTitle({ key, label }) { - return key === 'default' ? `Default baseline — ${label}` : `Profile ${key} — ${label}`; + return `Profile ${key} — ${label}`; } function updatePublishProfileDescription() { @@ -1719,7 +2103,7 @@

Console Logs

addBtn.style.marginBottom = '0'; addBtn.disabled = chainActive; addBtn.addEventListener('click', () => { - const last = chainedProfileKeys[chainedProfileKeys.length - 1] ?? 'default'; + const last = chainedProfileKeys[chainedProfileKeys.length - 1] ?? DEFAULT_PROFILE_KEY; chainedProfileKeys.push(last); renderChainedProfiles(); updatePublishProfileDescription(); @@ -1936,7 +2320,7 @@

Console Logs

if (!isConnected) return; addLog('Auto-starting benchmark (3s after remote stream)', 'info'); startBenchmark(); - }, 3000); + }, 1500); } async function connectPublisher() { @@ -1996,8 +2380,9 @@

Console Logs

scheduleAutoStartBenchmark(); if (chainActive) { if (chainAdvanceTimer) clearTimeout(chainAdvanceTimer); - chainAdvanceTimer = setTimeout(advanceChain, CHAIN_INTERVAL_MS); - addLog(`Chain advance armed in ${CHAIN_INTERVAL_MS / 1000}s`, 'info'); + const chainIntervalMs = getChainIntervalMs(); + chainAdvanceTimer = setTimeout(advanceChain, chainIntervalMs); + addLog(`Chain advance armed in ${chainIntervalMs / 1000}s`, 'info'); } }, onConnectionChange: (state) => { @@ -2121,7 +2506,8 @@

Console Logs

chainIndex = 0; if (chainedProfileKeys.length > 1) { setChainUiLocked(true); - addLog(`Starting chain of ${chainedProfileKeys.length} profiles × ${CHAIN_INTERVAL_MS / 1000}s = ${(chainedProfileKeys.length * CHAIN_INTERVAL_MS / 1000)}s total`, 'success'); + const chainIntervalMs = getChainIntervalMs(); + addLog(`Starting chain of ${chainedProfileKeys.length} profiles × ${chainIntervalMs / 1000}s = ${(chainedProfileKeys.length * chainIntervalMs / 1000)}s total`, 'success'); } else { setChainUiLocked(false); } diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 23704e02..7f5248b5 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -18,7 +18,9 @@ import { RealtimeObservability } from "./observability/realtime-observability"; import type { WebRTCStats } from "./observability/webrtc-stats"; import { StreamSession } from "./stream-session"; import type { ConnectionState, GenerationEnded, GenerationTick, ImageSetOptions, QueuePosition } from "./types"; -import type { RoomOptions, TrackPublishOptions, VideoReceiverStats, VideoSenderStats } from "livekit-client"; +import type { RoomOptions, TrackPublishOptions } from "livekit-client"; + +import type { RealtimeVideoStats } from "./media-channel"; export type RealTimeClientOptions = { baseUrl: string; @@ -89,12 +91,7 @@ export type RealTimeClient = { subscribeToken: string | null; getSubscribeToken: () => string | null; setImage: (image: Blob | File | string | null, options?: ImageSetOptions) => Promise; - getVideoStats: () => Promise<{ - sender: VideoSenderStats[]; - receiver: VideoReceiverStats[]; - keyFramesEncoded: number; - keyFramesDecoded: number; - }>; + getVideoStats: () => Promise; }; export const createRealTimeClient = (opts: RealTimeClientOptions) => { diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index ec0fc00c..3ffc53d1 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -21,6 +21,111 @@ import type { RealtimeObservability } from "./observability/realtime-observabili export type VideoCodec = "h264" | "vp8" | "vp9" | "av1"; +export interface RealtimeVideoSenderStats extends VideoSenderStats { + qpSum?: number; + framesEncoded?: number; + totalEncodeTime?: number; + keyFramesEncoded?: number; + codecMimeType?: string; +} + +export interface RealtimeVideoReceiverStats extends VideoReceiverStats { + qpSum?: number; + freezeCount?: number; + totalFreezesDuration?: number; + pauseCount?: number; + totalPausesDuration?: number; + keyFramesDecoded?: number; + framesPerSecond?: number; + codecMimeType?: string; +} + +export interface RealtimeVideoStats { + sender: RealtimeVideoSenderStats[]; + receiver: RealtimeVideoReceiverStats[]; +} + +type RawOutboundRtp = { + type?: string; + kind?: string; + rid?: string; + ssrc?: number; + qpSum?: number; + framesEncoded?: number; + totalEncodeTime?: number; + keyFramesEncoded?: number; + codecId?: string; +}; + +type RawInboundRtp = { + type?: string; + kind?: string; + qpSum?: number; + framesDropped?: number; + freezeCount?: number; + totalFreezesDuration?: number; + pauseCount?: number; + totalPausesDuration?: number; + keyFramesDecoded?: number; + framesPerSecond?: number; + codecId?: string; +}; + +type RawCodec = { type?: string; id?: string; mimeType?: string }; + +function shortCodecName(mimeType: string | undefined): string | undefined { + if (!mimeType) return undefined; + const slash = mimeType.indexOf("/"); + return slash >= 0 ? mimeType.slice(slash + 1).toUpperCase() : mimeType.toUpperCase(); +} + +function collectCodecMimeMap(report: RTCStatsReport | null): Map { + const map = new Map(); + if (!report) return map; + report.forEach((stat: unknown) => { + const c = stat as RawCodec; + if (c.type === "codec" && c.id && typeof c.mimeType === "string") { + map.set(c.id, c.mimeType); + } + }); + return map; +} + +function mergeSenderLayer( + layer: VideoSenderStats, + outbound: RawOutboundRtp | undefined, + codecMime: Map, +): RealtimeVideoSenderStats { + return { + ...layer, + qpSum: outbound?.qpSum, + framesEncoded: outbound?.framesEncoded, + totalEncodeTime: outbound?.totalEncodeTime, + keyFramesEncoded: outbound?.keyFramesEncoded, + codecMimeType: shortCodecName(outbound?.codecId ? codecMime.get(outbound.codecId) : undefined), + }; +} + +function mergeReceiver( + base: VideoReceiverStats, + inbound: RawInboundRtp | undefined, + codecMime: Map, +): RealtimeVideoReceiverStats { + const merged: RealtimeVideoReceiverStats = { + ...base, + qpSum: inbound?.qpSum, + freezeCount: inbound?.freezeCount, + totalFreezesDuration: inbound?.totalFreezesDuration, + pauseCount: inbound?.pauseCount, + totalPausesDuration: inbound?.totalPausesDuration, + keyFramesDecoded: inbound?.keyFramesDecoded, + framesPerSecond: inbound?.framesPerSecond, + codecMimeType: shortCodecName(inbound?.codecId ? codecMime.get(inbound.codecId) : undefined), + }; + if (typeof inbound?.framesDropped === "number") merged.framesDropped = inbound.framesDropped; + return merged; +} + export function getDefaultVideoPublishOptions( videoCodec?: VideoCodec, overrides?: Partial, @@ -125,69 +230,76 @@ export class MediaChannel { this.config.observability?.setLiveKitRoom(room); } - async getVideoStats(): Promise<{ - sender: VideoSenderStats[]; - receiver: VideoReceiverStats[]; - keyFramesEncoded: number; - keyFramesDecoded: number; - }> { + async getVideoStats(): Promise { const room = this.room; - if (!room) return { sender: [], receiver: [], keyFramesEncoded: 0, keyFramesDecoded: 0 }; + if (!room) return { sender: [], receiver: [] }; - const sender: VideoSenderStats[] = []; - let keyFramesEncoded = 0; + const sender: RealtimeVideoSenderStats[] = []; for (const pub of room.localParticipant.videoTrackPublications.values()) { const track = pub.track; - if (track instanceof LocalVideoTrack) { - try { - const layers = await track.getSenderStats(); - sender.push(...layers); - } catch (error) { - this.logger.debug("getSenderStats failed", { error: (error as Error).message }); - } - try { - const report = await track.getRTCStatsReport(); - report?.forEach((stat: unknown) => { - const s = stat as { type?: string; kind?: string; keyFramesEncoded?: number }; - if (s.type === "outbound-rtp" && s.kind === "video" && typeof s.keyFramesEncoded === "number") { - keyFramesEncoded += s.keyFramesEncoded; - } - }); - } catch (error) { - this.logger.debug("getRTCStatsReport (sender) failed", { error: (error as Error).message }); - } + if (!(track instanceof LocalVideoTrack)) continue; + + let layers: VideoSenderStats[] = []; + try { + layers = await track.getSenderStats(); + } catch (error) { + this.logger.debug("getSenderStats failed", { error: (error as Error).message }); + } + + let report: RTCStatsReport | null = null; + try { + report = (await track.getRTCStatsReport()) ?? null; + } catch (error) { + this.logger.debug("getRTCStatsReport (sender) failed", { error: (error as Error).message }); + } + + const codecMime = collectCodecMimeMap(report); + const outbounds: RawOutboundRtp[] = []; + report?.forEach((stat: unknown) => { + const s = stat as RawOutboundRtp; + if (s.type === "outbound-rtp" && s.kind === "video") outbounds.push(s); + }); + + for (const layer of layers) { + const match = outbounds.find((o) => (o.rid ?? "") === (layer.rid ?? "")) ?? outbounds[0]; + sender.push(mergeSenderLayer(layer, match, codecMime)); } } - const receiver: VideoReceiverStats[] = []; - let keyFramesDecoded = 0; + const receiver: RealtimeVideoReceiverStats[] = []; for (const participant of room.remoteParticipants.values()) { if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) continue; for (const pub of participant.videoTrackPublications.values()) { const track = pub.track; - if (track instanceof RemoteVideoTrack) { - try { - const stats = await track.getReceiverStats(); - if (stats) receiver.push(stats); - } catch (error) { - this.logger.debug("getReceiverStats failed", { error: (error as Error).message }); - } - try { - const report = await track.getRTCStatsReport(); - report?.forEach((stat: unknown) => { - const s = stat as { type?: string; kind?: string; keyFramesDecoded?: number }; - if (s.type === "inbound-rtp" && s.kind === "video" && typeof s.keyFramesDecoded === "number") { - keyFramesDecoded += s.keyFramesDecoded; - } - }); - } catch (error) { - this.logger.debug("getRTCStatsReport (receiver) failed", { error: (error as Error).message }); - } + if (!(track instanceof RemoteVideoTrack)) continue; + + let stats: VideoReceiverStats | undefined; + try { + stats = await track.getReceiverStats(); + } catch (error) { + this.logger.debug("getReceiverStats failed", { error: (error as Error).message }); } + if (!stats) continue; + + let report: RTCStatsReport | null = null; + try { + report = (await track.getRTCStatsReport()) ?? null; + } catch (error) { + this.logger.debug("getRTCStatsReport (receiver) failed", { error: (error as Error).message }); + } + + const codecMime = collectCodecMimeMap(report); + let inbound: RawInboundRtp | undefined; + report?.forEach((stat: unknown) => { + const s = stat as RawInboundRtp; + if (!inbound && s.type === "inbound-rtp" && s.kind === "video") inbound = s; + }); + + receiver.push(mergeReceiver(stats, inbound, codecMime)); } } - return { sender, receiver, keyFramesEncoded, keyFramesDecoded }; + return { sender, receiver }; } async publishLocalTracks(): Promise { diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts index 54a47126..2bca2978 100644 --- a/packages/sdk/src/realtime/stream-session.ts +++ b/packages/sdk/src/realtime/stream-session.ts @@ -4,8 +4,8 @@ import pRetry, { AbortError } from "p-retry"; import { createConsoleLogger, type Logger } from "../utils/logger"; import { REALTIME_CONFIG } from "./config-realtime"; import { InitialStateGate } from "./initial-state-gate"; -import { MediaChannel, type VideoCodec } from "./media-channel"; -import type { RoomOptions, TrackPublishOptions, VideoReceiverStats, VideoSenderStats } from "livekit-client"; +import { MediaChannel, type RealtimeVideoStats, type VideoCodec } from "./media-channel"; +import type { RoomOptions, TrackPublishOptions } from "livekit-client"; import type { RealtimeObservability } from "./observability/realtime-observability"; import { SignalingChannel } from "./signaling-channel"; import type { @@ -127,12 +127,7 @@ export class StreamSession { return this.signaling.sendPrompt(text, opts); } - async getVideoStats(): Promise<{ - sender: VideoSenderStats[]; - receiver: VideoReceiverStats[]; - keyFramesEncoded: number; - keyFramesDecoded: number; - }> { + async getVideoStats(): Promise { return this.media.getVideoStats(); } diff --git a/packages/sdk/src/realtime/subscribe-client.ts b/packages/sdk/src/realtime/subscribe-client.ts index 1b555e70..5a885d4f 100644 --- a/packages/sdk/src/realtime/subscribe-client.ts +++ b/packages/sdk/src/realtime/subscribe-client.ts @@ -6,13 +6,13 @@ import { Room, RoomEvent, Track, - type VideoReceiverStats, } from "livekit-client"; import { classifyWebrtcError, type DecartSDKError } from "../utils/errors"; import { createConsoleLogger, type Logger } from "../utils/logger"; import { REALTIME_CONFIG } from "./config-realtime"; import { createEventBuffer } from "./event-buffer"; +import type { RealtimeVideoReceiverStats, RealtimeVideoStats } from "./media-channel"; import type { DiagnosticEvent } from "./observability/diagnostics"; import { RealtimeObservability } from "./observability/realtime-observability"; import type { ConnectionState } from "./types"; @@ -57,7 +57,7 @@ export type RealTimeSubscribeClient = { disconnect: () => void; on: (event: K, listener: (data: SubscribeEvents[K]) => void) => void; off: (event: K, listener: (data: SubscribeEvents[K]) => void) => void; - getVideoStats: () => Promise<{ sender: never[]; receiver: VideoReceiverStats[] }>; + getVideoStats: () => Promise; }; export type SubscribeOptions = { @@ -190,18 +190,24 @@ export const createRealTimeSubscribeClient = (opts: RealTimeSubscribeClientOptio on: emitter.on, off: emitter.off, getVideoStats: async () => { - const receiver: VideoReceiverStats[] = []; + const receiver: RealtimeVideoReceiverStats[] = []; for (const participant of activeRoom.remoteParticipants.values()) { if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) continue; for (const pub of participant.videoTrackPublications.values()) { const track = pub.track; - if (track instanceof RemoteVideoTrack) { + if (!(track instanceof RemoteVideoTrack)) continue; + try { + const stats = await track.getReceiverStats(); + if (!stats) continue; + let report: RTCStatsReport | null = null; try { - const stats = await track.getReceiverStats(); - if (stats) receiver.push(stats); + report = (await track.getRTCStatsReport()) ?? null; } catch (error) { - logger.debug("getReceiverStats failed", { error: (error as Error).message }); + logger.debug("getRTCStatsReport (receiver) failed", { error: (error as Error).message }); } + receiver.push(enrichReceiverFromReport(stats, report)); + } catch (error) { + logger.debug("getReceiverStats failed", { error: (error as Error).message }); } } } @@ -224,3 +230,47 @@ export const createRealTimeSubscribeClient = (opts: RealTimeSubscribeClientOptio return { subscribe }; }; + +function enrichReceiverFromReport( + base: import("livekit-client").VideoReceiverStats, + report: RTCStatsReport | null, +): RealtimeVideoReceiverStats { + if (!report) return { ...base }; + const codecMime = new Map(); + type RawInbound = { + type?: string; + kind?: string; + qpSum?: number; + framesDropped?: number; + freezeCount?: number; + totalFreezesDuration?: number; + pauseCount?: number; + totalPausesDuration?: number; + keyFramesDecoded?: number; + codecId?: string; + id?: string; + mimeType?: string; + }; + let inbound: RawInbound | undefined; + report.forEach((stat: unknown) => { + const s = stat as { type?: string; id?: string; mimeType?: string } & RawInbound; + if (s.type === "codec" && s.id && typeof s.mimeType === "string") { + codecMime.set(s.id, s.mimeType); + } else if (!inbound && s.type === "inbound-rtp" && s.kind === "video") { + inbound = s; + } + }); + const mime = inbound?.codecId ? codecMime.get(inbound.codecId) : undefined; + const merged: RealtimeVideoReceiverStats = { + ...base, + qpSum: inbound?.qpSum, + freezeCount: inbound?.freezeCount, + totalFreezesDuration: inbound?.totalFreezesDuration, + pauseCount: inbound?.pauseCount, + totalPausesDuration: inbound?.totalPausesDuration, + keyFramesDecoded: inbound?.keyFramesDecoded, + codecMimeType: mime ? mime.slice(mime.indexOf("/") + 1).toUpperCase() : undefined, + }; + if (typeof inbound?.framesDropped === "number") merged.framesDropped = inbound.framesDropped; + return merged; +} From 9a3d59db093bc55f64b2ff7e7b18d1353ca9842a Mon Sep 17 00:00:00 2001 From: Verion1 Date: Sat, 23 May 2026 21:11:25 +0300 Subject: [PATCH 7/7] new way to profile --- packages/sdk/index.html | 363 ++++++++++++++++++++++++++++++---------- 1 file changed, 277 insertions(+), 86 deletions(-) diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 4755cf9f..1a612db6 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -283,6 +283,7 @@ +

🎭 Decart SDK Test Page

@@ -381,7 +382,7 @@

Configuration

-
+

Benchmark Dashboard

@@ -390,16 +391,17 @@

Benchmark Dashboard

- - - - + +
+ + +
+
Status: idle   Elapsed: 0s -   Phase: (none)
@@ -570,8 +572,8 @@

Console Logs

// ----------------------------------------------------------------------- const PUBLISH_PROFILES = { A: { - label: 'H.264 economy — 1.6 Mbps, maintain-resolution', - description: 'Weak-network champion. ~60% of F/G\'s budget at the same resolution and cadence target, biased to preserve pixel fidelity (drop fps before res). Tests how much bandwidth we can shave before the model\'s input quality breaks.', + label: 'H.264 economy — 1.6 Mbps @ 30 fps, maintain-resolution', + description: 'Bandwidth-shave candidate. ~89% of F\'s budget and 80% of G\'s, same resolution and 30 fps target, biased to preserve pixel fidelity (drop fps before res). Tests how much bitrate we can shave before model input quality breaks.', roomOptions: { adaptiveStream: false, dynacast: true, @@ -589,7 +591,7 @@

Console Logs

}, B: { label: 'H.264 sharper-frames — 2 Mbps @ 24 fps, maintain-resolution', - description: 'Bits-per-frame champion. Same budget as F/G but only 20 fps target, so each frame gets ~50% more bits and visibly sharper detail. Bets the inference model values per-frame fidelity over cadence — and that 20 fps is enough for its temporal needs.', + description: 'Bits-per-frame candidate. Same 2 Mbps as G but at 24 fps, so each frame gets ~25% more bits and visibly sharper detail. Bets the inference model values per-frame fidelity over cadence — and that 24 fps is enough for its temporal needs.', roomOptions: { adaptiveStream: false, dynacast: true, @@ -663,8 +665,8 @@

Console Logs

}, }, F: { - label: 'H.264 constrained, maintain-framerate', - description: 'H.264 single layer with a lower bitrate ceiling for weaker networks; drops resolution before FPS.', + label: 'H.264 constrained — 1.8 Mbps @ 24 fps, maintain-framerate', + description: 'H.264 single layer with a lower bitrate ceiling for weaker networks; drops resolution before FPS (preserves 24 fps cadence under pressure).', roomOptions: { adaptiveStream: false, dynacast: false, @@ -673,16 +675,16 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_000_000, - maxFramerate: 30, + maxBitrate: 1_800_000, + maxFramerate: 24, priority: 'high', }, degradationPreference: 'maintain-framerate', }, }, G: { - label: 'H.264 high bitrate, maintain-framerate', - description: 'H.264 single layer with more bitrate headroom for clean networks while still preserving resolution under pressure.', + label: 'H.264 high bitrate — 2 Mbps @ 30 fps, maintain-resolution', + description: 'H.264 single layer with more bitrate headroom for clean networks while still preserving resolution under pressure (drops fps before resolution).', roomOptions: { adaptiveStream: false, dynacast: false, @@ -699,8 +701,8 @@

Console Logs

}, }, H: { - label: 'H.264 balanced — like F/G but no degradation bias', - description: 'Identical to F/G except degradationPreference=balanced — lets WebRTC pick what to drop. Control profile: tells us whether explicit maintain-framerate or maintain-resolution actually moves the needle vs the engine\'s default behaviour.', + label: 'H.264 balanced — 1.8 Mbps @ 30 fps, default degradation', + description: 'H.264 single layer matching F\'s bitrate ceiling but at 30 fps, with no explicit degradationPreference so the browser picks its default trade-off (typically balanced) between resolution and framerate under pressure.', roomOptions: { adaptiveStream: false, dynacast: false, @@ -709,16 +711,42 @@

Console Logs

videoCodec: 'h264', simulcast: false, videoEncoding: { - maxBitrate: 2_000_000, + maxBitrate: 1_800_000, maxFramerate: 30, priority: 'high', }, - degradationPreference: 'balanced', + }, + }, + L: { + label: 'H.264 headroom — 2.8 Mbps @ 24 fps, default degradation', + description: 'H.264 single layer with the largest bitrate ceiling in the set, paired with a 24 fps target. No explicit degradationPreference; tests how the browser auto-balances when there\'s plenty of budget on a clean link.', + roomOptions: { + adaptiveStream: false, + dynacast: false, + }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { + maxBitrate: 2_800_000, + maxFramerate: 24, + priority: 'high', + }, }, }, }; const DEFAULT_PROFILE_KEY = Object.keys(PUBLISH_PROFILES)[0]; + const DEFAULT_CHAIN_PROFILE_KEYS = ['A', 'B', 'F', 'G', 'H', 'L']; + + function shuffledDefaultChain() { + const keys = DEFAULT_CHAIN_PROFILE_KEYS.filter((k) => k in PUBLISH_PROFILES); + for (let i = keys.length - 1; i > 0; i -= 1) { + const j = Math.floor(Math.random() * (i + 1)); + [keys[i], keys[j]] = [keys[j], keys[i]]; + } + return keys; + } // Chain state: array of profile keys to run sequentially, plus the index currently active. // Read lazily so it tracks the current bench-duration input value at chain time. @@ -728,7 +756,7 @@

Console Logs

? (durationSeconds * 1000) + 6000 : 127_000; } - const chainedProfileKeys = [DEFAULT_PROFILE_KEY]; + const chainedProfileKeys = shuffledDefaultChain(); let chainIndex = 0; let chainActive = false; let chainAdvanceTimer = null; @@ -741,14 +769,15 @@

Console Logs

} const PROFILE_SHORT_LABELS = { - A: 'A — H.264 economy 1.2 Mbps', - B: 'B — H.264 sharper 2 Mbps @ 20 fps', + A: 'A — H.264 1.6 Mbps @ 30 fps, maintain-resolution', + B: 'B — H.264 2 Mbps @ 24 fps, maintain-resolution', C: 'C — VP9 SVC L1T3', D: 'D — VP9 SVC L3T3_KEY', E: 'E — AV1 SVC L3T3_KEY', - F: 'F — H.264 2 Mbps, maintain-framerate', - G: 'G — H.264 2 Mbps, maintain-resolution', - H: 'H — H.264 2 Mbps, balanced', + F: 'F — H.264 1.8 Mbps @ 24 fps, maintain-framerate', + G: 'G — H.264 2 Mbps @ 30 fps, maintain-resolution', + H: 'H — H.264 1.8 Mbps @ 30 fps, default', + L: 'L — H.264 2.8 Mbps @ 24 fps, default', }; function buildProfileOptionsHtml() { @@ -778,14 +807,13 @@

Console Logs

benchDuration: document.getElementById('bench-duration'), benchStart: document.getElementById('bench-start'), benchStop: document.getElementById('bench-stop'), - benchPhaseName: document.getElementById('bench-phase-name'), - benchMarkPhase: document.getElementById('bench-mark-phase'), - benchExportJson: document.getElementById('bench-export-json'), - benchExportCsv: document.getElementById('bench-export-csv'), + recordingName: document.getElementById('recording-name'), + includeRecordingToggle: document.getElementById('include-recording-toggle'), + benchExportAll: document.getElementById('bench-export-all'), + benchDashboard: document.getElementById('bench-dashboard'), benchClear: document.getElementById('bench-clear'), benchStatusText: document.getElementById('bench-status-text'), benchElapsedText: document.getElementById('bench-elapsed-text'), - benchPhaseText: document.getElementById('bench-phase-text'), benchLive: document.getElementById('bench-live'), benchChartFps: document.getElementById('bench-chart-fps'), benchChartBitrate: document.getElementById('bench-chart-bitrate'), @@ -1074,8 +1102,6 @@

Console Logs

// --- Benchmark engine --- const BENCH_POLL_INTERVAL_MS = 1000; - const BENCH_WARMUP_FPS_THRESHOLD = 28; - const BENCH_WARMUP_REQUIRED_SAMPLES = 1; const BENCH_PALETTE = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']; const benchmarkRuns = []; @@ -1098,10 +1124,19 @@

Console Logs

} const QP_MAX = { H264: 51, VP8: 127, VP9: 255, AV1: 255 }; + function qpMaxForCodec(codec) { + const key = (codec || '').toUpperCase().replace(/[^A-Z0-9]/g, ''); + if (key.includes('H264') || key.includes('AVC')) return QP_MAX.H264; + if (key.includes('VP8')) return QP_MAX.VP8; + if (key.includes('VP9')) return QP_MAX.VP9; + if (key.includes('AV1')) return QP_MAX.AV1; + return null; + } + function normalizeQp(qp, codec) { if (qp == null || !Number.isFinite(qp)) return null; - const key = (codec || '').toUpperCase().replace(/[^A-Z0-9]/g, ''); - const max = QP_MAX[key] ?? 51; + const max = qpMaxForCodec(codec); + if (!max) return null; return Math.max(0, Math.min(100, (qp / max) * 100)); } @@ -1622,10 +1657,6 @@

Console Logs

return d; } - function currentPhase() { - if (!currentBenchRun || currentBenchRun.phases.length === 0) return '(none)'; - return currentBenchRun.phases[currentBenchRun.phases.length - 1].name; - } async function collectBenchSample() { if (!currentBenchRun || !decartRealtime?.getVideoStats) return; @@ -1644,15 +1675,6 @@

Console Logs

`in=${derived.inFrameWidth ?? '?'}×${derived.inFrameHeight ?? '?'} (${derived.inboundCodec ?? '?'}) ` + `qpOut=${derived.avgQpOut?.toFixed?.(1) ?? '?'} qpIn=${derived.avgQpIn?.toFixed?.(1) ?? '?'} ` + `bppOut=${derived.bppOut?.toFixed?.(3) ?? '?'} bppIn=${derived.bppIn?.toFixed?.(3) ?? '?'}`; - if (currentBenchRun.warmupMs === null && derived.receiverFps != null) { - currentBenchRun.warmupRun.push(derived.receiverFps); - if (currentBenchRun.warmupRun.length > BENCH_WARMUP_REQUIRED_SAMPLES) currentBenchRun.warmupRun.shift(); - if (currentBenchRun.warmupRun.length === BENCH_WARMUP_REQUIRED_SAMPLES && - currentBenchRun.warmupRun.every((f) => f >= BENCH_WARMUP_FPS_THRESHOLD)) { - currentBenchRun.warmupMs = tMs; - addLog(`Warmup reached: ${(tMs / 1000).toFixed(1)}s`, 'success'); - } - } if (currentBenchRun.samples.length % 2 === 0) refreshTimeSeriesCharts(); } @@ -1674,6 +1696,24 @@

Console Logs

return filtered.length ? filtered.reduce((a, b) => a + b, 0) / filtered.length : null; } + function averageFromRunDeltas(samples, totalKey, countKey) { + const first = samples.find((s) => + typeof s.derived[totalKey] === 'number' && + Number.isFinite(s.derived[totalKey]) && + typeof s.derived[countKey] === 'number' && + Number.isFinite(s.derived[countKey]) + )?.derived; + const last = [...samples].reverse().find((s) => + typeof s.derived[totalKey] === 'number' && + Number.isFinite(s.derived[totalKey]) && + typeof s.derived[countKey] === 'number' && + Number.isFinite(s.derived[countKey]) + )?.derived; + const totalDelta = safeDelta(last?.[totalKey], first?.[totalKey]); + const countDelta = safeDelta(last?.[countKey], first?.[countKey]); + return totalDelta != null && countDelta != null && countDelta > 0 ? totalDelta / countDelta : null; + } + function computeBenchSummary(run) { const fpsArr = run.samples.map((s) => s.derived.receiverFps).filter((v) => v != null && Number.isFinite(v)); const rxBr = run.samples.map((s) => s.derived.receiverBitrateBps).filter((v) => v != null && Number.isFinite(v)); @@ -1712,7 +1752,6 @@

Console Logs

return { durationMs: run.endedAtMs - run.startedAtMs, samples: run.samples.length, - warmupMs: run.warmupMs, fps: { avg: avgFps, min: fpsArr.length ? Math.min(...fpsArr) : null, @@ -1732,8 +1771,8 @@

Console Logs

inResolution, outboundCodec, inboundCodec, - avgQpOut: meanIgnoringNulls(run.samples.map((s) => s.derived.avgQpOut)), - avgQpIn: meanIgnoringNulls(run.samples.map((s) => s.derived.avgQpIn)), + avgQpOut: averageFromRunDeltas(run.samples, 'qpSumOutTotal', 'framesEncodedTotal'), + avgQpIn: averageFromRunDeltas(run.samples, 'qpSumInTotal', 'framesDecodedTotal'), avgBppOut: meanIgnoringNulls(run.samples.map((s) => s.derived.bppOut)), avgBppIn: meanIgnoringNulls(run.samples.map((s) => s.derived.bppIn)), avgEncodeMs: meanIgnoringNulls(run.samples.map((s) => s.derived.avgEncodeMs)), @@ -1755,20 +1794,14 @@

Console Logs

durationMs: durationSeconds * 1000, startedAtMs: nowMs(), endedAtMs: null, - phases: [{ name: 'start', tMs: 0 }], samples: [], - warmupMs: null, - warmupRun: [], prevRaw: null, }; elements.benchStart.disabled = true; elements.benchStop.disabled = false; - elements.benchMarkPhase.disabled = false; - elements.benchExportJson.disabled = false; - elements.benchExportCsv.disabled = false; + elements.benchExportAll.disabled = false; elements.benchClear.disabled = false; elements.benchStatusText.textContent = `running (profile=${profileKey})`; - elements.benchPhaseText.textContent = 'start'; elements.benchElapsedText.textContent = '0s'; benchPollTimer = setInterval(collectBenchSample, BENCH_POLL_INTERVAL_MS); benchRunTimer = setTimeout(stopBenchmark, durationSeconds * 1000); @@ -1788,26 +1821,20 @@

Console Logs

currentBenchRun.summary = computeBenchSummary(currentBenchRun); benchmarkRuns.push(currentBenchRun); console.log('[Benchmark] Run complete:', currentBenchRun); - addLog(`Benchmark stopped. profile=${currentBenchRun.profile} warmup=${currentBenchRun.summary.warmupMs != null ? (currentBenchRun.summary.warmupMs / 1000).toFixed(1) + 's' : 'n/a'} avgFps=${currentBenchRun.summary.fps.avg.toFixed(1)} p5Fps=${(currentBenchRun.summary.fps.p5 ?? 0).toFixed(1)}`, 'success'); + addLog(`Benchmark stopped. profile=${currentBenchRun.profile} avgFps=${currentBenchRun.summary.fps.avg.toFixed(1)} p5Fps=${(currentBenchRun.summary.fps.p5 ?? 0).toFixed(1)}`, 'success'); const finished = currentBenchRun; currentBenchRun = null; elements.benchStart.disabled = !decartRealtime?.getVideoStats; elements.benchStop.disabled = true; - elements.benchMarkPhase.disabled = true; elements.benchStatusText.textContent = `idle (last: ${finished.profile})`; refreshTimeSeriesCharts(); refreshSummaryChart(); } - function markBenchPhase() { - if (!currentBenchRun) return; - const raw = elements.benchPhaseName.value.trim(); - const name = raw || `phase-${currentBenchRun.phases.length}`; - const tMs = nowMs() - currentBenchRun.startedAtMs; - currentBenchRun.phases.push({ name, tMs }); - elements.benchPhaseName.value = ''; - elements.benchPhaseText.textContent = name; - addLog(`Phase "${name}" marked at ${(tMs / 1000).toFixed(1)}s`, 'info'); + function sanitizeFileName(raw) { + const trimmed = (raw || '').trim(); + if (!trimmed) return `decart_export_${Date.now()}`; + return trimmed.replace(/[\\/:*?"<>|]+/g, '_').replace(/\s+/g, '_').slice(0, 120); } function triggerDownload(blob, filename) { @@ -1821,14 +1848,25 @@

Console Logs

setTimeout(() => URL.revokeObjectURL(url), 1000); } - function exportBenchJson() { - const payload = { exportedAt: new Date().toISOString(), runs: benchmarkRuns }; - const blob = new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }); - triggerDownload(blob, `benchmark_${Date.now()}.json`); - addLog(`Exported ${benchmarkRuns.length} run(s) as JSON`, 'success'); + async function exportChartsBlob() { + if (typeof html2canvas !== 'function') { + throw new Error('html2canvas not loaded'); + } + const target = elements.benchDashboard; + if (!target) throw new Error('Benchmark dashboard element not found'); + const canvas = await html2canvas(target, { + backgroundColor: '#ffffff', + scale: window.devicePixelRatio > 1 ? 2 : 1, + useCORS: true, + logging: false, + }); + const blob = await new Promise((resolve, reject) => { + canvas.toBlob((b) => b ? resolve(b) : reject(new Error('toBlob returned null')), 'image/png'); + }); + return { blob, width: canvas.width, height: canvas.height }; } - function exportBenchCsv() { + function buildCsvBlob() { const header = [ 'run', 'profile', 'sessionId', 'tMs', 'outboundCodec', 'inboundCodec', @@ -1897,15 +1935,55 @@

Console Logs

}); }); const blob = new Blob([rows.join('\n')], { type: 'text/csv' }); - triggerDownload(blob, `benchmark_${Date.now()}.csv`); - addLog(`Exported ${benchmarkRuns.length} run(s) as CSV`, 'success'); + return blob; + } + + async function exportAll() { + const btn = elements.benchExportAll; + const prev = btn.textContent; + btn.disabled = true; + btn.textContent = 'Exporting…'; + const baseName = sanitizeFileName(elements.recordingName.value); + const ts = Date.now(); + try { + if (benchmarkRuns.length > 0) { + const csvBlob = buildCsvBlob(); + triggerDownload(csvBlob, `${baseName}_${ts}.csv`); + addLog(`Exported ${benchmarkRuns.length} run(s) as CSV → ${baseName}_${ts}.csv`, 'success'); + } else { + addLog('No benchmark runs to export as CSV', 'info'); + } + try { + const { blob: pngBlob, width, height } = await exportChartsBlob(); + triggerDownload(pngBlob, `${baseName}_${ts}.png`); + addLog(`Exported benchmark dashboard as PNG (${width}×${height}) → ${baseName}_${ts}.png`, 'success'); + } catch (err) { + addLog(`Chart export failed: ${err?.message ?? err}`, 'error'); + } + if (!elements.includeRecordingToggle.checked) { + addLog('Chain recording skipped in export (checkbox off)', 'info'); + } else if (chainRecorder.blobUrl) { + const ext = chainRecorder.lastExt || 'webm'; + const a = document.createElement('a'); + a.href = chainRecorder.blobUrl; + a.download = `${baseName}_${ts}.${ext}`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + addLog(`Exported chain recording → ${baseName}_${ts}.${ext}`, 'success'); + } else { + addLog('No chain recording available to export', 'info'); + } + } finally { + btn.disabled = false; + btn.textContent = prev; + } } function clearBenchRuns() { if (currentBenchRun) { addLog('Stop the current benchmark first', 'error'); return; } benchmarkRuns.length = 0; - elements.benchExportJson.disabled = true; - elements.benchExportCsv.disabled = true; + elements.benchExportAll.disabled = true; elements.benchClear.disabled = true; elements.benchLive.textContent = ''; elements.benchStatusText.textContent = 'idle'; @@ -1916,9 +1994,7 @@

Console Logs

elements.benchStart.addEventListener('click', startBenchmark); elements.benchStop.addEventListener('click', stopBenchmark); - elements.benchMarkPhase.addEventListener('click', markBenchPhase); - elements.benchExportJson.addEventListener('click', exportBenchJson); - elements.benchExportCsv.addEventListener('click', exportBenchCsv); + elements.benchExportAll.addEventListener('click', exportAll); elements.benchClear.addEventListener('click', clearBenchRuns); // --- MediaRecorder for the Decart remote stream --- @@ -1991,6 +2067,116 @@

Console Logs

} } + // --- Chain-scoped recorder: one continuous MediaRecorder for the whole chain. + // Mirrors the remote