From e6a734a25027d12264ecc23884d31b42d84e4e95 Mon Sep 17 00:00:00 2001
From: nagar-decart <eyal.nagar@decart.ai>
Date: Fri, 29 May 2026 13:33:56 +0300
Subject: [PATCH 1/3] feat(realtime): add playRemoteAudio connect option to
 gate audio playback
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

livekit-client's track.attach() auto-creates a hidden <audio> element for
every subscribed audio track and plays it — independent of any <video muted>
element the app wires up to onRemoteStream. After API#1764 enabled V2V
audio passthrough server-side, apps started hearing their own mic echoed
back even though they kept the same <video muted> playback element that
previously silenced everything.

Add an opt-in `playRemoteAudio: boolean` (default `false`). When false, the
TrackSubscribed handler skips both track.attach() and the addTrack into
remoteStream for audio tracks, so neither the SDK nor the app can
accidentally route remote audio to a speaker. Apps that want server-emitted
audio (avatars, V2V echo, voice-driven sessions) opt in with `true`.

Default is false to restore the prior implicit "no audio output" behavior
that apps relied on before the server-side passthrough flip — turning it on
silently would be a breaking change for every existing app.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 packages/sdk/src/realtime/client.ts         | 22 ++++++-
 packages/sdk/src/realtime/media-channel.ts  | 12 ++++
 packages/sdk/src/realtime/stream-session.ts |  2 +
 packages/sdk/tests/realtime.unit.test.ts    | 63 ++++++++++++++++++++-
 4 files changed, 94 insertions(+), 5 deletions(-)

diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts
index b35987d..8061e46 100644
--- a/packages/sdk/src/realtime/client.ts
+++ b/packages/sdk/src/realtime/client.ts
@@ -56,6 +56,16 @@ const realTimeClientConnectOptionsSchema = z.object({
   resolution: z.enum(["720p", "1080p"]).optional(),
   /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */
   preferredVideoCodec: z.enum(["h264", "vp9"]).optional(),
+  /**
+   * Play remote audio tracks published by the server. Default `false`.
+   *
+   * Set `true` for models that emit meaningful audio (avatars, voice
+   * sessions, V2V passthrough where input audio should round-trip). When
+   * `false`, audio tracks are dropped on the client — no hidden `<audio>`
+   * element is created by livekit-client and audio is not added to the
+   * stream passed to `onRemoteStream`.
+   */
+  playRemoteAudio: z.boolean().optional(),
 });
 export type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
   model: ModelDefinition | CustomModelDefinition;
@@ -102,8 +112,15 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
     const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
     if (!parsedOptions.success) throw parsedOptions.error;
 
-    const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution, preferredVideoCodec } =
-      parsedOptions.data;
+    const {
+      onRemoteStream,
+      onConnectionChange,
+      onQueuePosition,
+      initialState,
+      resolution,
+      preferredVideoCodec,
+      playRemoteAudio,
+    } = parsedOptions.data;
     const mirror = parsedOptions.data.mirror ?? false;
     let inputStream: MediaStream = stream ?? new MediaStream();
 
@@ -169,6 +186,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         initialPrompt,
         logger,
         videoCodec: publishCodec,
+        playRemoteAudio,
       });
 
       let sessionId: string | null = null;
diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts
index 725943d..04f8929 100644
--- a/packages/sdk/src/realtime/media-channel.ts
+++ b/packages/sdk/src/realtime/media-channel.ts
@@ -45,6 +45,17 @@ export interface MediaChannelConfig {
   localStream: MediaStream | null;
   logger?: Logger;
   videoCodec?: VideoCodec;
+  /**
+   * Play remote audio tracks published by the server. Default `false`.
+   *
+   * When `false`, audio tracks subscribed from the server are dropped on the
+   * client: livekit-client's `track.attach()` is skipped (so no hidden
+   * `<audio>` element is created) and the track is not added to the remote
+   * stream. Set to `true` for models that emit meaningful audio output
+   * (avatars, voice-driven sessions, or V2V where the input audio should
+   * round-trip to subscribers).
+   */
+  playRemoteAudio?: boolean;
 }
 
 export type MediaConnectOptions = {
@@ -81,6 +92,7 @@ export class MediaChannel {
     room.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => {
       if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
       if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
+      if (track.kind === Track.Kind.Audio && !this.config.playRemoteAudio) return;
 
       track.attach();
       const mediaStreamTrack = track.mediaStreamTrack;
diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts
index 014baee..4c7f665 100644
--- a/packages/sdk/src/realtime/stream-session.ts
+++ b/packages/sdk/src/realtime/stream-session.ts
@@ -61,6 +61,7 @@ interface StreamSessionConfig {
   initialPrompt?: InitialPrompt;
   logger?: Logger;
   videoCodec?: VideoCodec;
+  playRemoteAudio?: boolean;
 }
 
 export class StreamSession {
@@ -317,6 +318,7 @@ export class StreamSession {
       localStream: this.config.localStream,
       logger: this.logger,
       videoCodec: this.config.videoCodec,
+      playRemoteAudio: this.config.playRemoteAudio,
     });
     this.wireSignalingEvents();
     this.wireMediaEvents();
diff --git a/packages/sdk/tests/realtime.unit.test.ts b/packages/sdk/tests/realtime.unit.test.ts
index 270d1f4..b6b6b80 100644
--- a/packages/sdk/tests/realtime.unit.test.ts
+++ b/packages/sdk/tests/realtime.unit.test.ts
@@ -743,16 +743,18 @@ describe("StreamSession startup orchestration", () => {
     });
   };
 
-  const subscribeRemoteTrack = () => {
+  const subscribeRemoteTrack = (kind: "video" | "audio" = "video") => {
     const room = liveKitMock.roomInstances.at(-1) as InstanceType<typeof liveKitMock.MockRoom>;
-    const mediaStreamTrack = { id: "remote-video", kind: "video" };
+    const mediaStreamTrack = { id: `remote-${kind}`, kind };
+    const trackKind = kind === "video" ? liveKitMock.Track.Kind.Video : liveKitMock.Track.Kind.Audio;
     const track = {
-      kind: liveKitMock.Track.Kind.Video,
+      kind: trackKind,
       mediaStreamTrack,
       attach: vi.fn(),
       on: vi.fn(),
     };
     room.emit(liveKitMock.RoomEvent.TrackSubscribed, track, {}, { identity: "inference-server-1" });
+    return track;
   };
 
   const createLocalStream = () =>
@@ -976,6 +978,61 @@ describe("StreamSession startup orchestration", () => {
     expect(firstRoom.localParticipant.publishTrack).not.toHaveBeenCalled();
     await expect(connectPromise).rejects.toThrow("WebSocket closed: 1000 closed");
   });
+
+  it("drops remote audio tracks by default — no attach, not added to remoteStream", async () => {
+    const { StreamSession } = await import("../src/realtime/stream-session.js");
+    const session = new StreamSession({
+      url: "wss://example.test/realtime",
+      localStream: null,
+      initialPrompt: { text: "go" },
+    });
+    const remoteStreams: MediaStream[] = [];
+    session.on("remoteStream", (stream) => remoteStreams.push(stream));
+
+    session.connect().catch(() => {});
+    const ws = FakeWebSocket.instances[0];
+    ws.onopen?.();
+    await flushMicrotasks();
+    sendRoomInfo(ws);
+    await flushMicrotasks();
+
+    const audioTrack = subscribeRemoteTrack("audio");
+    expect(audioTrack.attach).not.toHaveBeenCalled();
+    expect(remoteStreams).toHaveLength(0);
+
+    const videoTrack = subscribeRemoteTrack("video");
+    expect(videoTrack.attach).toHaveBeenCalledTimes(1);
+    expect(remoteStreams).toHaveLength(1);
+    expect((remoteStreams[0] as unknown as FakeMediaStream).getTracks()).toEqual([
+      expect.objectContaining({ kind: "video" }),
+    ]);
+  });
+
+  it("plays remote audio tracks when playRemoteAudio is true", async () => {
+    const { StreamSession } = await import("../src/realtime/stream-session.js");
+    const session = new StreamSession({
+      url: "wss://example.test/realtime",
+      localStream: null,
+      initialPrompt: { text: "go" },
+      playRemoteAudio: true,
+    });
+    const remoteStreams: MediaStream[] = [];
+    session.on("remoteStream", (stream) => remoteStreams.push(stream));
+
+    session.connect().catch(() => {});
+    const ws = FakeWebSocket.instances[0];
+    ws.onopen?.();
+    await flushMicrotasks();
+    sendRoomInfo(ws);
+    await flushMicrotasks();
+
+    const audioTrack = subscribeRemoteTrack("audio");
+    expect(audioTrack.attach).toHaveBeenCalledTimes(1);
+    expect(remoteStreams).toHaveLength(1);
+    expect((remoteStreams[0] as unknown as FakeMediaStream).getTracks()).toEqual([
+      expect.objectContaining({ kind: "audio" }),
+    ]);
+  });
 });
 
 describe("WebRTC Error Classification", () => {

From 24245181ec42979ea522f385b91ace98c36f982a Mon Sep 17 00:00:00 2001
From: nagar-decart <eyal.nagar@decart.ai>
Date: Fri, 29 May 2026 13:58:34 +0300
Subject: [PATCH 2/3] feat(realtime): apply playRemoteAudio gate to
 subscribe-client too
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The publisher-side gate added in the parent commit only covers
realtime.connect(). realtime.subscribe() has the same unconditional
track.attach() on subscribed audio — viewer clients hit the same
surprise audio playback when the inference server publishes audio.

Mirror the gate exactly: SubscribeOptions gets `playRemoteAudio?: boolean`
(default false); the TrackSubscribed handler skips attach + remoteStream
addTrack for audio when unset/false. Same semantics, same default, same
opt-in story across both API surfaces.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 packages/sdk/src/realtime/subscribe-client.ts | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/packages/sdk/src/realtime/subscribe-client.ts b/packages/sdk/src/realtime/subscribe-client.ts
index 3782fd6..3109a90 100644
--- a/packages/sdk/src/realtime/subscribe-client.ts
+++ b/packages/sdk/src/realtime/subscribe-client.ts
@@ -61,6 +61,17 @@ export type SubscribeOptions = {
   token: string;
   onRemoteStream: (stream: MediaStream) => void;
   onConnectionChange?: (state: ConnectionState) => void;
+  /**
+   * Play remote audio tracks published by the server. Default `false`.
+   *
+   * Mirrors the publisher-side `realtime.connect()` option. When `false`,
+   * audio tracks subscribed from the inference server are dropped on the
+   * client: livekit-client's `track.attach()` is skipped (so no hidden
+   * `<audio>` element is created) and the track is not added to the stream
+   * passed to `onRemoteStream`. Set `true` for viewers of avatar / V2V /
+   * voice-driven sessions where the server-side audio is meaningful.
+   */
+  playRemoteAudio?: boolean;
 };
 
 export type RealTimeSubscribeClientOptions = {
@@ -148,6 +159,7 @@ export const createRealTimeSubscribeClient = (opts: RealTimeSubscribeClientOptio
       activeRoom.on(RoomEvent.TrackSubscribed, (track: RemoteTrack, _pub, participant: RemoteParticipant) => {
         if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
         if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
+        if (track.kind === Track.Kind.Audio && !options.playRemoteAudio) return;
 
         track.attach();
         const mediaStreamTrack = track.mediaStreamTrack;

From c89a1c1a0b368a1dc176cb3b51843e537991b757 Mon Sep 17 00:00:00 2001
From: nagar-decart <eyal.nagar@decart.ai>
Date: Fri, 29 May 2026 14:03:07 +0300
Subject: [PATCH 3/3] docs(realtime): rewrite playRemoteAudio comments for
 external readers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Drop internal-only context from the comments — server-side PR
numbers, internal model family names, the internal product name for
the upstream service, and livekit-client implementation details.
Describe only what the option does from a consumer's standpoint:
when set to false the track is dropped (no playback element attached,
not added to remoteStream); set true when the model emits audio you
want heard.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 packages/sdk/src/realtime/client.ts           | 11 +++++------
 packages/sdk/src/realtime/media-channel.ts    | 12 +++++-------
 packages/sdk/src/realtime/subscribe-client.ts | 12 +++++-------
 3 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts
index 8061e46..2a5ac5a 100644
--- a/packages/sdk/src/realtime/client.ts
+++ b/packages/sdk/src/realtime/client.ts
@@ -57,13 +57,12 @@ const realTimeClientConnectOptionsSchema = z.object({
   /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */
   preferredVideoCodec: z.enum(["h264", "vp9"]).optional(),
   /**
-   * Play remote audio tracks published by the server. Default `false`.
+   * Play remote audio tracks. Default `false`.
    *
-   * Set `true` for models that emit meaningful audio (avatars, voice
-   * sessions, V2V passthrough where input audio should round-trip). When
-   * `false`, audio tracks are dropped on the client — no hidden `<audio>`
-   * element is created by livekit-client and audio is not added to the
-   * stream passed to `onRemoteStream`.
+   * When `false`, any audio the model emits is dropped on the client — no
+   * playback element is attached and audio is not added to the stream
+   * passed to `onRemoteStream`. Set `true` when the model emits audio you
+   * want the user to hear.
    */
   playRemoteAudio: z.boolean().optional(),
 });
diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts
index 04f8929..a7ddf74 100644
--- a/packages/sdk/src/realtime/media-channel.ts
+++ b/packages/sdk/src/realtime/media-channel.ts
@@ -46,14 +46,12 @@ export interface MediaChannelConfig {
   logger?: Logger;
   videoCodec?: VideoCodec;
   /**
-   * Play remote audio tracks published by the server. Default `false`.
+   * Play remote audio tracks. Default `false`.
    *
-   * When `false`, audio tracks subscribed from the server are dropped on the
-   * client: livekit-client's `track.attach()` is skipped (so no hidden
-   * `<audio>` element is created) and the track is not added to the remote
-   * stream. Set to `true` for models that emit meaningful audio output
-   * (avatars, voice-driven sessions, or V2V where the input audio should
-   * round-trip to subscribers).
+   * When `false`, any audio tracks delivered to this room are dropped on the
+   * client: no playback element is attached and the track is not added to
+   * the stream emitted on `remoteStream`. Set to `true` when the model emits
+   * audio you want the user to hear.
    */
   playRemoteAudio?: boolean;
 }
diff --git a/packages/sdk/src/realtime/subscribe-client.ts b/packages/sdk/src/realtime/subscribe-client.ts
index 3109a90..7298dfc 100644
--- a/packages/sdk/src/realtime/subscribe-client.ts
+++ b/packages/sdk/src/realtime/subscribe-client.ts
@@ -62,14 +62,12 @@ export type SubscribeOptions = {
   onRemoteStream: (stream: MediaStream) => void;
   onConnectionChange?: (state: ConnectionState) => void;
   /**
-   * Play remote audio tracks published by the server. Default `false`.
+   * Play remote audio tracks. Default `false`.
    *
-   * Mirrors the publisher-side `realtime.connect()` option. When `false`,
-   * audio tracks subscribed from the inference server are dropped on the
-   * client: livekit-client's `track.attach()` is skipped (so no hidden
-   * `<audio>` element is created) and the track is not added to the stream
-   * passed to `onRemoteStream`. Set `true` for viewers of avatar / V2V /
-   * voice-driven sessions where the server-side audio is meaningful.
+   * When `false`, any audio the model emits is dropped on the client — no
+   * playback element is attached and audio is not added to the stream
+   * passed to `onRemoteStream`. Set `true` when the model emits audio you
+   * want the viewer to hear.
    */
   playRemoteAudio?: boolean;
 };