From f19001090c534935c4d5d53cf1da5d9fc48c9f29 Mon Sep 17 00:00:00 2001 From: Julien Lavigne du Cadet <203107797+julien-lottie@users.noreply.github.com> Date: Fri, 15 May 2026 16:50:36 +0100 Subject: [PATCH 1/3] fix(transcription): warn only when rotateSegment backlog grows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "rotateSegment called while previous segment is still being rotated" log fires at every turn boundary because playback-finished, output attach/detach, and new-utterance events naturally overlap with the prior segment's close-and-recreate task. The rotation is safely serialized — the new Task awaits oldTask.result before recreating the SegmentSynchronizerImpl — so a single overlap is the expected case and no transcript data is lost. Track the number of rotations queued behind the in-flight one and only warn when more than one is stacked. That's when the backlog is actually growing and an operator should pay attention. Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/quiet-rotate-segment.md | 5 +++ .../src/voice/transcription/synchronizer.ts | 36 ++++++++++++++----- 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 .changeset/quiet-rotate-segment.md diff --git a/.changeset/quiet-rotate-segment.md b/.changeset/quiet-rotate-segment.md new file mode 100644 index 000000000..bd91aabfb --- /dev/null +++ b/.changeset/quiet-rotate-segment.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +fix(transcription): quiet the `rotateSegment` overlap warning. The single-overlap case (one rotation queued behind another) is expected at normal turn boundaries — rotations are safely serialized via `oldTask.result`. Track the queue depth instead and only warn when more than one rotation is stacked behind the in-flight one, which would indicate a genuine backlog. diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index 39872b716..19d8b5bed 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -518,6 +518,9 @@ export class TranscriptionSynchronizer { private options: TextSyncOptions; private rotateSegmentTask: Task; + // number of rotations queued behind the currently-running one; used to warn only + // when the backlog grows beyond a single expected overlap + private queuedRotations: number = 0; private _outputsAttached: boolean = true; private closed: boolean = false; @@ -598,7 +601,16 @@ export class TranscriptionSynchronizer { } if (!this.rotateSegmentTask.done) { - this.logger.warn('rotateSegment called while previous segment is still being rotated'); + // The new task chains on the old one via `oldTask.result`, so rotations are + // serialized and no transcript data is lost. A single overlap is expected when + // turn-boundary events (playback finished, attach/detach, new utterance) fire + // back-to-back; only warn once the backlog grows beyond one queued rotation. + this.queuedRotations++; + if (this.queuedRotations > 1) { + this.logger.warn( + `rotateSegment backlog: ${this.queuedRotations} rotations queued behind the in-flight one`, + ); + } } this.rotateSegmentTask = Task.from((controller) => this.rotateSegmentTaskImpl(controller.signal, this.rotateSegmentTask), @@ -619,16 +631,22 @@ export class TranscriptionSynchronizer { } private async rotateSegmentTaskImpl(abort: AbortSignal, oldTask?: Task) { - if (oldTask) { - await oldTask.result; - } + try { + if (oldTask) { + await oldTask.result; + } - if (abort.aborted) { - return; - } + if (abort.aborted) { + return; + } - await this._impl.close(); - this._impl = new SegmentSynchronizerImpl(this.options, this.textOutput.nextInChain, true); + await this._impl.close(); + this._impl = new SegmentSynchronizerImpl(this.options, this.textOutput.nextInChain, true); + } finally { + if (this.queuedRotations > 0) { + this.queuedRotations--; + } + } } } From 6a791d8780e54324b53abe9395f91b6b4e160869 Mon Sep 17 00:00:00 2001 From: Julien Lavigne du Cadet <203107797+julien-lottie@users.noreply.github.com> Date: Sat, 16 May 2026 08:37:33 +0100 Subject: [PATCH 2/3] fix(transcription): suppress rotateSegment backlog warn during startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production data on Lottie's eliza-agent (dash0) shows every observed `rotateSegment backlog` warn fires within ~0.2–1.2s of session startup, before the first agent utterance — never at a mid-conversation turn boundary. The trigger is a race between the constructor-scheduled initial rotation task and the room's CONN_CONNECTED event, which stacks two extra rotateSegment calls onto the chain before the initial task drains. The chain settles long before any TTS frame is produced, so the caller-perceived latency is zero. Track when the initial task has resolved at least once via `Task.addDoneCallback`, and gate the warn behind that flag. The counter (`queuedRotations`) keeps incrementing during startup so the serialisation invariants are preserved; only the noisy log line is suppressed. Real mid-conversation backlogs (which is what the warn was designed to surface) still trip the warn once the synchronizer leaves the startup window. Add two regression tests covering both paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/quiet-rotate-segment.md | 2 +- .../voice/transcription/synchronizer.test.ts | 59 +++++++++++++++++++ .../src/voice/transcription/synchronizer.ts | 20 +++++-- 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/.changeset/quiet-rotate-segment.md b/.changeset/quiet-rotate-segment.md index bd91aabfb..6e9a3e5e4 100644 --- a/.changeset/quiet-rotate-segment.md +++ b/.changeset/quiet-rotate-segment.md @@ -2,4 +2,4 @@ '@livekit/agents': patch --- -fix(transcription): quiet the `rotateSegment` overlap warning. The single-overlap case (one rotation queued behind another) is expected at normal turn boundaries — rotations are safely serialized via `oldTask.result`. Track the queue depth instead and only warn when more than one rotation is stacked behind the in-flight one, which would indicate a genuine backlog. +fix(transcription): quiet the `rotateSegment` overlap warning. The single-overlap case (one rotation queued behind another) is expected at normal turn boundaries — rotations are safely serialized via `oldTask.result`. Track the queue depth instead and only warn when more than one rotation is stacked behind the in-flight one, and additionally suppress the warn during the synchronizer's startup window: production data shows the room-connection-state-changed event can stack two extra rotations onto the constructor-scheduled initial task, producing a benign depth=2 chain that drains before any audio is produced. After the initial task resolves, real mid-conversation backlogs still trip the warn. diff --git a/agents/src/voice/transcription/synchronizer.test.ts b/agents/src/voice/transcription/synchronizer.test.ts index 287cd3d3e..657512efc 100644 --- a/agents/src/voice/transcription/synchronizer.test.ts +++ b/agents/src/voice/transcription/synchronizer.test.ts @@ -293,3 +293,62 @@ describe('TranscriptionSynchronizer attachment warnings', () => { await synchronizer.close(); }); }); + +describe('TranscriptionSynchronizer rotateSegment backlog warn', () => { + const backlogWarnPrefix = 'rotateSegment backlog:'; + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('does not warn while the initial constructor-scheduled rotation is in flight', async () => { + const warn = vi.fn(); + vi.spyOn(logModule, 'log').mockReturnValue({ + warn, + debug: vi.fn(), + info: vi.fn(), + error: vi.fn(), + } as unknown as ReturnType); + + const synchronizer = new TranscriptionSynchronizer(new MockAudioOutput(), new MockTextOutput()); + // Force two extra rotateSegment calls synchronously, before the initial task can settle. + // This reproduces the production startup race (room connection + handoff stacking + // rotations onto the initial constructor-scheduled task). + synchronizer.audioOutput.onDetached(); + synchronizer.textOutput.onDetached(); + + expect( + warn.mock.calls.filter((c) => typeof c[0] === 'string' && c[0].startsWith(backlogWarnPrefix)), + ).toHaveLength(0); + + await synchronizer.close(); + }); + + it('warns when the chain stacks beyond depth 1 after the initial rotation has settled', async () => { + const warn = vi.fn(); + vi.spyOn(logModule, 'log').mockReturnValue({ + warn, + debug: vi.fn(), + info: vi.fn(), + error: vi.fn(), + } as unknown as ReturnType); + + const synchronizer = new TranscriptionSynchronizer(new MockAudioOutput(), new MockTextOutput()); + + // Let the initial constructor-scheduled task drain so we leave the startup window. + await synchronizer.barrier(); + + // Now simulate a real mid-conversation backlog: three rotateSegment calls back to back + // so two end up queued behind the in-flight one. + synchronizer.audioOutput.onDetached(); + synchronizer.audioOutput.onAttached(); + synchronizer.audioOutput.onDetached(); + + const backlogWarns = warn.mock.calls.filter( + (c) => typeof c[0] === 'string' && c[0].startsWith(backlogWarnPrefix), + ); + expect(backlogWarns.length).toBeGreaterThanOrEqual(1); + + await synchronizer.close(); + }); +}); diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index 19d8b5bed..12aeb061f 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -521,6 +521,13 @@ export class TranscriptionSynchronizer { // number of rotations queued behind the currently-running one; used to warn only // when the backlog grows beyond a single expected overlap private queuedRotations: number = 0; + // The constructor schedules an initial rotation task. During session startup the room + // connection + agent handoff can fire two more rotateSegment calls before that initial + // task drains, producing a benign depth=2 chain that does not affect the caller (the + // chain settles before any audio is produced). Suppress the backlog warn until the + // initial task has resolved at least once so it only fires on real mid-conversation + // backlogs. + private initialRotationDone: boolean = false; private _outputsAttached: boolean = true; private closed: boolean = false; @@ -555,9 +562,11 @@ export class TranscriptionSynchronizer { // initial segment/first segment, recreated for each new segment this._impl = new SegmentSynchronizerImpl(this.options, nextInChainText); - this.rotateSegmentTask = Task.from((controller) => - this.rotateSegmentTaskImpl(controller.signal), - ); + const initialTask = Task.from((controller) => this.rotateSegmentTaskImpl(controller.signal)); + initialTask.addDoneCallback(() => { + this.initialRotationDone = true; + }); + this.rotateSegmentTask = initialTask; } get outputsAttached(): boolean { @@ -604,9 +613,10 @@ export class TranscriptionSynchronizer { // The new task chains on the old one via `oldTask.result`, so rotations are // serialized and no transcript data is lost. A single overlap is expected when // turn-boundary events (playback finished, attach/detach, new utterance) fire - // back-to-back; only warn once the backlog grows beyond one queued rotation. + // back-to-back; only warn once the backlog grows beyond one queued rotation, and + // skip the warn during the synchronizer's startup window (see initialRotationDone). this.queuedRotations++; - if (this.queuedRotations > 1) { + if (this.queuedRotations > 1 && this.initialRotationDone) { this.logger.warn( `rotateSegment backlog: ${this.queuedRotations} rotations queued behind the in-flight one`, ); From 32951d1deaae56984b4a74fbc13930e421cea496 Mon Sep 17 00:00:00 2001 From: Julien Lavigne du Cadet <203107797+julien-lottie@users.noreply.github.com> Date: Sun, 17 May 2026 07:42:48 +0100 Subject: [PATCH 3/3] refactor(transcription): set initialRotationDone inside task body Devin Review flagged the previous `Task.addDoneCallback` approach as microtask-fragile: the callback fires from a `.finally()` chained off the result promise, which is queued *after* any direct `await result` continuation. The current test passes because of a favourable ordering, but a small refactor of the surrounding code could quietly flip it. Set the flag synchronously in `rotateSegmentTaskImpl`'s `finally` block instead. By the time `runTask` resolves the future, the finally has already executed, so any continuation (including `barrier()` and the test's `await synchronizer.barrier()`) observes `initialRotationDone = true` deterministically. No reliance on microtask scheduling. Co-Authored-By: Claude Opus 4.7 (1M context) --- agents/src/voice/transcription/synchronizer.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index 12aeb061f..703bdc36e 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -562,11 +562,9 @@ export class TranscriptionSynchronizer { // initial segment/first segment, recreated for each new segment this._impl = new SegmentSynchronizerImpl(this.options, nextInChainText); - const initialTask = Task.from((controller) => this.rotateSegmentTaskImpl(controller.signal)); - initialTask.addDoneCallback(() => { - this.initialRotationDone = true; - }); - this.rotateSegmentTask = initialTask; + this.rotateSegmentTask = Task.from((controller) => + this.rotateSegmentTaskImpl(controller.signal), + ); } get outputsAttached(): boolean { @@ -656,6 +654,11 @@ export class TranscriptionSynchronizer { if (this.queuedRotations > 0) { this.queuedRotations--; } + // Set synchronously inside the task body so that by the time `task.result` + // resolves, the flag is already true for any continuation (including + // `barrier()` callers). Using `Task.addDoneCallback` for this would be + // fragile against microtask reordering. + this.initialRotationDone = true; } } }