Audio EOT

examples/README.md

@@ -54,7 +54,7 @@ To run the examples, you'll need:
 
 - A [LiveKit Cloud](https://cloud.livekit.io) account or a local [LiveKit server](https://github.com/livekit/livekit)
 - API keys for the model providers you want to use in a `.env` file
-- Python 3.9 or higher
+- Python 3.10 or higher
 - [uv](https://docs.astral.sh/uv/)
 
 ### Environment file

examples/voice_agents/basic_agent.py

@@ -20,7 +20,7 @@
 from livekit.agents.beta import EndCallTool
 from livekit.agents.llm import function_tool
 from livekit.plugins import silero
-from livekit.plugins.turn_detector.multilingual import MultilingualModel
+from livekit.plugins.turn_detector.audio import AudioTurnDetector
 
 # uncomment to enable Krisp background voice/noise cancellation
 # from livekit.plugins import noise_cancellation
@@ -98,7 +98,7 @@ async def entrypoint(ctx: JobContext) -> None:
         turn_handling=TurnHandlingOptions(
             # VAD and turn detection are used to determine when the user is speaking and when the agent should respond
             # See more at https://docs.livekit.io/agents/build/turns
-            turn_detection=MultilingualModel(),
+            turn_detection=AudioTurnDetector(),
             interruption={
                 # sometimes background noise could interrupt the agent session, these are considered false positive interruptions
                 # when it's detected, you may resume the agent's speech

livekit-agents/livekit/agents/inference/_utils.py

@@ -14,6 +14,7 @@
 HEADER_USER_AGENT = "User-Agent"
 HEADER_ROOM_ID = "X-LiveKit-Room-ID"
 HEADER_JOB_ID = "X-LiveKit-Job-ID"
+HEADER_WORKER_TOKEN = "X-LiveKit-Worker-Token"
 HEADER_INFERENCE_PROVIDER = "X-LiveKit-Inference-Provider"
 HEADER_INFERENCE_PRIORITY = "X-LiveKit-Inference-Priority"
 
@@ -55,6 +56,8 @@ def get_inference_headers() -> dict[str, str]:
             headers[HEADER_ROOM_ID] = ctx.job.room.sid
         if ctx.job.id:
             headers[HEADER_JOB_ID] = ctx.job.id
+        if worker_token := os.getenv("LIVEKIT_WORKER_TOKEN"):
+            headers[HEADER_WORKER_TOKEN] = worker_token
     except RuntimeError:
         pass
     return headers

livekit-agents/livekit/agents/metrics/__init__.py

@@ -1,5 +1,6 @@
 from .base import (
     AgentMetrics,
+    EOTInferenceMetrics,
     EOUMetrics,
     InterruptionMetrics,
     LLMMetrics,
@@ -10,6 +11,7 @@
 )
 from .usage import (
     AgentSessionUsage,
+    EOTModelUsage,
     InterruptionModelUsage,
     LLMModelUsage,
     ModelUsage,
@@ -25,6 +27,7 @@
     "AgentMetrics",
     "VADMetrics",
     "EOUMetrics",
+    "EOTInferenceMetrics",
     "STTMetrics",
     "TTSMetrics",
     "RealtimeModelMetrics",
@@ -34,6 +37,7 @@
     "TTSModelUsage",
     "STTModelUsage",
     "InterruptionModelUsage",
+    "EOTModelUsage",
     "ModelUsage",
     "AgentSessionUsage",
     "ModelUsageCollector",

livekit-agents/livekit/agents/metrics/base.py

@@ -112,6 +112,22 @@ class EOUMetrics(_BaseMetrics):
     metadata: Metadata | None = None
 
 
+class EOTInferenceMetrics(_BaseMetrics):
+    """Per-inference metrics emitted by the EOT model on each prediction."""
+
+    type: Literal["eot_inference_metrics"] = "eot_inference_metrics"
+    timestamp: float
+    total_duration: float
+    """Earliest audio creation time in an inference to response receive time."""
+    detection_delay: float
+    """Latest audio creation time in an inference to response receive time."""
+    prediction_duration: float
+    """Server side model inference time."""
+    num_requests: int = 1
+    """Number of inference requests made during one inference."""
+    metadata: Metadata | None = None
+
+
 class RealtimeModelMetrics(_BaseMetrics):
     class CachedTokenDetails(BaseModel):
         audio_tokens: int = 0
@@ -199,6 +215,7 @@ class AvatarMetrics(_BaseMetrics):
     | TTSMetrics
     | VADMetrics
     | EOUMetrics
+    | EOTInferenceMetrics
     | RealtimeModelMetrics
     | InterruptionMetrics
     | AvatarMetrics

livekit-agents/livekit/agents/metrics/usage.py

@@ -7,6 +7,7 @@
 
 from .base import (
     AgentMetrics,
+    EOTInferenceMetrics,
     InterruptionMetrics,
     LLMMetrics,
     RealtimeModelMetrics,
@@ -108,7 +109,19 @@ class InterruptionModelUsage(_BaseModelUsage):
     """Total number of requests sent to the interruption detection model."""
 
 
-ModelUsage = LLMModelUsage | TTSModelUsage | STTModelUsage | InterruptionModelUsage
+class EOTModelUsage(_BaseModelUsage):
+    """Usage summary for end-of-turn detection models."""
+
+    type: Literal["eot_usage"] = "eot_usage"
+    provider: str
+    """The provider name (e.g., 'livekit')."""
+    model: str
+    """The model name (e.g., 'eot-audio')."""
+    total_requests: int = 0
+    """Total number of inference requests sent to the EOT model."""
+
+
+ModelUsage = LLMModelUsage | TTSModelUsage | STTModelUsage | InterruptionModelUsage | EOTModelUsage
 """Union type for all model usage types."""
 
 
@@ -125,13 +138,19 @@ def __init__(self) -> None:
         self._tts_usage: dict[tuple[str, str], TTSModelUsage] = {}
         self._stt_usage: dict[tuple[str, str], STTModelUsage] = {}
         self._interruption_usage: dict[tuple[str, str], InterruptionModelUsage] = {}
+        self._eot_usage: dict[tuple[str, str], EOTModelUsage] = {}
 
     def __call__(self, metrics: AgentMetrics) -> None:
         self.collect(metrics)
 
     def _extract_provider_model(
         self,
-        metrics: LLMMetrics | STTMetrics | TTSMetrics | RealtimeModelMetrics | InterruptionMetrics,
+        metrics: LLMMetrics
+        | STTMetrics
+        | TTSMetrics
+        | RealtimeModelMetrics
+        | InterruptionMetrics
+        | EOTInferenceMetrics,
     ) -> tuple[str, str]:
         """Extract provider and model from metrics metadata."""
         provider = ""
@@ -169,6 +188,13 @@ def _get_interruption_usage(self, provider: str, model: str) -> InterruptionMode
             self._interruption_usage[key] = InterruptionModelUsage(provider=provider, model=model)
         return self._interruption_usage[key]
 
+    def _get_eot_usage(self, provider: str, model: str) -> EOTModelUsage:
+        """Get or create an EOTModelUsage for the given provider/model combination."""
+        key = (provider, model)
+        if key not in self._eot_usage:
+            self._eot_usage[key] = EOTModelUsage(provider=provider, model=model)
+        return self._eot_usage[key]
+
     def collect(self, metrics: AgentMetrics) -> None:
         if isinstance(metrics, LLMMetrics):
             provider, model = self._extract_provider_model(metrics)
@@ -225,6 +251,10 @@ def collect(self, metrics: AgentMetrics) -> None:
             provider, model = self._extract_provider_model(metrics)
             interruption_usage = self._get_interruption_usage(provider, model)
             interruption_usage.total_requests += metrics.num_requests
+        elif isinstance(metrics, EOTInferenceMetrics):
+            provider, model = self._extract_provider_model(metrics)
+            eot_usage = self._get_eot_usage(provider, model)
+            eot_usage.total_requests += metrics.num_requests
 
     def flatten(self) -> list[ModelUsage]:
         """Returns a list of usage summaries, one per model/provider combination."""
@@ -233,4 +263,5 @@ def flatten(self) -> list[ModelUsage]:
         result.extend(u.model_copy(deep=True) for u in self._tts_usage.values())
         result.extend(u.model_copy(deep=True) for u in self._stt_usage.values())
         result.extend(u.model_copy(deep=True) for u in self._interruption_usage.values())
+        result.extend(u.model_copy(deep=True) for u in self._eot_usage.values())
         return result

livekit-agents/livekit/agents/telemetry/trace_types.py

@@ -59,6 +59,8 @@
 ATTR_TRANSCRIPT_CONFIDENCE = "lk.transcript_confidence"
 ATTR_TRANSCRIPTION_DELAY = "lk.transcription_delay"
 ATTR_END_OF_TURN_DELAY = "lk.end_of_turn_delay"
+ATTR_EOU_SOURCE = "lk.eou.source"
+ATTR_EOU_DETECTION_DELAY = "lk.eou.detection_delay"
 
 # metrics
 ATTR_LLM_METRICS = "lk.llm_metrics"

livekit-agents/livekit/agents/utils/__init__.py

@@ -4,6 +4,7 @@
 from .audio import AudioArrayBuffer, AudioBuffer, combine_frames, merge_frames
 from .bounded_dict import BoundedDict
 from .connection_pool import ConnectionPool
+from .env import resolve_env_var
 from .exp_filter import ExpFilter
 from .log import log_exceptions
 from .misc import is_dev_mode, is_given, is_hosted, nodename, shortuuid, time_ms
@@ -39,6 +40,7 @@
     "wait_for_agent",
     "wait_for_participant",
     "wait_for_track_publication",
+    "resolve_env_var",
 ]
 
 # Cleanup docs of unexported modules

livekit-agents/livekit/agents/utils/env.py

@@ -0,0 +1,33 @@
+import os
+
+from ..types import NotGivenOr
+from .misc import is_given
+
+
+def resolve_env_var(val: NotGivenOr[str], *env_vars: str, default: str = "") -> str:
+    """
+    Resolve an environment variable from a list of potential sources.
+
+    Args:
+        val: The value to resolve.
+        *env_vars: The environment variables to check. Order matters, the first non-None value will be returned.
+        default: The default value to return if no environment variables are set.
+
+    Returns:
+        The resolved environment variable.
+
+    Examples:
+    >>> resolve_env_var(
+    ...     NOT_GIVEN,
+    ...     "ABC_URL",
+    ...     default="https://agent-gateway.livekit.cloud/v1",
+    ... )
+    "https://agent-gateway.livekit.cloud/v1"
+    """
+    if is_given(val):
+        return val
+    for env_var in env_vars:
+        curr_val = os.getenv(env_var, None)
+        if curr_val is not None and curr_val != "":
+            return curr_val
+    return default

livekit-agents/livekit/agents/voice/agent_activity.py

@@ -78,7 +78,7 @@
     update_instructions,
 )
 from .speech_handle import DEFAULT_INPUT_DETAILS, InputDetails, SpeechHandle
-from .turn import EndpointingOptions, TurnDetectionMode
+from .turn import EndpointingOptions, TurnDetectionMode, _AudioTurnDetector
 
 if TYPE_CHECKING:
     from ..llm import mcp
@@ -232,6 +232,11 @@ def _validate_turn_detection(
         self, turn_detection: TurnDetectionMode | None
     ) -> TurnDetectionMode | None:
         if turn_detection is not None and not isinstance(turn_detection, str):
+            if isinstance(turn_detection, _AudioTurnDetector) and self.vad is None:
+                raise ValueError(
+                    "AudioTurnDetector requires a VAD model; pass vad=silero.VAD.load() "
+                    "(or another VAD) to AgentSession/Agent."
+                )
             # return directly if turn_detection is _TurnDetector
             return turn_detection
 
@@ -693,6 +698,9 @@ async def _start_session(self, *, reuse_resources: _ReusableResources | None = N
             self._interruption_detector.on("error", self._on_error)
             self._interruption_detector.on("overlapping_speech", self._on_overlap_speech_ended)
 
+        if isinstance(self._turn_detection, _AudioTurnDetector):
+            self._turn_detection.on("metrics_collected", self._on_metrics_collected)
+
         if self.mcp_servers:
             from ..llm.mcp import MCPToolset
 
@@ -963,6 +971,9 @@ async def _close_session(self) -> None:
             self._interruption_detector.off("error", self._on_error)
             self._interruption_detector.off("overlapping_speech", self._on_overlap_speech_ended)
 
+        if isinstance(self._turn_detection, _AudioTurnDetector):
+            self._turn_detection.off("metrics_collected", self._on_metrics_collected)
+
         if self._rt_session is not None:
             await self._rt_session.aclose()