voice.asyncapi.yaml

asyncapi: 3.0.0
info:
  title: DeepL Voice API - WebSocket Streaming
  version: '3.9.0'
  description: >
    WebSocket streaming API for real-time voice transcription and translation.
    After obtaining a streaming URL and token via the REST API, establish a WebSocket connection
    to stream audio data and receive real-time transcriptions and translations.
  contact:
    name: DeepL - Contact us
    url: https://www.deepl.com/contact-us

servers:
  production:
    host: api.deepl.com
    pathname: /v3/voice/realtime/connect
    protocol: wss
    description: DeepL Voice API WebSocket endpoint.
    variables:
      token:
        description: This is the ephemeral authentication token obtained from the REST API
        examples:
          - VGhpcyBpcyBhIGZha2UgdG9rZW4K

channels:
  voiceStream:
    address: /v3/voice/realtime/connect?token={token}
    description: >
        WebSocket channel for streaming audio and receiving transcriptions and translations.
        Messages are exchanged in JSON format and WebSocket frames must be of type TEXT.
    parameters:
      token:
        description: >
          This is the ephemeral authentication token obtained from the
          [Get Streaming URL](/api-reference/voice/get-streaming-url) endpoint.
          The token is valid for one-time use only and must be passed as a query parameter when establishing
          the WebSocket connection.
        examples:
          - VGhpcyBpcyBhIGZha2UgdG9rZW4K
    messages:
      SourceMediaChunk:
        $ref: '#/components/messages/SourceMediaChunk'
      EndOfSourceMedia:
        $ref: '#/components/messages/EndOfSourceMedia'
      SourceTranscriptUpdate:
        $ref: '#/components/messages/SourceTranscriptUpdate'
      TargetTranscriptUpdate:
        $ref: '#/components/messages/TargetTranscriptUpdate'
      EndOfSourceTranscript:
        $ref: '#/components/messages/EndOfSourceTranscript'
      EndOfTargetTranscript:
        $ref: '#/components/messages/EndOfTargetTranscript'
      EndOfStream:
        $ref: '#/components/messages/EndOfStream'
      Error:
        $ref: '#/components/messages/Error'

operations:
  sendAudioData:
    action: send
    channel:
      $ref: '#/channels/voiceStream'
    summary: Send audio data to the server
    description: |-
      Send audio chunks and control messages to the server.
      Audio data must be base64-encoded and match the configured format.
    messages:
      - $ref: '#/channels/voiceStream/messages/SourceTranscriptUpdate'
      - $ref: '#/channels/voiceStream/messages/TargetTranscriptUpdate'
      - $ref: '#/channels/voiceStream/messages/EndOfSourceTranscript'
      - $ref: '#/channels/voiceStream/messages/EndOfTargetTranscript'
      - $ref: '#/channels/voiceStream/messages/EndOfStream'
      - $ref: '#/channels/voiceStream/messages/Error'
  receiveTranscriptions:
    action: receive
    channel:
      $ref: '#/channels/voiceStream'
    summary: Receive transcriptions and translations
    description: |-
      Receive real-time transcription and translation messages from the server.
      The server sends transcript updates, end of transcript notifications and errors.
    messages:
      - $ref: '#/channels/voiceStream/messages/SourceMediaChunk'
      - $ref: '#/channels/voiceStream/messages/EndOfSourceMedia'

components:
  messages:
    SourceMediaChunk:
      name: SourceMediaChunk
      title: Source Media Chunk
      description: "
        The message contains a base64-encoded binary chunk of audio data.
        The audio encoding must be the same that was specified in the [Get Streaming URL](/api-reference/voice/get-streaming-url) request.
        \n\n
        The chunk size must not be more than 100 kilobyte or one second in duration. The recommended duration is
        50 - 250 milliseconds to achieve the best tradeoff between latency and quality.
        The interval between chunks must not be less than half of the duration of the preceding chunk and not
        exceed 30 seconds. Otherwise you will run into rate limits or the session will terminate due to timing out and the connection will be closed.
        \n\n
        For PCM data the chunk size must be a multiple of the frame size aka encoding unit."
      contentType: application/json
      payload:
        $ref: '#/components/schemas/SourceMediaChunkPayload'
      examples:
        - name: SourceMediaChunk
          summary: Source media chunk
          payload:
            source_media_chunk:
              data: VGhpcyBpcyBhIGZha2UgYXVkaW8gY2h1bmsK

    EndOfSourceMedia:
      name: EndOfSourceMedia
      title: End of Source Media
      description: >
        The message indicates the end of source media data. It causes the finalization of tentative transcript segments
        and triggers the emission of final transcript updates, end of transcript messages and the end of stream message.
        No more data chunks can be sent afterwards. It marks the end of the stream input.
      contentType: application/json
      payload:
        $ref: '#/components/schemas/EndOfSourceMediaPayload'
      examples:
        - name: EndStream
          summary: End of audio stream
          payload:
            end_of_source_media: {}

    SourceTranscriptUpdate:
      name: SourceTranscriptUpdate
      title: Source Transcript Update
      description: "
        The message contains an update to the transcription of the supplied media in the *source* language.
        \n\n
        Each message is an incremental addition to the already received updates of the *source* transcript with concluded
        and tentative text segments. Concluded segments are fixed and will only appear once, while tentative segments
        may be updated in subsequent messages as more audio is processed.
        \n\n
        Clients should merge the concluded segments into a final transcript and update the tentative segments as
        new updates arrive."
      contentType: application/json
      payload:
        $ref: '#/components/schemas/SourceTranscriptUpdatePayload'
      examples:
        - name: TranscriptUpdate
          summary: Source language transcript update
          payload:
            source_transcript_update:
              concluded:
                - language: en
                  text: 'Hello, how are you'
                  start_time: 0
                  end_time: 1500
              tentative:
                - language: en
                  text: ' today?'
                  start_time: 1500
                  end_time: 2000

    TargetTranscriptUpdate:
      name: TargetTranscriptUpdate
      title: Target Transcript Update
      description: "
        The message contains an update to the transcription of the supplied media in the *target* language.
        \n\n
        Each message is an incremental addition to the already received updates of the *target* transcript with concluded
        and tentative text segments. Concluded segments are fixed and will only appear once, while tentative segments
        may be updated in subsequent messages as more audio is processed.
        \n\n
        Clients should merge the concluded segments into a final transcript and update the tentative segments
        as new updates arrive."
      contentType: application/json
      payload:
        $ref: '#/components/schemas/TargetTranscriptUpdatePayload'
      examples:
        - name: TranslationUpdate
          summary: Target language translation update
          payload:
            target_transcript_update:
              language: es
              concluded:
                - text: 'Hola, ¿cómo estás'
                  start_time: 0
                  end_time: 1500
              tentative:
                - text: ' hoy?'
                  start_time: 1500
                  end_time: 2000

    EndOfSourceTranscript:
      name: EndOfSourceTranscript
      title: End of Source Transcript
      description: >
        The message indicates that the *source* transcript is complete and no further updates will be sent.
        It gets emitted after client sends End of Source Media.
      contentType: application/json
      payload:
        $ref: '#/components/schemas/EndOfSourceTranscriptPayload'
      examples:
        - name: EndSource
          summary: Source transcript complete
          payload:
            end_of_source_transcript: {}

    EndOfTargetTranscript:
      name: EndOfTargetTranscript
      title: End of Target Transcript
      description: >
        This message indicates that the *target* transcript is complete and no further updates will be sent.
        It gets emitted after client sends End of Source Media.
      contentType: application/json
      payload:
        $ref: '#/components/schemas/EndOfTargetTranscriptPayload'
      examples:
        - name: EndTarget
          summary: Target transcript complete
          payload:
            end_of_target_transcript:
              language: fr

    EndOfStream:
      name: EndOfStream
      title: End of Stream
      description: >
        This message indicates that all outputs are complete and the stream ended.
        It is the very last message the client will receive after it sends End of Source Media.
        You can safely close the connection after you received this message.
      contentType: application/json
      payload:
        $ref: '#/components/schemas/EndOfStreamPayload'
      examples:
        - name: EndStream
          summary: Stream ended
          payload:
            end_of_stream: {}

    Error:
      name: Error
      title: Error
      description: >
        This message reports errors encountered during audio processing or streaming.
        It includes an error code, reason code, and a human-readable message.
        After an error, the session is terminated and reconnection is not possible.
        You need to request a new session.
      contentType: application/json
      payload:
        $ref: '#/components/schemas/ErrorPayload'
      examples:
        - name: AudioFormatError
          summary: Audio format error
          payload:
            error:
              request_type: source_media_chunk
              error_code: 400
              reason_code: 4000403
              error_message: Audio format not supported

  schemas:
    SourceMediaChunkPayload:
      type: object
      required:
        - source_media_chunk
      properties:
        source_media_chunk:
          type: object
          required:
            - data
          properties:
            data:
              type: string
              format: byte
              description: Base64-encoded media data in the format specified during session initialization

    EndOfSourceMediaPayload:
      type: object
      required:
        - end_of_source_media
      properties:
        end_of_source_media:
          type: object
          description: Empty object signaling end of media stream

    TranscriptSegment:
      type: object
      required:
        - text
        - start_time
        - end_time
      properties:
        text:
          type: string
          description: Source or target transcript text
        start_time:
          type: integer
          description: Estimated start time of the segment in the input stream in milliseconds
          examples:
            - 1250
        end_time:
          type: integer
          description: Estimated end time of the segment in the input stream in milliseconds
          examples:
            - 1570

    SourceTranscriptSegment:
      allOf:
        - $ref: '#/components/schemas/TranscriptSegment'
        - type: object
          required:
            - language
          properties:
            language:
              type: string
              description: IETF BCP 47 language tag of the detected source language
              examples:
                - en

    SourceTranscriptUpdatePayload:
      type: object
      required:
        - source_transcript_update
      properties:
        source_transcript_update:
          type: object
          required:
            - concluded
            - tentative
          properties:
            concluded:
              type: array
              # list properties of referenced items here because there is a bug in mintlify that does not render them properly for array types
              description: Array of fixed transcript segments that will not change anymore. Array objects contain `language` property of type `string` (IETF BCP 47 language tag of the detected source language), `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).
              items:
                $ref: '#/components/schemas/SourceTranscriptSegment'
            tentative:
              type: array
              # list properties of referenced items here because there is a bug in mintlify that does not render them properly for array types
              description: Array of preliminary transcript segments that are subject to change. Array objects contain `language` property of type `string` (IETF BCP 47 language tag of the detected source language), `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).
              items:
                $ref: '#/components/schemas/SourceTranscriptSegment'

    TargetTranscriptUpdatePayload:
      type: object
      required:
        - target_transcript_update
      properties:
        target_transcript_update:
          type: object
          required:
            - language
            - concluded
            - tentative
          properties:
            language:
              type: string
              description: IETF BCP 47 language tag of the target language
              examples:
                - es
            concluded:
              type: array
              # list properties of referenced items here because there is a bug in mintlify that does not render them properly for array types
              description: Array of fixed transcript segments that will not change anymore. Array objects contain `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).
              items:
                $ref: '#/components/schemas/TranscriptSegment'
            tentative:
              type: array
              # list properties of referenced items here because there is a bug in mintlify that does not render them properly for array types
              description: Array of preliminary transcript segments that are subject to change. Array objects contain `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).
              items:
                $ref: '#/components/schemas/TranscriptSegment'

    EndOfSourceTranscriptPayload:
      type: object
      required:
        - end_of_source_transcript
      properties:
        end_of_source_transcript:
          type: object
          description: Empty object indicating source transcript is complete

    EndOfTargetTranscriptPayload:
      type: object
      required:
        - end_of_target_transcript
      properties:
        end_of_target_transcript:
          type: object
          required:
            - language
          properties:
            language:
              type: string
              description: IETF BCP 47 language tag indicating which target transcript has ended
              examples:
                - fr

    EndOfStreamPayload:
      type: object
      required:
        - end_of_stream

    ErrorPayload:
      type: object
      required:
        - error
      properties:
        error:
          type: object
          required:
            - request_type
            - error_code
            - reason_code
            - error_message
          properties:
            request_type:
              type: string
              description: The type of request that caused the error
              examples:
                - source_media_chunk
            error_code:
              type: integer
              description: HTTP-style error code
              examples:
                - 400
            reason_code:
              type: integer
              description: Detailed reason code for debugging
              examples:
                - 4000403
            error_message:
              type: string
              description: Human-readable error description
              examples:
                - Audio format not supported