voice.asyncapi.json

{
  "asyncapi": "3.0.0",
  "info": {
    "title": "DeepL Voice API - WebSocket Streaming",
    "version": "3.9.0",
    "description": "WebSocket streaming API for real-time voice transcription and translation. After obtaining a streaming URL and token via the REST API, establish a WebSocket connection to stream audio data and receive real-time transcriptions and translations.\n",
    "contact": {
      "name": "DeepL - Contact us",
      "url": "https://www.deepl.com/contact-us"
    }
  },
  "servers": {
    "production": {
      "host": "api.deepl.com",
      "pathname": "/v3/voice/realtime/connect",
      "protocol": "wss",
      "description": "DeepL Voice API WebSocket endpoint.",
      "variables": {
        "token": {
          "description": "This is the ephemeral authentication token obtained from the REST API",
          "examples": [
            "VGhpcyBpcyBhIGZha2UgdG9rZW4K"
          ]
        }
      }
    }
  },
  "channels": {
    "voiceStream": {
      "address": "/v3/voice/realtime/connect?token={token}",
      "description": "WebSocket channel for streaming audio and receiving transcriptions and translations. Messages are exchanged in JSON format and WebSocket frames must be of type TEXT.\n",
      "parameters": {
        "token": {
          "description": "This is the ephemeral authentication token obtained from the [Get Streaming URL](/api-reference/voice/get-streaming-url) endpoint. The token is valid for one-time use only and must be passed as a query parameter when establishing the WebSocket connection.\n",
          "examples": [
            "VGhpcyBpcyBhIGZha2UgdG9rZW4K"
          ]
        }
      },
      "messages": {
        "SourceMediaChunk": {
          "$ref": "#/components/messages/SourceMediaChunk"
        },
        "EndOfSourceMedia": {
          "$ref": "#/components/messages/EndOfSourceMedia"
        },
        "SourceTranscriptUpdate": {
          "$ref": "#/components/messages/SourceTranscriptUpdate"
        },
        "TargetTranscriptUpdate": {
          "$ref": "#/components/messages/TargetTranscriptUpdate"
        },
        "EndOfSourceTranscript": {
          "$ref": "#/components/messages/EndOfSourceTranscript"
        },
        "EndOfTargetTranscript": {
          "$ref": "#/components/messages/EndOfTargetTranscript"
        },
        "EndOfStream": {
          "$ref": "#/components/messages/EndOfStream"
        },
        "Error": {
          "$ref": "#/components/messages/Error"
        }
      }
    }
  },
  "operations": {
    "sendAudioData": {
      "action": "send",
      "channel": {
        "$ref": "#/channels/voiceStream"
      },
      "summary": "Send audio data to the server",
      "description": "Send audio chunks and control messages to the server.\nAudio data must be base64-encoded and match the configured format.",
      "messages": [
        {
          "$ref": "#/channels/voiceStream/messages/SourceTranscriptUpdate"
        },
        {
          "$ref": "#/channels/voiceStream/messages/TargetTranscriptUpdate"
        },
        {
          "$ref": "#/channels/voiceStream/messages/EndOfSourceTranscript"
        },
        {
          "$ref": "#/channels/voiceStream/messages/EndOfTargetTranscript"
        },
        {
          "$ref": "#/channels/voiceStream/messages/EndOfStream"
        },
        {
          "$ref": "#/channels/voiceStream/messages/Error"
        }
      ]
    },
    "receiveTranscriptions": {
      "action": "receive",
      "channel": {
        "$ref": "#/channels/voiceStream"
      },
      "summary": "Receive transcriptions and translations",
      "description": "Receive real-time transcription and translation messages from the server.\nThe server sends transcript updates, end of transcript notifications and errors.",
      "messages": [
        {
          "$ref": "#/channels/voiceStream/messages/SourceMediaChunk"
        },
        {
          "$ref": "#/channels/voiceStream/messages/EndOfSourceMedia"
        }
      ]
    }
  },
  "components": {
    "messages": {
      "SourceMediaChunk": {
        "name": "SourceMediaChunk",
        "title": "Source Media Chunk",
        "description": " The message contains a base64-encoded binary chunk of audio data. The audio encoding must be the same that was specified in the [Get Streaming URL](/api-reference/voice/get-streaming-url) request. \n\n The chunk size must not be more than 100 kilobyte or one second in duration. The recommended duration is 50 - 250 milliseconds to achieve the best tradeoff between latency and quality. The interval between chunks must not be less than half of the duration of the preceding chunk and not exceed 30 seconds. Otherwise you will run into rate limits or the session will terminate due to timing out and the connection will be closed. \n\n For PCM data the chunk size must be a multiple of the frame size aka encoding unit.",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/SourceMediaChunkPayload"
        },
        "examples": [
          {
            "name": "SourceMediaChunk",
            "summary": "Source media chunk",
            "payload": {
              "source_media_chunk": {
                "data": "VGhpcyBpcyBhIGZha2UgYXVkaW8gY2h1bmsK"
              }
            }
          }
        ]
      },
      "EndOfSourceMedia": {
        "name": "EndOfSourceMedia",
        "title": "End of Source Media",
        "description": "The message indicates the end of source media data. It causes the finalization of tentative transcript segments and triggers the emission of final transcript updates, end of transcript messages and the end of stream message. No more data chunks can be sent afterwards. It marks the end of the stream input.\n",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/EndOfSourceMediaPayload"
        },
        "examples": [
          {
            "name": "EndStream",
            "summary": "End of audio stream",
            "payload": {
              "end_of_source_media": {}
            }
          }
        ]
      },
      "SourceTranscriptUpdate": {
        "name": "SourceTranscriptUpdate",
        "title": "Source Transcript Update",
        "description": " The message contains an update to the transcription of the supplied media in the *source* language. \n\n Each message is an incremental addition to the already received updates of the *source* transcript with concluded and tentative text segments. Concluded segments are fixed and will only appear once, while tentative segments may be updated in subsequent messages as more audio is processed. \n\n Clients should merge the concluded segments into a final transcript and update the tentative segments as new updates arrive.",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/SourceTranscriptUpdatePayload"
        },
        "examples": [
          {
            "name": "TranscriptUpdate",
            "summary": "Source language transcript update",
            "payload": {
              "source_transcript_update": {
                "concluded": [
                  {
                    "language": "en",
                    "text": "Hello, how are you",
                    "start_time": 0,
                    "end_time": 1500
                  }
                ],
                "tentative": [
                  {
                    "language": "en",
                    "text": " today?",
                    "start_time": 1500,
                    "end_time": 2000
                  }
                ]
              }
            }
          }
        ]
      },
      "TargetTranscriptUpdate": {
        "name": "TargetTranscriptUpdate",
        "title": "Target Transcript Update",
        "description": " The message contains an update to the transcription of the supplied media in the *target* language. \n\n Each message is an incremental addition to the already received updates of the *target* transcript with concluded and tentative text segments. Concluded segments are fixed and will only appear once, while tentative segments may be updated in subsequent messages as more audio is processed. \n\n Clients should merge the concluded segments into a final transcript and update the tentative segments as new updates arrive.",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/TargetTranscriptUpdatePayload"
        },
        "examples": [
          {
            "name": "TranslationUpdate",
            "summary": "Target language translation update",
            "payload": {
              "target_transcript_update": {
                "language": "es",
                "concluded": [
                  {
                    "text": "Hola, \u00bfc\u00f3mo est\u00e1s",
                    "start_time": 0,
                    "end_time": 1500
                  }
                ],
                "tentative": [
                  {
                    "text": " hoy?",
                    "start_time": 1500,
                    "end_time": 2000
                  }
                ]
              }
            }
          }
        ]
      },
      "EndOfSourceTranscript": {
        "name": "EndOfSourceTranscript",
        "title": "End of Source Transcript",
        "description": "The message indicates that the *source* transcript is complete and no further updates will be sent. It gets emitted after client sends End of Source Media.\n",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/EndOfSourceTranscriptPayload"
        },
        "examples": [
          {
            "name": "EndSource",
            "summary": "Source transcript complete",
            "payload": {
              "end_of_source_transcript": {}
            }
          }
        ]
      },
      "EndOfTargetTranscript": {
        "name": "EndOfTargetTranscript",
        "title": "End of Target Transcript",
        "description": "This message indicates that the *target* transcript is complete and no further updates will be sent. It gets emitted after client sends End of Source Media.\n",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/EndOfTargetTranscriptPayload"
        },
        "examples": [
          {
            "name": "EndTarget",
            "summary": "Target transcript complete",
            "payload": {
              "end_of_target_transcript": {
                "language": "fr"
              }
            }
          }
        ]
      },
      "EndOfStream": {
        "name": "EndOfStream",
        "title": "End of Stream",
        "description": "This message indicates that all outputs are complete and the stream ended. It is the very last message the client will receive after it sends End of Source Media. You can safely close the connection after you received this message.\n",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/EndOfStreamPayload"
        },
        "examples": [
          {
            "name": "EndStream",
            "summary": "Stream ended",
            "payload": {
              "end_of_stream": {}
            }
          }
        ]
      },
      "Error": {
        "name": "Error",
        "title": "Error",
        "description": "This message reports errors encountered during audio processing or streaming. It includes an error code, reason code, and a human-readable message. After an error, the session is terminated and reconnection is not possible. You need to request a new session.\n",
        "contentType": "application/json",
        "payload": {
          "$ref": "#/components/schemas/ErrorPayload"
        },
        "examples": [
          {
            "name": "AudioFormatError",
            "summary": "Audio format error",
            "payload": {
              "error": {
                "request_type": "source_media_chunk",
                "error_code": 400,
                "reason_code": 4000403,
                "error_message": "Audio format not supported"
              }
            }
          }
        ]
      }
    },
    "schemas": {
      "SourceMediaChunkPayload": {
        "type": "object",
        "required": [
          "source_media_chunk"
        ],
        "properties": {
          "source_media_chunk": {
            "type": "object",
            "required": [
              "data"
            ],
            "properties": {
              "data": {
                "type": "string",
                "format": "byte",
                "description": "Base64-encoded media data in the format specified during session initialization"
              }
            }
          }
        }
      },
      "EndOfSourceMediaPayload": {
        "type": "object",
        "required": [
          "end_of_source_media"
        ],
        "properties": {
          "end_of_source_media": {
            "type": "object",
            "description": "Empty object signaling end of media stream"
          }
        }
      },
      "TranscriptSegment": {
        "type": "object",
        "required": [
          "text",
          "start_time",
          "end_time"
        ],
        "properties": {
          "text": {
            "type": "string",
            "description": "Source or target transcript text"
          },
          "start_time": {
            "type": "integer",
            "description": "Estimated start time of the segment in the input stream in milliseconds",
            "examples": [
              1250
            ]
          },
          "end_time": {
            "type": "integer",
            "description": "Estimated end time of the segment in the input stream in milliseconds",
            "examples": [
              1570
            ]
          }
        }
      },
      "SourceTranscriptSegment": {
        "allOf": [
          {
            "$ref": "#/components/schemas/TranscriptSegment"
          },
          {
            "type": "object",
            "required": [
              "language"
            ],
            "properties": {
              "language": {
                "type": "string",
                "description": "IETF BCP 47 language tag of the detected source language",
                "examples": [
                  "en"
                ]
              }
            }
          }
        ]
      },
      "SourceTranscriptUpdatePayload": {
        "type": "object",
        "required": [
          "source_transcript_update"
        ],
        "properties": {
          "source_transcript_update": {
            "type": "object",
            "required": [
              "concluded",
              "tentative"
            ],
            "properties": {
              "concluded": {
                "type": "array",
                "description": "Array of fixed transcript segments that will not change anymore. Array objects contain `language` property of type `string` (IETF BCP 47 language tag of the detected source language), `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).",
                "items": {
                  "$ref": "#/components/schemas/SourceTranscriptSegment"
                }
              },
              "tentative": {
                "type": "array",
                "description": "Array of preliminary transcript segments that are subject to change. Array objects contain `language` property of type `string` (IETF BCP 47 language tag of the detected source language), `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).",
                "items": {
                  "$ref": "#/components/schemas/SourceTranscriptSegment"
                }
              }
            }
          }
        }
      },
      "TargetTranscriptUpdatePayload": {
        "type": "object",
        "required": [
          "target_transcript_update"
        ],
        "properties": {
          "target_transcript_update": {
            "type": "object",
            "required": [
              "language",
              "concluded",
              "tentative"
            ],
            "properties": {
              "language": {
                "type": "string",
                "description": "IETF BCP 47 language tag of the target language",
                "examples": [
                  "es"
                ]
              },
              "concluded": {
                "type": "array",
                "description": "Array of fixed transcript segments that will not change anymore. Array objects contain `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).",
                "items": {
                  "$ref": "#/components/schemas/TranscriptSegment"
                }
              },
              "tentative": {
                "type": "array",
                "description": "Array of preliminary transcript segments that are subject to change. Array objects contain `text` property of type `string` (Source or target transcript text), `start_time` property of type `integer` (Estimated start time of the segment in the input stream in milliseconds) and `end_time` property of type `integer` (Estimated end time of the segment in the input stream in milliseconds).",
                "items": {
                  "$ref": "#/components/schemas/TranscriptSegment"
                }
              }
            }
          }
        }
      },
      "EndOfSourceTranscriptPayload": {
        "type": "object",
        "required": [
          "end_of_source_transcript"
        ],
        "properties": {
          "end_of_source_transcript": {
            "type": "object",
            "description": "Empty object indicating source transcript is complete"
          }
        }
      },
      "EndOfTargetTranscriptPayload": {
        "type": "object",
        "required": [
          "end_of_target_transcript"
        ],
        "properties": {
          "end_of_target_transcript": {
            "type": "object",
            "required": [
              "language"
            ],
            "properties": {
              "language": {
                "type": "string",
                "description": "IETF BCP 47 language tag indicating which target transcript has ended",
                "examples": [
                  "fr"
                ]
              }
            }
          }
        }
      },
      "EndOfStreamPayload": {
        "type": "object",
        "required": [
          "end_of_stream"
        ]
      },
      "ErrorPayload": {
        "type": "object",
        "required": [
          "error"
        ],
        "properties": {
          "error": {
            "type": "object",
            "required": [
              "request_type",
              "error_code",
              "reason_code",
              "error_message"
            ],
            "properties": {
              "request_type": {
                "type": "string",
                "description": "The type of request that caused the error",
                "examples": [
                  "source_media_chunk"
                ]
              },
              "error_code": {
                "type": "integer",
                "description": "HTTP-style error code",
                "examples": [
                  400
                ]
              },
              "reason_code": {
                "type": "integer",
                "description": "Detailed reason code for debugging",
                "examples": [
                  4000403
                ]
              },
              "error_message": {
                "type": "string",
                "description": "Human-readable error description",
                "examples": [
                  "Audio format not supported"
                ]
              }
            }
          }
        }
      }
    }
  }
}