feat: Add TwelveLabs Pegasus video-to-Markdown converter

README.md

@@ -110,6 +110,7 @@ At the moment, the following optional dependencies are available:
 * `[az-content-understanding]` Installs dependencies for Azure Content Understanding
 * `[audio-transcription]` Installs dependencies for audio transcription of wav and mp3 files
 * `[youtube-transcription]` Installs dependencies for fetching YouTube video transcription
+* `[twelvelabs]` Installs dependencies for video understanding with TwelveLabs Pegasus
 
 ### Plugins
 
@@ -280,6 +281,39 @@ result = md.convert("example.jpg")
 print(result.text_content)
 ```
 
+### TwelveLabs (Video Understanding)
+
+The built-in audio/video converter only transcribes speech. [TwelveLabs](https://twelvelabs.io)
+Pegasus is a video-understanding model that reads the actual frames and audio, so the
+resulting Markdown can also describe visual scenes, on-screen action, and structure — not
+just the spoken words. It is opt-in and only activates for video files (`.mp4`, `.mov`,
+`.webm`, `.mkv`, `.avi`, `.m4v`) when an API key is supplied. All other formats continue to
+use the existing converters unchanged.
+
+Install: `pip install 'markitdown[twelvelabs]'`
+
+From the command line (reads the `TWELVELABS_API_KEY` environment variable):
+
+```bash
+export TWELVELABS_API_KEY="<your_api_key>"
+markitdown talk.mp4 --use-twelvelabs -o talk.md
+```
+
+In Python (pass the key explicitly or via `TWELVELABS_API_KEY`):
+
+```python
+from markitdown import MarkItDown
+
+md = MarkItDown(
+    twelvelabs_api_key="<your_api_key>",
+    twelvelabs_prompt="Summarize this video as Markdown.",  # optional
+)
+result = md.convert("talk.mp4")
+print(result.text_content)
+```
+
+You can grab a free API key at [twelvelabs.io](https://twelvelabs.io) — there's a generous free tier.
+
 ### Docker
 
 ```sh

packages/markitdown/pyproject.toml

@@ -49,6 +49,7 @@ all = [
   "azure-ai-documentintelligence",
   "azure-ai-contentunderstanding>=1.2.0b1",
   "azure-identity",
+  "twelvelabs>=1.2.8",
 ]
 pptx = ["python-pptx"]
 docx = ["mammoth~=1.11.0", "lxml"]
@@ -61,6 +62,7 @@ youtube-transcription = ["youtube-transcript-api"]
 az-doc-intel = ["azure-ai-documentintelligence", "azure-identity"]
 # >=1.2.0b1 required for to_llm_input() helper used by ContentUnderstandingConverter
 az-content-understanding = ["azure-ai-contentunderstanding>=1.2.0b1", "azure-identity"]
+twelvelabs = ["twelvelabs>=1.2.8"]
 
 [project.urls]
 Documentation = "https://github.com/microsoft/markitdown#readme"

packages/markitdown/src/markitdown/__main__.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 import argparse
+import os
 import sys
 import codecs
 from typing import Any, Dict
@@ -94,6 +95,13 @@ def main():
         help="Use Azure Content Understanding to extract text. Requires --cu-endpoint.",
     )
 
+    cloud_group.add_argument(
+        "--use-twelvelabs",
+        action="store_true",
+        dest="use_twelvelabs",
+        help="Use TwelveLabs Pegasus to convert video to Markdown. Requires the TWELVELABS_API_KEY environment variable.",
+    )
+
     parser.add_argument(
         "-e",
         "--endpoint",
@@ -119,6 +127,18 @@ def main():
         help="Comma-separated list of file types to route to Content Understanding (e.g., pdf,jpeg,mp4). If omitted, all supported types are routed.",
     )
 
+    parser.add_argument(
+        "--twelvelabs-model",
+        type=str,
+        help="TwelveLabs Pegasus model name to use with --use-twelvelabs (e.g., pegasus1.5).",
+    )
+
+    parser.add_argument(
+        "--twelvelabs-prompt",
+        type=str,
+        help="Instruction sent to Pegasus describing how to summarize the video when using --use-twelvelabs.",
+    )
+
     parser.add_argument(
         "-p",
         "--use-plugins",
@@ -241,6 +261,22 @@ def main():
             cu_kwargs["cu_file_types"] = cu_types
 
         markitdown = MarkItDown(enable_plugins=args.use_plugins, **cu_kwargs)
+    elif args.use_twelvelabs:
+        twelvelabs_api_key = os.environ.get("TWELVELABS_API_KEY")
+        if twelvelabs_api_key is None:
+            _exit_with_error(
+                "The TWELVELABS_API_KEY environment variable is required when using --use-twelvelabs."
+            )
+        elif args.filename is None:
+            _exit_with_error("Filename is required when using TwelveLabs.")
+
+        tl_kwargs: Dict[str, Any] = {"twelvelabs_api_key": twelvelabs_api_key}
+        if args.twelvelabs_model is not None:
+            tl_kwargs["twelvelabs_model"] = args.twelvelabs_model
+        if args.twelvelabs_prompt is not None:
+            tl_kwargs["twelvelabs_prompt"] = args.twelvelabs_prompt
+
+        markitdown = MarkItDown(enable_plugins=args.use_plugins, **tl_kwargs)
     else:
         markitdown = MarkItDown(enable_plugins=args.use_plugins)
 

packages/markitdown/src/markitdown/_markitdown.py

@@ -40,6 +40,7 @@
     DocumentIntelligenceConverter,
     ContentUnderstandingConverter,
     CsvConverter,
+    TwelveLabsConverter,
 )
 
 from ._base_converter import DocumentConverter, DocumentConverterResult
@@ -248,6 +249,26 @@ def enable_builtins(self, **kwargs) -> None:
                     ContentUnderstandingConverter(**cu_args),
                 )
 
+            # Register the TwelveLabs (Pegasus) video converter at the top of the
+            # stack if an API key is provided (via kwarg or TWELVELABS_API_KEY).
+            twelvelabs_api_key = kwargs.get("twelvelabs_api_key") or os.getenv(
+                "TWELVELABS_API_KEY"
+            )
+            if twelvelabs_api_key is not None:
+                tl_args: Dict[str, Any] = {"api_key": twelvelabs_api_key}
+
+                tl_model = kwargs.get("twelvelabs_model")
+                if tl_model is not None:
+                    tl_args["model_name"] = tl_model
+
+                tl_prompt = kwargs.get("twelvelabs_prompt")
+                if tl_prompt is not None:
+                    tl_args["prompt"] = tl_prompt
+
+                self.register_converter(
+                    TwelveLabsConverter(**tl_args),
+                )
+
             self._builtins_enabled = True
         else:
             warn("Built-in converters are already enabled.", RuntimeWarning)

packages/markitdown/src/markitdown/converters/__init__.py

@@ -27,6 +27,7 @@
 )
 from ._epub_converter import EpubConverter
 from ._csv_converter import CsvConverter
+from ._twelvelabs_converter import TwelveLabsConverter
 
 __all__ = [
     "PlainTextConverter",
@@ -51,4 +52,5 @@
     "ContentUnderstandingFileType",
     "EpubConverter",
     "CsvConverter",
+    "TwelveLabsConverter",
 ]

packages/markitdown/src/markitdown/converters/_twelvelabs_converter.py

@@ -0,0 +1,163 @@
+import os
+import sys
+import base64
+from typing import Any, BinaryIO, List, Optional, Union
+
+from .._base_converter import DocumentConverter, DocumentConverterResult
+from .._stream_info import StreamInfo
+from .._exceptions import MissingDependencyException
+
+# Try loading the optional (but in this case, required) dependency.
+# Save reporting of any exceptions for later.
+_dependency_exc_info = None
+try:
+    from twelvelabs import TwelveLabs
+    from twelvelabs.types.video_context import VideoContext_Base64String
+except ImportError:
+    # Preserve the error and stack trace for later
+    _dependency_exc_info = sys.exc_info()
+
+    # Define these names for type hinting when the package is not available
+    class TwelveLabs:  # type: ignore[no-redef]
+        pass
+
+    class VideoContext_Base64String:  # type: ignore[no-redef]
+        pass
+
+
+# Video container formats understood by the Pegasus video-understanding model.
+ACCEPTED_MIME_TYPE_PREFIXES = [
+    "video/mp4",
+    "video/quicktime",
+    "video/webm",
+    "video/x-matroska",
+    "video/x-msvideo",
+    "video/x-m4v",
+]
+
+ACCEPTED_FILE_EXTENSIONS = [
+    ".mp4",
+    ".mov",
+    ".webm",
+    ".mkv",
+    ".avi",
+    ".m4v",
+]
+
+DEFAULT_PROMPT = (
+    "Describe this video in detail as Markdown. Summarize what happens, "
+    "transcribe any spoken words, and note key visual scenes in order."
+)
+
+
+class TwelveLabsConverter(DocumentConverter):
+    """Converts video files to Markdown using the TwelveLabs Pegasus video-understanding model.
+
+    This converter is opt-in: it is only registered when a TwelveLabs API key is
+    supplied (via the ``twelvelabs_api_key`` argument to ``MarkItDown`` or the
+    ``TWELVELABS_API_KEY`` environment variable). Pegasus reads the actual video
+    frames and audio, so the resulting Markdown can capture visual scenes in
+    addition to speech, unlike the speech-only built-in ``AudioConverter``.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: Optional[str] = None,
+        model_name: str = "pegasus1.5",
+        prompt: Optional[str] = None,
+        max_tokens: int = 2048,
+        file_extensions: Optional[List[str]] = None,
+        mime_type_prefixes: Optional[List[str]] = None,
+    ):
+        """
+        Initialize the TwelveLabsConverter.
+
+        Args:
+            api_key: TwelveLabs API key. Falls back to the ``TWELVELABS_API_KEY``
+                environment variable when not provided.
+            model_name: Pegasus model to use (e.g. ``"pegasus1.5"``).
+            prompt: Instruction sent to Pegasus describing how to summarize the
+                video. Defaults to a general "describe this video as Markdown" prompt.
+            max_tokens: Maximum number of tokens Pegasus may generate.
+            file_extensions: Override the list of accepted file extensions.
+            mime_type_prefixes: Override the list of accepted MIME-type prefixes.
+        """
+        super().__init__()
+
+        # Raise an error if the dependency is not available. This converter is
+        # only instantiated when explicitly requested, so failing here is correct.
+        if _dependency_exc_info is not None:
+            raise MissingDependencyException(
+                "TwelveLabsConverter requires the optional dependency [twelvelabs] (or [all]) to be installed. E.g., `pip install markitdown[twelvelabs]`"
+            ) from _dependency_exc_info[
+                1
+            ].with_traceback(  # type: ignore[union-attr]
+                _dependency_exc_info[2]
+            )
+
+        resolved_key = api_key or os.environ.get("TWELVELABS_API_KEY")
+        if not resolved_key:
+            raise ValueError(
+                "TwelveLabsConverter requires an API key. Pass twelvelabs_api_key=... "
+                "or set the TWELVELABS_API_KEY environment variable. "
+                "Grab a free key at https://twelvelabs.io."
+            )
+
+        self._model_name = model_name
+        self._prompt = prompt
+        self._max_tokens = max_tokens
+        self._file_extensions = (
+            file_extensions if file_extensions is not None else ACCEPTED_FILE_EXTENSIONS
+        )
+        self._mime_type_prefixes = (
+            mime_type_prefixes
+            if mime_type_prefixes is not None
+            else ACCEPTED_MIME_TYPE_PREFIXES
+        )
+        self._client = TwelveLabs(api_key=resolved_key)
+
+    def accepts(
+        self,
+        file_stream: BinaryIO,
+        stream_info: StreamInfo,
+        **kwargs: Any,  # Options to pass to the converter
+    ) -> bool:
+        mimetype = (stream_info.mimetype or "").lower()
+        extension = (stream_info.extension or "").lower()
+
+        if extension in self._file_extensions:
+            return True
+
+        for prefix in self._mime_type_prefixes:
+            if mimetype.startswith(prefix):
+                return True
+
+        return False
+
+    def convert(
+        self,
+        file_stream: BinaryIO,
+        stream_info: StreamInfo,
+        **kwargs: Any,  # Options to pass to the converter
+    ) -> DocumentConverterResult:
+        prompt = kwargs.get("twelvelabs_prompt") or self._prompt or DEFAULT_PROMPT
+
+        # Read the video bytes and base64-encode them for the analyze request.
+        cur_pos = file_stream.tell()
+        try:
+            base64_video = base64.b64encode(file_stream.read()).decode("ascii")
+        finally:
+            file_stream.seek(cur_pos)
+
+        response = self._client.analyze(
+            model_name=self._model_name,
+            video=VideoContext_Base64String(base_64_string=base64_video),
+            prompt=prompt,
+            max_tokens=self._max_tokens,
+        )
+
+        markdown = response.data or ""
+
+        title: Union[str, None] = stream_info.filename
+        return DocumentConverterResult(markdown=markdown.strip(), title=title)