Merge branch 'whoisthey/language-model-input-modalities' into whoisthey/text-file-attachments

Author: whoisthey

CHANGELOG.md

@@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added per-step token cost tracking and estimated tool call token usage to Ask Sourcebot chat history. [#1353](https://github.com/sourcebot-dev/sourcebot/pull/1353)
 - [EE] Added mermaid diagram rendering to Ask Sourcebot answers, with pan/zoom, copy/export, in-thread deep links, and an interleaved right-panel view. [#1369](https://github.com/sourcebot-dev/sourcebot/pull/1369)
 - [EE] Added a context-window usage gauge to the Ask Sourcebot chat details, showing how much of the selected model's context window each turn occupies. Window sizes are resolved from the models.dev catalog. [#1370](https://github.com/sourcebot-dev/sourcebot/pull/1370)
-- Added optional `inputModalities` and `supportedDocumentTypes` configuration for language models, exposing model input-modality and document capabilities (defaults to text-only, no documents). [#1372](https://github.com/sourcebot-dev/sourcebot/pull/1372)
+- Added language model input-modality and document capability resolution, automatically resolved from the models.dev catalog (falls back to text-only for uncatalogued/self-hosted models). [#1372](https://github.com/sourcebot-dev/sourcebot/pull/1372)
 
 ### Fixed
 - Send anonymous server-side PostHog events as personless so unauthenticated requests don't inflate person counts. [#1367](https://github.com/sourcebot-dev/sourcebot/pull/1367)

docs/snippets/schemas/v3/index.schema.mdx

@@ -762,14 +762,6 @@ export interface AmazonBedrockLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 /**
  * Optional headers to use with the model.
@@ -850,14 +842,6 @@ export interface AnthropicLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface AzureLanguageModel {
   /**
@@ -913,14 +897,6 @@ export interface AzureLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface DeepSeekLanguageModel {
   /**
@@ -960,14 +936,6 @@ export interface DeepSeekLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface GoogleGenerativeAILanguageModel {
   /**
@@ -1015,14 +983,6 @@ export interface GoogleGenerativeAILanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface GoogleVertexAnthropicLanguageModel {
   /**
@@ -1070,14 +1030,6 @@ export interface GoogleVertexAnthropicLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface GoogleVertexLanguageModel {
   /**
@@ -1133,14 +1085,6 @@ export interface GoogleVertexLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface MistralLanguageModel {
   /**
@@ -1180,14 +1124,6 @@ export interface MistralLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface OpenAILanguageModel {
   /**
@@ -1235,14 +1171,6 @@ export interface OpenAILanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface OpenAICompatibleLanguageModel {
   /**
@@ -1287,14 +1215,6 @@ export interface OpenAICompatibleLanguageModel {
    * Optional temperature setting to use with the model.
    */
   temperature?: number;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 /**
  * Optional query parameters to include in the request url.
@@ -1359,14 +1279,6 @@ export interface OpenRouterLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface XaiLanguageModel {
   /**
@@ -1406,14 +1318,6 @@ export interface XaiLanguageModel {
    */
   temperature?: number;
   headers?: LanguageModelHeaders;
-  /**
-   * Optional list of input modalities (perceptual channels the model ingests natively) this model can accept. Governs single-medium attachments by their content: images require `image`, audio requires `audio`, video requires `video`, and plain-text files (.txt, .md, source code) count as `text`. Rich container documents like PDF are gated separately via `supportedDocumentTypes`. When omitted, the model is treated as text-only (fail-closed).
-   */
-  inputModalities?: ("text" | "image" | "audio" | "video")[];
-  /**
-   * Optional list of rich document formats (e.g. PDF) the model can ingest natively. A document here means a compound container format that bundles text with embedded images and layout, which the provider parses server-side — NOT plain-text files (.txt, .md, which count as `text`) and NOT single-medium files (images/audio/video, which are governed by `inputModalities`). When omitted, no document formats are accepted (fail-closed); this does not restrict `text` or the modalities declared in `inputModalities`.
-   */
-  supportedDocumentTypes?: "pdf"[];
 }
 export interface GitHubAppConfig {
   /**