Added multi-modal support

README.md

@@ -31,6 +31,8 @@ TypeChat uses language models to translate user intent into JSON that conforms t
 TypeChat provides:
 * Json translation, validation, repair and deserialization.
 * Extensibility: interfaces for customizing schemas, validators and prompts.
+* **Multimodal prompts**: send images (by url or local file) alongside text using `MultimodalPromptSection`, so vision capable models (e.g. gpt-4o) can translate what they *see* into strongly typed objects.
+* **OpenAI Responses API**: `LanguageModel` supports both the Chat Completions and Responses APIs. The Responses API is auto-selected when the endpoint path ends with `/responses`, or set `OpenAIConfig.UseResponsesApi`.
 * Schema export: classes to generate schema for the .NET Type you want to translate to. Exported schema includes dependencies and base classes. The exported schema is specified using **Typescript**, which can concisely express schema for JSON objects. 
   * Support for common scenarios shown in TypeChat examples. When you encounter limitations (such as how generics are currently exported), you can supply schema text, such as Typescript authored by hand.  
   * Helper attributes for Vocabularies and Comments. Vocabularies are string tables that constrain the values that can be assigned to string properties. Dynamic loading of vocabularies enables scenarios where they vary at runtime.
@@ -131,6 +133,9 @@ The following examples demonstrate how to use JsonTranslator, Schemas and Vocabu
 * [Calendar](./examples/Calendar): Transform language into calendar actions
 * [Restaurant](./examples/Restaurant): Order processing at a pizza restaurant
 
+### Multimodal (images)
+* [Multimodal](./examples/Multimodal): Send an image to a vision capable model and translate what it sees into a strongly typed object. Requires a vision model such as gpt-4o.
+
 ### Hierarchical schemas
 * [MultiSchema](./examples/MultiSchema): dynamically route user intent to other 'sub-apps'
 * [SchemaHierarchy](./examples/SchemaHierarchy): A Json Translator than uses multiple child JsonTranslators. For each user request, it picks the semantically ***nearest*** child translator and routes the input to it. 

examples/Multimodal/ImageSchema.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+using Microsoft.TypeChat.Schema;
+
+namespace Multimodal;
+
+/// <summary>
+/// A strongly typed description of an image, produced by translating a multimodal prompt
+/// (text + image) into this type.
+/// </summary>
+public class ImageResponse
+{
+    [Comment("A single sentence caption that describes the image")]
+    [JsonPropertyName("caption")]
+    public string Caption { get; set; }
+
+    [Comment("The most prominent objects visible in the image")]
+    [JsonPropertyName("objects")]
+    public string[] Objects { get; set; }
+
+    [Comment("The dominant colors in the image")]
+    [JsonPropertyName("colors")]
+    public string[] Colors { get; set; }
+
+    [Comment("True if the image contains any text")]
+    [JsonPropertyName("containsText")]
+    public bool ContainsText { get; set; }
+}

examples/Multimodal/Multimodal.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <RollForward>LatestMajor</RollForward>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>annotations</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="..\appSettings.json" Link="appSettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup Condition="Exists('..\appSettings.Development.json')">
+    <Content Include="..\appSettings.Development.json" Link="appSettings.Development.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\typechat.examplesLib\TypeChat.ExamplesLib.csproj" />
+    <ProjectReference Include="..\..\src\typechat\TypeChat.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="..\..\assets\microsoft-logo.png" Link="microsoft-logo.png">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="input.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

examples/Multimodal/Program.cs

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft. All rights reserved.
+using Microsoft.TypeChat;
+
+namespace Multimodal;
+
+/// <summary>
+/// This example shows how to send images to a vision capable model and translate what the model
+/// "sees" into a strongly typed object.
+///
+/// Each input line is either:
+///   - a local image file path (e.g. C:\pics\cat.jpg), or
+///   - an http(s) url to an image (e.g. https://.../cat.jpg)
+///
+/// The image is placed into a multimodal prompt alongside an instruction, and JsonTranslator
+/// translates the response into an <see cref="ImageResponse"/>.
+///
+/// NOTE: This requires a vision capable model (e.g. gpt-4o). Set the model name in appSettings.
+/// </summary>
+public class MultimodalApp : ConsoleApp
+{
+    private readonly JsonTranslator<ImageResponse> _translator;
+
+    public MultimodalApp()
+    {
+        OpenAIConfig config = Config.LoadOpenAI();
+
+        // TIP: To use the OpenAI Responses API instead of Chat Completions, either point
+        // config.Endpoint at a url whose path ends with /responses, or force it explicitly:
+        //   config.UseResponsesApi = true;
+
+        _translator = new JsonTranslator<ImageResponse>(new LanguageModel(config));
+    }
+
+    public override async Task ProcessInputAsync(string input, CancellationToken cancelToken)
+    {
+        PromptImage image = IsUrl(input) ?
+                            new PromptImage(input, ImageDetail.Auto) :
+                            PromptImage.FromFile(input);
+
+        // Build a multimodal request: an instruction followed by the image.
+        Prompt request = new Prompt();
+        request.Add(new MultimodalPromptSection()
+            .AddText("Describe the following image.")
+            .AddImage(image));
+
+        ImageResponse response = await _translator.TranslateAsync(request, null, null, cancelToken);
+
+        Console.WriteLine($"Caption: {response.Caption}");
+        Console.WriteLine($"Contains text: {response.ContainsText}");
+        if (response.Objects is not null && response.Objects.Length > 0)
+        {
+            Console.WriteLine("Objects: " + string.Join(", ", response.Objects));
+        }
+        if (response.Colors is not null && response.Colors.Length > 0)
+        {
+            Console.WriteLine("Colors: " + string.Join(", ", response.Colors));
+        }
+    }
+
+    private static bool IsUrl(string input)
+    {
+        return input.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
+               input.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
+    }
+
+    public static async Task<int> Main(string[] args)
+    {
+        try
+        {
+            MultimodalApp app = new MultimodalApp();
+            // Pass an input file as arg[0] to run in batch mode.
+            await app.RunAsync("🖼️> ", args.GetOrNull(0));
+        }
+        catch (Exception ex)
+        {
+            WriteError(ex);
+            Console.ReadLine();
+            return -1;
+        }
+
+        return 0;
+    }
+}

examples/Multimodal/Readme.md

@@ -0,0 +1,45 @@
+# Multimodal
+
+Demonstrates sending **images** to a vision capable language model and translating what the model
+sees into a **strongly typed** object using `JsonTranslator`.
+
+## What it shows
+
+- Building a multimodal prompt with [`MultimodalPromptSection`](../../src/typechat/MultimodalPromptSection.cs),
+  mixing text and images.
+- Referencing images either by url or by local file (`PromptImage.FromFile`, which base64 encodes the image
+  into a data uri).
+- Translating the multimodal request into a typed [`ImageResponse`](./ImageSchema.cs) — TypeChat validates and
+  repairs the model's JSON exactly as it does for text-only requests.
+
+```cs
+Prompt request = new Prompt();
+request.Add(new MultimodalPromptSection()
+    .AddText("Describe the following image.")
+    .AddImage(PromptImage.FromFile("cat.jpg")));
+
+ImageResponse response = await translator.TranslateAsync(request, null);
+```
+
+## Requirements
+
+- A **vision capable** model such as `gpt-4o`. Set the model name in `appSettings.Development.json`.
+
+## Running
+
+The example ships with the official Microsoft logo (`microsoft-logo.png`, sourced from
+[../../assets](../../assets)) which is copied next to the built app, so it runs out of the box.
+
+- Interactive: run the project and paste an image file path or an http(s) url at the prompt.
+- Batch: pass an input file whose lines are image paths/urls, e.g. `dotnet run -- input.txt`.
+  The bundled [input.txt](./input.txt) points at `microsoft-logo.png`.
+
+## OpenAI Responses API
+
+The built-in `LanguageModel` also supports the OpenAI **Responses API**. It is selected automatically when
+the configured endpoint path ends with `/responses`, or you can force it:
+
+```cs
+OpenAIConfig config = Config.LoadOpenAI();
+config.UseResponsesApi = true; // or false to force Chat Completions
+```

examples/Multimodal/input.txt

@@ -0,0 +1,10 @@
+# Multimodal example input.
+#
+# Each non-comment line is processed as one image and described by the model.
+# A line is either a local image file path or an http(s) url to an image.
+#
+# This requires a vision capable model (e.g. gpt-4o) configured in appSettings.
+#
+# The Microsoft logo below ships with this example and is copied next to the built app.
+# Replace or add your own image paths / urls as needed.
+microsoft-logo.png

src/typechat.meai/ChatLanguageModel.cs

@@ -110,9 +110,56 @@ public static void Append(this List<ChatMessage> history, IEnumerable<IPromptSec
 
     public static void Append(this List<ChatMessage> history, IPromptSection message)
     {
+        if (message is IMultimodalPromptSection multimodal && HasImages(multimodal))
+        {
+            history.Add(new ChatMessage(multimodal.GetRole(), ToContents(multimodal)));
+            return;
+        }
         history.Add(new ChatMessage(role: message.GetRole(), content: message.GetText()));
     }
 
+    internal static bool HasImages(IMultimodalPromptSection section)
+    {
+        IReadOnlyList<PromptContentPart> parts = section.ContentParts;
+        for (int i = 0; i < parts.Count; ++i)
+        {
+            if (parts[i].IsImage)
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    internal static IList<AIContent> ToContents(IMultimodalPromptSection section)
+    {
+        IReadOnlyList<PromptContentPart> parts = section.ContentParts;
+        List<AIContent> contents = new List<AIContent>(parts.Count);
+        for (int i = 0; i < parts.Count; ++i)
+        {
+            PromptContentPart part = parts[i];
+            if (part.IsImage)
+            {
+                contents.Add(ToContent(part.Image!));
+            }
+            else if (!string.IsNullOrEmpty(part.Text))
+            {
+                contents.Add(new TextContent(part.Text));
+            }
+        }
+        return contents;
+    }
+
+    internal static AIContent ToContent(PromptImage image)
+    {
+        // A data uri (data:{mediaType};base64,{data}) carries its own media type and bytes.
+        if (image.IsDataUri)
+        {
+            return new DataContent(image.Url);
+        }
+        return new UriContent(image.Url, PromptImage.GetMediaType(image.Url));
+    }
+
     internal static bool IsRole(this ChatRole role, string label)
     {
         return role.Value.Equals(label, StringComparison.OrdinalIgnoreCase);