From 8f1f8e07c349d1dde59efb5ef0b72af5d465e135 Mon Sep 17 00:00:00 2001 From: robgruen Date: Tue, 2 Jun 2026 09:32:07 -0700 Subject: [PATCH 1/8] removed embedding specific endpoint --- src/typechat/OpenAIConfig.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/typechat/OpenAIConfig.cs b/src/typechat/OpenAIConfig.cs index bde1f88..2ef4bcd 100644 --- a/src/typechat/OpenAIConfig.cs +++ b/src/typechat/OpenAIConfig.cs @@ -149,11 +149,6 @@ public static OpenAIConfig FromEnvironment(bool isEmbedding = false) } if (isEmbedding) { - if (Environment.GetEnvironmentVariable(VariableNames.OPENAI_EMBEDDING_ENDPOINT) != null) - { - config.Endpoint = Environment.GetEnvironmentVariable(VariableNames.OPENAI_EMBEDDING_ENDPOINT); - } - config.Model = Environment.GetEnvironmentVariable(VariableNames.OPENAI_EMBEDDINGMODEL); } else From 1a445d833055e73e73bccae6c6c9e265e1de971b Mon Sep 17 00:00:00 2001 From: robgruen Date: Tue, 2 Jun 2026 10:03:32 -0700 Subject: [PATCH 2/8] removed embedding specific endpoint --- src/typechat/OpenAIConfig.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/typechat/OpenAIConfig.cs b/src/typechat/OpenAIConfig.cs index 2ef4bcd..6c4128b 100644 --- a/src/typechat/OpenAIConfig.cs +++ b/src/typechat/OpenAIConfig.cs @@ -39,11 +39,6 @@ public static class VariableNames /// public const string OPENAI_EMBEDDINGMODEL = "OPENAI_EMBEDDINGMODEL"; - /// - /// The embedding endpoint to use - /// - public const string OPENAI_EMBEDDING_ENDPOINT = "OPENAI_EMBEDDING_ENDPOINT"; - /// /// Api key to use for Azure OpenAI service /// From 9792c08374401676fa5f255827739593e64ecbbd Mon Sep 17 00:00:00 2001 From: robgruen Date: Tue, 2 Jun 2026 10:14:29 -0700 Subject: [PATCH 3/8] removed embedding endpoint --- .github/workflows/integration_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 07058d5..da26e28 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -99,6 +99,5 @@ jobs: AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_API_KEY }} OPENAI_MODEL: ${{ secrets.AZURE_COMPLETION_MODEL }} OPENAI_EMBEDDINGMODEL: ${{ secrets.AZURE_EMBEDDING_MODEL }} - OPENAI_EMBEDDING_ENDPOINT: ${{ secrets.OPENAI_EMBEDDING_ENDPOINT }} run: | dotnet test -c ${{ matrix.configuration }} tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj --no-build -v Normal --logger trx From ae7ff93ea8e840fa402243f2ccc8d15b0513273d Mon Sep 17 00:00:00 2001 From: robgruen Date: Tue, 2 Jun 2026 10:40:44 -0700 Subject: [PATCH 4/8] removed development environment --- .github/workflows/integration_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index da26e28..7c715b0 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -25,7 +25,6 @@ permissions: jobs: build-integrationtest: - environment: development strategy: fail-fast: false matrix: From c764199fccc6483c35ad51c8a9fea03e520aa685 Mon Sep 17 00:00:00 2001 From: robgruen Date: Tue, 2 Jun 2026 10:46:38 -0700 Subject: [PATCH 5/8] Add environment setting to integration tests workflow --- .github/workflows/integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 7c715b0..da26e28 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -25,6 +25,7 @@ permissions: jobs: build-integrationtest: + environment: development strategy: fail-fast: false matrix: From 6d5d104cf8a38bc589df4c32d3489280f9c3a97f Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 6 Jul 2026 10:52:10 -0700 Subject: [PATCH 6/8] Added support for IEnumerable (arraylike excluding string and dictionaries). --- src/typechat.schema/TypeEx.cs | 107 ++++++++++++++++++++++ src/typechat.schema/TypescriptExporter.cs | 77 ++++++++++++---- tests/TypeChat.TestLib/Schemas.cs | 39 ++++++++ tests/TypeChat.UnitTests/TestSchema.cs | 69 ++++++++++++++ 4 files changed, 275 insertions(+), 17 deletions(-) diff --git a/src/typechat.schema/TypeEx.cs b/src/typechat.schema/TypeEx.cs index c5fe980..04eafa5 100644 --- a/src/typechat.schema/TypeEx.cs +++ b/src/typechat.schema/TypeEx.cs @@ -149,6 +149,113 @@ internal static bool IsTask(this Type type) return (args.IsNullOrEmpty()) ? null : args[0]; } + /// + /// Is the given type one that serializes to a JSON array? This includes arrays and any type + /// that implements IEnumerable{T} (List{T}, IList{T}, ICollection{T}, HashSet{T}...). + /// Strings (which are IEnumerable{char}) and dictionaries are deliberately excluded: they do + /// not serialize to JSON arrays. + /// + /// type to inspect + /// the type of the items in the array + /// true if the type is array-like + internal static bool IsArrayLike(this Type type, out Type elementType) + { + elementType = null; + + // Strings are IEnumerable but must be exported as scalar strings, not char arrays + if (type.IsString()) + { + return false; + } + // Dictionaries are IEnumerable> but must be exported as maps, not arrays. + // Use a structural check so that even a self-mapping dictionary is excluded here. + if (type.IsDictionaryShaped()) + { + return false; + } + if (type.IsArray) + { + elementType = type.GetElementType(); + return true; + } + + Type? enumerableType = FindGenericInterface(type, typeof(IEnumerable<>)); + if (enumerableType is not null) + { + Type genericArg = enumerableType.GetGenericArguments()[0]; + // Guard against a type that enumerates itself (e.g. class Node : IEnumerable): + // there is no finite element type to unwrap to, so treat it as a plain object instead. + if (genericArg != type) + { + elementType = genericArg; + return true; + } + return false; + } + // Non-generic IEnumerable (e.g. ArrayList): the element type is unknown, so treat it as 'any' + if (typeof(IEnumerable).IsAssignableFrom(type)) + { + elementType = typeof(object); + return true; + } + return false; + } + + /// + /// Is the given type one that serializes to a JSON object/map? This includes any type that + /// implements IDictionary{K,V} or IReadOnlyDictionary{K,V}, as well as the non-generic IDictionary. + /// + /// type to inspect + /// the type of the map's keys + /// the type of the map's values + /// true if the type is a dictionary + internal static bool IsDictionary(this Type type, out Type keyType, out Type valueType) + { + keyType = typeof(string); + valueType = typeof(object); + + Type? dictionaryType = FindGenericInterface(type, typeof(IDictionary<,>)) ?? + FindGenericInterface(type, typeof(IReadOnlyDictionary<,>)); + if (dictionaryType is not null) + { + Type[] args = dictionaryType.GetGenericArguments(); + // Guard against a type that maps to itself (e.g. class D : IDictionary): + // there is no finite value type to unwrap to, so treat it as a plain object instead. + if (args[1] != type) + { + keyType = args[0]; + valueType = args[1]; + return true; + } + return false; + } + // Non-generic IDictionary (e.g. Hashtable): keys and values are untyped + return typeof(IDictionary).IsAssignableFrom(type); + } + + private static Type? FindGenericInterface(Type type, Type genericInterface) + { + if (type.IsGenericType && type.GetGenericTypeDefinition() == genericInterface) + { + return type; + } + foreach (Type iface in type.GetInterfaces()) + { + if (iface.IsGenericType && iface.GetGenericTypeDefinition() == genericInterface) + { + return iface; + } + } + return null; + } + + private static bool IsDictionaryShaped(this Type type) + { + return FindGenericInterface(type, typeof(IDictionary<,>)) is not null || + FindGenericInterface(type, typeof(IReadOnlyDictionary<,>)) is not null || + typeof(IDictionary).IsAssignableFrom(type); + } + internal static IEnumerable Subclasses(this Type type) { if (!type.IsClass) diff --git a/src/typechat.schema/TypescriptExporter.cs b/src/typechat.schema/TypescriptExporter.cs index 2d06173..3f6382d 100644 --- a/src/typechat.schema/TypescriptExporter.cs +++ b/src/typechat.schema/TypescriptExporter.cs @@ -175,20 +175,21 @@ public override void ExportPending() public override void ExportType(Type type) { - if (type.IsArray) + if (type.IsDictionary(out _, out Type valueType)) { - AddPending(type.GetElementType()); + AddPending(valueType); + } + else if (type.IsArrayLike(out Type elementType)) + { + AddPending(elementType); + } + else if (type.IsEnum) + { + ExportEnum(type); } else { - if (type.IsEnum) - { - ExportEnum(type); - } - else - { - ExportClass(type); - } + ExportClass(type); } } @@ -505,7 +506,7 @@ private TypescriptExporter ExportVariable(MemberInfo member, Type type) _writer.Variable( member.PropertyName(), DataType(actualType), - type.IsArray, + actualType.IsArrayLike(out _), isNullable ); } @@ -534,7 +535,7 @@ private TypescriptExporter ExportParameter(ParameterInfo parameter, int i, int c DataType(type), i, count, - type.IsArray, + type.IsArrayLike(out _), isNullable ); AddPending(type); @@ -658,9 +659,20 @@ private string DataType(Type type) type = type.GetGenericType() ?? typeof(void); } - if (type.IsArray) + Type? nullableValueType = type.GetNullableValueType(); + if (nullableValueType is not null) + { + type = nullableValueType; + } + + if (type.IsDictionary(out Type keyType, out Type valueType)) + { + return DataTypeMap(keyType, valueType); + } + + if (type.IsArrayLike(out Type elementType)) { - return DataType(type.GetElementType()); + return DataType(elementType); } string? typeName = null; @@ -680,6 +692,23 @@ private string DataType(Type type) return typeName; } + private string DataTypeMap(Type keyType, Type valueType) + { + string valueDataType = DataType(valueType); + if (valueType.IsArrayLike(out _)) + { + valueDataType += Typescript.Punctuation.Array; + } + return $"Record<{MapKeyType(keyType)}, {valueDataType}>"; + } + + private static string MapKeyType(Type keyType) + { + // JSON object keys are always strings; only numeric keys are represented as 'number' + string? primitive = Typescript.Types.ToPrimitive(keyType); + return primitive == Typescript.Types.Number ? Typescript.Types.Number : Typescript.Types.String; + } + private string InterfaceName(Type type, bool useMapper = true) { string? typeName = null; @@ -756,12 +785,26 @@ protected override bool ShouldExport(Type type, out Type typeToExport) return false; } - if (type.IsArray) + // Unwrap arrays, collections and dictionaries down to the underlying element/value type + while (true) { - type = type.GetElementType(); + if (type.IsDictionary(out _, out Type valueType)) + { + type = valueType; + } + else if (type.IsArrayLike(out Type elementType)) + { + type = elementType; + } + else + { + break; + } } typeToExport = type; - return !type.IsPrimitive && !_nonExportTypes.Contains(type); + return !type.IsPrimitive && + !type.IsNullableValueType() && + !_nonExportTypes.Contains(type); } } diff --git a/tests/TypeChat.TestLib/Schemas.cs b/tests/TypeChat.TestLib/Schemas.cs index f49236e..08599c5 100644 --- a/tests/TypeChat.TestLib/Schemas.cs +++ b/tests/TypeChat.TestLib/Schemas.cs @@ -336,6 +336,45 @@ public class FriendsOfPerson public Name[] FriendNames { get; set; } } +// For testing that IEnumerable exports as a JSON array (string[]) and not as a +// leaky List_String interface. See https://github.com/microsoft/typechat.net/issues/218 +public sealed class Pizza +{ + public IEnumerable Toppings { get; set; } + public string Size { get; set; } +} + +// For testing that the various collection and dictionary types all export as JSON arrays/maps +public class CollectionsObj +{ + public IEnumerable Tags { get; set; } + public List Names { get; set; } + public IList Scores { get; set; } + public ICollection Locations { get; set; } + public IReadOnlyList Ratings { get; set; } + public HashSet UniqueTags { get; set; } + public string[] Aliases { get; set; } + public Dictionary Counts { get; set; } + public IDictionary LocationsByCity { get; set; } + public Dictionary NamesById { get; set; } + public Dictionary> Buckets { get; set; } +} + +// A type that enumerates itself (composite pattern): it implements IEnumerable, +// so it has no finite "array of..." representation. Schema generation must still terminate. +public class Composite : IEnumerable +{ + public string Name { get; set; } + public IEnumerator GetEnumerator() => Enumerable.Empty().GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); +} + +public class CompositeHolder +{ + public List Nodes { get; set; } + public Composite Root { get; set; } +} + // For testing Generics public class Child { diff --git a/tests/TypeChat.UnitTests/TestSchema.cs b/tests/TypeChat.UnitTests/TestSchema.cs index ecd36d4..2e04b02 100644 --- a/tests/TypeChat.UnitTests/TestSchema.cs +++ b/tests/TypeChat.UnitTests/TestSchema.cs @@ -95,5 +95,74 @@ public void ExportGenerics() Assert.True(lines.ContainsSubstring("Child_Name[]")); Assert.True(lines.ContainsSubstring("Child_Location[]")); } + + [Fact] + public void ExportEnumerable_Issue218() + { + var schema = TypescriptExporter.GenerateSchema(typeof(Pizza)); + var lines = schema.Schema.Text.Lines(); + + // IEnumerable must be exported as a JSON array of strings... + Assert.True(lines.ContainsSubstring("Toppings", "string[]")); + Assert.True(lines.ContainsSubstring("Size", "string")); + // ...and NOT as a leaky List_String interface exposing Capacity/Count/Item + Assert.False(lines.ContainsSubstring("List_String")); + Assert.False(lines.ContainsSubstring("Capacity")); + } + + [Fact] + public void ExportCollections() + { + var schema = TypescriptExporter.GenerateSchema(typeof(CollectionsObj)); + var lines = schema.Schema.Text.Lines(); + + // IEnumerable, List, IList, ICollection, IReadOnlyList, HashSet and arrays => T[] + Assert.True(lines.ContainsSubstring("Tags", "string[]")); + Assert.True(lines.ContainsSubstring("Names", "Name[]")); + Assert.True(lines.ContainsSubstring("Scores", "number[]")); + Assert.True(lines.ContainsSubstring("Locations", "Location[]")); + Assert.True(lines.ContainsSubstring("Ratings", "number[]")); + Assert.True(lines.ContainsSubstring("UniqueTags", "string[]")); + Assert.True(lines.ContainsSubstring("Aliases", "string[]")); + + // Dictionaries => Record, including a dictionary whose value is itself a collection + Assert.True(lines.ContainsSubstring("Counts", "Record")); + Assert.True(lines.ContainsSubstring("LocationsByCity", "Record")); + Assert.True(lines.ContainsSubstring("NamesById", "Record")); + Assert.True(lines.ContainsSubstring("Buckets", "Record")); + + // Element/value types are still exported as interfaces + Assert.True(lines.ContainsSubstring("interface", "Name")); + Assert.True(lines.ContainsSubstring("interface", "Location")); + + // The internals of List/Dictionary<> must NOT leak into the schema + Assert.False(lines.ContainsSubstring("Capacity")); + Assert.False(lines.ContainsSubstring("KeyValuePair")); + Assert.False(lines.ContainsSubstring("interface", "List_")); + } + + [Fact] + public void SelfReferentialEnumerable_Terminates() + { + // Composite : IEnumerable has no finite "array of..." representation. + // Schema generation must terminate rather than loop forever while unwrapping it. + TypescriptSchema schema = null; + var worker = new System.Threading.Thread( + () => schema = TypescriptExporter.GenerateSchema(typeof(CompositeHolder))) + { + IsBackground = true + }; + worker.Start(); + bool finished = worker.Join(System.TimeSpan.FromSeconds(5)); + + Assert.True(finished, "Schema generation did not terminate for a self-referential IEnumerable type"); + + var lines = schema.Schema.Text.Lines(); + // A List is still a normal array... + Assert.True(lines.ContainsSubstring("Nodes", "Composite[]")); + // ...but the self-enumerating Composite itself falls back to a plain object interface + Assert.True(lines.ContainsSubstring("interface", "Composite")); + Assert.True(lines.ContainsSubstring("Name", "string")); + } } From 7f467c13bb0d6d1e157f1a826016e52e101d4a30 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 6 Jul 2026 11:00:47 -0700 Subject: [PATCH 7/8] Added support for struct serialization. --- src/typechat.schema/TypeEx.cs | 16 +++++++++++++- src/typechat.schema/Typescript.cs | 10 ++++++++- src/typechat.schema/TypescriptExporter.cs | 12 ++++++++--- tests/TypeChat.TestLib/Schemas.cs | 24 +++++++++++++++++++++ tests/TypeChat.UnitTests/TestSchema.cs | 26 +++++++++++++++++++++++ 5 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/typechat.schema/TypeEx.cs b/src/typechat.schema/TypeEx.cs index 04eafa5..5785077 100644 --- a/src/typechat.schema/TypeEx.cs +++ b/src/typechat.schema/TypeEx.cs @@ -40,7 +40,21 @@ public static bool IsBoolean(this Type type) public static bool IsDateTime(this Type type) { - return type == typeof(DateTime) || type == typeof(TimeSpan); + // These are value types that System.Text.Json serializes to/from a JSON *string* rather than + // to an object. They must be treated as the 'string' primitive by the exporter (see + // Typescript.Types.ToPrimitive); otherwise they would be exported as (mostly empty) interfaces. + if (type == typeof(DateTime) || type == typeof(TimeSpan) || type == typeof(DateTimeOffset)) + { + return true; + } +#if NET6_0_OR_GREATER + // DateOnly/TimeOnly only exist on net6.0+, but Json serializes them as strings too. + if (type == typeof(DateOnly) || type == typeof(TimeOnly)) + { + return true; + } +#endif + return false; } public static bool IsNumber(this Type type) diff --git a/src/typechat.schema/Typescript.cs b/src/typechat.schema/Typescript.cs index 57b6cbd..dc8df58 100644 --- a/src/typechat.schema/Typescript.cs +++ b/src/typechat.schema/Typescript.cs @@ -70,7 +70,15 @@ public static class Types } else if (type.IsDateTime()) { - return String; // Json does not have a primitive DateTime + return String; // Json has no DateTime primitive: these value types serialize as strings + } + else if (type == typeof(Guid)) + { + // Guid is a value type, but System.Text.Json writes it as a JSON string (e.g. + // "3f2504e0-4f89-11d3-9a0c-0305e82c3301") and it exposes no public fields/properties. + // Without this mapping it would fall through to the object path and be exported as an + // empty "interface Guid {}", which a model could never populate correctly. + return String; } else if (type.IsObject()) { diff --git a/src/typechat.schema/TypescriptExporter.cs b/src/typechat.schema/TypescriptExporter.cs index 3f6382d..3295374 100644 --- a/src/typechat.schema/TypescriptExporter.cs +++ b/src/typechat.schema/TypescriptExporter.cs @@ -196,9 +196,15 @@ public override void ExportType(Type type) public TypescriptExporter ExportClass(Type type) { ArgumentVerify.ThrowIfNull(type, nameof(type)); - if (!type.IsClass) - { - ArgumentVerify.Throw($"{type.Name} must be a class"); + // Export both reference types (classes) and non-enum value types (structs) as interfaces: + // System.Text.Json serializes a struct's public fields/properties into a JSON object exactly + // like a class (e.g. KeyValuePair => { Key, Value }, a custom Money => { Amount, Currency }). + // Enums are routed to ExportEnum before they reach here, and scalar value types that Json writes + // as a single value (Guid, DateTime, DateTimeOffset, ...) are mapped to primitives in + // Typescript.Types.ToPrimitive, so only genuine object-like structs arrive at this point. + if (!type.IsClass && (!type.IsValueType || type.IsPrimitive || type.IsEnum)) + { + ArgumentVerify.Throw($"{type.Name} must be a class or struct"); } if (IsExported(type)) diff --git a/tests/TypeChat.TestLib/Schemas.cs b/tests/TypeChat.TestLib/Schemas.cs index 08599c5..d50c366 100644 --- a/tests/TypeChat.TestLib/Schemas.cs +++ b/tests/TypeChat.TestLib/Schemas.cs @@ -375,6 +375,30 @@ public class CompositeHolder public Composite Root { get; set; } } +// For testing that structs export as interfaces: Json serializes a struct's public members as an +// object, just like a class. +public struct Money +{ + public decimal Amount { get; set; } + public string Currency { get; set; } +} + +public struct Coordinates +{ + public double Latitude { get; set; } + public double Longitude { get; set; } +} + +public class StructHolder +{ + public Money Price { get; set; } + public Coordinates Location { get; set; } + public KeyValuePair Pair { get; set; } + // Scalar value types that Json writes as a single string - must NOT become empty interfaces + public Guid Id { get; set; } + public DateTimeOffset When { get; set; } +} + // For testing Generics public class Child { diff --git a/tests/TypeChat.UnitTests/TestSchema.cs b/tests/TypeChat.UnitTests/TestSchema.cs index 2e04b02..db81aed 100644 --- a/tests/TypeChat.UnitTests/TestSchema.cs +++ b/tests/TypeChat.UnitTests/TestSchema.cs @@ -164,5 +164,31 @@ public void SelfReferentialEnumerable_Terminates() Assert.True(lines.ContainsSubstring("interface", "Composite")); Assert.True(lines.ContainsSubstring("Name", "string")); } + + [Fact] + public void ExportStructs() + { + var schema = TypescriptExporter.GenerateSchema(typeof(StructHolder)); + var lines = schema.Schema.Text.Lines(); + + // Structs export as interfaces, because Json serializes their public members as an object + Assert.True(lines.ContainsSubstring("Price", "Money")); + Assert.True(lines.ContainsSubstring("interface", "Money")); + Assert.True(lines.ContainsSubstring("Amount", "number")); + Assert.True(lines.ContainsSubstring("Currency", "string")); + + Assert.True(lines.ContainsSubstring("Location", "Coordinates")); + Assert.True(lines.ContainsSubstring("interface", "Coordinates")); + + Assert.True(lines.ContainsSubstring("Pair", "KeyValuePair")); + Assert.True(lines.ContainsSubstring("Key", "string")); + Assert.True(lines.ContainsSubstring("Value", "number")); + + // Scalar value types map to string, NOT to empty interfaces + Assert.True(lines.ContainsSubstring("Id", "string")); + Assert.True(lines.ContainsSubstring("When", "string")); + Assert.False(lines.ContainsSubstring("interface", "Guid")); + Assert.False(lines.ContainsSubstring("interface", "DateTimeOffset")); + } } From c8f64ee7c03ee2ed57c0b234d8ba0e2e4755501f Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 6 Jul 2026 13:01:58 -0700 Subject: [PATCH 8/8] Added multi-modal support. --- README.md | 5 + TypeChat.sln | 171 +++++++ assets/microsoft-logo.png | Bin 0 -> 17280 bytes examples/Multimodal/ImageSchema.cs | 29 ++ examples/Multimodal/Multimodal.csproj | 41 ++ examples/Multimodal/Program.cs | 83 ++++ examples/Multimodal/Readme.md | 45 ++ examples/Multimodal/input.txt | 10 + src/typechat.meai/ChatLanguageModel.cs | 47 ++ src/typechat/IMultimodalPromptSection.cs | 20 + src/typechat/ImageDetail.cs | 26 + src/typechat/JsonTranslatorPrompts.cs | 40 +- src/typechat/LanguageModel.cs | 256 +++++++++- src/typechat/MultimodalPromptSection.cs | 141 ++++++ src/typechat/OpenAIConfig.cs | 11 + src/typechat/PromptContentPart.cs | 96 ++++ src/typechat/PromptImage.cs | 109 +++++ tests/TypeChat.IntegrationTests/Readme.md | 13 +- .../TestMultimodalEndToEnd.cs | 134 ++++++ .../TypeChat.IntegrationTests.csproj | 6 + tests/TypeChat.TestLib/MockHttp.cs | 14 + tests/TypeChat.UnitTests/TestMultimodal.cs | 445 ++++++++++++++++++ .../TypeChat.UnitTests.csproj | 7 + 23 files changed, 1739 insertions(+), 10 deletions(-) create mode 100644 assets/microsoft-logo.png create mode 100644 examples/Multimodal/ImageSchema.cs create mode 100644 examples/Multimodal/Multimodal.csproj create mode 100644 examples/Multimodal/Program.cs create mode 100644 examples/Multimodal/Readme.md create mode 100644 examples/Multimodal/input.txt create mode 100644 src/typechat/IMultimodalPromptSection.cs create mode 100644 src/typechat/ImageDetail.cs create mode 100644 src/typechat/MultimodalPromptSection.cs create mode 100644 src/typechat/PromptContentPart.cs create mode 100644 src/typechat/PromptImage.cs create mode 100644 tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs create mode 100644 tests/TypeChat.UnitTests/TestMultimodal.cs diff --git a/README.md b/README.md index e487857..59b6e94 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ TypeChat uses language models to translate user intent into JSON that conforms t TypeChat provides: * Json translation, validation, repair and deserialization. * Extensibility: interfaces for customizing schemas, validators and prompts. +* **Multimodal prompts**: send images (by url or local file) alongside text using `MultimodalPromptSection`, so vision capable models (e.g. gpt-4o) can translate what they *see* into strongly typed objects. +* **OpenAI Responses API**: `LanguageModel` supports both the Chat Completions and Responses APIs. The Responses API is auto-selected when the endpoint path ends with `/responses`, or set `OpenAIConfig.UseResponsesApi`. * Schema export: classes to generate schema for the .NET Type you want to translate to. Exported schema includes dependencies and base classes. The exported schema is specified using **Typescript**, which can concisely express schema for JSON objects. * Support for common scenarios shown in TypeChat examples. When you encounter limitations (such as how generics are currently exported), you can supply schema text, such as Typescript authored by hand. * Helper attributes for Vocabularies and Comments. Vocabularies are string tables that constrain the values that can be assigned to string properties. Dynamic loading of vocabularies enables scenarios where they vary at runtime. @@ -131,6 +133,9 @@ The following examples demonstrate how to use JsonTranslator, Schemas and Vocabu * [Calendar](./examples/Calendar): Transform language into calendar actions * [Restaurant](./examples/Restaurant): Order processing at a pizza restaurant +### Multimodal (images) +* [Multimodal](./examples/Multimodal): Send an image to a vision capable model and translate what it sees into a strongly typed object. Requires a vision model such as gpt-4o. + ### Hierarchical schemas * [MultiSchema](./examples/MultiSchema): dynamically route user intent to other 'sub-apps' * [SchemaHierarchy](./examples/SchemaHierarchy): A Json Translator than uses multiple child JsonTranslators. For each user request, it picks the semantically ***nearest*** child translator and routes the input to it. diff --git a/TypeChat.sln b/TypeChat.sln index e96e373..cbf1ab1 100644 --- a/TypeChat.sln +++ b/TypeChat.sln @@ -64,88 +64,258 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.Schema", "src\type EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TypeChat.Extensions.AI", "src\typechat.meai\TypeChat.Extensions.AI.csproj", "{9F7809A0-7F9E-D22A-A697-F02E1613F65D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Multimodal", "examples\Multimodal\Multimodal.csproj", "{48848A65-8862-4376-81A2-9CE8DC3DCFD7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x64.ActiveCfg = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x64.Build.0 = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x86.ActiveCfg = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x86.Build.0 = Debug|Any CPU {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|Any CPU.ActiveCfg = Release|Any CPU {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|Any CPU.Build.0 = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x64.ActiveCfg = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x64.Build.0 = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x86.ActiveCfg = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x86.Build.0 = Release|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x64.ActiveCfg = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x64.Build.0 = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x86.ActiveCfg = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x86.Build.0 = Debug|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|Any CPU.ActiveCfg = Release|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|Any CPU.Build.0 = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x64.ActiveCfg = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x64.Build.0 = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x86.ActiveCfg = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x86.Build.0 = Release|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x64.ActiveCfg = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x64.Build.0 = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x86.ActiveCfg = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x86.Build.0 = Debug|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Release|Any CPU.ActiveCfg = Release|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Release|Any CPU.Build.0 = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x64.ActiveCfg = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x64.Build.0 = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x86.ActiveCfg = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x86.Build.0 = Release|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x64.ActiveCfg = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x64.Build.0 = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x86.ActiveCfg = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x86.Build.0 = Debug|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Release|Any CPU.ActiveCfg = Release|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Release|Any CPU.Build.0 = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x64.ActiveCfg = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x64.Build.0 = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x86.ActiveCfg = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x86.Build.0 = Release|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x64.ActiveCfg = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x64.Build.0 = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x86.ActiveCfg = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x86.Build.0 = Debug|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Release|Any CPU.ActiveCfg = Release|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Release|Any CPU.Build.0 = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x64.ActiveCfg = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x64.Build.0 = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x86.ActiveCfg = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x86.Build.0 = Release|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x64.ActiveCfg = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x64.Build.0 = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x86.ActiveCfg = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x86.Build.0 = Debug|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|Any CPU.ActiveCfg = Release|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|Any CPU.Build.0 = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x64.ActiveCfg = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x64.Build.0 = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x86.ActiveCfg = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x86.Build.0 = Release|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x64.ActiveCfg = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x64.Build.0 = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x86.ActiveCfg = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x86.Build.0 = Debug|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|Any CPU.ActiveCfg = Release|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|Any CPU.Build.0 = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x64.ActiveCfg = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x64.Build.0 = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x86.ActiveCfg = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x86.Build.0 = Release|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x64.ActiveCfg = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x64.Build.0 = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x86.ActiveCfg = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x86.Build.0 = Debug|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|Any CPU.ActiveCfg = Release|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|Any CPU.Build.0 = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x64.ActiveCfg = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x64.Build.0 = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x86.ActiveCfg = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x86.Build.0 = Release|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x64.ActiveCfg = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x64.Build.0 = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x86.ActiveCfg = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x86.Build.0 = Debug|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|Any CPU.ActiveCfg = Release|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|Any CPU.Build.0 = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x64.ActiveCfg = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x64.Build.0 = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x86.ActiveCfg = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x86.Build.0 = Release|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x64.ActiveCfg = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x64.Build.0 = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x86.ActiveCfg = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x86.Build.0 = Debug|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|Any CPU.ActiveCfg = Release|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|Any CPU.Build.0 = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x64.ActiveCfg = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x64.Build.0 = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x86.ActiveCfg = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x86.Build.0 = Release|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x64.ActiveCfg = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x64.Build.0 = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x86.ActiveCfg = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x86.Build.0 = Debug|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|Any CPU.ActiveCfg = Release|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|Any CPU.Build.0 = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x64.ActiveCfg = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x64.Build.0 = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x86.ActiveCfg = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x86.Build.0 = Release|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x64.Build.0 = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x86.ActiveCfg = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x86.Build.0 = Debug|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|Any CPU.ActiveCfg = Release|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|Any CPU.Build.0 = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x64.ActiveCfg = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x64.Build.0 = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x86.ActiveCfg = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x86.Build.0 = Release|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x64.ActiveCfg = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x64.Build.0 = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x86.ActiveCfg = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x86.Build.0 = Debug|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|Any CPU.ActiveCfg = Release|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|Any CPU.Build.0 = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x64.ActiveCfg = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x64.Build.0 = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x86.ActiveCfg = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x86.Build.0 = Release|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x64.ActiveCfg = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x64.Build.0 = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x86.ActiveCfg = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x86.Build.0 = Debug|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|Any CPU.ActiveCfg = Release|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|Any CPU.Build.0 = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x64.ActiveCfg = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x64.Build.0 = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x86.ActiveCfg = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x86.Build.0 = Release|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x64.ActiveCfg = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x64.Build.0 = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x86.ActiveCfg = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x86.Build.0 = Debug|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|Any CPU.ActiveCfg = Release|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|Any CPU.Build.0 = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x64.ActiveCfg = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x64.Build.0 = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x86.ActiveCfg = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x86.Build.0 = Release|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x64.ActiveCfg = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x64.Build.0 = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x86.ActiveCfg = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x86.Build.0 = Debug|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|Any CPU.ActiveCfg = Release|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|Any CPU.Build.0 = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x64.ActiveCfg = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x64.Build.0 = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x86.ActiveCfg = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x86.Build.0 = Release|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x64.ActiveCfg = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x64.Build.0 = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x86.ActiveCfg = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x86.Build.0 = Debug|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|Any CPU.ActiveCfg = Release|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|Any CPU.Build.0 = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x64.ActiveCfg = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x64.Build.0 = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x86.ActiveCfg = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x86.Build.0 = Release|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x64.ActiveCfg = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x64.Build.0 = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x86.ActiveCfg = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x86.Build.0 = Debug|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|Any CPU.ActiveCfg = Release|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|Any CPU.Build.0 = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x64.ActiveCfg = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x64.Build.0 = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x86.ActiveCfg = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x86.Build.0 = Release|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x64.ActiveCfg = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x64.Build.0 = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x86.ActiveCfg = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x86.Build.0 = Debug|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|Any CPU.ActiveCfg = Release|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|Any CPU.Build.0 = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x64.ActiveCfg = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x64.Build.0 = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x86.ActiveCfg = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x86.Build.0 = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x64.ActiveCfg = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x64.Build.0 = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x86.ActiveCfg = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x86.Build.0 = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|Any CPU.Build.0 = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x64.ActiveCfg = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x64.Build.0 = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x86.ActiveCfg = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -165,6 +335,7 @@ Global {6FDA463B-4A1E-4E3E-B508-A14910FAF579} = {179655A3-3581-45F4-BE46-1BF4FABA7F97} {C585B613-3BAF-4999-8FA5-7BF425AEB1E2} = {EAEA9E40-7370-407E-B2DA-3F6E265262BF} {80304E1F-5B1A-4142-B950-78383E1D1877} = {179655A3-3581-45F4-BE46-1BF4FABA7F97} + {48848A65-8862-4376-81A2-9CE8DC3DCFD7} = {179655A3-3581-45F4-BE46-1BF4FABA7F97} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8A9AF678-E669-4AF2-9644-7EBB970DAC41} diff --git a/assets/microsoft-logo.png b/assets/microsoft-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..c76d3fce220a53b6088c021336ccaebc1c70a8bb GIT binary patch literal 17280 zcmb_^2UJsQxaA2Tf`Sl1q$&mkR6u$!q5-8B3BC7@^bWZw7`lk`rYInxgN6=KE=>eP z2q2*;O*(>r^!an|d$ZQ8w`SIxHFI<+<(!<8zkJ{R_TFD&bhPf%o?|@+K@hF#1Ed}V zk>kPFeN>d--+-4aW8gmuMm;BA;?b% zf|jiyNG1b33e6q3uy zG%V$VU6wD_%B@TDvHMfi$Ea^eX504r{lLVob=zJ`+ss*=oIVVJP(FPsr_d`z4+qEf zq!0A}d;y0uSRwvBc<{)74-Iq-=AR=2Vj!IihYQ1a{yoQrOyl2k{-((I_Z$qh(0~7& z|GS(0KR(cZz1e?#=>K_6!I-NJ)xYnE)zZ>(+OAzFD^0*uf(6RT&Q_1M=Fbzd{%AwQ zssHET@CmySC71+E-nr9LG_gfjH(|qR@V?N{%~Z8p5Oe3xIDG3y`(D_KUVEdoLTHMz ziUM7X^<<9z#f)ZqSTdY|bH8o-pM&X_+G97aF?7_74JT|jKZccEN9YhxH&h^vl9QJQ zd--IW0t4I8wcl19&qaNF{hv?pb|E+o)l;Dsw9ltEd(3pK@+wWHnYP?bz+^hkefKm@9+kLd7C`%-C!F!-3Hx>9ke37 z7A;S9-#RD4k^GMc8aIF1`ok?P+WHPkJSry}l%=Pq=eSUO*6tz$kH)4Ou>$GE-@0+ih_AM7 z{!$11iL_?6tm0pBXctQCUEHp$Leax@A9J?_q56a`f}wgF4F=Y_!K{D8mdNJW{OcDa zXf04S5ziv3*Sc|gGc3Yepw3h&2a)X){hi@ASMM1bjs_#V&!czl-Sl>cH?-E(N%k?_1y`i> z&hMVtWcc?AZ3JT4j@cmiucJ4Z)tuk6P97d7$rz3E4PyeK(~2qOPcWG0Z%RvD(+Vbsy#Q@-h+Q| z-#0fm)0v|KxD#DVmXlQ#FARw#-SrJ6X}gjO_FxZY14Gi`YEN;))0abXNkC= zaxO$cz2#Wj8LJ@~HyLP?37!1wqHFlW`^cE3N{u18EBKsWS1@i8O^WF9=i%(;=H_b< z53)Hkt!^sVRUSJ1egi}wui!pL7klr_%u7$> z5i;aq-%$7Jx7z8s%zI$`Ui8>W=n=H<>mS9nZfp>zr%&%PkGetXqi^CfMIFHi-8_q3 z{*bpA2FIK1V5CVe{DPAz@#o5lY}ool!}%^^b%3vL2BhUt)!qoh*KzTmdcs3K?Rp|h#I^DVQ_B1FJuN5Ye1Qq|_2pv9y* z+kAzv10`&u*JF;6OK@KN=yusPuuk*{`iip;Yz0rd}{@yHE z_wjdzu!#z+^xwW6=RksaA%$&pmB^m7mgDfz)*k56-`P$Gbc?n&V$OXfvg*G@KI$h8 zV*-=%^<&ieI&z`wkp&SC`I*v3zCTkqZ?!g-Bs=yS!}m&yUHdE>g_7gz-EaG$C)Ut3 zUBj3nuCDGhqnH|8ECCmdPr5JWP@geq5B3MG3NrD%d&tK?sga6`iV+pEKg) zjeBo~mbl!B6`LCWjzAxRZ@15;`Val{V65DJQ?(y__yg|y-0gIIeO;4v9}>;{ zybJAK(N0}%T#r(scZ1ECBD~4b9UI%Fm1Om1s3eU-y{nE}>PRHD=#5B`)1InuAq6nq z+@c@~^Po4xS7EH3&a>4vrL~56bTdc&{rwC2H{bt-2}L4#4qH#Nu&q;IJ)ZlMb(w@Z zEC%mJr-Gd*j`Q^Q^*!1H>v`Jgtb}c6JR{n+E9A^TzG54VmD0{rkcD*LMVY_cIBcg` zIXipn3-X08n|7|uj!#0M704TZWYGQw!f?3mEF4bVq%kD6Pmw`-U6~;?|NGtZ@5Lk~uZ~&?%Lo)z zlb{jl7h(ojm#5}Mn4GVCzcR&LM`WDxVcHXV z(qb^F+tSjq#C^7-ri>hNW4Wsvr&jp&lA8kwot+lPT;nVVe#GurZEu_o%YfTdmw?~G zW-e1-(Bw<8ELlz^Cns-jF!LZYWrOSfv>PcicD-$ia}$A>qlG(!Xxj8jXxO5K*(pB! zu&J(pVg1kB`yKK6Z1Q=1Kb7u|yo_NAsgxc!ikW9ZpG0ULS1L*h5pleNX7!KA+l7iS z4;awC^b`!xJJd=9H9DD-%0g6gTM=vwMd(E`+f@)GN6D&QIcG7@wr9M>mSe{&utk`+ zm}c6S1y~Duie42J6$`nI=4~cn?+w3B7RR#b;o*0+m{0^&q&BA80V*OhYSX_d&enO_ zqQOD|OFQ*!7qT)=qno*e3e!U7!8ZBOPtyd=54r;r1Hh)H#5S&P`QJcjVjB(o%ygKN zm9dR9c)fO^=yZVa#Ds)|K8`#9*?{jCdsoosnz9hsDvOQYaS2%S?gxj5r^$FWGTx`d zw-pzp`j6eLW3;_#2exrc$=cZEU8v~x=`T6Y$$4^jl=)Qn$)2a-Q5`3hm9fDY{COyk zDLLZpG8gt>^zN(8(@tBPW;qZvFLpLA2D}QQ29DOgecFY?;bQHL>EgOMpmiq~ zmqyqA9BEH+BX-{4ygE_zBox{_E?5l9FlaH}OO$A+IdKg-1hM8Xd*j>W%omfdLE4l` zn`n&OOyl}Wvk5&ZHDsOtsDXyPEGbz+Lh{t?zPaprh}`<(woHwMIK;*AJt6?lKJ1H) zZzjom=DkB8DIL__rkrO0lw!zEQ}KN#+y}}oT~1G@lPXmm;D9_(1h(^09~HUY68n_x zil4=`RO;~Y8+9BziPFpp!KFvGn_FP-Dy9B}2jqaEc6n}8&A;-&SEeQ3o!Gma{Em)} zZsQu_DcP87U|^slJd;`P5JJB%Rg|sBcw&^ca}#|C9aZa<9Gy(~3><57SzthV@~4TM zp%~asB&!Oi?VN5ZU0#R0=e?yt$`(eo=psEH6c6o<;mlRX#N9rUvTiZflC0|KV$fAO z>47nNd^Gl;=MsZMZ%y8rM|5aKOm{wem$BK3Z5_X>Bx=&m@5BH_Y@fQ?F66N2-OxCp zqf6+!D_A3*-g}%ziZvj&hPyJW!0|8WWoJBn{r&T#BTkOI^?p=24Hi5WJ}{ZgG8rEp z{=M?%J=k6S%2Fv20-ofR`e!w~u8OhdEv}uJp@J+?gc9+osP`~-%9$#cyGK;$Ugf#; z#Mx%nPD&7Z3$PcS1Q*|#bA>uzKmh>(o;Fzv{E($-JKH%;Xg6?&VCX90zx?y3P_bq< zP7&QMKOn1*lrR%ks!UF@`aKkXFDol6+<}xVDkvX0bIFDE&U+u0E`R2vQZO74q{k;B zYPiq}1KU$I{3}JTUcIWMyKvz^IC1g~C)I_E$>8Mlkp z;-Sal&AQ_EkXjgB0z3VWgz8{dJ;DVL0EcyAJAUZ()(FBLsbrAjd}YEO`8`W}!3Hai z5_@@U{<=kN6a$Ktq1KK3)u1UB=CE>UL4S&>*Vynj8CCN^ze(N{=Gp%MGmco7%jgmA zwGdF&&3QM3KF1tgMr+!>bEQPLlqxEIa;Qx&S#IdC`4vOK)0(eCfT4~xXaYRXp)1|% zCjF-|wq$O`BfikI0pTIiuAF(z#B_%?=AQh34}Y)Bv`~Eh{Pwjk%GqqK%&^t6Ep8}cTmV-d>g2lrI$~j{`#w!US@s6J}lCwZf znuKK#@@8*#IUcPja_-kBvgGczjozHUOGUV+A(SkK9BxYqe2_*dBX53}!XGYOd}c zQczVdr*v)BcY1F??aq(GJc$CVC1dh{Y1+<;UWpwo4NWss7sq>kS5wH)##fJr^)(2f zw3hHt`)2}j zgD?paFbcYxtD}t6qTuvW@EF*Zq3?Ii;vy7VFfxcb_UG6bkBDOr&i=h3+pp3x_~->j z%1d!EeFc=zb^I&ds1`kBk+=@3OaO7&G|f*{Fg=^`DF;|mj<9-Wf?v&2J1A=Vy19 zH}%w%o!8$Pg~q!+T#IPXzX_^MqHOB>0RVR13U2KmosvIt?N;6SP8!IIU9*>xtMX11 zO!*i+?9az4dyASP1RX*(zu`#6OkN|^t-Y$!$*=r_MnTM(){T<#a=C~78OihWEu+>f{ z2r4RF76Ev-=c6Z7n%vsO6j{x>%qIi6poV0ZR1 zYAxTHTRTtWm;OAI(~HV zpN;jER4GFfjNa9j19l?SM&x>^_TU3qF)=Y;22+hNJCV%EQrFu)FZB<7e(FQjCl-~( z#g2}smKV0=spB}1pT&MFm0SB>%kd)^c8J7RSf8?^Bgg~K-=W_-*X>ZtOI9_;Xt><% zCK1Bg{-r*PnG6xj1#h^6qNAzqeoxab#g4CB2{KEwFE7P%T)VUv_u+$5etF?8i7{I~ zJQ1I98`!Dw%n1-o>3MfXX)}9yPJ8DaGkzOUqk)>Pn38j1NRB~>pirZwv#nwB>AmLj zQvKrAj3a}C*0J3;;2S_=s=A|vR-VhovUct%D||dP$m%po0ClqseBX&<@4GX!2m6zaRj9-c zoK@O%F*P-HCgC|EZgMrqqRBaHg+i%H8to3ANEf+Fy`NK%n!mXCa##E9C439yAx<-^ z1to=z9YJAXVJTtL(*`I-0mNa+^2yup7d5d5qP;cizUF&7;sUM*zn3ia2nk%{1URJV z;q06#8?w79sj!;vP85Xwi6G>+>7VuiRCgP?5B-F6(obH4YW;WE=?|HS92VH4TFk_S znF_VSt)!EalRCwpbp2AZ@jIp_CjL5vfw#*r$p8M@RK03u(f)ZNZm-T=79uguv03iK zR5QVVSJ?2|f7|S>EiLUEfsKIE>aq*W3vhYRaohl)p_O_`3pl9;8{XQPO$&h=)gip! z&q`G$T0Tn{rQ`R~kPo)ek(Riaf;~t9JLA^E=gjR9?9umz#k|9-@!=Qz z7|^w#oL(q$4!?(Tgp7(QC7pJS-#HD^K#Kb>rx}uYQ9BESiO+dd5uICSA)3*W^Aj9* zLb@fj?rvxY4RRHXor|17z@^#HEPBa$xil+f!R$$kLHXpH)jAiRq*xN7?2U&2D*Voq z9@AYJIvoHIm8dnNk-(xEw79#w8!ko_sfAjjsIq=+IL(Fd&c#j<0m^)`ap)o4Go}k6tI=*#kv}5 zt+lmPy;L&J;FDL59QtVaMSkymo<70-?iJavCa{@|ie2a0A^&sL-LWf;+aM$j*6Z>y zGJy)<{u5dC`Cl9y9Mk1Pb+y)-*k2p^H+Syio;CiF{@BuVtP~E29GnsV$t^7H=WNN1 zX=oLYG~WaS1ahH%iYg^?LONOfO{&O-4N1;2!IoW>okd}26nb45tCcI`cX>#RPM5$S z6m8fAc4J#1Hk&Y0@39kh`Ub50%LRPR@Z&s9#W_b6c8PwkMN1hHvV%A>sU8LGRZ~L&Xu*BVB_a zOJBk(fwYg@q z*jPcVmeIaPdG_ovprz2WQjQ^cbtRabzXppdZV|vyflY2RFTnbkkIHjm=l0CkY{vX1 zAHvv^1`Xk_Yb-%I?h;{Om9^jxUmbctPa&=z4Avz>#Y#9eeo;7>i)Zx&6ixDoAMHjw zh%Tu_9~1qajV7Dji7#Kizz}+vz=xBLIbNLy&F3VvJbD1muqkh8IUay4Xlg+*qJ`b@ zqELDy@-b+Ihd%lIHxZX!oI5#wmDO!I;%4e6t^Lqe!Cq78#M&`0c3G;(HP}SP@WI98 zQjMVrg5RXuW+}l1zf6{ePN6hm?ydd^T_+K&{7?~&t)fjSzhLQUtMf) zQXdt(H7)ukWg^*VBLtb^JVzSW8Y>RdLG_a>d@wBkxhH*rbG+CG1Pdv=SI#XLmP(L> zroGtD!$Wx+BlUfJd~R1hv0VvIsY=l#vsD~Hg`sqU>@)|_&v?;(qc)ijDgueGdbXRZ(iZFOQM`?>#3kkALgO*3^uHVgHH(LM zJHIc5@0d1`3-rpI=?U4{I09>YjnBa&SJHDz{np_gsW!Ti-kXU@IVhWzS{%Lg7Le0} zLqk4+^c17lvN8+ys`ylQ#Y|QL3SVGgYk*^Ua$L zP>x3c+_62^GemgD%u0#s%<{egxiAnZ%Q7I><#n*aF&(N9kz#Mm0x2dqF%(<>?peg1 z3u`e#9S5}AaYqYteF`VTSA$A#H~4DtO5-BHgzow~V^p#E2!tj1rnW3H*9r~a=>V~oIe6qL)ydN_e@6xu<%o&Q05Q$Bf3J(3(1%Q534kHQiYLyRT^_BM6c_J<39*0h z+5kC&r~Jn>vU;u`rm85(-#d;%7AX%a3;-c9>2Hbc%hI zv>=C(;)Aa4TBSAfeKI57Hy>n5esZCX?#ahz8~Z;x^lQwR8({fy+QEF z&?ncX{$PHe_PDKfo`w3yckkBS8+V!p$vcQZ%s*GB6N~3tlu=>pK2$->TNFp9_dMR0y<-3kt zidx}`QDs7QQP2a%`PgppiWLQxy_xKy{fEfS<3?<_^QBWeCFIkaj{y%O{xQteZGXRkw`ppRqjHmkeaAp9;DGYZNtawWzVRrd_d0X*htd3%EpYaR8AqAE};VWN(H0vao-Yb*nV}Ezqu?}-Fr}yHBw%J+ERY`H&1|S_cn3_ZVhewt5e+uFFxTFdqEQ!r=)}-*U)b3cNv#asXh;h__;n z5(kpS(`P)LaJa&8dDkVgxfES)Nk|YjdPLiMI=tgeKltz^CzZzoz3Wnm+fVF`J;{t@ z=cHC;Mkwl+JE<6~pfMfP?K=Kj03K;q8#T)>pcs@7^@qeNwZ{U#YYmCD3*riaMj8Uq z)v$%rwUTKf8KVG3D8kT08v|8@(Z<-i!F2LTAbG*(Z+_=-5I*X``>DV~|x zsp=Ki^a=T**5m;A6eOHFawhUw+>x2z*Jt8i0b$_`0f@K5*5$=ii(}(y*bjK66Bz%B z))}&8-Wzo#0vyPL&PDJYtdmOb6oboH)`K@5rH$d`D~;N)`nu?+Ep^zF4>e_7Ad`N> zt;KkiB!N`w6)(tp$zOi@Ilkg9ywCf=(6{rvlK*a0eLlu@*k)kHF)$sREMzhLI8@+m zQ>#x*yzmeekofW^5v=I;nyNhQ0*OiAAzGLQ*z5NhOs;Ax>ntp52rUeMAFCtZ^cmPb z0myh@ZGC;JX#H_W`xv62;IwmhzfF={f|67nVZ~9jJ2$E@IaNHp<^YqI^CB#1odd3X zTHL0qO>^~J9xfSzz2V|JfU8K3P>*JS8q^tn>6m+EHsWv(mFOjt+lot69XCOEH#%o& z8>RWVf>Exw=;b}5M6Y_tnB0{zCWYErqKdUAJ5uN)xkq%L058mKa{7f^AF~7V_P!C0 zXMtUV0roE1_rt=_w2ZK(D0Xf^S4C3n_ORa*-itjeNlNrbTrsJ-g6a85oSh5;y(R&u zlcGkR-%D>dvYE)N6j8^I7lX7x9)1W!d!5-`X@RT-7Va&|1gyeUrRt$OF?|96yX)HY z<;sFG&;ssVT`JC!=1z>p`bWQ`i|d`=nlJm_NSlUG^$4Ag$Qq(b&18S zJw%=`JjDTMDXWTP9ElQG?Www8nyVDM7-R`m89z9DWbb!JbqB6YR_0p%ex!|M+=$M} z@cIYLkDacWwYuqWfe(Z&7YZ+n^1}6Tp!6fTwbX~pZX-_3Z>HvfLMTjE5uW&yvaeUw zJp+5-LeZf`QT}fKpK>>2r7;3U5Nbs_2}KlGwjw8BuX)b>sJk*O+DxqJIqO z9Zq4lcZ-oatJ$hKu;8eSX1IT5k#mK?b8F&m4WNvv(3fqQiu_Tzvq4iNATWy1U>dO@ zJBr}Z&O1^8H+-O0(=|aN&ct6*<&ovr-)zePOfco<{4?Bf;47W_r1vJHyT6mEoSUdsq>^{Lc!W*qy$R9e=SsxVAz`Z#gJy75X4~wl0D2S`kgz7M{Ru1 zvkPfl10y+^OBQ!&+c5;%ak%O00!;VVg(|TU@a<^B!Qmu8!j-FK#RF z_pCjHJvs63ctXwaht>D4k2;6~F9H{H1EvsA+l5*UILnZoQEqN-A2!(ANLA0z!4+Hi z<*YciG2WV199F)@hAob((Vy$EoWyuGUclGz|qmshtUsCp(Ndt z@DSBk)$0a)C}hC{^NFOrmoT@ze;=B-YLmQ{x9Astt@`_%VFGU~FTR|{brxeIB!DQV zPEu7Lv!_&|cZhsdvBIQmX9(4Gh()iIEGGa5Pd%lovA*pgvTlH?|F&kK-qE}3uNtm{ zcjCef&}AX^Di}Ub&kX};_q?-@c2h+Nud6Uq7J{!f!P=cqmnJmRsyXBt>sN-EOVWwk z0i%9<49S%+Bl>nJEs948fT}glZM#ZJPBZs)F2}g%0|E0|bC67$sRPj;U$NPuNz&VO zX$$pAZo!&4e%!I!05+fb!{bvRiLe?tR$?zBj*M^K;OWuy(@{?l`N$B1fN34X*OXM9 zr`f08Y-0F69osE|*y2JZ5KGI-p^82`k?hKa>8Igx*sQVL{avDGF@i6xQju5cpEyk0 zj$6A93DWxH#O)bhya9MVnAEj5YcD>{Z&eySuCA7f6d83iVpC&$JnhF2{?MwWs#i?Y zN9t3UKakBqSsT~wi+T@hzViB+mN3n1EFp&@Yt82js8v>gsY1xf-j)aW8W7m`oK$zg zDnKSywkqtb|Mg85DthuR(-pWY>b2Z#c_kk@0G-ChiA{xD8+~|iz{4xd3{p-Jwt~UC zx-f=;w^c9hH~?^LKp#H#m610`ou0@02ve^K<}K!4N&(AT@|VIzrOfkpHmry^ZhK2o zG`x3%DNg&XOUCq950XT9yY28QO8i|5-bzv8!E(>?VL0RA38Lx4 z+(IhJLI>}os}(|pb-8|#mzS3%8szZ&P=RwHHZ zjlrp@6IsafiP0R%J$4#UZaKZ@_{3;8Do0hsKQD=|dMfIbJ99EkIG{6mUQu@%z!_y3 zpW2eJdVNI*7Nc=F`6}n%F-(TvDceU4=0R#~rtb(B6BoZ;6NX(jfDiZ3|KykooTVi} z6aCx3fJwgDD^z({H%EP-jb*Bfo z1WxDu2rrpCqkfhF?2Ao>dd@OvW5}ZX`$8Y!9*m==ljcC?^2q+O%M$$K3dIM}RqAPR z9T%4$q<9gmN7$zdFn+}V14ZIKC0g}bYEW22#A$C?`Plp1MXsB{l_cqcD<@Qm9=Y?| zvGPf@T@sZCr5tk?ISKXon+E`E);JlS9qM&?w=@gvr=4R8mYBDmNBUn`9sD|?Ioj9B z!1BHa?&`cX*F!?^`r1)WDMtWnsWmhQQY8-}G}wK1K^Y-?pRy0(4nn;Kk&S z*J|jd#>NrbCu|=d!{4`XF)%&-;<^0RhL^|}8?T^1>E-G2l_qY;B`@s4e&YXZPi|~0bsHqj2@5z9X|$1mC^tfq36#ubxki@V8?le*iSi`- z)9W0!BIl{6POX9KwS6*WZ)e9HTl{{jlg;xr!5Br>e-pwUgNw#L5oP7 zJ>s~d-&^G-=eZ@fjlmt6PL$;}anE&df@wXvt5T4;@YZisvT{=F!eI+L9b-DiZy zbJGFkyHl2mWRAyLq54RqJ@XH{hlJ_-`oD+2fB&BHBXkw;A9T%N%$`c=d$p4v(oSwpKr5Df}NS)u2fL1>oTPvUVJBC3W{(gV$o+rLc5aHrW zkp1qJ;j4z)xjXVe1P~x?ut1rQnycbr2fsH&zs7iQB|(AL3Vw_-CEKtk$!7Omq2v6$ zaYuP>fG}WiQ5(hOe0knGsK+QmGFNMw!pckk*ia%{ULdjhDho?yf>Yj!g^oE{fC9&ZPPJ_=z|5@R5bp6UcZZ?Ik{V142m6oK7FV=fOofhSvDD~Pc(*aU)7 zV&d#C(=_qPnzC%op^4;j>v>=tr^i2d;=VZnoOTiHiGH2&EH~f{7w^bK{>TS7rIc`= zE!r!|-Z*Oc@B7s+vASh@<~8miOE*vYv=MTj4Io3~G;#S0!UGPqrYe5-)R2@MT|uex zQ)eYZpz~aOdk=vJnvE15Oj=;7lRWJglSlPRhToS(*?F-$3Zu8VrZgDPy}--CCD`Ro z#90H&Hgtu&O40P=q3qs-&nY?Qr;QHN$DH**D_ngzFuS_C>gM6(YLw2-7EB4Z>(s;N0)rq46hK(GXj_kn4TX&@WdX zcZ!SzNk`gf4vF0}kH`Y(D1q%f6aSwS>;=M&{no#6YZqOw1<|xKLo>=PEYNY9y}8+j zcqEXFD)JU5&(1hP#N3Op zhQOK-0;czUe+>Q=obhnjWnF!^hvCtex;WGciJ|Xt)uxQ-yMnn0fMDeLCdwR2S9H3ENF1#e2QHT zOS7MJF&?odl@z*?k!FIDJwoxfgbqyn$4+V*8)`AXM`qZaKmp%d<5A7;x`H7tB71Aj zDbY_{uzE}->8A_e5HB!^ONbHL?i(E$gU$C$T;L5*zt3CDVVuk|0YTpe|303(@mM>8 z!NbbR3W=-{pLhdFeHDt20EN%AaoP^clRx^`0i*@o2r#_tL4go{l#8`0i4AalCD_ZO zPCmd=X9;RQ4w4HG=l)b8Zv~jUzzWsPTa=IfY5ABA$^Gzgq5UrTBTboH3y>7HNYGBD zI$49KHWNEVJ~InvJoN|fo4oX}3ut$gGY72i2acg5uyd#FYQO2ES7ibx!_Dlz1-XHc z3wL;ags&v1724DWu*nXIfgTl87Z6ZN+-6xu~>;pBVLIKv4Y;1hHdD2B} zlcNsQ)wh5RI2vnVv4X3XtKOFL{JW;?DeQ;=?*m!Dlj~sow2^n9cd^}!s7W29#F;8_ zbT0I6I50oofRMR^y#>#tw2H>|17;jpM3cztFXJC7(aTLvOdK5n6auWDG?(K#3Q2t& zDLk4Y_f=FzU{#=l<#O^Gw0hp^GV+wNgH`+D2(Wx7nF;Q3pmGgoA8kNZXW^MEK~r~o zYff$=2C&OMh%FU*igrW|)$2-~I-B2=UppZQSXEeq*_8jgPjAu=LzO|WdEPRP&6E$> zWdmC(naVNF(!(8foM+8EoejDM9!X?}=_9A`KWn0Z*<2qL096gDG58r=f+t7<3q2Tw zR&0Mvu&EOLqqS*D@2Q|g0T9)sfzBJUoTCwo_25Df`A^THI<{axwodg?1||v8i9Ue# zcnZB89q%uPcFRthUvrdY+!>b246KXAuwQ+h{kU_Bn--#JM{gWDxv+BZn5Lb9_|kiY z;u}Nd|J*UEr7zaIF}mG+4_Y-ZHqxeigvMUkTo1pcVsRmIhPc{iu>FG@^a$m;*7x|s zgS!4szUsd=D+v~Zg*gr9vVF|@EvC36(cy&az=}|YbribzI~)*nZM0S}u3xP+VABF$ zgY>JP=S%sCIN%)Bf~qjd<|S~Hd;tL1JvwFnAuT-Xhur`YbzTSj|CvYbFNiuQg=2Re z03FGo)4m5^K7Gy^w2 z^izVnlamuA)W4;`|hGgd~iAaeg;TS^FYoYB$JazqV!eu|lD_nX|-e}DFM zrD5V|;w}$d`X$f<-EkP<>@}(Wjn`8TKy%O~{H0mBk)TmO*{*GAR@`z!xFD(4V|H;Ha zK*Emb-{UuF3w>sL>SX}mZ(V6@(Fku(*D7AL@x38I8VN5ElJy43y=rJQ;Aq!1eFr!o zuE3Yh;0@)R?NvAJLE8-lp6&MH6Kp>RE33>$+?u@`%yu`3%WUI@mxW&Z} z%21*YdzV8>Tr^+=pwAfFY*G`$XEXiVRMqu`z{vXHZK8JK?A&9{KaA+h%(TIEBykn9 z3e8MWDh`jLc3$3nCFJwy(X$kE!Y@7jFxH?NccNcr?l-R-(8qWI-^M04q8;0vSFv)E z)hB2I=l{=}uDe)mwYi0Pm)O{9mFnA3ID&g~`NV9WSc?{Q4; zmkB0{4~bEC|5}S!m4hrR6%xu;hHAhnQwMc09PdXPobo-Algb}daT{*_?~~OETjMe` znMkexCw;yY>liBDHU#M3Yn+t=T~d^I^?0iCw+EB<`It~9{4A}v(QR@mP+%SdvAWHu z*LD{TSc-;QZh$BYb~9VoNv~H9=m?v*i*n>T>Or{9`=R7X zD}EGA0&hs}L55;Pcmr@gNI{B7nh_OGn^byug4Q?C?Uf3(Hc)n$bwBv3_B^Kp>R(=4 zJI>rVEGGX+Atc>@$d9NjStjvVQ6NnseV<#dBP&^!+^pA8=%SqKQNn`8BFk`PoZ}y4r>scCZ zOESQ{bSw44{P`vNb+%(&<1Folllw$;W7+ZE~0cQae4G1r!91 ztkGTtm{2yngYX35Q2C8wuGVQk&b`0;>OEi&@F(KxcH}lpf!}8{nRVl194N?t-YZ#F zhwhq=`U8(&bPmRBbLl}HVoMQR*H_PxvT+DBY@pw223||wdx8nPmbRzz znIShEItKopkRh=)EY|YO<}aDm#IwLT79pa!jRgX>vhd+bsrBwv!h!o(@z?Fef!pNu z?D%etAog9t+KkzopgqXJCq+Fnj$Zt2={c`ii6J0D4<$QP?f;CGI0Gy z9`=9&ef?TG%b|E3$=HpcD$gmQT1=7aYK3~*Bd)5iO?vG_h@g{Z--UR@jn+hr>)QvN zqTKWA9({e^Ddaj*<_D}5i?G#O2-#B5xnp8jodMy=AYedb0OSZ8#fQnLyVf6K+d!`t zwGddR16D7l!SUue^YLeQ`%0)SVKAmt4!RxJeIByV zCtQOD>fXJRlOx6UPIY!bSeXF-SRGl_d#neI6<2)g^9BRI&0NZ=dYLMV$ibeM77q!d zP_qYfr#klF-D1#yLH^W`li8@Ni_3V2SYKc1n$P|~G{OHHnbp^(lh0j06B-K%yZaZ> zmrK{S&p-q3>gp72h`Se6>VNt9`%Bc-)oG&$2}nCE7FX)>Nhsa{(ji<&)VvAqW24-( zDbWuO2O;quZ;l<=X9`j?0bG}S`hhXd?b{n_1|I&3R{sZ##kZCo{e}PEpBexn@xKQtNB(n=ShRl*(&PSf zkYdk2$N%AD4F2aQ0{nN*A@%D0_iy(9_(1>lW=^3lkuwI-ulgB16PETU2JRYL`t1tc zyF+@U7aFHm!)D3?MP0MHz(-VE_f2me7LLLw3ZLg256zObl_sF;kn r$Solu86lzb1uF6X%LVS9cFqn#|N9GqPfvuv1(2$;7P8{5RpkEwUsAHh literal 0 HcmV?d00001 diff --git a/examples/Multimodal/ImageSchema.cs b/examples/Multimodal/ImageSchema.cs new file mode 100644 index 0000000..e0ea8d1 --- /dev/null +++ b/examples/Multimodal/ImageSchema.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; +using Microsoft.TypeChat.Schema; + +namespace Multimodal; + +/// +/// A strongly typed description of an image, produced by translating a multimodal prompt +/// (text + image) into this type. +/// +public class ImageResponse +{ + [Comment("A single sentence caption that describes the image")] + [JsonPropertyName("caption")] + public string Caption { get; set; } + + [Comment("The most prominent objects visible in the image")] + [JsonPropertyName("objects")] + public string[] Objects { get; set; } + + [Comment("The dominant colors in the image")] + [JsonPropertyName("colors")] + public string[] Colors { get; set; } + + [Comment("True if the image contains any text")] + [JsonPropertyName("containsText")] + public bool ContainsText { get; set; } +} diff --git a/examples/Multimodal/Multimodal.csproj b/examples/Multimodal/Multimodal.csproj new file mode 100644 index 0000000..7593aae --- /dev/null +++ b/examples/Multimodal/Multimodal.csproj @@ -0,0 +1,41 @@ + + + + Exe + net8.0 + latest + LatestMajor + enable + annotations + + + + + Always + + + + + + Always + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + + diff --git a/examples/Multimodal/Program.cs b/examples/Multimodal/Program.cs new file mode 100644 index 0000000..6a0fb4d --- /dev/null +++ b/examples/Multimodal/Program.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft. All rights reserved. +using Microsoft.TypeChat; + +namespace Multimodal; + +/// +/// This example shows how to send images to a vision capable model and translate what the model +/// "sees" into a strongly typed object. +/// +/// Each input line is either: +/// - a local image file path (e.g. C:\pics\cat.jpg), or +/// - an http(s) url to an image (e.g. https://.../cat.jpg) +/// +/// The image is placed into a multimodal prompt alongside an instruction, and JsonTranslator +/// translates the response into an . +/// +/// NOTE: This requires a vision capable model (e.g. gpt-4o). Set the model name in appSettings. +/// +public class MultimodalApp : ConsoleApp +{ + private readonly JsonTranslator _translator; + + public MultimodalApp() + { + OpenAIConfig config = Config.LoadOpenAI(); + + // TIP: To use the OpenAI Responses API instead of Chat Completions, either point + // config.Endpoint at a url whose path ends with /responses, or force it explicitly: + // config.UseResponsesApi = true; + + _translator = new JsonTranslator(new LanguageModel(config)); + } + + public override async Task ProcessInputAsync(string input, CancellationToken cancelToken) + { + PromptImage image = IsUrl(input) ? + new PromptImage(input, ImageDetail.Auto) : + PromptImage.FromFile(input); + + // Build a multimodal request: an instruction followed by the image. + Prompt request = new Prompt(); + request.Add(new MultimodalPromptSection() + .AddText("Describe the following image.") + .AddImage(image)); + + ImageResponse response = await _translator.TranslateAsync(request, null, null, cancelToken); + + Console.WriteLine($"Caption: {response.Caption}"); + Console.WriteLine($"Contains text: {response.ContainsText}"); + if (response.Objects is not null && response.Objects.Length > 0) + { + Console.WriteLine("Objects: " + string.Join(", ", response.Objects)); + } + if (response.Colors is not null && response.Colors.Length > 0) + { + Console.WriteLine("Colors: " + string.Join(", ", response.Colors)); + } + } + + private static bool IsUrl(string input) + { + return input.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + input.StartsWith("https://", StringComparison.OrdinalIgnoreCase); + } + + public static async Task Main(string[] args) + { + try + { + MultimodalApp app = new MultimodalApp(); + // Pass an input file as arg[0] to run in batch mode. + await app.RunAsync("🖼️> ", args.GetOrNull(0)); + } + catch (Exception ex) + { + WriteError(ex); + Console.ReadLine(); + return -1; + } + + return 0; + } +} diff --git a/examples/Multimodal/Readme.md b/examples/Multimodal/Readme.md new file mode 100644 index 0000000..c7da325 --- /dev/null +++ b/examples/Multimodal/Readme.md @@ -0,0 +1,45 @@ +# Multimodal + +Demonstrates sending **images** to a vision capable language model and translating what the model +sees into a **strongly typed** object using `JsonTranslator`. + +## What it shows + +- Building a multimodal prompt with [`MultimodalPromptSection`](../../src/typechat/MultimodalPromptSection.cs), + mixing text and images. +- Referencing images either by url or by local file (`PromptImage.FromFile`, which base64 encodes the image + into a data uri). +- Translating the multimodal request into a typed [`ImageResponse`](./ImageSchema.cs) — TypeChat validates and + repairs the model's JSON exactly as it does for text-only requests. + +```cs +Prompt request = new Prompt(); +request.Add(new MultimodalPromptSection() + .AddText("Describe the following image.") + .AddImage(PromptImage.FromFile("cat.jpg"))); + +ImageResponse response = await translator.TranslateAsync(request, null); +``` + +## Requirements + +- A **vision capable** model such as `gpt-4o`. Set the model name in `appSettings.Development.json`. + +## Running + +The example ships with the official Microsoft logo (`microsoft-logo.png`, sourced from +[../../assets](../../assets)) which is copied next to the built app, so it runs out of the box. + +- Interactive: run the project and paste an image file path or an http(s) url at the prompt. +- Batch: pass an input file whose lines are image paths/urls, e.g. `dotnet run -- input.txt`. + The bundled [input.txt](./input.txt) points at `microsoft-logo.png`. + +## OpenAI Responses API + +The built-in `LanguageModel` also supports the OpenAI **Responses API**. It is selected automatically when +the configured endpoint path ends with `/responses`, or you can force it: + +```cs +OpenAIConfig config = Config.LoadOpenAI(); +config.UseResponsesApi = true; // or false to force Chat Completions +``` diff --git a/examples/Multimodal/input.txt b/examples/Multimodal/input.txt new file mode 100644 index 0000000..b7284fd --- /dev/null +++ b/examples/Multimodal/input.txt @@ -0,0 +1,10 @@ +# Multimodal example input. +# +# Each non-comment line is processed as one image and described by the model. +# A line is either a local image file path or an http(s) url to an image. +# +# This requires a vision capable model (e.g. gpt-4o) configured in appSettings. +# +# The Microsoft logo below ships with this example and is copied next to the built app. +# Replace or add your own image paths / urls as needed. +microsoft-logo.png diff --git a/src/typechat.meai/ChatLanguageModel.cs b/src/typechat.meai/ChatLanguageModel.cs index d8bda2c..d7632af 100644 --- a/src/typechat.meai/ChatLanguageModel.cs +++ b/src/typechat.meai/ChatLanguageModel.cs @@ -110,9 +110,56 @@ public static void Append(this List history, IEnumerable history, IPromptSection message) { + if (message is IMultimodalPromptSection multimodal && HasImages(multimodal)) + { + history.Add(new ChatMessage(multimodal.GetRole(), ToContents(multimodal))); + return; + } history.Add(new ChatMessage(role: message.GetRole(), content: message.GetText())); } + internal static bool HasImages(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + if (parts[i].IsImage) + { + return true; + } + } + return false; + } + + internal static IList ToContents(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + List contents = new List(parts.Count); + for (int i = 0; i < parts.Count; ++i) + { + PromptContentPart part = parts[i]; + if (part.IsImage) + { + contents.Add(ToContent(part.Image!)); + } + else if (!string.IsNullOrEmpty(part.Text)) + { + contents.Add(new TextContent(part.Text)); + } + } + return contents; + } + + internal static AIContent ToContent(PromptImage image) + { + // A data uri (data:{mediaType};base64,{data}) carries its own media type and bytes. + if (image.IsDataUri) + { + return new DataContent(image.Url); + } + return new UriContent(image.Url, PromptImage.GetMediaType(image.Url)); + } + internal static bool IsRole(this ChatRole role, string label) { return role.Value.Equals(label, StringComparison.OrdinalIgnoreCase); diff --git a/src/typechat/IMultimodalPromptSection.cs b/src/typechat/IMultimodalPromptSection.cs new file mode 100644 index 0000000..96169a5 --- /dev/null +++ b/src/typechat/IMultimodalPromptSection.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// A prompt section whose content is multimodal: an ordered list of , +/// each of which is either text or an image. +/// +/// This extends so that language models which do not support images can +/// continue to consume the section as text via , while models that +/// support images can read the individual . +/// +/// +public interface IMultimodalPromptSection : IPromptSection +{ + /// + /// The ordered content parts (text and images) that make up this section. + /// + IReadOnlyList ContentParts { get; } +} diff --git a/src/typechat/ImageDetail.cs b/src/typechat/ImageDetail.cs new file mode 100644 index 0000000..fd16bcc --- /dev/null +++ b/src/typechat/ImageDetail.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// Controls how a model processes an image supplied in a multimodal prompt and generates +/// its textual understanding of the image. +/// Mirrors the OpenAI image "detail" setting. +/// +public enum ImageDetail +{ + /// + /// Let the model choose how to process the image. + /// + Auto, + + /// + /// The model treats the image as a low resolution 512x512px image. + /// + Low, + + /// + /// The model considers the image at full size. + /// + High, +} diff --git a/src/typechat/JsonTranslatorPrompts.cs b/src/typechat/JsonTranslatorPrompts.cs index 8c86a7b..6987eb4 100644 --- a/src/typechat/JsonTranslatorPrompts.cs +++ b/src/typechat/JsonTranslatorPrompts.cs @@ -43,7 +43,7 @@ public static Prompt AddContextAndRequest(Prompt prompt, Prompt request, IList= 1) { - prompt += RequestSection(request[0].GetText()); + prompt.Append(RequestSection(request[0])); return prompt; } @@ -77,6 +77,44 @@ public static PromptSection RequestSection(string request) return requestSection; } + /// + /// Wrap a user request section for the model. When the request carries images (a multimodal + /// section), the images are preserved so that vision capable models receive them. Otherwise this + /// behaves exactly like . + /// + /// the user request section + /// a prompt section to send to the model + public static IPromptSection RequestSection(IPromptSection request) + { + ArgumentVerify.ThrowIfNull(request, nameof(request)); + if (request is IMultimodalPromptSection multimodal && HasImages(multimodal)) + { + MultimodalPromptSection section = new MultimodalPromptSection(request.Source); + section.AddText("The following is a user request:"); + IReadOnlyList parts = multimodal.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + section.Add(parts[i]); + } + section.AddText("The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:\n"); + return section; + } + return RequestSection(request.GetText()); + } + + private static bool HasImages(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + if (parts[i].IsImage) + { + return true; + } + } + return false; + } + public static string RepairPrompt(string validationError) { validationError ??= string.Empty; diff --git a/src/typechat/LanguageModel.cs b/src/typechat/LanguageModel.cs index cb88153..4622c4d 100644 --- a/src/typechat/LanguageModel.cs +++ b/src/typechat/LanguageModel.cs @@ -13,6 +13,7 @@ public class LanguageModel : ILanguageModel, IDisposable private readonly ModelInfo _model; private HttpClient _client; private string _endPoint; + private bool _useResponsesApi; /// /// Create an OpenAILanguageModel object using the given OpenAIConfig @@ -48,8 +49,15 @@ public async Task CompleteAsync(Prompt prompt, TranslationSettings? sett { ArgumentVerify.ThrowIfNullOrEmpty(prompt, nameof(prompt)); - var request = CreateRequest(prompt, settings); string apiToken = _config.HasTokenProvider ? await _config.ApiTokenProvider.GetAccessTokenAsync(cancelToken) : null; + if (_useResponsesApi) + { + var responsesRequest = CreateResponsesRequest(prompt, settings); + var responsesResponse = await _client.GetJsonResponseAsync(_endPoint, responsesRequest, _config.MaxRetries, _config.MaxPauseMs, apiToken).ConfigureAwait(false); + return responsesResponse.GetText(); + } + + var request = CreateRequest(prompt, settings); var response = await _client.GetJsonResponseAsync(_endPoint, request, _config.MaxRetries, _config.MaxPauseMs, apiToken).ConfigureAwait(false); return response.GetText(); } @@ -64,12 +72,26 @@ private Request CreateRequest(Prompt prompt, TranslationSettings? settings = nul return request; } + private ResponsesRequest CreateResponsesRequest(Prompt prompt, TranslationSettings? settings = null) + { + // The Responses API always carries the model/deployment in the request body. + return ResponsesRequest.Create(_model.Name, prompt, settings ?? s_defaultSettings); + } + private void ConfigureClient() { if (_config.Azure) { - if (_config.Endpoint.IndexOf(@"chat/completions", StringComparison.OrdinalIgnoreCase) >= 0) + bool wantResponses = _config.UseResponsesApi ?? EndpointTargetsResponses(_config.Endpoint); + if (wantResponses) + { + // Route to the Responses API regardless of how the endpoint is written (base url, + // a chat/completions url, or an explicit /responses url). + _endPoint = BuildAzureResponsesEndpoint(_config.Endpoint, _config.ApiVersion); + } + else if (_config.Endpoint.IndexOf(@"chat/completions", StringComparison.OrdinalIgnoreCase) >= 0) { + // A fully qualified chat/completions endpoint was supplied - use it as-is. _endPoint = _config.Endpoint; } else @@ -84,19 +106,85 @@ private void ConfigureClient() } else { - _endPoint = _config.Endpoint; + bool wantResponses = _config.UseResponsesApi ?? EndpointTargetsResponses(_config.Endpoint); + _endPoint = wantResponses ? BuildResponsesEndpoint(_config.Endpoint) : _config.Endpoint; _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _config.ApiKey); if (!string.IsNullOrEmpty(_config.Organization)) { _client.DefaultRequestHeaders.Add("OpenAI-Organization", _config.Organization); } } + _useResponsesApi = _config.UseResponsesApi ?? EndpointTargetsResponses(_endPoint); if (_config.TimeoutMs > 0) { _client.Timeout = TimeSpan.FromMilliseconds(_config.TimeoutMs); } } + /// + /// Build an Azure OpenAI Responses API endpoint from any Azure endpoint. The Responses route is + /// resource scoped (not deployment scoped): "{scheme}://{host}/openai/responses?api-version=...". + /// The model/deployment is carried in the request body. + /// + private static string BuildAzureResponsesEndpoint(string endpoint, string apiVersion) + { + if (EndpointTargetsResponses(endpoint)) + { + return endpoint; + } + Uri authority = new Uri(new Uri(endpoint).GetLeftPart(UriPartial.Authority)); + string path = string.IsNullOrEmpty(apiVersion) ? "openai/responses" : $"openai/responses?api-version={apiVersion}"; + return new Uri(authority, path).AbsoluteUri; + } + + /// + /// Build an OpenAI (non-Azure) Responses API endpoint. If the endpoint targets chat/completions, + /// it is rewritten to the sibling "responses" route; otherwise "responses" is appended. + /// + private static string BuildResponsesEndpoint(string endpoint) + { + if (EndpointTargetsResponses(endpoint)) + { + return endpoint; + } + const string chatCompletions = "chat/completions"; + int index = endpoint.IndexOf(chatCompletions, StringComparison.OrdinalIgnoreCase); + if (index >= 0) + { + return endpoint.Substring(0, index) + "responses" + endpoint.Substring(index + chatCompletions.Length); + } + return endpoint.TrimEnd('/') + "/responses"; + } + + /// + /// Returns true when the given endpoint targets the OpenAI Responses API. + /// Detection is based on whether the url path ends with "/responses" (ignoring any query string). + /// This covers both the standard OpenAI endpoint (https://api.openai.com/v1/responses) and + /// Azure OpenAI endpoints that end with "/responses?api-version=...". + /// + internal static bool EndpointTargetsResponses(string endpoint) + { + if (string.IsNullOrEmpty(endpoint)) + { + return false; + } + string path = endpoint; + if (Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uri)) + { + path = uri.AbsolutePath; + } + else + { + int query = path.IndexOf('?'); + if (query >= 0) + { + path = path.Substring(0, query); + } + } + path = path.TrimEnd('/'); + return path.EndsWith("/responses", StringComparison.OrdinalIgnoreCase); + } + private struct Request { public string? model { get; set; } @@ -117,7 +205,7 @@ public static Request Create(Prompt prompt, TranslationSettings settings) private struct Response { - public Choice[] choices { get; set; } + public ResponseChoice[] choices { get; set; } public string GetText() { @@ -130,25 +218,177 @@ public string GetText() } } + // A request message. 'content' is either a string (text only) or an array of + // ContentPart (multimodal). System.Text.Json serializes the object by its runtime type. private struct Message { public string role { get; set; } - public string content { get; set; } + public object content { get; set; } public static Message[] Create(Prompt prompt) { Message[] messages = new Message[prompt.Count]; for (int i = 0; i < prompt.Count; ++i) { - messages[i] = new Message { role = prompt[i].Source, content = prompt[i].GetText() }; + messages[i] = Create(prompt[i]); } return messages; } + + public static Message Create(IPromptSection section) + { + return new Message { role = section.Source, content = CreateContent(section) }; + } + + private static object CreateContent(IPromptSection section) + { + if (section is IMultimodalPromptSection multimodal && HasImages(multimodal)) + { + IReadOnlyList parts = multimodal.ContentParts; + ContentPart[] content = new ContentPart[parts.Count]; + for (int i = 0; i < parts.Count; ++i) + { + content[i] = ContentPart.Create(parts[i]); + } + return content; + } + return section.GetText(); + } + + private static bool HasImages(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + if (parts[i].IsImage) + { + return true; + } + } + return false; + } + } + + // A single content part for a Chat Completions multimodal message. + private struct ContentPart + { + public string type { get; set; } + public string? text { get; set; } + public ImageUrl? image_url { get; set; } + + public static ContentPart Create(PromptContentPart part) + { + if (part.IsImage) + { + PromptImage image = part.Image!; + return new ContentPart + { + type = "image_url", + image_url = new ImageUrl + { + url = image.Url, + detail = DetailToString(image.Detail) + } + }; + } + return new ContentPart { type = "text", text = part.Text ?? string.Empty }; + } + + // Auto is the service default, so it is omitted from the wire format. + private static string? DetailToString(ImageDetail detail) + { + switch (detail) + { + case ImageDetail.Low: + return "low"; + case ImageDetail.High: + return "high"; + default: + return null; + } + } + } + + private struct ImageUrl + { + public string url { get; set; } + public string? detail { get; set; } + } + + private struct ResponseChoice + { + public ResponseMessage message { get; set; } + } + + private struct ResponseMessage + { + public string role { get; set; } + public string content { get; set; } + } + + // Request for the OpenAI Responses API. Uses 'input' (array of messages) instead of 'messages', + // and 'max_output_tokens' instead of 'max_tokens'. + private struct ResponsesRequest + { + public string? model { get; set; } + public Message[] input { get; set; } + public double? temperature { get; set; } + public int? max_output_tokens { get; set; } + + public static ResponsesRequest Create(string model, Prompt prompt, TranslationSettings settings) + { + return new ResponsesRequest + { + model = model, + input = Message.Create(prompt), + temperature = (settings.Temperature > 0) ? settings.Temperature : 0, + max_output_tokens = (settings.MaxTokens > 0) ? settings.MaxTokens : null + }; + } + } + + // Response from the OpenAI Responses API. The text is returned inside output[n].content[m].text + // where the matching output item has type == "message" and the content item has type == "output_text". + private struct ResponsesResponse + { + public OutputItem[] output { get; set; } + + public string GetText() + { + if (output is not null) + { + for (int i = 0; i < output.Length; ++i) + { + OutputItem item = output[i]; + if (!string.Equals(item.type, "message", StringComparison.Ordinal) || item.content is null) + { + continue; + } + for (int j = 0; j < item.content.Length; ++j) + { + OutputContent content = item.content[j]; + if (string.Equals(content.type, "output_text", StringComparison.Ordinal) && content.text is not null) + { + return content.text; + } + } + } + } + return string.Empty; + } + } + + private struct OutputItem + { + public string type { get; set; } + public string? role { get; set; } + public OutputContent[] content { get; set; } } - private struct Choice + private struct OutputContent { - public Message message { get; set; } + public string type { get; set; } + public string text { get; set; } } protected virtual void Dispose(bool disposing) diff --git a/src/typechat/MultimodalPromptSection.cs b/src/typechat/MultimodalPromptSection.cs new file mode 100644 index 0000000..519c337 --- /dev/null +++ b/src/typechat/MultimodalPromptSection.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// A prompt section that can contain a mix of text and images. +/// +/// Build a section by adding text and image parts in the order they should appear: +/// +/// var section = new MultimodalPromptSection() +/// .AddText("What is in this image?") +/// .AddImage(PromptImage.FromFile("cat.png")); +/// +/// +/// Language models that support images (such as GPT-4-vision, GPT-4-omni and GPT-4-turbo) will receive +/// the images. Models that only support text will receive the concatenated text via . +/// +public class MultimodalPromptSection : IMultimodalPromptSection +{ + private readonly string _source; + private readonly List _parts; + + /// + /// Create a new, empty multimodal prompt section. + /// + /// The source of the section. Defaults to + public MultimodalPromptSection(string? source = null) + { + _source = string.IsNullOrEmpty(source) ? PromptSection.Sources.User : source!; + _parts = new List(); + } + + /// + /// Create a new multimodal prompt section from a set of content parts. + /// + /// The source of the section + /// The content parts + public MultimodalPromptSection(string source, IEnumerable parts) + : this(source) + { + ArgumentVerify.ThrowIfNull(parts, nameof(parts)); + foreach (var part in parts) + { + Add(part); + } + } + + /// + /// The source of this section. Typical sources are defined in . + /// + public string? Source => _source; + + /// + /// The ordered content parts (text and images) that make up this section. + /// + public IReadOnlyList ContentParts => _parts; + + /// + /// True when this section contains at least one image part. + /// + public bool HasImages + { + get + { + for (int i = 0; i < _parts.Count; ++i) + { + if (_parts[i].IsImage) + { + return true; + } + } + return false; + } + } + + /// + /// Add a content part to the section. + /// + /// the part to add + /// this section, to allow chaining + public MultimodalPromptSection Add(PromptContentPart part) + { + ArgumentVerify.ThrowIfNull(part, nameof(part)); + _parts.Add(part); + return this; + } + + /// + /// Add a text part to the section. + /// + /// the text to add + /// this section, to allow chaining + public MultimodalPromptSection AddText(string text) => Add(PromptContentPart.FromText(text)); + + /// + /// Add an image part to the section. + /// + /// the image to add + /// this section, to allow chaining + public MultimodalPromptSection AddImage(PromptImage image) => Add(PromptContentPart.FromImage(image)); + + /// + /// Add an image part to the section. + /// + /// An http(s) url to a hosted image, or a data uri containing base64 encoded image bytes + /// Controls how the model should process the image + /// this section, to allow chaining + public MultimodalPromptSection AddImage(string url, ImageDetail detail = ImageDetail.Auto) => AddImage(new PromptImage(url, detail)); + + /// + /// Return the concatenation of all text parts in this section. Image parts are ignored. + /// + /// string + public string GetText() + { + if (_parts.Count == 0) + { + return string.Empty; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < _parts.Count; ++i) + { + PromptContentPart part = _parts[i]; + if (part.IsText && !string.IsNullOrEmpty(part.Text)) + { + if (sb.Length > 0) + { + sb.Append('\n'); + } + sb.Append(part.Text); + } + } + return sb.ToString(); + } + + /// + /// Create a new multimodal section whose source is . + /// + /// MultimodalPromptSection + public static MultimodalPromptSection FromUser() => new MultimodalPromptSection(PromptSection.Sources.User); +} diff --git a/src/typechat/OpenAIConfig.cs b/src/typechat/OpenAIConfig.cs index 6c4128b..832f048 100644 --- a/src/typechat/OpenAIConfig.cs +++ b/src/typechat/OpenAIConfig.cs @@ -83,6 +83,17 @@ public OpenAIConfig() { } /// public string ApiVersion { get; set; } = "2023-05-15"; + /// + /// Selects the OpenAI REST API used by : + /// + /// null (default): auto-detect. The Responses API is used when + /// targets a path that ends with /responses; otherwise the Chat Completions API is used. + /// true: always use the Responses API. + /// false: always use the Chat Completions API. + /// + /// + public bool? UseResponsesApi { get; set; } + /// /// Http Settings /// diff --git a/src/typechat/PromptContentPart.cs b/src/typechat/PromptContentPart.cs new file mode 100644 index 0000000..a71bcf3 --- /dev/null +++ b/src/typechat/PromptContentPart.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// The kind of content carried by a . +/// +public enum PromptContentType +{ + /// + /// The part contains text. + /// + Text, + + /// + /// The part contains an image. + /// + Image, +} + +/// +/// A single part of a multimodal prompt section. A part is either text or an image. +/// A is an ordered collection of these parts, allowing +/// text and images to be interleaved within one prompt section. +/// +public sealed class PromptContentPart +{ + private readonly string? _text; + private readonly PromptImage? _image; + + private PromptContentPart(PromptContentType type, string? text, PromptImage? image) + { + Type = type; + _text = text; + _image = image; + } + + /// + /// The kind of content this part carries. + /// + public PromptContentType Type { get; } + + /// + /// True when this part contains text. + /// + public bool IsText => Type == PromptContentType.Text; + + /// + /// True when this part contains an image. + /// + public bool IsImage => Type == PromptContentType.Image; + + /// + /// The text of this part, or null when this is not a text part. + /// + public string? Text => _text; + + /// + /// The image of this part, or null when this is not an image part. + /// + public PromptImage? Image => _image; + + /// + /// Create a text content part. + /// + /// text + /// PromptContentPart + public static PromptContentPart FromText(string text) + { + ArgumentVerify.ThrowIfNull(text, nameof(text)); + return new PromptContentPart(PromptContentType.Text, text, null); + } + + /// + /// Create an image content part. + /// + /// image + /// PromptContentPart + public static PromptContentPart FromImage(PromptImage image) + { + ArgumentVerify.ThrowIfNull(image, nameof(image)); + return new PromptContentPart(PromptContentType.Image, null, image); + } + + /// + /// Implicitly create a text content part from a string. + /// + /// text + public static implicit operator PromptContentPart(string text) => FromText(text); + + /// + /// Implicitly create an image content part from a . + /// + /// image + public static implicit operator PromptContentPart(PromptImage image) => FromImage(image); +} diff --git a/src/typechat/PromptImage.cs b/src/typechat/PromptImage.cs new file mode 100644 index 0000000..edfe652 --- /dev/null +++ b/src/typechat/PromptImage.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// An image referenced by a multimodal prompt section. +/// The image is identified by a , which may be either: +/// +/// An http(s) url to a hosted image +/// A data uri that embeds base64 encoded image bytes: data:{mediaType};base64,{data} +/// +/// Models such as GPT-4-vision, GPT-4-omni and GPT-4-turbo can accept images as part of a prompt. +/// +public class PromptImage +{ + /// + /// Create a new PromptImage + /// + /// An http(s) url to a hosted image, or a data uri containing base64 encoded image bytes + /// Controls how the model should process the image + public PromptImage(string url, ImageDetail detail = ImageDetail.Auto) + { + ArgumentVerify.ThrowIfNullOrEmpty(url, nameof(url)); + Url = url; + Detail = detail; + } + + /// + /// An http(s) url to a hosted image, or a data uri containing base64 encoded image bytes. + /// + public string Url { get; } + + /// + /// Controls how the model should process the image. + /// + public ImageDetail Detail { get; } + + /// + /// True when is a data uri that embeds the image bytes, rather than a remote url. + /// + public bool IsDataUri => Url.StartsWith("data:", StringComparison.OrdinalIgnoreCase); + + /// + /// Create a PromptImage from raw image bytes. The bytes are encoded as a base64 data uri. + /// + /// image bytes + /// the image media (MIME) type, e.g. "image/png" + /// Controls how the model should process the image + /// PromptImage + public static PromptImage FromBytes(byte[] bytes, string mediaType = "image/png", ImageDetail detail = ImageDetail.Auto) + { + ArgumentVerify.ThrowIfNull(bytes, nameof(bytes)); + ArgumentVerify.ThrowIfNullOrEmpty(mediaType, nameof(mediaType)); + string dataUri = $"data:{mediaType};base64,{Convert.ToBase64String(bytes)}"; + return new PromptImage(dataUri, detail); + } + + /// + /// Create a PromptImage by loading an image file and encoding its bytes as a base64 data uri. + /// The media type is inferred from the file extension unless is supplied. + /// + /// path to the image file + /// (optional) the image media (MIME) type. Inferred from the file extension when null + /// Controls how the model should process the image + /// PromptImage + public static PromptImage FromFile(string filePath, string? mediaType = null, ImageDetail detail = ImageDetail.Auto) + { + ArgumentVerify.ThrowIfNullOrEmpty(filePath, nameof(filePath)); + mediaType ??= GetMediaType(filePath); + byte[] bytes = File.ReadAllBytes(filePath); + return FromBytes(bytes, mediaType, detail); + } + + /// + /// Infer an image media (MIME) type from a file name or url extension. + /// Returns the wildcard type "image/*" when the extension is not recognized. + /// + /// a file path or url + /// media type + public static string GetMediaType(string pathOrUrl) + { + if (string.IsNullOrEmpty(pathOrUrl)) + { + return "image/*"; + } + string ext = Path.GetExtension(pathOrUrl).ToLowerInvariant(); + switch (ext) + { + case ".png": + return "image/png"; + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".gif": + return "image/gif"; + case ".webp": + return "image/webp"; + case ".bmp": + return "image/bmp"; + case ".tif": + case ".tiff": + return "image/tiff"; + case ".svg": + return "image/svg+xml"; + default: + return "image/*"; + } + } +} diff --git a/tests/TypeChat.IntegrationTests/Readme.md b/tests/TypeChat.IntegrationTests/Readme.md index c422dec..5afb616 100644 --- a/tests/TypeChat.IntegrationTests/Readme.md +++ b/tests/TypeChat.IntegrationTests/Readme.md @@ -8,4 +8,15 @@ The integration tests **require an Azure or OpenAI key** and will just skip if o - Build ### About API Keys -Read about API Keys in the [Project README](../../README.md) \ No newline at end of file +Read about API Keys in the [Project README](../../README.md) + +## Multimodal (image) and Responses API tests + +`TestMultimodalEndToEnd` covers image input and the OpenAI Responses API: + +- `DescribeMicrosoftLogo_FromFile` sends the bundled `microsoft-logo.png` to the model and validates + the strongly typed result. It requires a **vision capable** model (e.g. gpt-4o); the api-version is + set to a vision capable value automatically. +- `TranslateSentiment_ResponsesApi` exercises the `/responses` route. The api-version is set to a + Responses-capable value automatically; the configured resource/deployment must support the + Responses API. \ No newline at end of file diff --git a/tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs b/tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs new file mode 100644 index 0000000..8db5332 --- /dev/null +++ b/tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat.Tests; + +/// +/// Online (end to end) tests for multimodal (image) prompts. +/// +/// These use the built-in (which supports image content) and require a +/// vision capable model such as gpt-4o. The CI pipeline is configured with a gpt-4o deployment. +/// The Microsoft logo image ships with the test project (see the csproj) and is copied next to the +/// test binaries. +/// +public class TestMultimodalEndToEnd : TypeChatTest, IClassFixture +{ + // Image (vision) input on Azure OpenAI requires a recent api-version. The default OpenAIConfig + // api-version predates vision, so use a vision capable version for these tests. + private const string VisionApiVersion = "2024-08-01-preview"; + // The OpenAI Responses API requires a recent (preview) api-version on Azure. + private const string ResponsesApiVersion = "2025-04-01-preview"; + private const string LogoImagePath = "microsoft-logo.png"; + + private readonly Config _config; + + public TestMultimodalEndToEnd(Config config, ITestOutputHelper output) + : base(output) + { + _config = config; + } + + [SkippableFact] + public async Task DescribeMicrosoftLogo_FromFile() + { + Skip.If(!CanRunEndToEndTest(_config)); + + LogoInfo info = await DescribeImage(PromptImage.FromFile(LogoImagePath, "image/png", ImageDetail.High)); + + Assert.NotNull(info); + WriteLine($"Brand='{info.Brand}' ContainsText={info.ContainsText} Colors=[{string.Join(", ", info.Colors ?? Array.Empty())}]"); + + // The Microsoft logo contains the word "Microsoft". + Assert.True(info.ContainsText); + Assert.False(string.IsNullOrEmpty(info.Brand)); + Assert.Contains("microsoft", info.Brand, StringComparison.OrdinalIgnoreCase); + } + + [SkippableFact] + public async Task TranslateSentiment_ResponsesApi() + { + Skip.If(!CanRunEndToEndTest(_config)); + + LanguageModel model = new LanguageModel(ResponsesConfig(_config.OpenAI)); + JsonTranslator translator = new JsonTranslator( + model, + new TypeValidator() + ); + + SentimentResponse response = await translator.TranslateAsync("Tonights gonna be a good night! A good good night!"); + Assert.NotNull(response); + Assert.False(string.IsNullOrEmpty(response.Sentiment)); + WriteLine($"[Responses API] Sentiment='{response.Sentiment}'"); + } + + private async Task DescribeImage(PromptImage image) + { + LanguageModel model = new LanguageModel(VisionConfig(_config.OpenAI)); + JsonTranslator translator = new JsonTranslator(model); + + Prompt request = new Prompt(); + request.Add(new MultimodalPromptSection() + .AddText("Describe the following image.") + .AddImage(image)); + + return await translator.TranslateAsync(request, null); + } + + /// + /// Copy the shared fixture config (so we don't mutate it) and make it suitable for image input: + /// a vision capable Azure api-version and a longer timeout, since vision calls can be slower. + /// + private static OpenAIConfig VisionConfig(OpenAIConfig src) + { + return new OpenAIConfig + { + Azure = src.Azure, + Endpoint = src.Endpoint, + ApiKey = src.ApiKey, + Organization = src.Organization, + Model = src.Model, + ApiVersion = src.Azure ? VisionApiVersion : src.ApiVersion, + TimeoutMs = Math.Max(src.TimeoutMs, 60 * 1000), + MaxRetries = src.MaxRetries, + MaxPauseMs = src.MaxPauseMs, + ApiTokenProvider = src.ApiTokenProvider, + UseResponsesApi = src.UseResponsesApi, + }; + } + + /// + /// Copy the shared fixture config and switch it to the OpenAI Responses API with an api-version + /// that supports it on Azure. + /// + private static OpenAIConfig ResponsesConfig(OpenAIConfig src) + { + return new OpenAIConfig + { + Azure = src.Azure, + Endpoint = src.Endpoint, + ApiKey = src.ApiKey, + Organization = src.Organization, + Model = src.Model, + ApiVersion = src.Azure ? ResponsesApiVersion : src.ApiVersion, + TimeoutMs = Math.Max(src.TimeoutMs, 60 * 1000), + MaxRetries = src.MaxRetries, + MaxPauseMs = src.MaxPauseMs, + ApiTokenProvider = src.ApiTokenProvider, + UseResponsesApi = true, + }; + } +} + +/// +/// A strongly typed description of a logo image. +/// +public class LogoInfo +{ + [Comment("The name of the company or brand shown in the image, if identifiable")] + public string Brand { get; set; } + + [Comment("True if the image contains any text")] + public bool ContainsText { get; set; } + + [Comment("The dominant colors in the image")] + public string[] Colors { get; set; } +} diff --git a/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj b/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj index 7bbbf9c..66542f0 100644 --- a/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj +++ b/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj @@ -46,4 +46,10 @@ + + + Always + + + diff --git a/tests/TypeChat.TestLib/MockHttp.cs b/tests/TypeChat.TestLib/MockHttp.cs index e1e1e1a..1c04a3b 100644 --- a/tests/TypeChat.TestLib/MockHttp.cs +++ b/tests/TypeChat.TestLib/MockHttp.cs @@ -21,17 +21,31 @@ public MockHttpHandler(int statusCode = 0) public int RequestCount { get; set; } = 0; public HttpRequestMessage? LastRequest { get; set; } + public string? LastRequestBody { get; set; } public void Clear() { RequestCount = 0; LastRequest = null; + LastRequestBody = null; } protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { LastRequest = request; RequestCount++; + // Capture the request body now: the caller disposes the request (and its content) after this + // returns. The content may already be disposed on retry attempts (the caller reuses it), so guard. + if (request.Content is not null) + { + try + { + LastRequestBody = request.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + } + catch (ObjectDisposedException) + { + } + } HttpResponseMessage response = Response; if (response is null) { diff --git a/tests/TypeChat.UnitTests/TestMultimodal.cs b/tests/TypeChat.UnitTests/TestMultimodal.cs new file mode 100644 index 0000000..1f37dc9 --- /dev/null +++ b/tests/TypeChat.UnitTests/TestMultimodal.cs @@ -0,0 +1,445 @@ +// Copyright (c) Microsoft. All rights reserved. + +extern alias ExtensionsAI; +using System.Text.Json; +using System.Threading; +using Microsoft.Extensions.AI; +using Microsoft.TypeChat.Tests; +using MeaiChatModel = ExtensionsAI::Microsoft.TypeChat.ChatLanguageModel; + +namespace Microsoft.TypeChat.UnitTests; + +public class TestMultimodal : TypeChatTest +{ + #region Content types + + [Fact] + public void PromptImage_FromBytes_ProducesDataUri() + { + byte[] bytes = new byte[] { 1, 2, 3, 4 }; + PromptImage image = PromptImage.FromBytes(bytes, "image/png", ImageDetail.Low); + + Assert.True(image.IsDataUri); + Assert.Equal(ImageDetail.Low, image.Detail); + Assert.StartsWith("data:image/png;base64,", image.Url); + Assert.Equal(Convert.ToBase64String(bytes), image.Url.Substring("data:image/png;base64,".Length)); + } + + [Fact] + public void PromptImage_Url_IsNotDataUri() + { + PromptImage image = new PromptImage("https://example.com/cat.png"); + Assert.False(image.IsDataUri); + Assert.Equal(ImageDetail.Auto, image.Detail); + } + + [Fact] + public void PromptImage_FromFile_LoadsMicrosoftLogo() + { + // The shared Microsoft logo asset is copied next to the test binaries (see csproj). + PromptImage image = PromptImage.FromFile("microsoft-logo.png"); + Assert.True(image.IsDataUri); + Assert.StartsWith("data:image/png;base64,", image.Url); + Assert.True(image.Url.Length > 1000); // real image bytes were embedded + } + + [Theory] + [InlineData("cat.png", "image/png")] + [InlineData("cat.PNG", "image/png")] + [InlineData("cat.jpg", "image/jpeg")] + [InlineData("cat.jpeg", "image/jpeg")] + [InlineData("cat.gif", "image/gif")] + [InlineData("cat.webp", "image/webp")] + [InlineData("https://example.com/photos/cat.bmp", "image/bmp")] + [InlineData("cat", "image/*")] + [InlineData("https://example.com/image", "image/*")] + public void PromptImage_GetMediaType(string pathOrUrl, string expected) + { + Assert.Equal(expected, PromptImage.GetMediaType(pathOrUrl)); + } + + [Fact] + public void PromptContentPart_ImplicitConversions() + { + PromptContentPart text = "hello"; + Assert.True(text.IsText); + Assert.False(text.IsImage); + Assert.Equal("hello", text.Text); + + PromptContentPart image = new PromptImage("https://example.com/cat.png"); + Assert.True(image.IsImage); + Assert.False(image.IsText); + Assert.NotNull(image.Image); + } + + [Fact] + public void MultimodalSection_BuildsOrderedParts() + { + MultimodalPromptSection section = new MultimodalPromptSection() + .AddText("a") + .AddImage("https://example.com/cat.png", ImageDetail.High) + .AddText("b"); + + Assert.Equal(PromptSection.Sources.User, section.Source); + Assert.True(section.HasImages); + Assert.Equal(3, section.ContentParts.Count); + Assert.True(section.ContentParts[0].IsText); + Assert.True(section.ContentParts[1].IsImage); + Assert.Equal(ImageDetail.High, section.ContentParts[1].Image!.Detail); + Assert.True(section.ContentParts[2].IsText); + + // GetText concatenates text parts and ignores images, so text-only models still work. + Assert.Equal("a\nb", section.GetText()); + } + + [Fact] + public void MultimodalSection_TextOnly_HasNoImages() + { + MultimodalPromptSection section = new MultimodalPromptSection().AddText("just text"); + Assert.False(section.HasImages); + Assert.Equal("just text", section.GetText()); + } + + #endregion + + #region LanguageModel - Chat Completions wire format + + [Fact] + public async Task LanguageModel_TextOnly_SendsStringContent() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + JsonArray messages = body["messages"]!.AsArray(); + Assert.Single(messages); + JsonNode content = messages[0]!["content"]!; + Assert.Equal(JsonValueKind.String, content.GetValueKind()); + Assert.Equal("Hello", content.GetValue()); + Assert.Null(body["input"]); + } + + [Fact] + public async Task LanguageModel_Multimodal_SendsContentParts() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Model = "gpt-4o"; + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + Prompt prompt = new Prompt(); + prompt.Add(new MultimodalPromptSection() + .AddText("What is in this image?") + .AddImage("https://example.com/cat.png", ImageDetail.High)); + + string text = await model.CompleteAsync(prompt); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + JsonNode content = body["messages"]!.AsArray()[0]!["content"]!; + Assert.Equal(JsonValueKind.Array, content.GetValueKind()); + + JsonArray parts = content.AsArray(); + Assert.Equal(2, parts.Count); + Assert.Equal("text", parts[0]!["type"]!.GetValue()); + Assert.Equal("What is in this image?", parts[0]!["text"]!.GetValue()); + Assert.Equal("image_url", parts[1]!["type"]!.GetValue()); + Assert.Equal("https://example.com/cat.png", parts[1]!["image_url"]!["url"]!.GetValue()); + Assert.Equal("high", parts[1]!["image_url"]!["detail"]!.GetValue()); + } + + [Fact] + public async Task LanguageModel_Multimodal_AutoDetail_IsOmitted() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + Prompt prompt = new Prompt(); + prompt.Add(new MultimodalPromptSection().AddImage("https://example.com/cat.png")); // Auto + + await model.CompleteAsync(prompt); + + JsonNode body = ParseBody(handler); + JsonNode imageUrl = body["messages"]!.AsArray()[0]!["content"]!.AsArray()[0]!["image_url"]!; + Assert.Null(imageUrl["detail"]); + } + + #endregion + + #region LanguageModel - Responses API + + [Fact] + public async Task LanguageModel_ResponsesEndpoint_UsesInputAndParsesOutput() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Endpoint = "https://api.openai.com/v1/responses"; + config.Model = "gpt-4o"; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + Assert.Null(body["messages"]); + Assert.NotNull(body["input"]); + Assert.Equal("gpt-4o", body["model"]!.GetValue()); + + JsonArray input = body["input"]!.AsArray(); + Assert.Equal("user", input[0]!["role"]!.GetValue()); + Assert.Equal("Hello", input[0]!["content"]!.GetValue()); + } + + [Fact] + public async Task LanguageModel_UseResponsesApiTrue_OverridesChatCompletionsEndpoint() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); // endpoint targets /chat/completions + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + Assert.NotNull(body["input"]); + Assert.Null(body["messages"]); + } + + [Fact] + public async Task LanguageModel_UseResponsesApiFalse_OverridesResponsesEndpoint() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Endpoint = "https://api.openai.com/v1/responses"; + config.UseResponsesApi = false; + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + Assert.NotNull(body["messages"]); + Assert.Null(body["input"]); + } + + [Fact] + public async Task LanguageModel_Azure_Responses_RoutesToResponsesUrl() + { + OpenAIConfig config = MockOpenAIConfig(azure: true); // base Azure resource url + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + await model.CompleteAsync("Hello"); + + string url = handler.LastRequest!.RequestUri!.AbsoluteUri; + Assert.Contains("/openai/responses", url, StringComparison.OrdinalIgnoreCase); + Assert.Contains("api-version=", url, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("chat/completions", url, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task LanguageModel_Azure_Responses_NormalizesChatCompletionsUrl() + { + OpenAIConfig config = MockOpenAIConfig(azure: true); + config.Endpoint = "https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-08-01-preview"; + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + await model.CompleteAsync("Hello"); + + string url = handler.LastRequest!.RequestUri!.AbsoluteUri; + Assert.Contains("/openai/responses", url, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("chat/completions", url, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("deployments", url, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task LanguageModel_OpenAI_Responses_NormalizesChatCompletionsUrl() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); // endpoint targets /v1/chat/completions + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + await model.CompleteAsync("Hello"); + + string url = handler.LastRequest!.RequestUri!.AbsoluteUri; + Assert.EndsWith("/v1/responses", url, StringComparison.OrdinalIgnoreCase); + } + + #endregion + + #region JsonTranslator integration + + [Fact] + public void JsonTranslatorPrompts_RequestSection_PreservesImages() + { + MultimodalPromptSection request = new MultimodalPromptSection() + .AddText("Describe this") + .AddImage("https://example.com/cat.png"); + + IPromptSection section = JsonTranslatorPrompts.RequestSection(request); + MultimodalPromptSection multimodal = Assert.IsType(section); + Assert.True(multimodal.HasImages); + } + + [Fact] + public void JsonTranslatorPrompts_RequestSection_TextOnly_StaysText() + { + IPromptSection section = JsonTranslatorPrompts.RequestSection(PromptSection.FromUser("hello")); + Assert.IsType(section); + } + + [Fact] + public async Task JsonTranslator_Multimodal_SendsImageToModel() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Model = "gpt-4o"; + MockHttpHandler handler = new MockHttpHandler(TranslatorCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + JsonTranslator translator = new JsonTranslator(model); + + Prompt request = new Prompt(); + request.Add(new MultimodalPromptSection() + .AddText("Describe this image") + .AddImage("https://example.com/cat.png", ImageDetail.High)); + + ImageDescription result = await translator.TranslateAsync(request, null); + Assert.Equal("A cat", result.Description); + + // The image must have survived all the way to the wire request. + JsonNode body = ParseBody(handler); + JsonArray messages = body["messages"]!.AsArray(); + bool sentImage = messages.Any(m => + m!["content"]!.GetValueKind() == JsonValueKind.Array && + m["content"]!.AsArray().Any(p => p!["type"]!.GetValue() == "image_url")); + Assert.True(sentImage); + } + + #endregion + + #region Microsoft.Extensions.AI mapping + + [Fact] + public async Task Meai_Multimodal_MapsToAIContent() + { + FakeChatClient client = new FakeChatClient("Hello there!"); + MeaiChatModel model = new MeaiChatModel(client, "gpt-4o"); + + Prompt prompt = new Prompt(); + prompt.Add(new MultimodalPromptSection() + .AddText("What is this?") + .AddImage(PromptImage.FromBytes(new byte[] { 1, 2, 3 }, "image/png")) // -> DataContent + .AddImage("https://example.com/cat.jpg") // -> UriContent + .AddImage("https://example.com/image")); // -> UriContent (image/*) + + string text = await model.CompleteAsync(prompt); + Assert.Equal("Hello there!", text); + + ChatMessage message = Assert.Single(client.LastMessages!); + Assert.Equal(ChatRole.User, message.Role); + Assert.Contains(message.Contents, c => c is TextContent t && t.Text == "What is this?"); + Assert.Contains(message.Contents, c => c is DataContent); + Assert.Equal(2, message.Contents.Count(c => c is UriContent)); + } + + [Fact] + public async Task Meai_TextOnly_MapsToText() + { + FakeChatClient client = new FakeChatClient("ok"); + MeaiChatModel model = new MeaiChatModel(client, "gpt-4o"); + + await model.CompleteAsync("Hello"); + + ChatMessage message = Assert.Single(client.LastMessages!); + Assert.Equal(ChatRole.User, message.Role); + Assert.Equal("Hello", message.Text); + } + + #endregion + + #region Helpers + + private static JsonNode ParseBody(MockHttpHandler handler) + { + Assert.NotNull(handler.LastRequestBody); + return JsonNode.Parse(handler.LastRequestBody!)!; + } + + private static string ChatCompletionsCanned() + { + return @"{ + ""choices"": [{ + ""index"": 0, + ""message"": { ""role"": ""assistant"", ""content"": ""Hello there!"" }, + ""finish_reason"": ""stop"" + }] + }"; + } + + private static string ResponsesCanned() + { + return @"{ + ""id"": ""resp_123"", + ""output"": [{ + ""type"": ""message"", + ""role"": ""assistant"", + ""content"": [{ ""type"": ""output_text"", ""text"": ""Hello there!"" }] + }] + }"; + } + + private static string TranslatorCanned() + { + // The assistant message content is itself a JSON object matching ImageDescription. + return @"{ + ""choices"": [{ + ""message"": { ""role"": ""assistant"", ""content"": ""{\""Description\"": \""A cat\""}"" } + }] + }"; + } + + private sealed class FakeChatClient : IChatClient + { + private readonly string _responseText; + + public FakeChatClient(string responseText) + { + _responseText = responseText; + } + + public IReadOnlyList? LastMessages { get; private set; } + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + LastMessages = messages.ToList(); + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, _responseText))); + } + + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + throw new NotSupportedException(); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + } + + #endregion +} + +public class ImageDescription +{ + public string Description { get; set; } +} diff --git a/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj b/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj index 0e9816a..f8832c4 100644 --- a/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj +++ b/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj @@ -29,7 +29,14 @@ + + + + PreserveNewest + + +