diff --git a/README.md b/README.md index e487857..59b6e94 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ TypeChat uses language models to translate user intent into JSON that conforms t TypeChat provides: * Json translation, validation, repair and deserialization. * Extensibility: interfaces for customizing schemas, validators and prompts. +* **Multimodal prompts**: send images (by url or local file) alongside text using `MultimodalPromptSection`, so vision capable models (e.g. gpt-4o) can translate what they *see* into strongly typed objects. +* **OpenAI Responses API**: `LanguageModel` supports both the Chat Completions and Responses APIs. The Responses API is auto-selected when the endpoint path ends with `/responses`, or set `OpenAIConfig.UseResponsesApi`. * Schema export: classes to generate schema for the .NET Type you want to translate to. Exported schema includes dependencies and base classes. The exported schema is specified using **Typescript**, which can concisely express schema for JSON objects. * Support for common scenarios shown in TypeChat examples. When you encounter limitations (such as how generics are currently exported), you can supply schema text, such as Typescript authored by hand. * Helper attributes for Vocabularies and Comments. Vocabularies are string tables that constrain the values that can be assigned to string properties. Dynamic loading of vocabularies enables scenarios where they vary at runtime. @@ -131,6 +133,9 @@ The following examples demonstrate how to use JsonTranslator, Schemas and Vocabu * [Calendar](./examples/Calendar): Transform language into calendar actions * [Restaurant](./examples/Restaurant): Order processing at a pizza restaurant +### Multimodal (images) +* [Multimodal](./examples/Multimodal): Send an image to a vision capable model and translate what it sees into a strongly typed object. Requires a vision model such as gpt-4o. + ### Hierarchical schemas * [MultiSchema](./examples/MultiSchema): dynamically route user intent to other 'sub-apps' * [SchemaHierarchy](./examples/SchemaHierarchy): A Json Translator than uses multiple child JsonTranslators. For each user request, it picks the semantically ***nearest*** child translator and routes the input to it. diff --git a/TypeChat.sln b/TypeChat.sln index e96e373..cbf1ab1 100644 --- a/TypeChat.sln +++ b/TypeChat.sln @@ -64,88 +64,258 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TypeChat.Schema", "src\type EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TypeChat.Extensions.AI", "src\typechat.meai\TypeChat.Extensions.AI.csproj", "{9F7809A0-7F9E-D22A-A697-F02E1613F65D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Multimodal", "examples\Multimodal\Multimodal.csproj", "{48848A65-8862-4376-81A2-9CE8DC3DCFD7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x64.ActiveCfg = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x64.Build.0 = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x86.ActiveCfg = Debug|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Debug|x86.Build.0 = Debug|Any CPU {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|Any CPU.ActiveCfg = Release|Any CPU {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|Any CPU.Build.0 = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x64.ActiveCfg = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x64.Build.0 = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x86.ActiveCfg = Release|Any CPU + {3E986252-C73E-4FD5-99A1-CE2FAB8899FC}.Release|x86.Build.0 = Release|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x64.ActiveCfg = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x64.Build.0 = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x86.ActiveCfg = Debug|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Debug|x86.Build.0 = Debug|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|Any CPU.ActiveCfg = Release|Any CPU {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|Any CPU.Build.0 = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x64.ActiveCfg = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x64.Build.0 = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x86.ActiveCfg = Release|Any CPU + {68E9AF34-7982-4BA4-864D-D24DF937A6A2}.Release|x86.Build.0 = Release|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x64.ActiveCfg = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x64.Build.0 = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x86.ActiveCfg = Debug|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Debug|x86.Build.0 = Debug|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Release|Any CPU.ActiveCfg = Release|Any CPU {AB618076-FA46-4014-871A-B78179E248FF}.Release|Any CPU.Build.0 = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x64.ActiveCfg = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x64.Build.0 = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x86.ActiveCfg = Release|Any CPU + {AB618076-FA46-4014-871A-B78179E248FF}.Release|x86.Build.0 = Release|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x64.ActiveCfg = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x64.Build.0 = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x86.ActiveCfg = Debug|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Debug|x86.Build.0 = Debug|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Release|Any CPU.ActiveCfg = Release|Any CPU {67AC4986-54B8-490F-8582-657651E3E62C}.Release|Any CPU.Build.0 = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x64.ActiveCfg = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x64.Build.0 = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x86.ActiveCfg = Release|Any CPU + {67AC4986-54B8-490F-8582-657651E3E62C}.Release|x86.Build.0 = Release|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x64.ActiveCfg = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x64.Build.0 = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x86.ActiveCfg = Debug|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Debug|x86.Build.0 = Debug|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Release|Any CPU.ActiveCfg = Release|Any CPU {C223E3DB-9B2E-4839-A56E-838030300938}.Release|Any CPU.Build.0 = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x64.ActiveCfg = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x64.Build.0 = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x86.ActiveCfg = Release|Any CPU + {C223E3DB-9B2E-4839-A56E-838030300938}.Release|x86.Build.0 = Release|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x64.ActiveCfg = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x64.Build.0 = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x86.ActiveCfg = Debug|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Debug|x86.Build.0 = Debug|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|Any CPU.ActiveCfg = Release|Any CPU {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|Any CPU.Build.0 = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x64.ActiveCfg = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x64.Build.0 = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x86.ActiveCfg = Release|Any CPU + {131566E6-2B78-486C-AFF5-7202A437E79F}.Release|x86.Build.0 = Release|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x64.ActiveCfg = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x64.Build.0 = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x86.ActiveCfg = Debug|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Debug|x86.Build.0 = Debug|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|Any CPU.ActiveCfg = Release|Any CPU {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|Any CPU.Build.0 = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x64.ActiveCfg = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x64.Build.0 = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x86.ActiveCfg = Release|Any CPU + {D78553D9-4CBD-4C3A-A11B-58EA0077FD84}.Release|x86.Build.0 = Release|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x64.ActiveCfg = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x64.Build.0 = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x86.ActiveCfg = Debug|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Debug|x86.Build.0 = Debug|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|Any CPU.ActiveCfg = Release|Any CPU {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|Any CPU.Build.0 = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x64.ActiveCfg = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x64.Build.0 = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x86.ActiveCfg = Release|Any CPU + {A6E4FA5A-116E-4770-8BD2-31185FA13632}.Release|x86.Build.0 = Release|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x64.ActiveCfg = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x64.Build.0 = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x86.ActiveCfg = Debug|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Debug|x86.Build.0 = Debug|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|Any CPU.ActiveCfg = Release|Any CPU {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|Any CPU.Build.0 = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x64.ActiveCfg = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x64.Build.0 = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x86.ActiveCfg = Release|Any CPU + {5F8B1030-2D87-4942-BCB3-271B4541BAF3}.Release|x86.Build.0 = Release|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x64.ActiveCfg = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x64.Build.0 = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x86.ActiveCfg = Debug|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Debug|x86.Build.0 = Debug|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|Any CPU.ActiveCfg = Release|Any CPU {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|Any CPU.Build.0 = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x64.ActiveCfg = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x64.Build.0 = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x86.ActiveCfg = Release|Any CPU + {9598D98A-3688-45E5-9C51-1C4F385BD693}.Release|x86.Build.0 = Release|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x64.ActiveCfg = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x64.Build.0 = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x86.ActiveCfg = Debug|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Debug|x86.Build.0 = Debug|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|Any CPU.ActiveCfg = Release|Any CPU {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|Any CPU.Build.0 = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x64.ActiveCfg = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x64.Build.0 = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x86.ActiveCfg = Release|Any CPU + {F4B127FE-CAB9-483E-AA79-6A2F8665C5FC}.Release|x86.Build.0 = Release|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x64.Build.0 = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x86.ActiveCfg = Debug|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Debug|x86.Build.0 = Debug|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|Any CPU.ActiveCfg = Release|Any CPU {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|Any CPU.Build.0 = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x64.ActiveCfg = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x64.Build.0 = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x86.ActiveCfg = Release|Any CPU + {EF1231C5-56C4-439F-84F0-FE5D5315D4AA}.Release|x86.Build.0 = Release|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x64.ActiveCfg = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x64.Build.0 = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x86.ActiveCfg = Debug|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Debug|x86.Build.0 = Debug|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|Any CPU.ActiveCfg = Release|Any CPU {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|Any CPU.Build.0 = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x64.ActiveCfg = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x64.Build.0 = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x86.ActiveCfg = Release|Any CPU + {918FF1BA-C7F8-4A9C-9459-0247AC2BD3FE}.Release|x86.Build.0 = Release|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x64.ActiveCfg = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x64.Build.0 = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x86.ActiveCfg = Debug|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Debug|x86.Build.0 = Debug|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|Any CPU.ActiveCfg = Release|Any CPU {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|Any CPU.Build.0 = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x64.ActiveCfg = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x64.Build.0 = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x86.ActiveCfg = Release|Any CPU + {B7F278E8-CE85-4987-B292-F15238A80A75}.Release|x86.Build.0 = Release|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x64.ActiveCfg = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x64.Build.0 = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x86.ActiveCfg = Debug|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Debug|x86.Build.0 = Debug|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|Any CPU.ActiveCfg = Release|Any CPU {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|Any CPU.Build.0 = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x64.ActiveCfg = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x64.Build.0 = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x86.ActiveCfg = Release|Any CPU + {6FDA463B-4A1E-4E3E-B508-A14910FAF579}.Release|x86.Build.0 = Release|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x64.ActiveCfg = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x64.Build.0 = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x86.ActiveCfg = Debug|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Debug|x86.Build.0 = Debug|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|Any CPU.ActiveCfg = Release|Any CPU {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|Any CPU.Build.0 = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x64.ActiveCfg = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x64.Build.0 = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x86.ActiveCfg = Release|Any CPU + {C585B613-3BAF-4999-8FA5-7BF425AEB1E2}.Release|x86.Build.0 = Release|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x64.ActiveCfg = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x64.Build.0 = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x86.ActiveCfg = Debug|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Debug|x86.Build.0 = Debug|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|Any CPU.ActiveCfg = Release|Any CPU {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|Any CPU.Build.0 = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x64.ActiveCfg = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x64.Build.0 = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x86.ActiveCfg = Release|Any CPU + {80304E1F-5B1A-4142-B950-78383E1D1877}.Release|x86.Build.0 = Release|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x64.ActiveCfg = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x64.Build.0 = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x86.ActiveCfg = Debug|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Debug|x86.Build.0 = Debug|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|Any CPU.ActiveCfg = Release|Any CPU {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|Any CPU.Build.0 = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x64.ActiveCfg = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x64.Build.0 = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x86.ActiveCfg = Release|Any CPU + {3B193D73-6A6F-4666-B377-B08CCCC92A8B}.Release|x86.Build.0 = Release|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x64.ActiveCfg = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x64.Build.0 = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x86.ActiveCfg = Debug|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Debug|x86.Build.0 = Debug|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|Any CPU.ActiveCfg = Release|Any CPU {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|Any CPU.Build.0 = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x64.ActiveCfg = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x64.Build.0 = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x86.ActiveCfg = Release|Any CPU + {9F7809A0-7F9E-D22A-A697-F02E1613F65D}.Release|x86.Build.0 = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x64.ActiveCfg = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x64.Build.0 = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x86.ActiveCfg = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Debug|x86.Build.0 = Debug|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|Any CPU.Build.0 = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x64.ActiveCfg = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x64.Build.0 = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x86.ActiveCfg = Release|Any CPU + {48848A65-8862-4376-81A2-9CE8DC3DCFD7}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -165,6 +335,7 @@ Global {6FDA463B-4A1E-4E3E-B508-A14910FAF579} = {179655A3-3581-45F4-BE46-1BF4FABA7F97} {C585B613-3BAF-4999-8FA5-7BF425AEB1E2} = {EAEA9E40-7370-407E-B2DA-3F6E265262BF} {80304E1F-5B1A-4142-B950-78383E1D1877} = {179655A3-3581-45F4-BE46-1BF4FABA7F97} + {48848A65-8862-4376-81A2-9CE8DC3DCFD7} = {179655A3-3581-45F4-BE46-1BF4FABA7F97} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8A9AF678-E669-4AF2-9644-7EBB970DAC41} diff --git a/assets/microsoft-logo.png b/assets/microsoft-logo.png new file mode 100644 index 0000000..c76d3fc Binary files /dev/null and b/assets/microsoft-logo.png differ diff --git a/examples/Multimodal/ImageSchema.cs b/examples/Multimodal/ImageSchema.cs new file mode 100644 index 0000000..e0ea8d1 --- /dev/null +++ b/examples/Multimodal/ImageSchema.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; +using Microsoft.TypeChat.Schema; + +namespace Multimodal; + +/// +/// A strongly typed description of an image, produced by translating a multimodal prompt +/// (text + image) into this type. +/// +public class ImageResponse +{ + [Comment("A single sentence caption that describes the image")] + [JsonPropertyName("caption")] + public string Caption { get; set; } + + [Comment("The most prominent objects visible in the image")] + [JsonPropertyName("objects")] + public string[] Objects { get; set; } + + [Comment("The dominant colors in the image")] + [JsonPropertyName("colors")] + public string[] Colors { get; set; } + + [Comment("True if the image contains any text")] + [JsonPropertyName("containsText")] + public bool ContainsText { get; set; } +} diff --git a/examples/Multimodal/Multimodal.csproj b/examples/Multimodal/Multimodal.csproj new file mode 100644 index 0000000..7593aae --- /dev/null +++ b/examples/Multimodal/Multimodal.csproj @@ -0,0 +1,41 @@ + + + + Exe + net8.0 + latest + LatestMajor + enable + annotations + + + + + Always + + + + + + Always + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + + diff --git a/examples/Multimodal/Program.cs b/examples/Multimodal/Program.cs new file mode 100644 index 0000000..6a0fb4d --- /dev/null +++ b/examples/Multimodal/Program.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft. All rights reserved. +using Microsoft.TypeChat; + +namespace Multimodal; + +/// +/// This example shows how to send images to a vision capable model and translate what the model +/// "sees" into a strongly typed object. +/// +/// Each input line is either: +/// - a local image file path (e.g. C:\pics\cat.jpg), or +/// - an http(s) url to an image (e.g. https://.../cat.jpg) +/// +/// The image is placed into a multimodal prompt alongside an instruction, and JsonTranslator +/// translates the response into an . +/// +/// NOTE: This requires a vision capable model (e.g. gpt-4o). Set the model name in appSettings. +/// +public class MultimodalApp : ConsoleApp +{ + private readonly JsonTranslator _translator; + + public MultimodalApp() + { + OpenAIConfig config = Config.LoadOpenAI(); + + // TIP: To use the OpenAI Responses API instead of Chat Completions, either point + // config.Endpoint at a url whose path ends with /responses, or force it explicitly: + // config.UseResponsesApi = true; + + _translator = new JsonTranslator(new LanguageModel(config)); + } + + public override async Task ProcessInputAsync(string input, CancellationToken cancelToken) + { + PromptImage image = IsUrl(input) ? + new PromptImage(input, ImageDetail.Auto) : + PromptImage.FromFile(input); + + // Build a multimodal request: an instruction followed by the image. + Prompt request = new Prompt(); + request.Add(new MultimodalPromptSection() + .AddText("Describe the following image.") + .AddImage(image)); + + ImageResponse response = await _translator.TranslateAsync(request, null, null, cancelToken); + + Console.WriteLine($"Caption: {response.Caption}"); + Console.WriteLine($"Contains text: {response.ContainsText}"); + if (response.Objects is not null && response.Objects.Length > 0) + { + Console.WriteLine("Objects: " + string.Join(", ", response.Objects)); + } + if (response.Colors is not null && response.Colors.Length > 0) + { + Console.WriteLine("Colors: " + string.Join(", ", response.Colors)); + } + } + + private static bool IsUrl(string input) + { + return input.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + input.StartsWith("https://", StringComparison.OrdinalIgnoreCase); + } + + public static async Task Main(string[] args) + { + try + { + MultimodalApp app = new MultimodalApp(); + // Pass an input file as arg[0] to run in batch mode. + await app.RunAsync("🖼️> ", args.GetOrNull(0)); + } + catch (Exception ex) + { + WriteError(ex); + Console.ReadLine(); + return -1; + } + + return 0; + } +} diff --git a/examples/Multimodal/Readme.md b/examples/Multimodal/Readme.md new file mode 100644 index 0000000..c7da325 --- /dev/null +++ b/examples/Multimodal/Readme.md @@ -0,0 +1,45 @@ +# Multimodal + +Demonstrates sending **images** to a vision capable language model and translating what the model +sees into a **strongly typed** object using `JsonTranslator`. + +## What it shows + +- Building a multimodal prompt with [`MultimodalPromptSection`](../../src/typechat/MultimodalPromptSection.cs), + mixing text and images. +- Referencing images either by url or by local file (`PromptImage.FromFile`, which base64 encodes the image + into a data uri). +- Translating the multimodal request into a typed [`ImageResponse`](./ImageSchema.cs) — TypeChat validates and + repairs the model's JSON exactly as it does for text-only requests. + +```cs +Prompt request = new Prompt(); +request.Add(new MultimodalPromptSection() + .AddText("Describe the following image.") + .AddImage(PromptImage.FromFile("cat.jpg"))); + +ImageResponse response = await translator.TranslateAsync(request, null); +``` + +## Requirements + +- A **vision capable** model such as `gpt-4o`. Set the model name in `appSettings.Development.json`. + +## Running + +The example ships with the official Microsoft logo (`microsoft-logo.png`, sourced from +[../../assets](../../assets)) which is copied next to the built app, so it runs out of the box. + +- Interactive: run the project and paste an image file path or an http(s) url at the prompt. +- Batch: pass an input file whose lines are image paths/urls, e.g. `dotnet run -- input.txt`. + The bundled [input.txt](./input.txt) points at `microsoft-logo.png`. + +## OpenAI Responses API + +The built-in `LanguageModel` also supports the OpenAI **Responses API**. It is selected automatically when +the configured endpoint path ends with `/responses`, or you can force it: + +```cs +OpenAIConfig config = Config.LoadOpenAI(); +config.UseResponsesApi = true; // or false to force Chat Completions +``` diff --git a/examples/Multimodal/input.txt b/examples/Multimodal/input.txt new file mode 100644 index 0000000..b7284fd --- /dev/null +++ b/examples/Multimodal/input.txt @@ -0,0 +1,10 @@ +# Multimodal example input. +# +# Each non-comment line is processed as one image and described by the model. +# A line is either a local image file path or an http(s) url to an image. +# +# This requires a vision capable model (e.g. gpt-4o) configured in appSettings. +# +# The Microsoft logo below ships with this example and is copied next to the built app. +# Replace or add your own image paths / urls as needed. +microsoft-logo.png diff --git a/src/typechat.meai/ChatLanguageModel.cs b/src/typechat.meai/ChatLanguageModel.cs index d8bda2c..d7632af 100644 --- a/src/typechat.meai/ChatLanguageModel.cs +++ b/src/typechat.meai/ChatLanguageModel.cs @@ -110,9 +110,56 @@ public static void Append(this List history, IEnumerable history, IPromptSection message) { + if (message is IMultimodalPromptSection multimodal && HasImages(multimodal)) + { + history.Add(new ChatMessage(multimodal.GetRole(), ToContents(multimodal))); + return; + } history.Add(new ChatMessage(role: message.GetRole(), content: message.GetText())); } + internal static bool HasImages(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + if (parts[i].IsImage) + { + return true; + } + } + return false; + } + + internal static IList ToContents(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + List contents = new List(parts.Count); + for (int i = 0; i < parts.Count; ++i) + { + PromptContentPart part = parts[i]; + if (part.IsImage) + { + contents.Add(ToContent(part.Image!)); + } + else if (!string.IsNullOrEmpty(part.Text)) + { + contents.Add(new TextContent(part.Text)); + } + } + return contents; + } + + internal static AIContent ToContent(PromptImage image) + { + // A data uri (data:{mediaType};base64,{data}) carries its own media type and bytes. + if (image.IsDataUri) + { + return new DataContent(image.Url); + } + return new UriContent(image.Url, PromptImage.GetMediaType(image.Url)); + } + internal static bool IsRole(this ChatRole role, string label) { return role.Value.Equals(label, StringComparison.OrdinalIgnoreCase); diff --git a/src/typechat.schema/TypeEx.cs b/src/typechat.schema/TypeEx.cs index c5fe980..5785077 100644 --- a/src/typechat.schema/TypeEx.cs +++ b/src/typechat.schema/TypeEx.cs @@ -40,7 +40,21 @@ public static bool IsBoolean(this Type type) public static bool IsDateTime(this Type type) { - return type == typeof(DateTime) || type == typeof(TimeSpan); + // These are value types that System.Text.Json serializes to/from a JSON *string* rather than + // to an object. They must be treated as the 'string' primitive by the exporter (see + // Typescript.Types.ToPrimitive); otherwise they would be exported as (mostly empty) interfaces. + if (type == typeof(DateTime) || type == typeof(TimeSpan) || type == typeof(DateTimeOffset)) + { + return true; + } +#if NET6_0_OR_GREATER + // DateOnly/TimeOnly only exist on net6.0+, but Json serializes them as strings too. + if (type == typeof(DateOnly) || type == typeof(TimeOnly)) + { + return true; + } +#endif + return false; } public static bool IsNumber(this Type type) @@ -149,6 +163,113 @@ internal static bool IsTask(this Type type) return (args.IsNullOrEmpty()) ? null : args[0]; } + /// + /// Is the given type one that serializes to a JSON array? This includes arrays and any type + /// that implements IEnumerable{T} (List{T}, IList{T}, ICollection{T}, HashSet{T}...). + /// Strings (which are IEnumerable{char}) and dictionaries are deliberately excluded: they do + /// not serialize to JSON arrays. + /// + /// type to inspect + /// the type of the items in the array + /// true if the type is array-like + internal static bool IsArrayLike(this Type type, out Type elementType) + { + elementType = null; + + // Strings are IEnumerable but must be exported as scalar strings, not char arrays + if (type.IsString()) + { + return false; + } + // Dictionaries are IEnumerable> but must be exported as maps, not arrays. + // Use a structural check so that even a self-mapping dictionary is excluded here. + if (type.IsDictionaryShaped()) + { + return false; + } + if (type.IsArray) + { + elementType = type.GetElementType(); + return true; + } + + Type? enumerableType = FindGenericInterface(type, typeof(IEnumerable<>)); + if (enumerableType is not null) + { + Type genericArg = enumerableType.GetGenericArguments()[0]; + // Guard against a type that enumerates itself (e.g. class Node : IEnumerable): + // there is no finite element type to unwrap to, so treat it as a plain object instead. + if (genericArg != type) + { + elementType = genericArg; + return true; + } + return false; + } + // Non-generic IEnumerable (e.g. ArrayList): the element type is unknown, so treat it as 'any' + if (typeof(IEnumerable).IsAssignableFrom(type)) + { + elementType = typeof(object); + return true; + } + return false; + } + + /// + /// Is the given type one that serializes to a JSON object/map? This includes any type that + /// implements IDictionary{K,V} or IReadOnlyDictionary{K,V}, as well as the non-generic IDictionary. + /// + /// type to inspect + /// the type of the map's keys + /// the type of the map's values + /// true if the type is a dictionary + internal static bool IsDictionary(this Type type, out Type keyType, out Type valueType) + { + keyType = typeof(string); + valueType = typeof(object); + + Type? dictionaryType = FindGenericInterface(type, typeof(IDictionary<,>)) ?? + FindGenericInterface(type, typeof(IReadOnlyDictionary<,>)); + if (dictionaryType is not null) + { + Type[] args = dictionaryType.GetGenericArguments(); + // Guard against a type that maps to itself (e.g. class D : IDictionary): + // there is no finite value type to unwrap to, so treat it as a plain object instead. + if (args[1] != type) + { + keyType = args[0]; + valueType = args[1]; + return true; + } + return false; + } + // Non-generic IDictionary (e.g. Hashtable): keys and values are untyped + return typeof(IDictionary).IsAssignableFrom(type); + } + + private static Type? FindGenericInterface(Type type, Type genericInterface) + { + if (type.IsGenericType && type.GetGenericTypeDefinition() == genericInterface) + { + return type; + } + foreach (Type iface in type.GetInterfaces()) + { + if (iface.IsGenericType && iface.GetGenericTypeDefinition() == genericInterface) + { + return iface; + } + } + return null; + } + + private static bool IsDictionaryShaped(this Type type) + { + return FindGenericInterface(type, typeof(IDictionary<,>)) is not null || + FindGenericInterface(type, typeof(IReadOnlyDictionary<,>)) is not null || + typeof(IDictionary).IsAssignableFrom(type); + } + internal static IEnumerable Subclasses(this Type type) { if (!type.IsClass) diff --git a/src/typechat.schema/Typescript.cs b/src/typechat.schema/Typescript.cs index 57b6cbd..dc8df58 100644 --- a/src/typechat.schema/Typescript.cs +++ b/src/typechat.schema/Typescript.cs @@ -70,7 +70,15 @@ public static class Types } else if (type.IsDateTime()) { - return String; // Json does not have a primitive DateTime + return String; // Json has no DateTime primitive: these value types serialize as strings + } + else if (type == typeof(Guid)) + { + // Guid is a value type, but System.Text.Json writes it as a JSON string (e.g. + // "3f2504e0-4f89-11d3-9a0c-0305e82c3301") and it exposes no public fields/properties. + // Without this mapping it would fall through to the object path and be exported as an + // empty "interface Guid {}", which a model could never populate correctly. + return String; } else if (type.IsObject()) { diff --git a/src/typechat.schema/TypescriptExporter.cs b/src/typechat.schema/TypescriptExporter.cs index 2d06173..3295374 100644 --- a/src/typechat.schema/TypescriptExporter.cs +++ b/src/typechat.schema/TypescriptExporter.cs @@ -175,29 +175,36 @@ public override void ExportPending() public override void ExportType(Type type) { - if (type.IsArray) + if (type.IsDictionary(out _, out Type valueType)) { - AddPending(type.GetElementType()); + AddPending(valueType); + } + else if (type.IsArrayLike(out Type elementType)) + { + AddPending(elementType); + } + else if (type.IsEnum) + { + ExportEnum(type); } else { - if (type.IsEnum) - { - ExportEnum(type); - } - else - { - ExportClass(type); - } + ExportClass(type); } } public TypescriptExporter ExportClass(Type type) { ArgumentVerify.ThrowIfNull(type, nameof(type)); - if (!type.IsClass) + // Export both reference types (classes) and non-enum value types (structs) as interfaces: + // System.Text.Json serializes a struct's public fields/properties into a JSON object exactly + // like a class (e.g. KeyValuePair => { Key, Value }, a custom Money => { Amount, Currency }). + // Enums are routed to ExportEnum before they reach here, and scalar value types that Json writes + // as a single value (Guid, DateTime, DateTimeOffset, ...) are mapped to primitives in + // Typescript.Types.ToPrimitive, so only genuine object-like structs arrive at this point. + if (!type.IsClass && (!type.IsValueType || type.IsPrimitive || type.IsEnum)) { - ArgumentVerify.Throw($"{type.Name} must be a class"); + ArgumentVerify.Throw($"{type.Name} must be a class or struct"); } if (IsExported(type)) @@ -505,7 +512,7 @@ private TypescriptExporter ExportVariable(MemberInfo member, Type type) _writer.Variable( member.PropertyName(), DataType(actualType), - type.IsArray, + actualType.IsArrayLike(out _), isNullable ); } @@ -534,7 +541,7 @@ private TypescriptExporter ExportParameter(ParameterInfo parameter, int i, int c DataType(type), i, count, - type.IsArray, + type.IsArrayLike(out _), isNullable ); AddPending(type); @@ -658,9 +665,20 @@ private string DataType(Type type) type = type.GetGenericType() ?? typeof(void); } - if (type.IsArray) + Type? nullableValueType = type.GetNullableValueType(); + if (nullableValueType is not null) + { + type = nullableValueType; + } + + if (type.IsDictionary(out Type keyType, out Type valueType)) + { + return DataTypeMap(keyType, valueType); + } + + if (type.IsArrayLike(out Type elementType)) { - return DataType(type.GetElementType()); + return DataType(elementType); } string? typeName = null; @@ -680,6 +698,23 @@ private string DataType(Type type) return typeName; } + private string DataTypeMap(Type keyType, Type valueType) + { + string valueDataType = DataType(valueType); + if (valueType.IsArrayLike(out _)) + { + valueDataType += Typescript.Punctuation.Array; + } + return $"Record<{MapKeyType(keyType)}, {valueDataType}>"; + } + + private static string MapKeyType(Type keyType) + { + // JSON object keys are always strings; only numeric keys are represented as 'number' + string? primitive = Typescript.Types.ToPrimitive(keyType); + return primitive == Typescript.Types.Number ? Typescript.Types.Number : Typescript.Types.String; + } + private string InterfaceName(Type type, bool useMapper = true) { string? typeName = null; @@ -756,12 +791,26 @@ protected override bool ShouldExport(Type type, out Type typeToExport) return false; } - if (type.IsArray) + // Unwrap arrays, collections and dictionaries down to the underlying element/value type + while (true) { - type = type.GetElementType(); + if (type.IsDictionary(out _, out Type valueType)) + { + type = valueType; + } + else if (type.IsArrayLike(out Type elementType)) + { + type = elementType; + } + else + { + break; + } } typeToExport = type; - return !type.IsPrimitive && !_nonExportTypes.Contains(type); + return !type.IsPrimitive && + !type.IsNullableValueType() && + !_nonExportTypes.Contains(type); } } diff --git a/src/typechat/IMultimodalPromptSection.cs b/src/typechat/IMultimodalPromptSection.cs new file mode 100644 index 0000000..96169a5 --- /dev/null +++ b/src/typechat/IMultimodalPromptSection.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// A prompt section whose content is multimodal: an ordered list of , +/// each of which is either text or an image. +/// +/// This extends so that language models which do not support images can +/// continue to consume the section as text via , while models that +/// support images can read the individual . +/// +/// +public interface IMultimodalPromptSection : IPromptSection +{ + /// + /// The ordered content parts (text and images) that make up this section. + /// + IReadOnlyList ContentParts { get; } +} diff --git a/src/typechat/ImageDetail.cs b/src/typechat/ImageDetail.cs new file mode 100644 index 0000000..fd16bcc --- /dev/null +++ b/src/typechat/ImageDetail.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// Controls how a model processes an image supplied in a multimodal prompt and generates +/// its textual understanding of the image. +/// Mirrors the OpenAI image "detail" setting. +/// +public enum ImageDetail +{ + /// + /// Let the model choose how to process the image. + /// + Auto, + + /// + /// The model treats the image as a low resolution 512x512px image. + /// + Low, + + /// + /// The model considers the image at full size. + /// + High, +} diff --git a/src/typechat/JsonTranslatorPrompts.cs b/src/typechat/JsonTranslatorPrompts.cs index 8c86a7b..6987eb4 100644 --- a/src/typechat/JsonTranslatorPrompts.cs +++ b/src/typechat/JsonTranslatorPrompts.cs @@ -43,7 +43,7 @@ public static Prompt AddContextAndRequest(Prompt prompt, Prompt request, IList= 1) { - prompt += RequestSection(request[0].GetText()); + prompt.Append(RequestSection(request[0])); return prompt; } @@ -77,6 +77,44 @@ public static PromptSection RequestSection(string request) return requestSection; } + /// + /// Wrap a user request section for the model. When the request carries images (a multimodal + /// section), the images are preserved so that vision capable models receive them. Otherwise this + /// behaves exactly like . + /// + /// the user request section + /// a prompt section to send to the model + public static IPromptSection RequestSection(IPromptSection request) + { + ArgumentVerify.ThrowIfNull(request, nameof(request)); + if (request is IMultimodalPromptSection multimodal && HasImages(multimodal)) + { + MultimodalPromptSection section = new MultimodalPromptSection(request.Source); + section.AddText("The following is a user request:"); + IReadOnlyList parts = multimodal.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + section.Add(parts[i]); + } + section.AddText("The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:\n"); + return section; + } + return RequestSection(request.GetText()); + } + + private static bool HasImages(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + if (parts[i].IsImage) + { + return true; + } + } + return false; + } + public static string RepairPrompt(string validationError) { validationError ??= string.Empty; diff --git a/src/typechat/LanguageModel.cs b/src/typechat/LanguageModel.cs index cb88153..4622c4d 100644 --- a/src/typechat/LanguageModel.cs +++ b/src/typechat/LanguageModel.cs @@ -13,6 +13,7 @@ public class LanguageModel : ILanguageModel, IDisposable private readonly ModelInfo _model; private HttpClient _client; private string _endPoint; + private bool _useResponsesApi; /// /// Create an OpenAILanguageModel object using the given OpenAIConfig @@ -48,8 +49,15 @@ public async Task CompleteAsync(Prompt prompt, TranslationSettings? sett { ArgumentVerify.ThrowIfNullOrEmpty(prompt, nameof(prompt)); - var request = CreateRequest(prompt, settings); string apiToken = _config.HasTokenProvider ? await _config.ApiTokenProvider.GetAccessTokenAsync(cancelToken) : null; + if (_useResponsesApi) + { + var responsesRequest = CreateResponsesRequest(prompt, settings); + var responsesResponse = await _client.GetJsonResponseAsync(_endPoint, responsesRequest, _config.MaxRetries, _config.MaxPauseMs, apiToken).ConfigureAwait(false); + return responsesResponse.GetText(); + } + + var request = CreateRequest(prompt, settings); var response = await _client.GetJsonResponseAsync(_endPoint, request, _config.MaxRetries, _config.MaxPauseMs, apiToken).ConfigureAwait(false); return response.GetText(); } @@ -64,12 +72,26 @@ private Request CreateRequest(Prompt prompt, TranslationSettings? settings = nul return request; } + private ResponsesRequest CreateResponsesRequest(Prompt prompt, TranslationSettings? settings = null) + { + // The Responses API always carries the model/deployment in the request body. + return ResponsesRequest.Create(_model.Name, prompt, settings ?? s_defaultSettings); + } + private void ConfigureClient() { if (_config.Azure) { - if (_config.Endpoint.IndexOf(@"chat/completions", StringComparison.OrdinalIgnoreCase) >= 0) + bool wantResponses = _config.UseResponsesApi ?? EndpointTargetsResponses(_config.Endpoint); + if (wantResponses) + { + // Route to the Responses API regardless of how the endpoint is written (base url, + // a chat/completions url, or an explicit /responses url). + _endPoint = BuildAzureResponsesEndpoint(_config.Endpoint, _config.ApiVersion); + } + else if (_config.Endpoint.IndexOf(@"chat/completions", StringComparison.OrdinalIgnoreCase) >= 0) { + // A fully qualified chat/completions endpoint was supplied - use it as-is. _endPoint = _config.Endpoint; } else @@ -84,19 +106,85 @@ private void ConfigureClient() } else { - _endPoint = _config.Endpoint; + bool wantResponses = _config.UseResponsesApi ?? EndpointTargetsResponses(_config.Endpoint); + _endPoint = wantResponses ? BuildResponsesEndpoint(_config.Endpoint) : _config.Endpoint; _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _config.ApiKey); if (!string.IsNullOrEmpty(_config.Organization)) { _client.DefaultRequestHeaders.Add("OpenAI-Organization", _config.Organization); } } + _useResponsesApi = _config.UseResponsesApi ?? EndpointTargetsResponses(_endPoint); if (_config.TimeoutMs > 0) { _client.Timeout = TimeSpan.FromMilliseconds(_config.TimeoutMs); } } + /// + /// Build an Azure OpenAI Responses API endpoint from any Azure endpoint. The Responses route is + /// resource scoped (not deployment scoped): "{scheme}://{host}/openai/responses?api-version=...". + /// The model/deployment is carried in the request body. + /// + private static string BuildAzureResponsesEndpoint(string endpoint, string apiVersion) + { + if (EndpointTargetsResponses(endpoint)) + { + return endpoint; + } + Uri authority = new Uri(new Uri(endpoint).GetLeftPart(UriPartial.Authority)); + string path = string.IsNullOrEmpty(apiVersion) ? "openai/responses" : $"openai/responses?api-version={apiVersion}"; + return new Uri(authority, path).AbsoluteUri; + } + + /// + /// Build an OpenAI (non-Azure) Responses API endpoint. If the endpoint targets chat/completions, + /// it is rewritten to the sibling "responses" route; otherwise "responses" is appended. + /// + private static string BuildResponsesEndpoint(string endpoint) + { + if (EndpointTargetsResponses(endpoint)) + { + return endpoint; + } + const string chatCompletions = "chat/completions"; + int index = endpoint.IndexOf(chatCompletions, StringComparison.OrdinalIgnoreCase); + if (index >= 0) + { + return endpoint.Substring(0, index) + "responses" + endpoint.Substring(index + chatCompletions.Length); + } + return endpoint.TrimEnd('/') + "/responses"; + } + + /// + /// Returns true when the given endpoint targets the OpenAI Responses API. + /// Detection is based on whether the url path ends with "/responses" (ignoring any query string). + /// This covers both the standard OpenAI endpoint (https://api.openai.com/v1/responses) and + /// Azure OpenAI endpoints that end with "/responses?api-version=...". + /// + internal static bool EndpointTargetsResponses(string endpoint) + { + if (string.IsNullOrEmpty(endpoint)) + { + return false; + } + string path = endpoint; + if (Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uri)) + { + path = uri.AbsolutePath; + } + else + { + int query = path.IndexOf('?'); + if (query >= 0) + { + path = path.Substring(0, query); + } + } + path = path.TrimEnd('/'); + return path.EndsWith("/responses", StringComparison.OrdinalIgnoreCase); + } + private struct Request { public string? model { get; set; } @@ -117,7 +205,7 @@ public static Request Create(Prompt prompt, TranslationSettings settings) private struct Response { - public Choice[] choices { get; set; } + public ResponseChoice[] choices { get; set; } public string GetText() { @@ -130,25 +218,177 @@ public string GetText() } } + // A request message. 'content' is either a string (text only) or an array of + // ContentPart (multimodal). System.Text.Json serializes the object by its runtime type. private struct Message { public string role { get; set; } - public string content { get; set; } + public object content { get; set; } public static Message[] Create(Prompt prompt) { Message[] messages = new Message[prompt.Count]; for (int i = 0; i < prompt.Count; ++i) { - messages[i] = new Message { role = prompt[i].Source, content = prompt[i].GetText() }; + messages[i] = Create(prompt[i]); } return messages; } + + public static Message Create(IPromptSection section) + { + return new Message { role = section.Source, content = CreateContent(section) }; + } + + private static object CreateContent(IPromptSection section) + { + if (section is IMultimodalPromptSection multimodal && HasImages(multimodal)) + { + IReadOnlyList parts = multimodal.ContentParts; + ContentPart[] content = new ContentPart[parts.Count]; + for (int i = 0; i < parts.Count; ++i) + { + content[i] = ContentPart.Create(parts[i]); + } + return content; + } + return section.GetText(); + } + + private static bool HasImages(IMultimodalPromptSection section) + { + IReadOnlyList parts = section.ContentParts; + for (int i = 0; i < parts.Count; ++i) + { + if (parts[i].IsImage) + { + return true; + } + } + return false; + } + } + + // A single content part for a Chat Completions multimodal message. + private struct ContentPart + { + public string type { get; set; } + public string? text { get; set; } + public ImageUrl? image_url { get; set; } + + public static ContentPart Create(PromptContentPart part) + { + if (part.IsImage) + { + PromptImage image = part.Image!; + return new ContentPart + { + type = "image_url", + image_url = new ImageUrl + { + url = image.Url, + detail = DetailToString(image.Detail) + } + }; + } + return new ContentPart { type = "text", text = part.Text ?? string.Empty }; + } + + // Auto is the service default, so it is omitted from the wire format. + private static string? DetailToString(ImageDetail detail) + { + switch (detail) + { + case ImageDetail.Low: + return "low"; + case ImageDetail.High: + return "high"; + default: + return null; + } + } + } + + private struct ImageUrl + { + public string url { get; set; } + public string? detail { get; set; } + } + + private struct ResponseChoice + { + public ResponseMessage message { get; set; } + } + + private struct ResponseMessage + { + public string role { get; set; } + public string content { get; set; } + } + + // Request for the OpenAI Responses API. Uses 'input' (array of messages) instead of 'messages', + // and 'max_output_tokens' instead of 'max_tokens'. + private struct ResponsesRequest + { + public string? model { get; set; } + public Message[] input { get; set; } + public double? temperature { get; set; } + public int? max_output_tokens { get; set; } + + public static ResponsesRequest Create(string model, Prompt prompt, TranslationSettings settings) + { + return new ResponsesRequest + { + model = model, + input = Message.Create(prompt), + temperature = (settings.Temperature > 0) ? settings.Temperature : 0, + max_output_tokens = (settings.MaxTokens > 0) ? settings.MaxTokens : null + }; + } + } + + // Response from the OpenAI Responses API. The text is returned inside output[n].content[m].text + // where the matching output item has type == "message" and the content item has type == "output_text". + private struct ResponsesResponse + { + public OutputItem[] output { get; set; } + + public string GetText() + { + if (output is not null) + { + for (int i = 0; i < output.Length; ++i) + { + OutputItem item = output[i]; + if (!string.Equals(item.type, "message", StringComparison.Ordinal) || item.content is null) + { + continue; + } + for (int j = 0; j < item.content.Length; ++j) + { + OutputContent content = item.content[j]; + if (string.Equals(content.type, "output_text", StringComparison.Ordinal) && content.text is not null) + { + return content.text; + } + } + } + } + return string.Empty; + } + } + + private struct OutputItem + { + public string type { get; set; } + public string? role { get; set; } + public OutputContent[] content { get; set; } } - private struct Choice + private struct OutputContent { - public Message message { get; set; } + public string type { get; set; } + public string text { get; set; } } protected virtual void Dispose(bool disposing) diff --git a/src/typechat/MultimodalPromptSection.cs b/src/typechat/MultimodalPromptSection.cs new file mode 100644 index 0000000..519c337 --- /dev/null +++ b/src/typechat/MultimodalPromptSection.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// A prompt section that can contain a mix of text and images. +/// +/// Build a section by adding text and image parts in the order they should appear: +/// +/// var section = new MultimodalPromptSection() +/// .AddText("What is in this image?") +/// .AddImage(PromptImage.FromFile("cat.png")); +/// +/// +/// Language models that support images (such as GPT-4-vision, GPT-4-omni and GPT-4-turbo) will receive +/// the images. Models that only support text will receive the concatenated text via . +/// +public class MultimodalPromptSection : IMultimodalPromptSection +{ + private readonly string _source; + private readonly List _parts; + + /// + /// Create a new, empty multimodal prompt section. + /// + /// The source of the section. Defaults to + public MultimodalPromptSection(string? source = null) + { + _source = string.IsNullOrEmpty(source) ? PromptSection.Sources.User : source!; + _parts = new List(); + } + + /// + /// Create a new multimodal prompt section from a set of content parts. + /// + /// The source of the section + /// The content parts + public MultimodalPromptSection(string source, IEnumerable parts) + : this(source) + { + ArgumentVerify.ThrowIfNull(parts, nameof(parts)); + foreach (var part in parts) + { + Add(part); + } + } + + /// + /// The source of this section. Typical sources are defined in . + /// + public string? Source => _source; + + /// + /// The ordered content parts (text and images) that make up this section. + /// + public IReadOnlyList ContentParts => _parts; + + /// + /// True when this section contains at least one image part. + /// + public bool HasImages + { + get + { + for (int i = 0; i < _parts.Count; ++i) + { + if (_parts[i].IsImage) + { + return true; + } + } + return false; + } + } + + /// + /// Add a content part to the section. + /// + /// the part to add + /// this section, to allow chaining + public MultimodalPromptSection Add(PromptContentPart part) + { + ArgumentVerify.ThrowIfNull(part, nameof(part)); + _parts.Add(part); + return this; + } + + /// + /// Add a text part to the section. + /// + /// the text to add + /// this section, to allow chaining + public MultimodalPromptSection AddText(string text) => Add(PromptContentPart.FromText(text)); + + /// + /// Add an image part to the section. + /// + /// the image to add + /// this section, to allow chaining + public MultimodalPromptSection AddImage(PromptImage image) => Add(PromptContentPart.FromImage(image)); + + /// + /// Add an image part to the section. + /// + /// An http(s) url to a hosted image, or a data uri containing base64 encoded image bytes + /// Controls how the model should process the image + /// this section, to allow chaining + public MultimodalPromptSection AddImage(string url, ImageDetail detail = ImageDetail.Auto) => AddImage(new PromptImage(url, detail)); + + /// + /// Return the concatenation of all text parts in this section. Image parts are ignored. + /// + /// string + public string GetText() + { + if (_parts.Count == 0) + { + return string.Empty; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < _parts.Count; ++i) + { + PromptContentPart part = _parts[i]; + if (part.IsText && !string.IsNullOrEmpty(part.Text)) + { + if (sb.Length > 0) + { + sb.Append('\n'); + } + sb.Append(part.Text); + } + } + return sb.ToString(); + } + + /// + /// Create a new multimodal section whose source is . + /// + /// MultimodalPromptSection + public static MultimodalPromptSection FromUser() => new MultimodalPromptSection(PromptSection.Sources.User); +} diff --git a/src/typechat/OpenAIConfig.cs b/src/typechat/OpenAIConfig.cs index bde1f88..832f048 100644 --- a/src/typechat/OpenAIConfig.cs +++ b/src/typechat/OpenAIConfig.cs @@ -39,11 +39,6 @@ public static class VariableNames /// public const string OPENAI_EMBEDDINGMODEL = "OPENAI_EMBEDDINGMODEL"; - /// - /// The embedding endpoint to use - /// - public const string OPENAI_EMBEDDING_ENDPOINT = "OPENAI_EMBEDDING_ENDPOINT"; - /// /// Api key to use for Azure OpenAI service /// @@ -88,6 +83,17 @@ public OpenAIConfig() { } /// public string ApiVersion { get; set; } = "2023-05-15"; + /// + /// Selects the OpenAI REST API used by : + /// + /// null (default): auto-detect. The Responses API is used when + /// targets a path that ends with /responses; otherwise the Chat Completions API is used. + /// true: always use the Responses API. + /// false: always use the Chat Completions API. + /// + /// + public bool? UseResponsesApi { get; set; } + /// /// Http Settings /// @@ -149,11 +155,6 @@ public static OpenAIConfig FromEnvironment(bool isEmbedding = false) } if (isEmbedding) { - if (Environment.GetEnvironmentVariable(VariableNames.OPENAI_EMBEDDING_ENDPOINT) != null) - { - config.Endpoint = Environment.GetEnvironmentVariable(VariableNames.OPENAI_EMBEDDING_ENDPOINT); - } - config.Model = Environment.GetEnvironmentVariable(VariableNames.OPENAI_EMBEDDINGMODEL); } else diff --git a/src/typechat/PromptContentPart.cs b/src/typechat/PromptContentPart.cs new file mode 100644 index 0000000..a71bcf3 --- /dev/null +++ b/src/typechat/PromptContentPart.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// The kind of content carried by a . +/// +public enum PromptContentType +{ + /// + /// The part contains text. + /// + Text, + + /// + /// The part contains an image. + /// + Image, +} + +/// +/// A single part of a multimodal prompt section. A part is either text or an image. +/// A is an ordered collection of these parts, allowing +/// text and images to be interleaved within one prompt section. +/// +public sealed class PromptContentPart +{ + private readonly string? _text; + private readonly PromptImage? _image; + + private PromptContentPart(PromptContentType type, string? text, PromptImage? image) + { + Type = type; + _text = text; + _image = image; + } + + /// + /// The kind of content this part carries. + /// + public PromptContentType Type { get; } + + /// + /// True when this part contains text. + /// + public bool IsText => Type == PromptContentType.Text; + + /// + /// True when this part contains an image. + /// + public bool IsImage => Type == PromptContentType.Image; + + /// + /// The text of this part, or null when this is not a text part. + /// + public string? Text => _text; + + /// + /// The image of this part, or null when this is not an image part. + /// + public PromptImage? Image => _image; + + /// + /// Create a text content part. + /// + /// text + /// PromptContentPart + public static PromptContentPart FromText(string text) + { + ArgumentVerify.ThrowIfNull(text, nameof(text)); + return new PromptContentPart(PromptContentType.Text, text, null); + } + + /// + /// Create an image content part. + /// + /// image + /// PromptContentPart + public static PromptContentPart FromImage(PromptImage image) + { + ArgumentVerify.ThrowIfNull(image, nameof(image)); + return new PromptContentPart(PromptContentType.Image, null, image); + } + + /// + /// Implicitly create a text content part from a string. + /// + /// text + public static implicit operator PromptContentPart(string text) => FromText(text); + + /// + /// Implicitly create an image content part from a . + /// + /// image + public static implicit operator PromptContentPart(PromptImage image) => FromImage(image); +} diff --git a/src/typechat/PromptImage.cs b/src/typechat/PromptImage.cs new file mode 100644 index 0000000..edfe652 --- /dev/null +++ b/src/typechat/PromptImage.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// An image referenced by a multimodal prompt section. +/// The image is identified by a , which may be either: +/// +/// An http(s) url to a hosted image +/// A data uri that embeds base64 encoded image bytes: data:{mediaType};base64,{data} +/// +/// Models such as GPT-4-vision, GPT-4-omni and GPT-4-turbo can accept images as part of a prompt. +/// +public class PromptImage +{ + /// + /// Create a new PromptImage + /// + /// An http(s) url to a hosted image, or a data uri containing base64 encoded image bytes + /// Controls how the model should process the image + public PromptImage(string url, ImageDetail detail = ImageDetail.Auto) + { + ArgumentVerify.ThrowIfNullOrEmpty(url, nameof(url)); + Url = url; + Detail = detail; + } + + /// + /// An http(s) url to a hosted image, or a data uri containing base64 encoded image bytes. + /// + public string Url { get; } + + /// + /// Controls how the model should process the image. + /// + public ImageDetail Detail { get; } + + /// + /// True when is a data uri that embeds the image bytes, rather than a remote url. + /// + public bool IsDataUri => Url.StartsWith("data:", StringComparison.OrdinalIgnoreCase); + + /// + /// Create a PromptImage from raw image bytes. The bytes are encoded as a base64 data uri. + /// + /// image bytes + /// the image media (MIME) type, e.g. "image/png" + /// Controls how the model should process the image + /// PromptImage + public static PromptImage FromBytes(byte[] bytes, string mediaType = "image/png", ImageDetail detail = ImageDetail.Auto) + { + ArgumentVerify.ThrowIfNull(bytes, nameof(bytes)); + ArgumentVerify.ThrowIfNullOrEmpty(mediaType, nameof(mediaType)); + string dataUri = $"data:{mediaType};base64,{Convert.ToBase64String(bytes)}"; + return new PromptImage(dataUri, detail); + } + + /// + /// Create a PromptImage by loading an image file and encoding its bytes as a base64 data uri. + /// The media type is inferred from the file extension unless is supplied. + /// + /// path to the image file + /// (optional) the image media (MIME) type. Inferred from the file extension when null + /// Controls how the model should process the image + /// PromptImage + public static PromptImage FromFile(string filePath, string? mediaType = null, ImageDetail detail = ImageDetail.Auto) + { + ArgumentVerify.ThrowIfNullOrEmpty(filePath, nameof(filePath)); + mediaType ??= GetMediaType(filePath); + byte[] bytes = File.ReadAllBytes(filePath); + return FromBytes(bytes, mediaType, detail); + } + + /// + /// Infer an image media (MIME) type from a file name or url extension. + /// Returns the wildcard type "image/*" when the extension is not recognized. + /// + /// a file path or url + /// media type + public static string GetMediaType(string pathOrUrl) + { + if (string.IsNullOrEmpty(pathOrUrl)) + { + return "image/*"; + } + string ext = Path.GetExtension(pathOrUrl).ToLowerInvariant(); + switch (ext) + { + case ".png": + return "image/png"; + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".gif": + return "image/gif"; + case ".webp": + return "image/webp"; + case ".bmp": + return "image/bmp"; + case ".tif": + case ".tiff": + return "image/tiff"; + case ".svg": + return "image/svg+xml"; + default: + return "image/*"; + } + } +} diff --git a/tests/TypeChat.IntegrationTests/Readme.md b/tests/TypeChat.IntegrationTests/Readme.md index c422dec..5afb616 100644 --- a/tests/TypeChat.IntegrationTests/Readme.md +++ b/tests/TypeChat.IntegrationTests/Readme.md @@ -8,4 +8,15 @@ The integration tests **require an Azure or OpenAI key** and will just skip if o - Build ### About API Keys -Read about API Keys in the [Project README](../../README.md) \ No newline at end of file +Read about API Keys in the [Project README](../../README.md) + +## Multimodal (image) and Responses API tests + +`TestMultimodalEndToEnd` covers image input and the OpenAI Responses API: + +- `DescribeMicrosoftLogo_FromFile` sends the bundled `microsoft-logo.png` to the model and validates + the strongly typed result. It requires a **vision capable** model (e.g. gpt-4o); the api-version is + set to a vision capable value automatically. +- `TranslateSentiment_ResponsesApi` exercises the `/responses` route. The api-version is set to a + Responses-capable value automatically; the configured resource/deployment must support the + Responses API. \ No newline at end of file diff --git a/tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs b/tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs new file mode 100644 index 0000000..8db5332 --- /dev/null +++ b/tests/TypeChat.IntegrationTests/TestMultimodalEndToEnd.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat.Tests; + +/// +/// Online (end to end) tests for multimodal (image) prompts. +/// +/// These use the built-in (which supports image content) and require a +/// vision capable model such as gpt-4o. The CI pipeline is configured with a gpt-4o deployment. +/// The Microsoft logo image ships with the test project (see the csproj) and is copied next to the +/// test binaries. +/// +public class TestMultimodalEndToEnd : TypeChatTest, IClassFixture +{ + // Image (vision) input on Azure OpenAI requires a recent api-version. The default OpenAIConfig + // api-version predates vision, so use a vision capable version for these tests. + private const string VisionApiVersion = "2024-08-01-preview"; + // The OpenAI Responses API requires a recent (preview) api-version on Azure. + private const string ResponsesApiVersion = "2025-04-01-preview"; + private const string LogoImagePath = "microsoft-logo.png"; + + private readonly Config _config; + + public TestMultimodalEndToEnd(Config config, ITestOutputHelper output) + : base(output) + { + _config = config; + } + + [SkippableFact] + public async Task DescribeMicrosoftLogo_FromFile() + { + Skip.If(!CanRunEndToEndTest(_config)); + + LogoInfo info = await DescribeImage(PromptImage.FromFile(LogoImagePath, "image/png", ImageDetail.High)); + + Assert.NotNull(info); + WriteLine($"Brand='{info.Brand}' ContainsText={info.ContainsText} Colors=[{string.Join(", ", info.Colors ?? Array.Empty())}]"); + + // The Microsoft logo contains the word "Microsoft". + Assert.True(info.ContainsText); + Assert.False(string.IsNullOrEmpty(info.Brand)); + Assert.Contains("microsoft", info.Brand, StringComparison.OrdinalIgnoreCase); + } + + [SkippableFact] + public async Task TranslateSentiment_ResponsesApi() + { + Skip.If(!CanRunEndToEndTest(_config)); + + LanguageModel model = new LanguageModel(ResponsesConfig(_config.OpenAI)); + JsonTranslator translator = new JsonTranslator( + model, + new TypeValidator() + ); + + SentimentResponse response = await translator.TranslateAsync("Tonights gonna be a good night! A good good night!"); + Assert.NotNull(response); + Assert.False(string.IsNullOrEmpty(response.Sentiment)); + WriteLine($"[Responses API] Sentiment='{response.Sentiment}'"); + } + + private async Task DescribeImage(PromptImage image) + { + LanguageModel model = new LanguageModel(VisionConfig(_config.OpenAI)); + JsonTranslator translator = new JsonTranslator(model); + + Prompt request = new Prompt(); + request.Add(new MultimodalPromptSection() + .AddText("Describe the following image.") + .AddImage(image)); + + return await translator.TranslateAsync(request, null); + } + + /// + /// Copy the shared fixture config (so we don't mutate it) and make it suitable for image input: + /// a vision capable Azure api-version and a longer timeout, since vision calls can be slower. + /// + private static OpenAIConfig VisionConfig(OpenAIConfig src) + { + return new OpenAIConfig + { + Azure = src.Azure, + Endpoint = src.Endpoint, + ApiKey = src.ApiKey, + Organization = src.Organization, + Model = src.Model, + ApiVersion = src.Azure ? VisionApiVersion : src.ApiVersion, + TimeoutMs = Math.Max(src.TimeoutMs, 60 * 1000), + MaxRetries = src.MaxRetries, + MaxPauseMs = src.MaxPauseMs, + ApiTokenProvider = src.ApiTokenProvider, + UseResponsesApi = src.UseResponsesApi, + }; + } + + /// + /// Copy the shared fixture config and switch it to the OpenAI Responses API with an api-version + /// that supports it on Azure. + /// + private static OpenAIConfig ResponsesConfig(OpenAIConfig src) + { + return new OpenAIConfig + { + Azure = src.Azure, + Endpoint = src.Endpoint, + ApiKey = src.ApiKey, + Organization = src.Organization, + Model = src.Model, + ApiVersion = src.Azure ? ResponsesApiVersion : src.ApiVersion, + TimeoutMs = Math.Max(src.TimeoutMs, 60 * 1000), + MaxRetries = src.MaxRetries, + MaxPauseMs = src.MaxPauseMs, + ApiTokenProvider = src.ApiTokenProvider, + UseResponsesApi = true, + }; + } +} + +/// +/// A strongly typed description of a logo image. +/// +public class LogoInfo +{ + [Comment("The name of the company or brand shown in the image, if identifiable")] + public string Brand { get; set; } + + [Comment("True if the image contains any text")] + public bool ContainsText { get; set; } + + [Comment("The dominant colors in the image")] + public string[] Colors { get; set; } +} diff --git a/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj b/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj index 7bbbf9c..66542f0 100644 --- a/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj +++ b/tests/TypeChat.IntegrationTests/TypeChat.IntegrationTests.csproj @@ -46,4 +46,10 @@ + + + Always + + + diff --git a/tests/TypeChat.TestLib/MockHttp.cs b/tests/TypeChat.TestLib/MockHttp.cs index e1e1e1a..1c04a3b 100644 --- a/tests/TypeChat.TestLib/MockHttp.cs +++ b/tests/TypeChat.TestLib/MockHttp.cs @@ -21,17 +21,31 @@ public MockHttpHandler(int statusCode = 0) public int RequestCount { get; set; } = 0; public HttpRequestMessage? LastRequest { get; set; } + public string? LastRequestBody { get; set; } public void Clear() { RequestCount = 0; LastRequest = null; + LastRequestBody = null; } protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { LastRequest = request; RequestCount++; + // Capture the request body now: the caller disposes the request (and its content) after this + // returns. The content may already be disposed on retry attempts (the caller reuses it), so guard. + if (request.Content is not null) + { + try + { + LastRequestBody = request.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + } + catch (ObjectDisposedException) + { + } + } HttpResponseMessage response = Response; if (response is null) { diff --git a/tests/TypeChat.TestLib/Schemas.cs b/tests/TypeChat.TestLib/Schemas.cs index f49236e..d50c366 100644 --- a/tests/TypeChat.TestLib/Schemas.cs +++ b/tests/TypeChat.TestLib/Schemas.cs @@ -336,6 +336,69 @@ public class FriendsOfPerson public Name[] FriendNames { get; set; } } +// For testing that IEnumerable exports as a JSON array (string[]) and not as a +// leaky List_String interface. See https://github.com/microsoft/typechat.net/issues/218 +public sealed class Pizza +{ + public IEnumerable Toppings { get; set; } + public string Size { get; set; } +} + +// For testing that the various collection and dictionary types all export as JSON arrays/maps +public class CollectionsObj +{ + public IEnumerable Tags { get; set; } + public List Names { get; set; } + public IList Scores { get; set; } + public ICollection Locations { get; set; } + public IReadOnlyList Ratings { get; set; } + public HashSet UniqueTags { get; set; } + public string[] Aliases { get; set; } + public Dictionary Counts { get; set; } + public IDictionary LocationsByCity { get; set; } + public Dictionary NamesById { get; set; } + public Dictionary> Buckets { get; set; } +} + +// A type that enumerates itself (composite pattern): it implements IEnumerable, +// so it has no finite "array of..." representation. Schema generation must still terminate. +public class Composite : IEnumerable +{ + public string Name { get; set; } + public IEnumerator GetEnumerator() => Enumerable.Empty().GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); +} + +public class CompositeHolder +{ + public List Nodes { get; set; } + public Composite Root { get; set; } +} + +// For testing that structs export as interfaces: Json serializes a struct's public members as an +// object, just like a class. +public struct Money +{ + public decimal Amount { get; set; } + public string Currency { get; set; } +} + +public struct Coordinates +{ + public double Latitude { get; set; } + public double Longitude { get; set; } +} + +public class StructHolder +{ + public Money Price { get; set; } + public Coordinates Location { get; set; } + public KeyValuePair Pair { get; set; } + // Scalar value types that Json writes as a single string - must NOT become empty interfaces + public Guid Id { get; set; } + public DateTimeOffset When { get; set; } +} + // For testing Generics public class Child { diff --git a/tests/TypeChat.UnitTests/TestMultimodal.cs b/tests/TypeChat.UnitTests/TestMultimodal.cs new file mode 100644 index 0000000..1f37dc9 --- /dev/null +++ b/tests/TypeChat.UnitTests/TestMultimodal.cs @@ -0,0 +1,445 @@ +// Copyright (c) Microsoft. All rights reserved. + +extern alias ExtensionsAI; +using System.Text.Json; +using System.Threading; +using Microsoft.Extensions.AI; +using Microsoft.TypeChat.Tests; +using MeaiChatModel = ExtensionsAI::Microsoft.TypeChat.ChatLanguageModel; + +namespace Microsoft.TypeChat.UnitTests; + +public class TestMultimodal : TypeChatTest +{ + #region Content types + + [Fact] + public void PromptImage_FromBytes_ProducesDataUri() + { + byte[] bytes = new byte[] { 1, 2, 3, 4 }; + PromptImage image = PromptImage.FromBytes(bytes, "image/png", ImageDetail.Low); + + Assert.True(image.IsDataUri); + Assert.Equal(ImageDetail.Low, image.Detail); + Assert.StartsWith("data:image/png;base64,", image.Url); + Assert.Equal(Convert.ToBase64String(bytes), image.Url.Substring("data:image/png;base64,".Length)); + } + + [Fact] + public void PromptImage_Url_IsNotDataUri() + { + PromptImage image = new PromptImage("https://example.com/cat.png"); + Assert.False(image.IsDataUri); + Assert.Equal(ImageDetail.Auto, image.Detail); + } + + [Fact] + public void PromptImage_FromFile_LoadsMicrosoftLogo() + { + // The shared Microsoft logo asset is copied next to the test binaries (see csproj). + PromptImage image = PromptImage.FromFile("microsoft-logo.png"); + Assert.True(image.IsDataUri); + Assert.StartsWith("data:image/png;base64,", image.Url); + Assert.True(image.Url.Length > 1000); // real image bytes were embedded + } + + [Theory] + [InlineData("cat.png", "image/png")] + [InlineData("cat.PNG", "image/png")] + [InlineData("cat.jpg", "image/jpeg")] + [InlineData("cat.jpeg", "image/jpeg")] + [InlineData("cat.gif", "image/gif")] + [InlineData("cat.webp", "image/webp")] + [InlineData("https://example.com/photos/cat.bmp", "image/bmp")] + [InlineData("cat", "image/*")] + [InlineData("https://example.com/image", "image/*")] + public void PromptImage_GetMediaType(string pathOrUrl, string expected) + { + Assert.Equal(expected, PromptImage.GetMediaType(pathOrUrl)); + } + + [Fact] + public void PromptContentPart_ImplicitConversions() + { + PromptContentPart text = "hello"; + Assert.True(text.IsText); + Assert.False(text.IsImage); + Assert.Equal("hello", text.Text); + + PromptContentPart image = new PromptImage("https://example.com/cat.png"); + Assert.True(image.IsImage); + Assert.False(image.IsText); + Assert.NotNull(image.Image); + } + + [Fact] + public void MultimodalSection_BuildsOrderedParts() + { + MultimodalPromptSection section = new MultimodalPromptSection() + .AddText("a") + .AddImage("https://example.com/cat.png", ImageDetail.High) + .AddText("b"); + + Assert.Equal(PromptSection.Sources.User, section.Source); + Assert.True(section.HasImages); + Assert.Equal(3, section.ContentParts.Count); + Assert.True(section.ContentParts[0].IsText); + Assert.True(section.ContentParts[1].IsImage); + Assert.Equal(ImageDetail.High, section.ContentParts[1].Image!.Detail); + Assert.True(section.ContentParts[2].IsText); + + // GetText concatenates text parts and ignores images, so text-only models still work. + Assert.Equal("a\nb", section.GetText()); + } + + [Fact] + public void MultimodalSection_TextOnly_HasNoImages() + { + MultimodalPromptSection section = new MultimodalPromptSection().AddText("just text"); + Assert.False(section.HasImages); + Assert.Equal("just text", section.GetText()); + } + + #endregion + + #region LanguageModel - Chat Completions wire format + + [Fact] + public async Task LanguageModel_TextOnly_SendsStringContent() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + JsonArray messages = body["messages"]!.AsArray(); + Assert.Single(messages); + JsonNode content = messages[0]!["content"]!; + Assert.Equal(JsonValueKind.String, content.GetValueKind()); + Assert.Equal("Hello", content.GetValue()); + Assert.Null(body["input"]); + } + + [Fact] + public async Task LanguageModel_Multimodal_SendsContentParts() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Model = "gpt-4o"; + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + Prompt prompt = new Prompt(); + prompt.Add(new MultimodalPromptSection() + .AddText("What is in this image?") + .AddImage("https://example.com/cat.png", ImageDetail.High)); + + string text = await model.CompleteAsync(prompt); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + JsonNode content = body["messages"]!.AsArray()[0]!["content"]!; + Assert.Equal(JsonValueKind.Array, content.GetValueKind()); + + JsonArray parts = content.AsArray(); + Assert.Equal(2, parts.Count); + Assert.Equal("text", parts[0]!["type"]!.GetValue()); + Assert.Equal("What is in this image?", parts[0]!["text"]!.GetValue()); + Assert.Equal("image_url", parts[1]!["type"]!.GetValue()); + Assert.Equal("https://example.com/cat.png", parts[1]!["image_url"]!["url"]!.GetValue()); + Assert.Equal("high", parts[1]!["image_url"]!["detail"]!.GetValue()); + } + + [Fact] + public async Task LanguageModel_Multimodal_AutoDetail_IsOmitted() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + Prompt prompt = new Prompt(); + prompt.Add(new MultimodalPromptSection().AddImage("https://example.com/cat.png")); // Auto + + await model.CompleteAsync(prompt); + + JsonNode body = ParseBody(handler); + JsonNode imageUrl = body["messages"]!.AsArray()[0]!["content"]!.AsArray()[0]!["image_url"]!; + Assert.Null(imageUrl["detail"]); + } + + #endregion + + #region LanguageModel - Responses API + + [Fact] + public async Task LanguageModel_ResponsesEndpoint_UsesInputAndParsesOutput() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Endpoint = "https://api.openai.com/v1/responses"; + config.Model = "gpt-4o"; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + Assert.Null(body["messages"]); + Assert.NotNull(body["input"]); + Assert.Equal("gpt-4o", body["model"]!.GetValue()); + + JsonArray input = body["input"]!.AsArray(); + Assert.Equal("user", input[0]!["role"]!.GetValue()); + Assert.Equal("Hello", input[0]!["content"]!.GetValue()); + } + + [Fact] + public async Task LanguageModel_UseResponsesApiTrue_OverridesChatCompletionsEndpoint() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); // endpoint targets /chat/completions + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + Assert.NotNull(body["input"]); + Assert.Null(body["messages"]); + } + + [Fact] + public async Task LanguageModel_UseResponsesApiFalse_OverridesResponsesEndpoint() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Endpoint = "https://api.openai.com/v1/responses"; + config.UseResponsesApi = false; + MockHttpHandler handler = new MockHttpHandler(ChatCompletionsCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + string text = await model.CompleteAsync("Hello"); + Assert.Equal("Hello there!", text.Trim()); + + JsonNode body = ParseBody(handler); + Assert.NotNull(body["messages"]); + Assert.Null(body["input"]); + } + + [Fact] + public async Task LanguageModel_Azure_Responses_RoutesToResponsesUrl() + { + OpenAIConfig config = MockOpenAIConfig(azure: true); // base Azure resource url + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + await model.CompleteAsync("Hello"); + + string url = handler.LastRequest!.RequestUri!.AbsoluteUri; + Assert.Contains("/openai/responses", url, StringComparison.OrdinalIgnoreCase); + Assert.Contains("api-version=", url, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("chat/completions", url, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task LanguageModel_Azure_Responses_NormalizesChatCompletionsUrl() + { + OpenAIConfig config = MockOpenAIConfig(azure: true); + config.Endpoint = "https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-08-01-preview"; + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + await model.CompleteAsync("Hello"); + + string url = handler.LastRequest!.RequestUri!.AbsoluteUri; + Assert.Contains("/openai/responses", url, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("chat/completions", url, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("deployments", url, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task LanguageModel_OpenAI_Responses_NormalizesChatCompletionsUrl() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); // endpoint targets /v1/chat/completions + config.UseResponsesApi = true; + MockHttpHandler handler = new MockHttpHandler(ResponsesCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + + await model.CompleteAsync("Hello"); + + string url = handler.LastRequest!.RequestUri!.AbsoluteUri; + Assert.EndsWith("/v1/responses", url, StringComparison.OrdinalIgnoreCase); + } + + #endregion + + #region JsonTranslator integration + + [Fact] + public void JsonTranslatorPrompts_RequestSection_PreservesImages() + { + MultimodalPromptSection request = new MultimodalPromptSection() + .AddText("Describe this") + .AddImage("https://example.com/cat.png"); + + IPromptSection section = JsonTranslatorPrompts.RequestSection(request); + MultimodalPromptSection multimodal = Assert.IsType(section); + Assert.True(multimodal.HasImages); + } + + [Fact] + public void JsonTranslatorPrompts_RequestSection_TextOnly_StaysText() + { + IPromptSection section = JsonTranslatorPrompts.RequestSection(PromptSection.FromUser("hello")); + Assert.IsType(section); + } + + [Fact] + public async Task JsonTranslator_Multimodal_SendsImageToModel() + { + OpenAIConfig config = MockOpenAIConfig(azure: false); + config.Model = "gpt-4o"; + MockHttpHandler handler = new MockHttpHandler(TranslatorCanned()); + using LanguageModel model = new LanguageModel(config, null, new HttpClient(handler)); + JsonTranslator translator = new JsonTranslator(model); + + Prompt request = new Prompt(); + request.Add(new MultimodalPromptSection() + .AddText("Describe this image") + .AddImage("https://example.com/cat.png", ImageDetail.High)); + + ImageDescription result = await translator.TranslateAsync(request, null); + Assert.Equal("A cat", result.Description); + + // The image must have survived all the way to the wire request. + JsonNode body = ParseBody(handler); + JsonArray messages = body["messages"]!.AsArray(); + bool sentImage = messages.Any(m => + m!["content"]!.GetValueKind() == JsonValueKind.Array && + m["content"]!.AsArray().Any(p => p!["type"]!.GetValue() == "image_url")); + Assert.True(sentImage); + } + + #endregion + + #region Microsoft.Extensions.AI mapping + + [Fact] + public async Task Meai_Multimodal_MapsToAIContent() + { + FakeChatClient client = new FakeChatClient("Hello there!"); + MeaiChatModel model = new MeaiChatModel(client, "gpt-4o"); + + Prompt prompt = new Prompt(); + prompt.Add(new MultimodalPromptSection() + .AddText("What is this?") + .AddImage(PromptImage.FromBytes(new byte[] { 1, 2, 3 }, "image/png")) // -> DataContent + .AddImage("https://example.com/cat.jpg") // -> UriContent + .AddImage("https://example.com/image")); // -> UriContent (image/*) + + string text = await model.CompleteAsync(prompt); + Assert.Equal("Hello there!", text); + + ChatMessage message = Assert.Single(client.LastMessages!); + Assert.Equal(ChatRole.User, message.Role); + Assert.Contains(message.Contents, c => c is TextContent t && t.Text == "What is this?"); + Assert.Contains(message.Contents, c => c is DataContent); + Assert.Equal(2, message.Contents.Count(c => c is UriContent)); + } + + [Fact] + public async Task Meai_TextOnly_MapsToText() + { + FakeChatClient client = new FakeChatClient("ok"); + MeaiChatModel model = new MeaiChatModel(client, "gpt-4o"); + + await model.CompleteAsync("Hello"); + + ChatMessage message = Assert.Single(client.LastMessages!); + Assert.Equal(ChatRole.User, message.Role); + Assert.Equal("Hello", message.Text); + } + + #endregion + + #region Helpers + + private static JsonNode ParseBody(MockHttpHandler handler) + { + Assert.NotNull(handler.LastRequestBody); + return JsonNode.Parse(handler.LastRequestBody!)!; + } + + private static string ChatCompletionsCanned() + { + return @"{ + ""choices"": [{ + ""index"": 0, + ""message"": { ""role"": ""assistant"", ""content"": ""Hello there!"" }, + ""finish_reason"": ""stop"" + }] + }"; + } + + private static string ResponsesCanned() + { + return @"{ + ""id"": ""resp_123"", + ""output"": [{ + ""type"": ""message"", + ""role"": ""assistant"", + ""content"": [{ ""type"": ""output_text"", ""text"": ""Hello there!"" }] + }] + }"; + } + + private static string TranslatorCanned() + { + // The assistant message content is itself a JSON object matching ImageDescription. + return @"{ + ""choices"": [{ + ""message"": { ""role"": ""assistant"", ""content"": ""{\""Description\"": \""A cat\""}"" } + }] + }"; + } + + private sealed class FakeChatClient : IChatClient + { + private readonly string _responseText; + + public FakeChatClient(string responseText) + { + _responseText = responseText; + } + + public IReadOnlyList? LastMessages { get; private set; } + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + LastMessages = messages.ToList(); + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, _responseText))); + } + + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + throw new NotSupportedException(); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + } + + #endregion +} + +public class ImageDescription +{ + public string Description { get; set; } +} diff --git a/tests/TypeChat.UnitTests/TestSchema.cs b/tests/TypeChat.UnitTests/TestSchema.cs index ecd36d4..db81aed 100644 --- a/tests/TypeChat.UnitTests/TestSchema.cs +++ b/tests/TypeChat.UnitTests/TestSchema.cs @@ -95,5 +95,100 @@ public void ExportGenerics() Assert.True(lines.ContainsSubstring("Child_Name[]")); Assert.True(lines.ContainsSubstring("Child_Location[]")); } + + [Fact] + public void ExportEnumerable_Issue218() + { + var schema = TypescriptExporter.GenerateSchema(typeof(Pizza)); + var lines = schema.Schema.Text.Lines(); + + // IEnumerable must be exported as a JSON array of strings... + Assert.True(lines.ContainsSubstring("Toppings", "string[]")); + Assert.True(lines.ContainsSubstring("Size", "string")); + // ...and NOT as a leaky List_String interface exposing Capacity/Count/Item + Assert.False(lines.ContainsSubstring("List_String")); + Assert.False(lines.ContainsSubstring("Capacity")); + } + + [Fact] + public void ExportCollections() + { + var schema = TypescriptExporter.GenerateSchema(typeof(CollectionsObj)); + var lines = schema.Schema.Text.Lines(); + + // IEnumerable, List, IList, ICollection, IReadOnlyList, HashSet and arrays => T[] + Assert.True(lines.ContainsSubstring("Tags", "string[]")); + Assert.True(lines.ContainsSubstring("Names", "Name[]")); + Assert.True(lines.ContainsSubstring("Scores", "number[]")); + Assert.True(lines.ContainsSubstring("Locations", "Location[]")); + Assert.True(lines.ContainsSubstring("Ratings", "number[]")); + Assert.True(lines.ContainsSubstring("UniqueTags", "string[]")); + Assert.True(lines.ContainsSubstring("Aliases", "string[]")); + + // Dictionaries => Record, including a dictionary whose value is itself a collection + Assert.True(lines.ContainsSubstring("Counts", "Record")); + Assert.True(lines.ContainsSubstring("LocationsByCity", "Record")); + Assert.True(lines.ContainsSubstring("NamesById", "Record")); + Assert.True(lines.ContainsSubstring("Buckets", "Record")); + + // Element/value types are still exported as interfaces + Assert.True(lines.ContainsSubstring("interface", "Name")); + Assert.True(lines.ContainsSubstring("interface", "Location")); + + // The internals of List/Dictionary<> must NOT leak into the schema + Assert.False(lines.ContainsSubstring("Capacity")); + Assert.False(lines.ContainsSubstring("KeyValuePair")); + Assert.False(lines.ContainsSubstring("interface", "List_")); + } + + [Fact] + public void SelfReferentialEnumerable_Terminates() + { + // Composite : IEnumerable has no finite "array of..." representation. + // Schema generation must terminate rather than loop forever while unwrapping it. + TypescriptSchema schema = null; + var worker = new System.Threading.Thread( + () => schema = TypescriptExporter.GenerateSchema(typeof(CompositeHolder))) + { + IsBackground = true + }; + worker.Start(); + bool finished = worker.Join(System.TimeSpan.FromSeconds(5)); + + Assert.True(finished, "Schema generation did not terminate for a self-referential IEnumerable type"); + + var lines = schema.Schema.Text.Lines(); + // A List is still a normal array... + Assert.True(lines.ContainsSubstring("Nodes", "Composite[]")); + // ...but the self-enumerating Composite itself falls back to a plain object interface + Assert.True(lines.ContainsSubstring("interface", "Composite")); + Assert.True(lines.ContainsSubstring("Name", "string")); + } + + [Fact] + public void ExportStructs() + { + var schema = TypescriptExporter.GenerateSchema(typeof(StructHolder)); + var lines = schema.Schema.Text.Lines(); + + // Structs export as interfaces, because Json serializes their public members as an object + Assert.True(lines.ContainsSubstring("Price", "Money")); + Assert.True(lines.ContainsSubstring("interface", "Money")); + Assert.True(lines.ContainsSubstring("Amount", "number")); + Assert.True(lines.ContainsSubstring("Currency", "string")); + + Assert.True(lines.ContainsSubstring("Location", "Coordinates")); + Assert.True(lines.ContainsSubstring("interface", "Coordinates")); + + Assert.True(lines.ContainsSubstring("Pair", "KeyValuePair")); + Assert.True(lines.ContainsSubstring("Key", "string")); + Assert.True(lines.ContainsSubstring("Value", "number")); + + // Scalar value types map to string, NOT to empty interfaces + Assert.True(lines.ContainsSubstring("Id", "string")); + Assert.True(lines.ContainsSubstring("When", "string")); + Assert.False(lines.ContainsSubstring("interface", "Guid")); + Assert.False(lines.ContainsSubstring("interface", "DateTimeOffset")); + } } diff --git a/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj b/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj index 0e9816a..f8832c4 100644 --- a/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj +++ b/tests/TypeChat.UnitTests/TypeChat.UnitTests.csproj @@ -29,7 +29,14 @@ + + + + PreserveNewest + + +