From b8a3ef0fc4ec54c4e36792e433125ed06f5c2742 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 20 Apr 2026 17:30:34 -0500 Subject: [PATCH 01/23] add kg orchestrator --- ...edgeCollectionType.cs => KnowledgeType.cs} | 3 +- .../Knowledges/IKnowledgeOrchestrator.cs | 40 +++ .../Knowledges/IKnowledgeService.cs | 4 +- .../Models/CollectionCreateOptions.cs | 10 + .../Models/KnowledgeCollectionConfigModel.cs | 7 + .../Models/KnowledgeCollectionData.cs | 14 ++ .../Models/KnowledgeCollectionSnapshot.cs | 27 ++ .../Knowledges/Models/KnowledgeCreateModel.cs | 10 + .../Knowledges/Models/KnowledgeFilter.cs | 34 +++ .../Models/KnowledgeQueryOptions.cs | 12 + .../Models/KnowledgeSearchOptions.cs | 27 ++ .../Models/KnowledgeSearchResult.cs | 20 ++ .../Knowledges/Models/KnowledgeUpdateModel.cs | 6 + .../Options/KnowledgeCollectionOptions.cs | 5 + .../Options/KnowledgeIndexOptions.cs | 8 + .../Knowledges/Options/KnowledgeOptionBase.cs | 6 + .../Options/KnowledgeSnapshotOptions.cs | 5 + .../VectorStorage/IVectorDb.cs | 4 +- .../Options/VectorCollectionIndexOptions.cs | 12 +- .../Request/VectorCollectionIndexRequest.cs | 4 +- .../KnowledgeBasePlugin.cs | 4 + .../Base/VectorOrchestratorBase.Collection.cs | 133 ++++++++++ .../Base/VectorOrchestratorBase.Data.cs | 230 ++++++++++++++++++ .../Base/VectorOrchestratorBase.Index.cs | 82 +++++++ .../Base/VectorOrchestratorBase.Snapshot.cs | 91 +++++++ .../Services/Base/VectorOrchestratorBase.cs | 40 +++ .../Document/DocumentKnowledgeOrchestrator.cs | 14 ++ .../Services/KnowledgeService.Index.cs | 4 +- .../QuestionAnswerKnowledgeOrchestrator.cs | 14 ++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 4 +- 30 files changed, 853 insertions(+), 21 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/{KnowledgeCollectionType.cs => KnowledgeType.cs} (66%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Collection.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Data.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Index.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Snapshot.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeType.cs similarity index 66% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeType.cs index b63488964..d1aee93a3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeType.cs @@ -1,7 +1,8 @@ namespace BotSharp.Abstraction.Knowledges.Enums; -public static class KnowledgeCollectionType +public static class KnowledgeType { public static string QuestionAnswer = "question-answer"; public static string Document = "document"; + public static string Taxonomy = "taxonomy"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs new file mode 100644 index 000000000..bb78913d4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs @@ -0,0 +1,40 @@ +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Knowledges.Options; + +namespace BotSharp.Abstraction.Knowledges; + +public interface IKnowledgeOrchestrator +{ + string Type { get; } + + #region Collection + Task ExistCollection(string collectionName, KnowledgeCollectionOptions options); + Task CreateCollection(string collectionName, CollectionCreateOptions options); + Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options); + Task> GetCollections(KnowledgeCollectionOptions options); + #endregion + + #region Data + Task> Search(string query, string collectionName, KnowledgeSearchOptions options); + Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter); + Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null); + Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options); + Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options); + Task CreateCollectionData(string collectionName, KnowledgeCreateModel create); + Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update); + Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update); + #endregion + + #region Index + Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options); + Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options); + #endregion + + #region Snapshot + Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null); + Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null); + Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null); + Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null); + Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 4af6d7e6c..c01879cb6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -73,8 +73,8 @@ Task ImportDocumentContentToKnowledge(string collectionName, string fileNa #endregion #region Index - Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); - Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs new file mode 100644 index 000000000..6fada86dd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Knowledges.Options; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class CollectionCreateOptions : KnowledgeOptionBase +{ + public int EmbeddingDimension { get; set; } + public string LlmProvider { get; set; } = null!; + public string LlmModel { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs new file mode 100644 index 000000000..633e1bc3d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs @@ -0,0 +1,7 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionConfig : VectorCollectionConfig +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs new file mode 100644 index 000000000..59ff18fac --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -0,0 +1,14 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionData +{ + public string Id { get; set; } + public Dictionary Payload { get; set; } = new(); + public Dictionary Data => Payload?.ToDictionary(x => x.Key, x => x.Value.DataValue) ?? []; + public double? Score { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs new file mode 100644 index 000000000..ba0b340b8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionSnapshot +{ + public string Name { get; set; } = default!; + public long Size { get; set; } + public DateTime CreatedTime { get; set; } + public string? CheckSum { get; set; } + + public static KnowledgeCollectionSnapshot? CopyFrom(VectorCollectionSnapshot? model) + { + if (model == null) + { + return null; + } + + return new KnowledgeCollectionSnapshot + { + Name = model.Name, + Size = model.Size, + CreatedTime = model.CreatedTime, + CheckSum = model.CheckSum + }; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs new file mode 100644 index 000000000..5e9008585 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Knowledges.Options; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCreateModel : KnowledgeOptionBase +{ + public string Text { get; set; } + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs new file mode 100644 index 000000000..430d2ae12 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -0,0 +1,34 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFilter : StringIdPagination +{ + [JsonPropertyName("db_provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } + + /// + /// Filter group: each item contains a logical operator and a list of key-value pairs + /// + [JsonPropertyName("filter_groups")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? FilterGroups { get; set; } + + /// + /// Order by a specific field + /// + [JsonPropertyName("order_by")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public VectorSort? OrderBy { get; set; } + + /// + /// Included payload fields + /// + [JsonPropertyName("fields")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Fields { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs new file mode 100644 index 000000000..413e6b11a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeQueryOptions +{ + public bool WithPayload { get; set; } + public bool WithVector { get; set; } + + public static KnowledgeQueryOptions Default() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs new file mode 100644 index 000000000..10b073dbf --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchOptions +{ + public string? DbProvider { get; set; } + public IEnumerable? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer]; + public IEnumerable? FilterGroups { get; set; } + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } + public VectorSearchParamModel? SearchParam { get; set; } + + public static KnowledgeSearchOptions Default() + { + return new() + { + Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer], + FilterGroups = null, + Limit = 5, + Confidence = 0.5f, + WithVector = false + }; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs new file mode 100644 index 000000000..65592cd3d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchResult : KnowledgeCollectionData +{ + public KnowledgeSearchResult() + { + + } + + public static KnowledgeSearchResult CopyFrom(KnowledgeCollectionData data) + { + return new KnowledgeSearchResult + { + Id = data.Id, + Payload = data.Payload, + Score = data.Score, + Vector = data.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs new file mode 100644 index 000000000..214ab9c01 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeUpdateModel : KnowledgeCreateModel +{ + public string Id { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs new file mode 100644 index 000000000..19aae5714 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs @@ -0,0 +1,5 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeCollectionOptions : KnowledgeOptionBase +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs new file mode 100644 index 000000000..cbf69b697 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.VectorStorage.Options; + +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeIndexOptions : KnowledgeOptionBase +{ + public IEnumerable Items { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs new file mode 100644 index 000000000..29a5fb8a2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeOptionBase +{ + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs new file mode 100644 index 000000000..9c02cb2fa --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs @@ -0,0 +1,5 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeSnapshotOptions : KnowledgeOptionBase +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index c862785a1..bd32e7d22 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -40,8 +40,8 @@ Task RecoverCollectionFromShapshot(string collectionName, string snapshotF Task DeleteCollectionShapshot(string collectionName, string snapshotName) => throw new NotImplementedException(); - Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) + Task CreateCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) => throw new NotImplementedException(); - Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) + Task DeleteCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs index 1f49a0cff..47d5a5df7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs @@ -1,18 +1,10 @@ namespace BotSharp.Abstraction.VectorStorage.Options; -public class VectorCollectionIndexOptions +public class CollectionIndexOptions { [JsonPropertyName("field_name")] public string FieldName { get; set; } = null!; -} -public class CreateVectorCollectionIndexOptions : VectorCollectionIndexOptions -{ [JsonPropertyName("field_schema_type")] - public string FieldSchemaType { get; set; } = null!; -} - -public class DeleteVectorCollectionIndexOptions : VectorCollectionIndexOptions -{ - + public string? FieldSchemaType { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs index 5ff787fa6..3cfd365d0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs @@ -4,10 +4,10 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class CreateVectorCollectionIndexRequest { - public IEnumerable Options { get; set; } = []; + public IEnumerable Options { get; set; } = []; } public class DeleteVectorCollectionIndexRequest { - public IEnumerable Options { get; set; } = []; + public IEnumerable Options { get; set; } = []; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index dd974d6d3..ac86046ee 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -23,6 +23,10 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddSingleton(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Collection.cs new file mode 100644 index 000000000..55f8f53b8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Collection.cs @@ -0,0 +1,133 @@ +using BotSharp.Abstraction.VectorStorage.Filters; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorOrchestratorBase +{ + #region Collection + public async Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) + { + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var exist = await vectorDb.DoesCollectionExist(collectionName); + if (exist) + { + return true; + } + + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = [collectionName], + VectorStroageProviders = [_settings.VectorDb.Provider] + }); + + return !configs.IsNullOrEmpty(); + } + + public async Task CreateCollection(string collectionName, CollectionCreateOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = _services.GetRequiredService(); + var created = await db.AddKnowledgeCollectionConfigs(new List + { + new VectorCollectionConfig + { + Name = collectionName, + Type = Type, + VectorStore = new VectorStoreConfig + { + Provider = options.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider)! + }, + TextEmbedding = new KnowledgeEmbeddingConfig + { + Provider = options.LlmProvider, + Model = options.LlmModel, + Dimension = options.EmbeddingDimension + } + } + }); + + if (!created) + { + return false; + } + + var vectorDb = GetVectorDb(options.DbProvider); + if (vectorDb == null) + { + return false; + } + + created = await vectorDb.CreateCollection(collectionName, options: new() + { + Provider = options.LlmProvider, + Model = options.LlmProvider, + Dimension = options.EmbeddingDimension + }); + + return created; + } + + public async Task> GetCollections(KnowledgeCollectionOptions options) + { + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionTypes = !string.IsNullOrEmpty(Type) ? [Type] : null, + VectorStroageProviders = [_settings.VectorDb.Provider] + }); + + var vectorDb = GetVectorDb(); + if (vectorDb == null) + { + return []; + } + var dbCollections = await vectorDb.GetCollections(); + return configs.Where(x => dbCollections.Contains(x.Name)).Select(x => new KnowledgeCollectionConfig + { + Name = x.Name, + Type = x.Type, + VectorStore = x.VectorStore, + TextEmbedding = x.TextEmbedding + }); + } + + public async Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var deleted = await vectorDb.DeleteCollection(collectionName); + + if (deleted) + { + var db = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var vectorStoreProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; + + await db.DeleteKnowledgeCollectionConfig(collectionName); + fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider); + await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider); + } + + return deleted; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Data.cs new file mode 100644 index 000000000..bfb56662f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Data.cs @@ -0,0 +1,230 @@ +using BotSharp.Abstraction.VectorStorage.Options; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorOrchestratorBase +{ + #region Collection data + public async Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) + { + return false; + } + + var vectorDb = GetVectorDb(create.DbProvider); + if (vectorDb == null) + { + return false; + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(create.Text); + + var guid = Guid.NewGuid(); + var payload = create.Payload ?? new(); + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); + } + + return await vectorDb.Upsert(collectionName, guid, vector, create.Text, payload); + } + + public async Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) + { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + return await vectorDb.DeleteCollectionData(collectionName, [guid]); + } + + public async Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) + { + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + return await vectorDb.DeleteCollectionAllData(collectionName); + } + + public async Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty()) + { + return []; + } + + var pointIds = ids.Select(x => new { Id = x, IsValid = Guid.TryParse(x, out var guid), ParseResult = guid }) + .Where(x => x.IsValid) + .Select(x => x.ParseResult) + .ToList(); + + var vectorDb = GetVectorDb(); + if (vectorDb == null) + { + return []; + } + + var queryOptions = options != null ? new VectorQueryOptions + { + WithPayload = options.WithPayload, + WithVector = options.WithVector + } : null; + + var points = await vectorDb.GetCollectionData(collectionName, pointIds, queryOptions); + return points.Select(x => new KnowledgeCollectionData + { + Id = x.Id, + Payload = x.Payload, + Score = x.Score, + Vector = x.Vector + }); + } + + public async Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + { + var vectorDb = GetVectorDb(filter.DbProvider); + if (vectorDb == null) + { + return new StringIdPagedItems(); + } + + var vectorFilter = new VectorFilter + { + WithVector = filter.WithVector, + FilterGroups = filter.FilterGroups, + OrderBy = filter.OrderBy, + Fields = filter.Fields, + Size = filter.Size, + StartId = filter.StartId + }; + + var pagedResult = await vectorDb.GetPagedCollectionData(collectionName, vectorFilter); + return new StringIdPagedItems + { + Count = pagedResult.Count, + Items = pagedResult.Items.Select(x => KnowledgeSearchResult.CopyFrom(new KnowledgeCollectionData + { + Id = x.Id, + Payload = x.Payload, + Score = x.Score, + Vector = x.Vector + })), + NextId = pagedResult.NextId, + }; + } + + public async Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(update.Text) + || !Guid.TryParse(update.Id, out var guid)) + { + return false; + } + + var vectorDb = GetVectorDb(update.DbProvider); + if (vectorDb == null) + { + return false; + } + + var found = await vectorDb.GetCollectionData(collectionName, [guid]); + if (found.IsNullOrEmpty()) + { + return false; + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(update.Text); + var payload = update.Payload ?? new(); + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); + } + + return await vectorDb.Upsert(collectionName, guid, vector, update.Text, payload); + } + + public async Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(update.Text) + || !Guid.TryParse(update.Id, out var guid)) + { + return false; + } + + var vectorDb = GetVectorDb(update.DbProvider); + if (vectorDb == null) + { + return false; + } + + var found = await vectorDb.GetCollectionData(collectionName, [guid], options: new() { WithVector = true, WithPayload = true }); + if (!found.IsNullOrEmpty()) + { + if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text) + { + // Only update payload + return await vectorDb.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload); + } + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(update.Text); + var payload = update.Payload ?? new(); + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); + } + + return await vectorDb.Upsert(collectionName, guid, vector, update.Text, payload); + } + #endregion + + public async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + { + var vectorDb = GetVectorDb(options.DbProvider); + if (vectorDb == null) + { + return []; + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(query); + + var searchOptions = new VectorSearchOptions + { + Fields = options.Fields, + FilterGroups = options.FilterGroups, + Limit = options.Limit, + Confidence = options.Confidence, + WithVector = options.WithVector, + SearchParam = options.SearchParam + }; + + var found = await vectorDb.Search(collectionName, vector, searchOptions); + return found.Select(x => KnowledgeSearchResult.CopyFrom(new KnowledgeCollectionData + { + Id = x.Id, + Payload = x.Payload, + Score = x.Score, + Vector = x.Vector + })).ToList(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Index.cs new file mode 100644 index 000000000..19de772d7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Index.cs @@ -0,0 +1,82 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorOrchestratorBase +{ + #region Index + public async Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return new(); + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return new(); + } + + var response = new SuccessFailResponse(); + var innerOptions = options?.Items?.DistinctBy(x => x.FieldName)?.ToList(); + if (innerOptions.IsNullOrEmpty()) + { + return new(); + } + + foreach (var option in innerOptions!) + { + var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option); + var field = $"{option.FieldName} ({option.FieldSchemaType})"; + if (created) + { + response.Success.Add(field); + } + else + { + response.Fail.Add(field); + } + } + + return response; + } + + public async Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return new(); + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return new(); + } + + var response = new SuccessFailResponse(); + var innerOptions = options?.Items?.DistinctBy(x => x.FieldName)?.ToList(); + if (innerOptions.IsNullOrEmpty()) + { + return new(); + } + + foreach (var option in innerOptions!) + { + var deleted = await vectorDb.DeleteCollectionPayloadIndex(collectionName, option); + var field = $"{option.FieldName}"; + if (deleted) + { + response.Success.Add(field); + } + else + { + response.Fail.Add(field); + } + } + + return response; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Snapshot.cs new file mode 100644 index 000000000..fc10fd9db --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Snapshot.cs @@ -0,0 +1,91 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorOrchestratorBase +{ + #region Snapshot + public async Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return []; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return []; + } + + var snapshots = await vectorDb.GetCollectionSnapshots(collectionName); + return snapshots.Select(x => KnowledgeCollectionSnapshot.CopyFrom(x)!); + } + + public async Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return null; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return null; + } + + var snapshot = await vectorDb.CreateCollectionShapshot(collectionName); + return KnowledgeCollectionSnapshot.CopyFrom(snapshot); + } + + public async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) + { + return BinaryData.Empty; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return BinaryData.Empty; + } + + var snapshot = await vectorDb.DownloadCollectionSnapshot(collectionName, snapshotFileName); + return snapshot; + } + + public async Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var done = await vectorDb.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData); + return done; + } + + public async Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var done = await vectorDb.DeleteCollectionShapshot(collectionName, snapshotName); + return done; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.cs new file mode 100644 index 000000000..c0c2b0880 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.cs @@ -0,0 +1,40 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorOrchestratorBase : IKnowledgeOrchestrator +{ + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + protected readonly KnowledgeBaseSettings _settings; + + public abstract string Type { get; } + + protected VectorOrchestratorBase( + IServiceProvider services, + ILogger logger, + KnowledgeBaseSettings settings) + { + _services = services; + _logger = logger; + _settings = settings; + } + + protected IVectorDb? GetVectorDb(string? dbProvider = null) + { + var provider = dbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + return db; + } + + protected async Task GetTextEmbedding(string collectionName) + { + return await KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); + } + + protected async Task GetUserId() + { + var userIdentity = _services.GetRequiredService(); + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(userIdentity.Id); + return user.Id; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs new file mode 100644 index 000000000..f495594b5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class DocumentKnowledgeOrchestrator : VectorOrchestratorBase, IKnowledgeOrchestrator +{ + public override string Type => KnowledgeType.Document; + + public DocumentKnowledgeOrchestrator( + IServiceProvider services, + ILogger logger, + KnowledgeBaseSettings settings) + : base(services, logger, settings) + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs index 4ea034487..d55198d9c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) + public async Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) { try { @@ -40,7 +40,7 @@ public async Task> CreateVectorCollectionPayloadInde } } - public async Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) + public async Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) { try { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs new file mode 100644 index 000000000..cbfcc0fe7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public class QuestionAnswerKnowledgeOrchestrator : VectorOrchestratorBase, IKnowledgeOrchestrator +{ + public override string Type => KnowledgeType.QuestionAnswer; + + public QuestionAnswerKnowledgeOrchestrator( + IServiceProvider services, + ILogger logger, + KnowledgeBaseSettings settings) + : base(services, logger, settings) + { + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index cbca32ae5..fab3b9cb2 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -346,7 +346,7 @@ public async Task DeleteCollectionAllData(string collectionName) #endregion #region Payload index - public async Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) + public async Task CreateCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) { var exist = await DoesCollectionExist(collectionName); if (!exist) @@ -360,7 +360,7 @@ public async Task CreateCollectionPayloadIndex(string collectionName, Crea return result.Status == UpdateStatus.Completed; } - public async Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) + public async Task DeleteCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) { var exist = await DoesCollectionExist(collectionName); if (!exist) From f8b19feaeab810af3c225367e10977ef05654752 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 20 Apr 2026 17:37:02 -0500 Subject: [PATCH 02/23] relocate --- .../Base/{ => Vector}/VectorOrchestratorBase.Collection.cs | 0 .../Services/Base/{ => Vector}/VectorOrchestratorBase.Data.cs | 0 .../Services/Base/{ => Vector}/VectorOrchestratorBase.Index.cs | 0 .../Services/Base/{ => Vector}/VectorOrchestratorBase.Snapshot.cs | 0 .../Services/Base/{ => Vector}/VectorOrchestratorBase.cs | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/{ => Vector}/VectorOrchestratorBase.Collection.cs (100%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/{ => Vector}/VectorOrchestratorBase.Data.cs (100%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/{ => Vector}/VectorOrchestratorBase.Index.cs (100%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/{ => Vector}/VectorOrchestratorBase.Snapshot.cs (100%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/{ => Vector}/VectorOrchestratorBase.cs (100%) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Collection.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Data.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Index.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.Snapshot.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/VectorOrchestratorBase.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs From 5ac0cdd1b4a4ea488e79dc46993cf6cb06229440 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 21 Apr 2026 12:46:05 -0500 Subject: [PATCH 03/23] migrate knowledge file --- .../Files/IFileStorageService.cs | 16 +- .../Knowledges/Filters/KnowledgeFileFilter.cs | 4 +- .../Knowledges/IKnowledgeDocOrchestrator.cs | 53 ++ .../Knowledges/IKnowledgeService.cs | 2 +- ...ocMetaData.cs => KnowledgeFileMetaData.cs} | 2 +- .../Knowledges/Options/KnowledgeDocOptions.cs | 6 - .../Options/KnowledgeFileHandleOptions.cs | 7 + .../Options/KnowledgeFileOptions.cs | 5 + .../Repositories/IBotSharpRepository.cs | 15 +- .../Filters/VectorCollectionConfigFilter.cs | 2 +- .../LocalFileStorageService.KnowledgeBase.cs | 30 +- .../FileRepository.KnowledgeBase.cs | 32 +- .../KnowledgeBaseController.Document.cs | 2 +- .../Request/VectorKnowledgeUploadRequest.cs | 2 +- .../KnowledgeBasePlugin.cs | 1 + .../VectorOrchestratorBase.Collection.cs | 4 +- .../Services/File/KnowledgeDocOrchestrator.cs | 476 ++++++++++++++++++ .../Services/KnowledgeService.Document.cs | 6 +- .../Services/KnowledgeService.Vector.cs | 4 +- .../MongoRepository.KnowledgeBase.cs | 28 +- .../TencentCosService.KnowledgeBase.cs | 28 +- .../Core/NullFileStorageService.cs | 8 +- 22 files changed, 626 insertions(+), 107 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/{KnowledgeDocMetaData.cs => KnowledgeFileMetaData.cs} (97%) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 72e8fddb4..7f785d188 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -62,17 +62,9 @@ public interface IFileStorageService #endregion #region Knowledge - bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData); - - /// - /// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection. - /// - /// - /// - /// - /// - bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null); - string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); - BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); + bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData); + bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null); + string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName); + BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs index ec2504181..676a74d21 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs @@ -2,12 +2,10 @@ namespace BotSharp.Abstraction.Knowledges.Filters; public class KnowledgeFileFilter : Pagination { + public string? DbProvider { get; set; } public IEnumerable? FileIds { get; set; } - public IEnumerable? FileNames { get; set; } - public IEnumerable? ContentTypes { get; set; } - public IEnumerable? FileSources { get; set; } public KnowledgeFileFilter() diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs new file mode 100644 index 000000000..1c58ff4fb --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs @@ -0,0 +1,53 @@ +using BotSharp.Abstraction.Knowledges.Filters; +using BotSharp.Abstraction.Knowledges.Options; +using BotSharp.Abstraction.Knowledges.Responses; + +namespace BotSharp.Abstraction.Knowledges; + +public interface IKnowledgeDocOrchestrator +{ + string Provider { get; } + + /// + /// Save documents and their contents to knowledgebase + /// + /// + /// + /// + /// + Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); + + /// + /// Delete one document and its related knowledge in the collection + /// + /// + /// + /// + /// + Task DeleteKnowledgeDocument(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); + + /// + /// Delete all documents and their related knowledge in the collection + /// + /// + /// + /// + Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); + + /// + /// Get knowlege documents by pagination + /// + /// + /// + /// + Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); + + /// + /// Get knowledge document binary data + /// + /// + /// + /// + /// + Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index c01879cb6..a75f10297 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -34,7 +34,7 @@ public interface IKnowledgeService /// /// /// - Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeDocOptions? options = null); + Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); /// /// Save document content to knowledgebase without saving the document /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs similarity index 97% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs index a1139c901..810e0ef96 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeDocMetaData +public class KnowledgeFileMetaData { [JsonPropertyName("collection")] public string Collection { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs deleted file mode 100644 index 52c123426..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Options; - -public class KnowledgeDocOptions : FileKnowledgeHandleOptions -{ - public string? Processor { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs new file mode 100644 index 000000000..48c505344 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeFileHandleOptions : FileKnowledgeHandleOptions +{ + public string? DbProvider { get; set; } + public string? Processor { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs new file mode 100644 index 000000000..b66e37431 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs @@ -0,0 +1,5 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeFileOptions : KnowledgeOptionBase +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 437e6cdd7..fef9c9684 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -239,21 +239,14 @@ Task DeleteKnowledgeCollectionConfig(string collectionName) => throw new NotImplementedException(); Task> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) => throw new NotImplementedException(); - Task GetKnowledgeCollectionConfig(string collectionName, string vectorStroageProvider) + Task GetKnowledgeCollectionConfig(string collectionName, string knowledgebaseProvider) => throw new NotImplementedException(); - Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + Task SaveKnolwedgeBaseFileMeta(KnowledgeFileMetaData metaData) => throw new NotImplementedException(); - /// - /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection. - /// - /// - /// - /// - /// - Task DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + Task DeleteKnolwedgeBaseFileMeta(string collectionName, string knowledgebaseProvider, Guid? fileId = null) => throw new NotImplementedException(); - Task> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + Task> GetKnowledgeBaseFileMeta(string collectionName, string knowledgebaseProvider, KnowledgeFileFilter filter) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs index 14d5126d4..10b27e203 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs @@ -4,7 +4,7 @@ public class VectorCollectionConfigFilter { public IEnumerable? CollectionNames { get; set; } public IEnumerable? CollectionTypes { get; set; } - public IEnumerable? VectorStroageProviders { get; set; } + public IEnumerable? VectorStorageProviders { get; set; } public static VectorCollectionConfigFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 72170fea0..e860c9ce1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -4,17 +4,17 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } try { - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var dir = Path.Combine(docDir, fileId.ToString()); if (ExistDirectory(dir)) { @@ -32,20 +32,20 @@ public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvi catch (Exception ex) { _logger.LogWarning(ex, $"Error when saving knowledge file " + - $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})."); + $"(Vector store provider: {knowledgebaseProvider}, Collection: {collectionName}, File name: {fileName})."); return false; } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!ExistDirectory(dir)) { return false; @@ -67,16 +67,16 @@ public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvide return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return string.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var file = Path.Combine(docDir, fileId.ToString(), fileName); if (!File.Exists(file)) { @@ -86,16 +86,16 @@ public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreP return $"/knowledge/document/{collectionName}/file/{fileId}"; } - public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return BinaryData.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var file = Path.Combine(docDir, fileId.ToString(), fileName); if (!File.Exists(file)) @@ -111,9 +111,9 @@ public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string v #region Private methods - private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) + private string BuildKnowledgeCollectionFileDir(string collectionName, string knowledgebaseProvider) { - return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr()); + return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, knowledgebaseProvider.CleanStr(), collectionName.CleanStr()); } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index 94b5aa0c5..1a4a7f853 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -103,9 +103,9 @@ public async Task> GetKnowledgeCollectionCon configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList(); } - if (!filter.VectorStroageProviders.IsNullOrEmpty()) + if (!filter.VectorStorageProviders.IsNullOrEmpty()) { - configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStore?.Provider)).ToList(); + configs = configs.Where(x => filter.VectorStorageProviders.Contains(x.VectorStore?.Provider)).ToList(); } return configs; @@ -114,12 +114,12 @@ public async Task> GetKnowledgeCollectionCon #if !DEBUG [SharpCache(10)] #endif - public async Task GetKnowledgeCollectionConfig(string collectionName, string vectorStroageProvider) + public async Task GetKnowledgeCollectionConfig(string collectionName, string knowledgebaseProvider) { var configs = await GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStroageProviders = [vectorStroageProvider] + VectorStorageProviders = [knowledgebaseProvider] }); return configs?.FirstOrDefault(); } @@ -127,7 +127,7 @@ public async Task GetKnowledgeCollectionConfig(string co #region Documents - public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + public async Task SaveKnolwedgeBaseFileMeta(KnowledgeFileMetaData metaData) { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) @@ -149,15 +149,15 @@ public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) return true; } - public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!Directory.Exists(dir)) { return false; @@ -179,21 +179,21 @@ public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, strin return await Task.FromResult(true); } - public async Task> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + public async Task> GetKnowledgeBaseFileMeta(string collectionName, string knowledgebaseProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { - return new PagedItems(); + return new PagedItems(); } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!Directory.Exists(dir)) { - return new PagedItems(); + return new PagedItems(); } - var records = new List(); + var records = new List(); foreach (var folder in Directory.EnumerateDirectories(dir)) { var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE); @@ -203,7 +203,7 @@ public async Task> GetKnowledgeBaseFileMeta(str } var content = await File.ReadAllTextAsync(metaFile); - var metaData = JsonSerializer.Deserialize(content, _options); + var metaData = JsonSerializer.Deserialize(content, _options); if (metaData == null) { continue; @@ -244,7 +244,7 @@ public async Task> GetKnowledgeBaseFileMeta(str records.Add(metaData); } - return new PagedItems + return new PagedItems { Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size), Count = records.Count diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs index cdd91c2dd..458e1c4cf 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs @@ -26,7 +26,7 @@ public async Task UploadKnowledgeDocuments([FromRoute] public async Task UploadKnowledgeDocuments( [FromRoute] string collection, [FromForm] IEnumerable files, - [FromForm] KnowledgeDocOptions? options = null) + [FromForm] KnowledgeFileHandleOptions? options = null) { if (files.IsNullOrEmpty()) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs index 2a97733e4..f7c34e37b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs @@ -9,5 +9,5 @@ public class VectorKnowledgeUploadRequest public IEnumerable Files { get; set; } = new List(); [JsonPropertyName("options")] - public KnowledgeDocOptions? Options { get; set; } + public KnowledgeFileHandleOptions? Options { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index ac86046ee..5310852e9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -26,6 +26,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs index 55f8f53b8..05fad46e6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs @@ -23,7 +23,7 @@ public async Task ExistCollection(string collectionName, KnowledgeCollecti var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStroageProviders = [_settings.VectorDb.Provider] + VectorStorageProviders = [_settings.VectorDb.Provider] }); return !configs.IsNullOrEmpty(); @@ -83,7 +83,7 @@ public async Task> GetCollections(Knowled var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionTypes = !string.IsNullOrEmpty(Type) ? [Type] : null, - VectorStroageProviders = [_settings.VectorDb.Provider] + VectorStorageProviders = [_settings.VectorDb.Provider] }); var vectorDb = GetVectorDb(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs new file mode 100644 index 000000000..15b3c8727 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs @@ -0,0 +1,476 @@ +using System.Net.Http; +using BotSharp.Abstraction.VectorStorage.Filters; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public class KnowledgeDocOrchestrator : IKnowledgeDocOrchestrator +{ + private readonly IServiceProvider _services; + private readonly KnowledgeBaseSettings _settings; + private readonly ILogger _logger; + + public string Provider => "botsharp-knowledge-doc"; + + public KnowledgeDocOrchestrator( + IServiceProvider services, + KnowledgeBaseSettings settings, + ILogger logger) + { + _services = services; + _settings = settings; + _logger = logger; + } + + public async Task UploadDocumentsToKnowledge( + string collectionName, + IEnumerable files, + KnowledgeFileHandleOptions? options = null) + { + var res = new UploadKnowledgeResponse + { + Success = [], + Failed = files?.Select(x => x.FileName) ?? [] + }; + + if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty()) + { + return res; + } + + var knowledgebaseProvider = options?.DbProvider ?? _settings.VectorDb.Provider; + var exist = await ExistCollection(collectionName, knowledgebaseProvider); + if (!exist) + { + return res; + } + + var knowledgeFiles = new List(); + var successFiles = new List(); + var failedFiles = new List(); + + foreach (var file in files!) + { + if (string.IsNullOrWhiteSpace(file.FileData) + && string.IsNullOrWhiteSpace(file.FileUrl)) + { + continue; + } + + try + { + // Get document info + var (contentType, binary) = await GetFileInfo(file); + var fileData = new FileBinaryDataModel + { + FileName = file.FileName, + ContentType = contentType, + FileBinaryData = binary + }; + var knowledges = await GetFileKnowledge(fileData, options); + if (knowledges.IsNullOrEmpty()) + { + failedFiles.Add(file.FileName); + continue; + } + + var fileId = Guid.NewGuid(); + var payload = new Dictionary() + { + { KnowledgePayloadName.DataSource, (VectorPayloadValue)VectorDataSource.File }, + { KnowledgePayloadName.FileId, (VectorPayloadValue)fileId.ToString() }, + { KnowledgePayloadName.FileName, (VectorPayloadValue)file.FileName }, + { KnowledgePayloadName.FileSource, (VectorPayloadValue)file.FileSource } + }; + + if (!string.IsNullOrWhiteSpace(file.FileUrl)) + { + payload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)file.FileUrl; + } + + foreach (var kg in knowledges) + { + var kgPayload = new Dictionary(kg.Payload ?? new Dictionary()); + foreach (var pair in payload) + { + kgPayload[pair.Key] = pair.Value; + } + kg.Payload = kgPayload; + } + + knowledgeFiles.Add(new() + { + FileId = fileId, + FileData = fileData, + FileSource = VectorDataSource.File, + FileKnowledges = knowledges + }); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when processing knowledge file ({file.FileName})."); + failedFiles.Add(file.FileName); + continue; + } + } + + var response = await HandleKnowledgeFiles(collectionName, knowledgebaseProvider, knowledgeFiles, saveFile: true); + return new UploadKnowledgeResponse + { + Success = successFiles.Concat(response.Success).Distinct(), + Failed = failedFiles.Concat(response.Failed).Distinct() + }; + } + + public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + try + { + var db = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var knowledgebaseProvider = options?.DbProvider ?? _settings.VectorDb.Provider; + var vectorDb = GetVectorDb(knowledgebaseProvider); + if (vectorDb == null) + { + return false; + } + + // Get doc meta data + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, knowledgebaseProvider, new KnowledgeFileFilter + { + Size = 1, + FileIds = [fileId] + }); + + // Delete doc + fileStorage.DeleteKnowledgeFile(collectionName, knowledgebaseProvider, fileId); + + var found = pageData?.Items?.FirstOrDefault(); + if (found != null && !found.VectorDataIds.IsNullOrEmpty()) + { + var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); + await vectorDb.DeleteCollectionData(collectionName, guids); + } + + await db.DeleteKnolwedgeBaseFileMeta(collectionName, knowledgebaseProvider, fileId); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when deleting knowledge document " + + $"(Collection: {collectionName}, File id: {fileId})"); + return false; + } + } + + public async Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + { + if (string.IsNullOrWhiteSpace(collectionName)) return false; + + var pageSize = filter.Size; + var innerFilter = new KnowledgeFileFilter + { + Page = 1, + Size = pageSize, + FileIds = filter.FileIds, + FileNames = filter.FileNames, + FileSources = filter.FileSources, + ContentTypes = filter.ContentTypes + }; + + var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + + var total = pageData.Count; + if (total == 0) return false; + + var page = 1; + var totalPages = total % pageSize == 0 ? total / pageSize : total / pageSize + 1; + + while (page <= totalPages) + { + if (page > 1) + { + pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + } + + var fileIds = pageData.Items.Select(x => x.FileId).ToList(); + foreach (var fileId in fileIds) + { + try + { + await DeleteKnowledgeDocument(collectionName, fileId); + } + catch + { + continue; + } + } + + page++; + } + + return true; + } + + public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return new PagedItems(); + } + + var db = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var knowledgebaseProvider = filter?.DbProvider ?? _settings.VectorDb.Provider; + + // Get doc meta data + var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, knowledgebaseProvider, filter); + + var files = pagedData.Items?.Select(x => new KnowledgeFileModel + { + FileId = x.FileId, + FileName = x.FileName, + FileSource = x.FileSource, + FileExtension = Path.GetExtension(x.FileName), + ContentType = x.ContentType, + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, knowledgebaseProvider, x.FileId, x.FileName), + RefData = x.RefData + })?.ToList() ?? []; + + return new PagedItems + { + Items = files, + Count = pagedData.Count + }; + } + + public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) + { + var db = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var vectorStoreProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; + + // Get doc binary data + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + { + Size = 1, + FileIds = [fileId] + }); + + var metaData = pageData?.Items?.FirstOrDefault(); + if (metaData == null) + { + return new FileBinaryDataModel + { + FileName = "error.txt", + ContentType = "text/plain", + FileBinaryData = BinaryData.Empty + }; + }; + + var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); + return new FileBinaryDataModel + { + FileName = metaData.FileName, + ContentType = metaData.ContentType, + FileBinaryData = binaryData + }; + } + + + #region Private methods + private IVectorDb? GetVectorDb(string? dbProvider = null) + { + var provider = dbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + return db; + } + + private async Task GetTextEmbedding(string collectionName) + { + return await KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); + } + + private async Task GetUserId() + { + var userIdentity = _services.GetRequiredService(); + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(userIdentity.Id); + return user.Id; + } + + private async Task ExistCollection(string collectionName, string? dbProvider) + { + var vectorDb = GetVectorDb(dbProvider); + if (vectorDb == null) + { + return false; + } + + var exist = await vectorDb.DoesCollectionExist(collectionName); + if (exist) return true; + + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = [collectionName], + VectorStorageProviders = [vectorDb.Provider] + }); + + return !configs.IsNullOrEmpty(); + } + + private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file) + { + if (file == null) + { + return (string.Empty, BinaryData.Empty); + } + + if (!string.IsNullOrWhiteSpace(file.FileUrl)) + { + var http = _services.GetRequiredService(); + var contentType = FileUtility.GetFileContentType(file.FileName); + using var client = http.CreateClient(); + var bytes = await client.GetByteArrayAsync(file.FileUrl); + return (contentType, BinaryData.FromBytes(bytes)); + } + else if (!string.IsNullOrWhiteSpace(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + return (contentType, binary); + } + + return (string.Empty, BinaryData.Empty); + } + + private async Task> GetFileKnowledge(FileBinaryDataModel file, KnowledgeFileHandleOptions? options) + { + var processor = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(options?.Processor)); + if (processor == null) + { + return Enumerable.Empty(); + } + + var response = await processor.GetFileKnowledgeAsync(file, options: options); + return response?.Success == true ? response.Knowledges ?? [] : []; + } + + private bool SaveDocument(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData binary) + { + var fileStorage = _services.GetRequiredService(); + var saved = fileStorage.SaveKnowledgeBaseFile(collectionName, knowledgebaseProvider, fileId, fileName, binary); + return saved; + } + + private async Task> SaveToKnowledgebase(string collectionName, string knowledgebaseProvider, IEnumerable contents, Dictionary? payload = null) + { + if (contents.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + var dataIds = new List(); + var vectorDb = GetVectorDb(knowledgebaseProvider); + var textEmbedding = await GetTextEmbedding(collectionName); + + for (int i = 0; i < contents.Count(); i++) + { + var content = contents.ElementAt(i); + + try + { + var vector = await textEmbedding.GetVectorAsync(content); + var dataId = Guid.NewGuid(); + var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload ?? []); + + if (!saved) + { + continue; + } + + dataIds.Add(dataId.ToString()); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when saving file knowledge to vector db collection {collectionName}. (Content: {content.SubstringMax(20)})"); + } + } + + return dataIds; + } + + private async Task HandleKnowledgeFiles( + string collectionName, + string knowledgebaseProvider, + IEnumerable knowledgeFiles, + bool saveFile = false) + { + if (knowledgeFiles.IsNullOrEmpty()) + { + return new(); + } + + var successFiles = new List(); + var failedFiles = new List(); + var db = _services.GetRequiredService(); + + var userId = await GetUserId(); + foreach (var item in knowledgeFiles) + { + var file = item.FileData; + + // Save document + if (saveFile) + { + var saved = SaveDocument(collectionName, knowledgebaseProvider, item.FileId, file.FileName, file.FileBinaryData); + if (!saved) + { + _logger.LogWarning($"Failed to save knowledge file: {file.FileName} to collection {collectionName}."); + failedFiles.Add(file.FileName); + continue; + } + } + + // Save to vector db + var dataIds = new List(); + foreach (var kg in item.FileKnowledges) + { + var ids = await SaveToKnowledgebase(collectionName, knowledgebaseProvider, kg.Contents, kg.Payload?.ToDictionary()); + dataIds.AddRange(ids); + } + + if (!dataIds.IsNullOrEmpty()) + { + await db.SaveKnolwedgeBaseFileMeta(new KnowledgeFileMetaData + { + Collection = collectionName, + FileId = item.FileId, + FileName = file.FileName, + FileSource = item.FileSource ?? VectorDataSource.File, + ContentType = file.ContentType, + VectorStoreProvider = knowledgebaseProvider, + VectorDataIds = dataIds, + CreateDate = DateTime.UtcNow, + CreateUserId = userId + }); + successFiles.Add(file.FileName); + } + else + { + failedFiles.Add(file.FileName); + } + } + + return new UploadKnowledgeResponse + { + Success = successFiles, + Failed = failedFiles + }; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 5f2222eb9..31b45a476 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -7,7 +7,7 @@ public partial class KnowledgeService public async Task UploadDocumentsToKnowledge( string collectionName, IEnumerable files, - KnowledgeDocOptions? options = null) + KnowledgeFileHandleOptions? options = null) { var res = new UploadKnowledgeResponse { @@ -358,7 +358,7 @@ public async Task GetKnowledgeDocumentBinaryData(string col } #region Read doc content - private async Task> GetFileKnowledge(FileBinaryDataModel file, KnowledgeDocOptions? options) + private async Task> GetFileKnowledge(FileBinaryDataModel file, KnowledgeFileHandleOptions? options) { var processor = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(options?.Processor)); if (processor == null) @@ -458,7 +458,7 @@ private async Task HandleKnowledgeFiles( if (!dataIds.IsNullOrEmpty()) { - await db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData + await db.SaveKnolwedgeBaseFileMeta(new KnowledgeFileMetaData { Collection = collectionName, FileId = item.FileId, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index 85f81a1e1..f26436340 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -17,7 +17,7 @@ public async Task ExistVectorCollection(string collectionName) var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStroageProviders = [_settings.VectorDb.Provider] + VectorStorageProviders = [_settings.VectorDb.Provider] }); return !configs.IsNullOrEmpty(); @@ -75,7 +75,7 @@ public async Task> GetVectorCollections(stri var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionTypes = !string.IsNullOrEmpty(type) ? [type] : null, - VectorStroageProviders = [_settings.VectorDb.Provider] + VectorStorageProviders = [_settings.VectorDb.Provider] }); var vectorDb = GetVectorDb(); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index e4aa6797f..1318824fd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -99,9 +99,9 @@ public async Task> GetKnowledgeCollectionCon filters.Add(builder.In(x => x.Type, filter.CollectionTypes)); } - if (!filter.VectorStroageProviders.IsNullOrEmpty()) + if (!filter.VectorStorageProviders.IsNullOrEmpty()) { - filters.Add(builder.In(x => x.VectorStore.Provider, filter.VectorStroageProviders)); + filters.Add(builder.In(x => x.VectorStore.Provider, filter.VectorStorageProviders)); } // Get data @@ -119,19 +119,19 @@ public async Task> GetKnowledgeCollectionCon #if !DEBUG [SharpCache(10)] #endif - public async Task GetKnowledgeCollectionConfig(string collectionName, string vectorStroageProvider) + public async Task GetKnowledgeCollectionConfig(string collectionName, string knowledgebaseProvider) { var configs = await GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStroageProviders = [vectorStroageProvider] + VectorStorageProviders = [knowledgebaseProvider] }); return configs?.FirstOrDefault(); } #endregion #region Documents - public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + public async Task SaveKnolwedgeBaseFileMeta(KnowledgeFileMetaData metaData) { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) @@ -159,10 +159,10 @@ public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) return true; } - public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } @@ -171,7 +171,7 @@ public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, strin var filters = new List>() { builder.Eq(x => x.Collection, collectionName), - builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + builder.Eq(x => x.VectorStoreProvider, knowledgebaseProvider) }; if (fileId != null) @@ -183,19 +183,19 @@ public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, strin return res.DeletedCount > 0; } - public async Task> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + public async Task> GetKnowledgeBaseFileMeta(string collectionName, string knowledgebaseProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { - return new PagedItems(); + return new PagedItems(); } var builder = Builders.Filter; var docFilters = new List>() { builder.Eq(x => x.Collection, collectionName), - builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + builder.Eq(x => x.VectorStoreProvider, knowledgebaseProvider) }; // Apply filters @@ -237,7 +237,7 @@ public async Task> GetKnowledgeBaseFileMeta(str var docs = docsTask.Result.ToList(); var count = countTask.Result; - var files = docs?.Select(x => new KnowledgeDocMetaData + var files = docs?.Select(x => new KnowledgeFileMetaData { Collection = x.Collection, FileId = x.FileId, @@ -251,7 +251,7 @@ public async Task> GetKnowledgeBaseFileMeta(str CreateUserId = x.CreateUserId })?.ToList() ?? []; - return new PagedItems + return new PagedItems { Items = files, Count = count diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index 4ae82e9e2..bc81f9af9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -2,17 +2,17 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } try { - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var dir = $"{docDir}/{fileId}"; if (ExistDirectory(dir)) { @@ -26,20 +26,20 @@ public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvi catch (Exception ex) { _logger.LogWarning(ex, $"Error when saving knowledge file " + - $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})."); + $"(Vector store provider: {knowledgebaseProvider}, Collection: {collectionName}, File name: {fileName})."); return false; } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!ExistDirectory(dir)) return false; if (fileId == null) @@ -58,16 +58,16 @@ public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvide return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return string.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); + var docDir = BuildKnowledgeCollectionFileDir(knowledgebaseProvider, collectionName); var fileDir = $"{docDir}/{fileId}"; if (!ExistDirectory(fileDir)) { @@ -77,10 +77,10 @@ public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreP return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ; } - public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return BinaryData.Empty; @@ -88,7 +88,7 @@ public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string v try { - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var fileDir = $"{docDir}/{fileId}"; if (!ExistDirectory(fileDir)) { @@ -106,7 +106,7 @@ public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string v } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})"); + _logger.LogWarning(ex, $"Error when downloading collection file ({collectionName}-{knowledgebaseProvider}-{fileId}-{fileName})"); return BinaryData.Empty; } } diff --git a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs index 10236d0c5..4c87ff2e5 100644 --- a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs +++ b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs @@ -114,23 +114,23 @@ public BinaryData GetSpeechFile(string conversationId, string fileName) return BinaryData.FromBytes(new byte[] { 0x03, 0x04, 0x05 }); } - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, + public bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData) { return true; } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { return $"https://fakeurl.com/{fileName}"; } - public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { return BinaryData.FromBytes(new byte[] { 0x06, 0x07, 0x08 }); From d1d7d00d2feba15f9afcac52aa5eed915b5cd614 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 21 Apr 2026 12:51:44 -0500 Subject: [PATCH 04/23] rename --- ...estrator.cs => IKnowledgeFileOrchestrator.cs} | 2 +- .../KnowledgeBasePlugin.cs | 2 +- .../Vector/VectorOrchestratorBase.Collection.cs | 8 ++++---- .../Base/Vector/VectorOrchestratorBase.Data.cs | 16 ++++++++-------- .../Base/Vector/VectorOrchestratorBase.Index.cs | 4 ++-- .../Vector/VectorOrchestratorBase.Snapshot.cs | 10 +++++----- ...hestrator.cs => KnowledgeFileOrchestrator.cs} | 8 ++++---- 7 files changed, 25 insertions(+), 25 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/{IKnowledgeDocOrchestrator.cs => IKnowledgeFileOrchestrator.cs} (97%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/{KnowledgeDocOrchestrator.cs => KnowledgeFileOrchestrator.cs} (98%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs similarity index 97% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs index 1c58ff4fb..9f9f29224 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeDocOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Knowledges; -public interface IKnowledgeDocOrchestrator +public interface IKnowledgeFileOrchestrator { string Provider { get; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 5310852e9..a8eb90fdf 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -26,7 +26,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs index 05fad46e6..8c75b1838 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public abstract partial class VectorOrchestratorBase { #region Collection - public async Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) + public virtual async Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) { var vectorDb = GetVectorDb(options?.DbProvider); if (vectorDb == null) @@ -29,7 +29,7 @@ public async Task ExistCollection(string collectionName, KnowledgeCollecti return !configs.IsNullOrEmpty(); } - public async Task CreateCollection(string collectionName, CollectionCreateOptions options) + public virtual async Task CreateCollection(string collectionName, CollectionCreateOptions options) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -77,7 +77,7 @@ public async Task CreateCollection(string collectionName, CollectionCreate return created; } - public async Task> GetCollections(KnowledgeCollectionOptions options) + public virtual async Task> GetCollections(KnowledgeCollectionOptions options) { var db = _services.GetRequiredService(); var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter @@ -101,7 +101,7 @@ public async Task> GetCollections(Knowled }); } - public async Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) + public virtual async Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) { if (string.IsNullOrWhiteSpace(collectionName)) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs index bfb56662f..aaba36290 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public abstract partial class VectorOrchestratorBase { #region Collection data - public async Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) + public virtual async Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) { @@ -32,7 +32,7 @@ public async Task CreateCollectionData(string collectionName, KnowledgeCre return await vectorDb.Upsert(collectionName, guid, vector, create.Text, payload); } - public async Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) + public virtual async Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) { if (!Guid.TryParse(id, out var guid)) { @@ -48,7 +48,7 @@ public async Task DeleteCollectionData(string collectionName, string id, K return await vectorDb.DeleteCollectionData(collectionName, [guid]); } - public async Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) + public virtual async Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) { var vectorDb = GetVectorDb(options?.DbProvider); if (vectorDb == null) @@ -59,7 +59,7 @@ public async Task DeleteCollectionData(string collectionName, KnowledgeCol return await vectorDb.DeleteCollectionAllData(collectionName); } - public async Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) + public virtual async Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty()) { @@ -93,7 +93,7 @@ public async Task> GetCollectionData(string }); } - public async Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + public virtual async Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) { var vectorDb = GetVectorDb(filter.DbProvider); if (vectorDb == null) @@ -126,7 +126,7 @@ public async Task> GetPagedCollectionD }; } - public async Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) + public virtual async Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) @@ -159,7 +159,7 @@ public async Task UpdateCollectionData(string collectionName, KnowledgeUpd return await vectorDb.Upsert(collectionName, guid, vector, update.Text, payload); } - public async Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) + public virtual async Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) @@ -197,7 +197,7 @@ public async Task UpsertCollectionData(string collectionName, KnowledgeUpd } #endregion - public async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + public virtual async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) { var vectorDb = GetVectorDb(options.DbProvider); if (vectorDb == null) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs index 19de772d7..43a5c3465 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public abstract partial class VectorOrchestratorBase { #region Index - public async Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) + public virtual async Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -42,7 +42,7 @@ public async Task> CreateIndexes(string collectionNa return response; } - public async Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) + public virtual async Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) { if (string.IsNullOrWhiteSpace(collectionName)) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs index fc10fd9db..e7eb717c8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs @@ -3,7 +3,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public abstract partial class VectorOrchestratorBase { #region Snapshot - public async Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) + public virtual async Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -20,7 +20,7 @@ public async Task> GetCollectionSnapsho return snapshots.Select(x => KnowledgeCollectionSnapshot.CopyFrom(x)!); } - public async Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) + public virtual async Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -37,7 +37,7 @@ public async Task> GetCollectionSnapsho return KnowledgeCollectionSnapshot.CopyFrom(snapshot); } - public async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) + public virtual async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) { @@ -54,7 +54,7 @@ public async Task DownloadCollectionSnapshot(string collectionName, return snapshot; } - public async Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) + public virtual async Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -71,7 +71,7 @@ public async Task RecoverCollectionFromSnapshot(string collectionName, str return done; } - public async Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) + public virtual async Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs similarity index 98% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index 15b3c8727..16add6749 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeDocOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -3,18 +3,18 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public class KnowledgeDocOrchestrator : IKnowledgeDocOrchestrator +public class KnowledgeFileOrchestrator : IKnowledgeFileOrchestrator { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly ILogger _logger; + private readonly ILogger _logger; public string Provider => "botsharp-knowledge-doc"; - public KnowledgeDocOrchestrator( + public KnowledgeFileOrchestrator( IServiceProvider services, KnowledgeBaseSettings settings, - ILogger logger) + ILogger logger) { _services = services; _settings = settings; From ab107abe5d4fb93c3b4ff4f502ddcaeed3c17c78 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 21 Apr 2026 16:03:29 -0500 Subject: [PATCH 05/23] refine kg controller --- ...{KnowledgeType.cs => KnowledgeBaseType.cs} | 2 +- .../Knowledges/IKnowledgeFileOrchestrator.cs | 22 +-- .../Knowledges/IKnowledgeOrchestrator.cs | 2 +- .../Knowledges/IKnowledgeService.cs | 99 ++++++------- .../Models/VectorCollectionConfigModel.cs | 6 - .../KnowledgeBaseController.Document.cs | 87 ----------- .../KnowledgeBaseController.File.cs | 103 +++++++++++++ .../KnowledgeBase/KnowledgeBaseController.cs | 139 +++++++++++++----- .../Request/CollectionIndexRequest.cs | 20 +++ ...nRequest.cs => CreateCollectionRequest.cs} | 2 +- ....cs => DeleteCollectionSnapshotRequest.cs} | 5 +- .../Request/GetKnowledgeDocsRequest.cs | 7 - .../Request/GetKnowledgeFilesRequest.cs | 8 + .../KnowledgeCollectionConfigsRequest.cs | 10 ++ ...teRequest.cs => KnowledgeCreateRequest.cs} | 2 +- .../Request/KnowledgeFileRequest.cs | 7 + ...teRequest.cs => KnowledgeUpdateRequest.cs} | 2 +- ...adRequest.cs => KnowledgeUploadRequest.cs} | 8 +- .../Request/VectorCollectionIndexRequest.cs | 13 -- .../View/VectorCollectionSnapshotViewModel.cs | 17 +++ .../VectorOrchestratorBase.Collection.cs | 40 ++--- .../Base/Vector/VectorOrchestratorBase.cs | 2 +- .../Document/DocumentKnowledgeOrchestrator.cs | 2 +- .../File/KnowledgeFileOrchestrator.cs | 22 +-- .../Services/KnowledgeService.Common.cs | 12 -- .../QuestionAnswerKnowledgeOrchestrator.cs | 2 +- 26 files changed, 379 insertions(+), 262 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/{KnowledgeType.cs => KnowledgeBaseType.cs} (84%) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{CreateVectorCollectionRequest.cs => CreateCollectionRequest.cs} (92%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{DeleteVectorCollectionSnapshotRequest.cs => DeleteCollectionSnapshotRequest.cs} (57%) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{VectorKnowledgeCreateRequest.cs => KnowledgeCreateRequest.cs} (88%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{VectorKnowledgeUpdateRequest.cs => KnowledgeUpdateRequest.cs} (68%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{VectorKnowledgeUploadRequest.cs => KnowledgeUploadRequest.cs} (63%) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeType.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs similarity index 84% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeType.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs index d1aee93a3..c2aa125f3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Enums; -public static class KnowledgeType +public static class KnowledgeBaseType { public static string QuestionAnswer = "question-answer"; public static string Document = "document"; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs index 9f9f29224..beb40e320 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs @@ -9,45 +9,45 @@ public interface IKnowledgeFileOrchestrator string Provider { get; } /// - /// Save documents and their contents to knowledgebase + /// Save files and their contents to knowledgebase /// /// /// - /// + /// /// - Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); + Task UploadFilesToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); /// - /// Delete one document and its related knowledge in the collection + /// Delete one file and its related knowledge in the collection /// /// /// /// /// - Task DeleteKnowledgeDocument(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); + Task DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); /// - /// Delete all documents and their related knowledge in the collection + /// Delete all files and their related knowledge in the collection /// /// /// /// - Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); + Task DeleteKnowledgeFiles(string collectionName, KnowledgeFileFilter filter); /// - /// Get knowlege documents by pagination + /// Get knowledge files by pagination /// /// /// /// - Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); + Task> GetPagedKnowledgeFiles(string collectionName, KnowledgeFileFilter filter); /// - /// Get knowledge document binary data + /// Get knowledge file binary data /// /// /// /// /// - Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); + Task GetKnowledgeFileBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs index bb78913d4..600566d08 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs @@ -5,7 +5,7 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeOrchestrator { - string Type { get; } + string KnowledgeType { get; } #region Collection Task ExistCollection(string collectionName, KnowledgeCollectionOptions options); diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index a75f10297..abba17465 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -26,58 +26,55 @@ public interface IKnowledgeService Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update); #endregion - #region Document - /// - /// Save documents and their contents to knowledgebase - /// - /// - /// - /// - /// - Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); - /// - /// Save document content to knowledgebase without saving the document - /// - /// - /// - /// - /// - /// - /// - Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, - DocMetaRefData? refData = null, Dictionary? payload = null); - /// - /// Delete one document and its related knowledge in the collection - /// - /// - /// - /// - Task DeleteKnowledgeDocument(string collectionName, Guid fileId); - /// - /// Delete all documents and their related knowledge in the collection - /// - /// - /// - /// - Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); - #endregion + //#region Document + ///// + ///// Save documents and their contents to knowledgebase + ///// + ///// + ///// + ///// + ///// + //Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); + ///// + ///// Save document content to knowledgebase without saving the document + ///// + ///// + ///// + ///// + ///// + ///// + ///// + //Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, + // DocMetaRefData? refData = null, Dictionary? payload = null); + ///// + ///// Delete one document and its related knowledge in the collection + ///// + ///// + ///// + ///// + //Task DeleteKnowledgeDocument(string collectionName, Guid fileId); + ///// + ///// Delete all documents and their related knowledge in the collection + ///// + ///// + ///// + ///// + //Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); + //Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); + //Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); + //#endregion - #region Snapshot - Task> GetVectorCollectionSnapshots(string collectionName); - Task CreateVectorCollectionSnapshot(string collectionName); - Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName); - Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData); - Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); - #endregion + //#region Snapshot + //Task> GetVectorCollectionSnapshots(string collectionName); + //Task CreateVectorCollectionSnapshot(string collectionName); + //Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName); + //Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData); + //Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); + //#endregion - #region Index - Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); - Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); - #endregion + //#region Index + //Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + //Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + //#endregion - #region Common - Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs); - #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs index 01ffcaf19..7a9b2e0cf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs @@ -1,11 +1,5 @@ namespace BotSharp.Abstraction.VectorStorage.Models; -public class VectorCollectionConfigsModel -{ - [JsonPropertyName("collections")] - public List Collections { get; set; } = new(); -} - public class VectorCollectionConfig { /// diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs deleted file mode 100644 index 458e1c4cf..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs +++ /dev/null @@ -1,87 +0,0 @@ -using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Knowledges.Options; -using BotSharp.Abstraction.Knowledges.Processors; -using BotSharp.Abstraction.Knowledges.Responses; -using BotSharp.OpenAPI.ViewModels.Knowledges; - -namespace BotSharp.OpenAPI.Controllers; - -public partial class KnowledgeBaseController -{ - #region Document - [HttpGet("/knowledge/processors")] - public IEnumerable GetKnowledgeProcessors() - { - return _services.GetServices().Select(x => x.Provider); - } - - [HttpPost("/knowledge/document/{collection}/upload")] - public async Task UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) - { - var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, request.Files, request.Options); - return response; - } - - [HttpPost("/knowledge/document/{collection}/form")] - public async Task UploadKnowledgeDocuments( - [FromRoute] string collection, - [FromForm] IEnumerable files, - [FromForm] KnowledgeFileHandleOptions? options = null) - { - if (files.IsNullOrEmpty()) - { - return new UploadKnowledgeResponse(); - } - - var docs = new List(); - foreach (var file in files) - { - var data = FileUtility.BuildFileDataFromFile(file); - docs.Add(new ExternalFileModel - { - FileName = file.FileName, - FileData = data - }); - } - - var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, docs, options); - return response; - } - - [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")] - public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) - { - var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId); - return response; - } - - [HttpDelete("/knowledge/document/{collection}/delete")] - public async Task DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) - { - var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request); - return response; - } - - [HttpPost("/knowledge/document/{collection}/page")] - public async Task> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) - { - var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request); - - return new PagedItems - { - Items = data.Items.Select(x => KnowledgeFileViewModel.From(x)), - Count = data.Count - }; - } - - [HttpGet("/knowledge/document/{collection}/file/{fileId}")] - public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) - { - var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId); - var stream = file.FileBinaryData.ToStream(); - stream.Position = 0; - - return new FileStreamResult(stream, file.ContentType) { FileDownloadName = file.FileName }; - } - #endregion -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs new file mode 100644 index 000000000..3c114f670 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -0,0 +1,103 @@ +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Knowledges.Options; +using BotSharp.Abstraction.Knowledges.Processors; +using BotSharp.Abstraction.Knowledges.Responses; +using BotSharp.OpenAPI.ViewModels.Knowledges; + +namespace BotSharp.OpenAPI.Controllers; + +public partial class KnowledgeBaseController +{ + #region File + [HttpGet("/knowledge/processors")] + public IEnumerable GetKnowledgeProcessors() + { + return _services.GetServices().Select(x => x.Provider); + } + + [HttpPost("/knowledge/file/{collection}/upload")] + public async Task UploadKnowledgeFiles([FromRoute] string collection, [FromBody] KnowledgeUploadRequest request) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); + var response = await fileOrchestrator.UploadFilesToKnowledge(collection, request.Files, request.Options); + return response; + } + + [HttpPost("/knowledge/file/{collection}/form")] + public async Task UploadKnowledgeFiles( + [FromRoute] string collection, + [FromForm] IEnumerable files, + [FromForm] string? orchestrator = null, + [FromForm] KnowledgeFileHandleOptions? options = null) + { + if (files.IsNullOrEmpty()) + { + return new UploadKnowledgeResponse(); + } + + var docs = new List(); + foreach (var file in files) + { + var data = FileUtility.BuildFileDataFromFile(file); + docs.Add(new ExternalFileModel + { + FileName = file.FileName, + FileData = data + }); + } + + var fileOrchestrator = GetKnowledgeFileOrchestrator(orchestrator); + var response = await fileOrchestrator.UploadFilesToKnowledge(collection, docs, options); + return response; + } + + [HttpDelete("/knowledge/file/{collection}/delete/{fileId}")] + public async Task DeleteKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.Orchestrator); + var options = !string.IsNullOrWhiteSpace(request?.DbProvider) ? new KnowledgeFileOptions { DbProvider = request.DbProvider } : null; + var response = await fileOrchestrator.DeleteKnowledgeFile(collection, fileId, options); + return response; + } + + [HttpDelete("/knowledge/file/{collection}/delete")] + public async Task DeleteKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); + var response = await fileOrchestrator.DeleteKnowledgeFiles(collection, request); + return response; + } + + [HttpPost("/knowledge/file/{collection}/page")] + public async Task> GetPagedKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); + var data = await fileOrchestrator.GetPagedKnowledgeFiles(collection, request); + + return new PagedItems + { + Items = data.Items.Select(x => KnowledgeFileViewModel.From(x)), + Count = data.Count + }; + } + + [HttpGet("/knowledge/file/{collection}/{fileId}")] + public async Task GetKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.Orchestrator); + var options = !string.IsNullOrWhiteSpace(request?.DbProvider) ? new KnowledgeFileOptions { DbProvider = request.DbProvider } : null; + var file = await fileOrchestrator.GetKnowledgeFileBinaryData(collection, fileId, options); + var stream = file.FileBinaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, file.ContentType) { FileDownloadName = file.FileName }; + } + + private IKnowledgeFileOrchestrator? GetKnowledgeFileOrchestrator(string? provider) + { + provider ??= "botsharp-knowledge-doc"; + var found = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(provider)); + return found; + } + #endregion +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 1ab9278b9..3ed55a6ca 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Graph; using BotSharp.Abstraction.Graph.Options; +using BotSharp.Abstraction.Knowledges.Options; +using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.Abstraction.VectorStorage.Options; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -47,7 +49,7 @@ public async Task> GetVectorCollect } [HttpPost("knowledge/vector/create-collection")] - public async Task CreateVectorCollection([FromBody] CreateVectorCollectionRequest request) + public async Task CreateVectorCollection([FromBody] CreateCollectionRequest request) { var options = new VectorCollectionCreateOptions { @@ -65,7 +67,7 @@ public async Task DeleteVectorCollection([FromRoute] string collection) } [HttpPost("/knowledge/vector/{collection}/search")] - public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { var options = new VectorSearchOptions { @@ -82,7 +84,7 @@ public async Task> SearchVectorKnowledge([ } [HttpPost("/knowledge/vector/{collection}/page")] - public async Task> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) + public async Task> GetPagedCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) { var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter); var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? []; @@ -96,7 +98,7 @@ public async Task> GetPagedVectorCo } [HttpPost("/knowledge/vector/{collection}/create")] - public async Task CreateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeCreateRequest request) + public async Task CreateVectorKnowledge([FromRoute] string collection, [FromBody] KnowledgeCreateRequest request) { var create = new VectorCreateModel { @@ -109,7 +111,7 @@ public async Task CreateVectorKnowledge([FromRoute] string collection, [Fr } [HttpGet("/knowledge/vector/{collection}/points")] - public async Task> GetVectorCollectionData([FromRoute] string collection, [FromQuery] QueryVectorDataRequest request) + public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] QueryVectorDataRequest request) { var options = new VectorQueryOptions { @@ -122,7 +124,7 @@ public async Task> GetVectorCollectionData } [HttpPut("/knowledge/vector/{collection}/update")] - public async Task UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request) + public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeUpdateRequest request) { var update = new VectorUpdateModel { @@ -136,13 +138,13 @@ public async Task UpdateVectorKnowledge([FromRoute] string collection, [Fr } [HttpDelete("/knowledge/vector/{collection}/data/{id}")] - public async Task DeleteVectorCollectionData([FromRoute] string collection, [FromRoute] string id) + public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id) { return await _knowledgeService.DeleteVectorCollectionData(collection, id); } [HttpDelete("/knowledge/vector/{collection}/data")] - public async Task DeleteVectorCollectionAllData([FromRoute] string collection) + public async Task DeleteCollectionAllData([FromRoute] string collection) { return await _knowledgeService.DeleteVectorCollectionAllData(collection); } @@ -150,55 +152,119 @@ public async Task DeleteVectorCollectionAllData([FromRoute] string collect #region Index - [HttpPost("/knowledge/vector/{collection}/payload/indexes")] - public async Task> CreateCollectionPayloadIndexes([FromRoute] string collection, [FromBody] CreateVectorCollectionIndexRequest request) + [HttpPost("/knowledge/{collection}/indexes")] + public async Task> CreateCollectionIndexes([FromRoute] string collection, [FromBody] CreateCollectionIndexRequest request) { - return await _knowledgeService.CreateVectorCollectionPayloadIndexes(collection, request.Options); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) + { + return new(); + } + + var options = new KnowledgeIndexOptions + { + Items = request.Options + }; + return await orchestrator.CreateIndexes(collection, options); } - [HttpDelete("/knowledge/vector/{collection}/payload/indexes")] - public async Task> DeleteCollectionPayloadIndexes([FromRoute] string collection, [FromBody] DeleteVectorCollectionIndexRequest request) + [HttpDelete("/knowledge/{collection}/indexes")] + public async Task> DeleteCollectionIndexes([FromRoute] string collection, [FromBody] DeleteCollectionIndexRequest request) { - return await _knowledgeService.DeleteVectorCollectionPayloadIndexes(collection, request.Options); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) + { + return new(); + } + + var options = new KnowledgeIndexOptions + { + Items = request.Options + }; + return await orchestrator.DeleteIndexes(collection, options); } #endregion #region Snapshot - [HttpGet("/knowledge/vector/{collection}/snapshots")] - public async Task> GetVectorCollectionSnapshots([FromRoute] string collection) + [HttpGet("/knowledge/{collection}/snapshots")] + public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType) { - var snapshots = await _knowledgeService.GetVectorCollectionSnapshots(collection); - return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x)); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return []; + } + + var snapshots = await orchestrator.GetCollectionSnapshots(collection); + return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x)!); } - [HttpPost("/knowledge/vector/{collection}/snapshot")] - public async Task CreateVectorCollectionSnapshot([FromRoute] string collection) + [HttpPost("/knowledge/{collection}/snapshot")] + public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromQuery] string knowledgeType) { - var snapshot = await _knowledgeService.CreateVectorCollectionSnapshot(collection); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return null; + } + + var snapshot = await orchestrator.CreateCollectionSnapshot(collection); return VectorCollectionSnapshotViewModel.From(snapshot); } - [HttpGet("/knowledge/vector/{collection}/snapshot")] - public async Task GetVectorCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName) + [HttpGet("/knowledge/{collection}/snapshot")] + public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType) { - var snapshot = await _knowledgeService.DownloadVectorCollectionSnapshot(collection, snapshotFileName); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return BuildFileResult(snapshotFileName, BinaryData.Empty); + } + + var snapshot = await orchestrator.DownloadCollectionSnapshot(collection, snapshotFileName); return BuildFileResult(snapshotFileName, snapshot); } - [HttpPost("/knowledge/vector/{collection}/snapshot/recover")] - public async Task RecoverVectorCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile) + [HttpPost("/knowledge/{collection}/snapshot/recover")] + public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromQuery] string knowledgeType) { + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return false; + } + var fileName = snapshotFile.FileName; var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile); - var done = await _knowledgeService.RecoverVectorCollectionFromSnapshot(collection, fileName, binary); + var done = await orchestrator.RecoverCollectionFromSnapshot(collection, fileName, binary); return done; } - [HttpDelete("/knowledge/vector/{collection}/snapshot")] - public async Task DeleteVectorCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteVectorCollectionSnapshotRequest request) + [HttpDelete("/knowledge/{collection}/snapshot")] + public async Task DeleteCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteCollectionSnapshotRequest request) { - var done = await _knowledgeService.DeleteVectorCollectionSnapshot(collection, request.SnapshotName); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) + { + return false; + } + + var done = await orchestrator.DeleteCollectionSnapshot(collection, request.SnapshotName); return done; } #endregion @@ -226,11 +292,16 @@ public async Task SearchGraphKnowledge([FromBody] Searc #region Common - [HttpPost("/knowledge/vector/refresh-configs")] - public async Task RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request) + [HttpPost("/knowledge/refresh-configs")] + public async Task RefreshCollectionConfigs([FromBody] KnowledgeCollectionConfigsRequest request) { - var saved = await _knowledgeService.RefreshVectorKnowledgeConfigs(request); - return saved ? "Success" : "Fail"; + var db = _services.GetRequiredService(); + if (request.Collections.IsNullOrEmpty()) + { + return false; + } + var saved = await db.AddKnowledgeCollectionConfigs(request.Collections, reset: true); + return saved; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs new file mode 100644 index 000000000..aefa77089 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs @@ -0,0 +1,20 @@ +using BotSharp.Abstraction.VectorStorage.Options; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class CreateCollectionIndexRequest +{ + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } = null!; + + public IEnumerable Options { get; set; } = []; +} + +public class DeleteCollectionIndexRequest +{ + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } = null!; + + public IEnumerable Options { get; set; } = []; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs similarity index 92% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs index 7df04941f..83f32fdfa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs @@ -2,7 +2,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class CreateVectorCollectionRequest +public class CreateCollectionRequest { [JsonPropertyName("collection_name")] public string CollectionName { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs similarity index 57% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs index 30398b2c0..6d171dd71 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs @@ -2,8 +2,11 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class DeleteVectorCollectionSnapshotRequest +public class DeleteCollectionSnapshotRequest { + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } = null!; + [JsonPropertyName("snapshot_name")] public string SnapshotName { get; set; } = default!; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs deleted file mode 100644 index 52022b70b..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs +++ /dev/null @@ -1,7 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Filters; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class GetKnowledgeDocsRequest : KnowledgeFileFilter -{ -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs new file mode 100644 index 000000000..328a2357a --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Knowledges.Filters; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GetKnowledgeFilesRequest : KnowledgeFileFilter +{ + public string? Orchestrator { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs new file mode 100644 index 000000000..ed1541129 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionConfigsRequest +{ + [JsonPropertyName("collections")] + public List Collections { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs similarity index 88% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs index c2117b69f..1afeb17e0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs @@ -3,7 +3,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeCreateRequest +public class KnowledgeCreateRequest { [JsonPropertyName("text")] public string Text { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs new file mode 100644 index 000000000..62fc16c60 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeFileRequest +{ + public string? Orchestrator { get; set; } + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs similarity index 68% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs index 355ae7760..fdff63182 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs @@ -2,7 +2,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeUpdateRequest : VectorKnowledgeCreateRequest +public class KnowledgeUpdateRequest : KnowledgeCreateRequest { [JsonPropertyName("id")] public string Id { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs similarity index 63% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs index f7c34e37b..9b2f27b7b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs @@ -3,8 +3,14 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeUploadRequest +public class KnowledgeUploadRequest { + /// + /// Provider for knowledge file orchestrator + /// + [JsonPropertyName("orchestrator")] + public string? Orchestrator { get; set; } + [JsonPropertyName("files")] public IEnumerable Files { get; set; } = new List(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs deleted file mode 100644 index 3cfd365d0..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs +++ /dev/null @@ -1,13 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Options; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class CreateVectorCollectionIndexRequest -{ - public IEnumerable Options { get; set; } = []; -} - -public class DeleteVectorCollectionIndexRequest -{ - public IEnumerable Options { get; set; } = []; -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs index 95b8a3251..949507a40 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; using System.Text.Json.Serialization; @@ -32,4 +33,20 @@ public class VectorCollectionSnapshotViewModel CheckSum = model.CheckSum }; } + + public static VectorCollectionSnapshotViewModel? From(KnowledgeCollectionSnapshot? model) + { + if (model == null) + { + return null; + } + + return new VectorCollectionSnapshotViewModel + { + Name = model.Name, + Size = model.Size, + CreatedTime = model.CreatedTime, + CheckSum = model.CheckSum + }; + } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs index 8c75b1838..3f50166f6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs @@ -23,7 +23,7 @@ public virtual async Task ExistCollection(string collectionName, Knowledge var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStorageProviders = [_settings.VectorDb.Provider] + VectorStorageProviders = [vectorDb.Provider] }); return !configs.IsNullOrEmpty(); @@ -36,16 +36,22 @@ public virtual async Task CreateCollection(string collectionName, Collecti return false; } + var vectorDb = GetVectorDb(options.DbProvider); + if (vectorDb == null) + { + return false; + } + var db = _services.GetRequiredService(); var created = await db.AddKnowledgeCollectionConfigs(new List { new VectorCollectionConfig { Name = collectionName, - Type = Type, + Type = KnowledgeType, VectorStore = new VectorStoreConfig { - Provider = options.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider)! + Provider = vectorDb.Provider }, TextEmbedding = new KnowledgeEmbeddingConfig { @@ -61,12 +67,6 @@ public virtual async Task CreateCollection(string collectionName, Collecti return false; } - var vectorDb = GetVectorDb(options.DbProvider); - if (vectorDb == null) - { - return false; - } - created = await vectorDb.CreateCollection(collectionName, options: new() { Provider = options.LlmProvider, @@ -79,18 +79,19 @@ public virtual async Task CreateCollection(string collectionName, Collecti public virtual async Task> GetCollections(KnowledgeCollectionOptions options) { - var db = _services.GetRequiredService(); - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionTypes = !string.IsNullOrEmpty(Type) ? [Type] : null, - VectorStorageProviders = [_settings.VectorDb.Provider] - }); - - var vectorDb = GetVectorDb(); + var vectorDb = GetVectorDb(options?.DbProvider); if (vectorDb == null) { return []; } + + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionTypes = !string.IsNullOrEmpty(KnowledgeType) ? [KnowledgeType] : null, + VectorStorageProviders = [vectorDb.Provider] + }); + var dbCollections = await vectorDb.GetCollections(); return configs.Where(x => dbCollections.Contains(x.Name)).Select(x => new KnowledgeCollectionConfig { @@ -120,11 +121,10 @@ public virtual async Task DeleteCollection(string collectionName, Knowledg { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; await db.DeleteKnowledgeCollectionConfig(collectionName); - fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider); - await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider); + fileStorage.DeleteKnowledgeFile(collectionName, vectorDb.Provider); + await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorDb.Provider); } return deleted; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs index c0c2b0880..482112980 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs @@ -6,7 +6,7 @@ public abstract partial class VectorOrchestratorBase : IKnowledgeOrchestrator protected readonly ILogger _logger; protected readonly KnowledgeBaseSettings _settings; - public abstract string Type { get; } + public abstract string KnowledgeType { get; } protected VectorOrchestratorBase( IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs index f495594b5..e88196b6a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class DocumentKnowledgeOrchestrator : VectorOrchestratorBase, IKnowledgeOrchestrator { - public override string Type => KnowledgeType.Document; + public override string KnowledgeType => KnowledgeBaseType.Document; public DocumentKnowledgeOrchestrator( IServiceProvider services, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index 16add6749..e680b1c9c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -21,7 +21,7 @@ public KnowledgeFileOrchestrator( _logger = logger; } - public async Task UploadDocumentsToKnowledge( + public async Task UploadFilesToKnowledge( string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null) @@ -121,7 +121,7 @@ public async Task UploadDocumentsToKnowledge( }; } - public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) + public async Task DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -167,7 +167,7 @@ public async Task DeleteKnowledgeDocument(string collectionName, Guid file } } - public async Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + public async Task DeleteKnowledgeFiles(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) return false; @@ -182,7 +182,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg ContentTypes = filter.ContentTypes }; - var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + var pageData = await GetPagedKnowledgeFiles(collectionName, innerFilter); var total = pageData.Count; if (total == 0) return false; @@ -194,7 +194,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg { if (page > 1) { - pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + pageData = await GetPagedKnowledgeFiles(collectionName, innerFilter); } var fileIds = pageData.Items.Select(x => x.FileId).ToList(); @@ -202,7 +202,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg { try { - await DeleteKnowledgeDocument(collectionName, fileId); + await DeleteKnowledgeFile(collectionName, fileId); } catch { @@ -216,7 +216,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg return true; } - public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + public async Task> GetPagedKnowledgeFiles(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -248,14 +248,14 @@ public async Task> GetPagedKnowledgeDocuments(str }; } - public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) + public async Task GetKnowledgeFileBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; + var knwoledgebaseProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; // Get doc binary data - var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, knwoledgebaseProvider, new KnowledgeFileFilter { Size = 1, FileIds = [fileId] @@ -272,7 +272,7 @@ public async Task GetKnowledgeDocumentBinaryData(string col }; }; - var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); + var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, knwoledgebaseProvider, fileId, metaData.FileName); return new FileBinaryDataModel { FileName = metaData.FileName, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs deleted file mode 100644 index aa758d796..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs) - { - var db = _services.GetRequiredService(); - var collections = configs.Collections ?? new(); - var saved = await db.AddKnowledgeCollectionConfigs(collections, reset: true); - return saved; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs index cbfcc0fe7..6b9f4a7d0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class QuestionAnswerKnowledgeOrchestrator : VectorOrchestratorBase, IKnowledgeOrchestrator { - public override string Type => KnowledgeType.QuestionAnswer; + public override string KnowledgeType => KnowledgeBaseType.QuestionAnswer; public QuestionAnswerKnowledgeOrchestrator( IServiceProvider services, From 12fbf1d4292401a2c992cb187d4816b74cf897e7 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 21 Apr 2026 17:21:28 -0500 Subject: [PATCH 06/23] clean collection from kg service --- .../Knowledges/IKnowledgeService.cs | 26 +-- .../Options/KnowledgeCollectionOptions.cs | 1 + .../KnowledgeBase/KnowledgeBaseController.cs | 212 ++++++++++++------ .../Request/CreateCollectionRequest.cs | 4 +- .../Request/KnowledgeCreateRequest.cs | 3 + .../Request/SearchVectorKnowledgeRequest.cs | 3 + ... => KnowledgeCollectionConfigViewModel.cs} | 6 +- ...=> KnowledgeCollectionDetailsViewModel.cs} | 6 +- ...> KnowledgeCollectionSnapshotViewModel.cs} | 10 +- ...odel.cs => KnowledgeKnowledgeViewModel.cs} | 34 ++- .../Functions/MemorizeKnowledgeFn.cs | 17 +- .../Hooks/KnowledgeHook.cs | 57 +++-- .../VectorOrchestratorBase.Collection.cs | 2 +- .../Services/DbKnowledgeService.cs | 12 +- 14 files changed, 260 insertions(+), 133 deletions(-) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/{VectorCollectionConfigViewModel.cs => KnowledgeCollectionConfigViewModel.cs} (57%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/{VectorCollectionDetailsViewModel.cs => KnowledgeCollectionDetailsViewModel.cs} (73%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/{VectorCollectionSnapshotViewModel.cs => KnowledgeCollectionSnapshotViewModel.cs} (74%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/{VectorKnowledgeViewModel.cs => KnowledgeKnowledgeViewModel.cs} (52%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index abba17465..35053c629 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -11,19 +11,19 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { #region Vector - Task ExistVectorCollection(string collectionName); - Task CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options); - Task DeleteVectorCollection(string collectionName); - Task> GetVectorCollections(string? type = null); - Task GetVectorCollectionDetails(string collectionName); - Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); - Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); - Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null); - Task DeleteVectorCollectionData(string collectionName, string id); - Task DeleteVectorCollectionAllData(string collectionName); - Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); - Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); - Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update); + //Task ExistVectorCollection(string collectionName); + //Task CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options); + //Task DeleteVectorCollection(string collectionName); + //Task> GetVectorCollections(string? type = null); + //Task GetVectorCollectionDetails(string collectionName); + //Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); + //Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); + //Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null); + //Task DeleteVectorCollectionData(string collectionName, string id); + //Task DeleteVectorCollectionAllData(string collectionName); + //Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); + //Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); + //Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update); #endregion //#region Document diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs index 19aae5714..69bdb9260 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs @@ -2,4 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Options; public class KnowledgeCollectionOptions : KnowledgeOptionBase { + public bool IncludeAllTypes { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 3ed55a6ca..aaf0d8d7c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -3,8 +3,6 @@ using BotSharp.Abstraction.Graph.Options; using BotSharp.Abstraction.Knowledges.Options; using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Abstraction.VectorStorage.Options; using BotSharp.OpenAPI.ViewModels.Knowledges; namespace BotSharp.OpenAPI.Controllers; @@ -13,63 +11,93 @@ namespace BotSharp.OpenAPI.Controllers; [ApiController] public partial class KnowledgeBaseController : ControllerBase { - private readonly IKnowledgeService _knowledgeService; private readonly IGraphKnowledgeService _graphKnowledgeService; private readonly IServiceProvider _services; public KnowledgeBaseController( - IKnowledgeService knowledgeService, IGraphKnowledgeService graphKnowledgeService, IServiceProvider services) { - _knowledgeService = knowledgeService; _graphKnowledgeService = graphKnowledgeService; _services = services; } - #region Vector - [HttpGet("knowledge/vector/{collection}/exist")] - public async Task ExistVectorCollection([FromRoute] string collection) + #region Collection + [HttpGet("knowledge/collection/{collection}/exist")] + public async Task ExistCollection([FromRoute] string collection, [FromQuery] string knowledgeType) { - return await _knowledgeService.ExistVectorCollection(collection); - } + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - [HttpGet("knowledge/vector/collections")] - public async Task> GetVectorCollections([FromQuery] string? type = null) - { - var collections = await _knowledgeService.GetVectorCollections(type); - return collections.Select(x => VectorCollectionConfigViewModel.From(x)); + if (orchestrator == null) + { + return false; + } + + return await orchestrator.ExistCollection(collection, new KnowledgeCollectionOptions()); } - [HttpGet("knowledge/vector/{collection}/details")] - public async Task GetVectorCollectionDetails([FromRoute] string collection) + [HttpGet("knowledge/collections")] + public async Task> GetCollections([FromQuery] string knowledgeType) { - var details = await _knowledgeService.GetVectorCollectionDetails(collection); - return VectorCollectionDetailsViewModel.From(details); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return []; + } + + var collections = await orchestrator.GetCollections(new KnowledgeCollectionOptions()); + return collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)); } - [HttpPost("knowledge/vector/create-collection")] - public async Task CreateVectorCollection([FromBody] CreateCollectionRequest request) + [HttpPost("knowledge/collection")] + public async Task CreateCollection([FromBody] CreateCollectionRequest request) { - var options = new VectorCollectionCreateOptions + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) { - Provider = request.Provider, - Model = request.Model, - Dimension = request.Dimension + return false; + } + + var options = new CollectionCreateOptions + { + LlmProvider = request.Provider, + LlmModel = request.Model, + EmbeddingDimension = request.Dimension }; - return await _knowledgeService.CreateVectorCollection(request.CollectionName, request.CollectionType, options); + return await orchestrator.CreateCollection(request.CollectionName, options); } - [HttpDelete("knowledge/vector/{collection}/delete-collection")] - public async Task DeleteVectorCollection([FromRoute] string collection) + [HttpDelete("knowledge/collection/{collection}")] + public async Task DeleteCollection([FromRoute] string collection, [FromQuery] string knowledgeType) { - return await _knowledgeService.DeleteVectorCollection(collection); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return false; + } + + return await orchestrator.DeleteCollection(collection, new KnowledgeCollectionOptions()); } - [HttpPost("/knowledge/vector/{collection}/search")] - public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) + [HttpPost("/knowledge/collection/{collection}/search")] + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { - var options = new VectorSearchOptions + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) + { + return []; + } + + var options = new KnowledgeSearchOptions { Fields = request?.Fields, FilterGroups = request?.FilterGroups, @@ -79,17 +107,25 @@ public async Task> SearchKnowledge([FromRo SearchParam = request?.SearchParam }; - var results = await _knowledgeService.SearchVectorKnowledge(request?.Text ?? string.Empty, collection, options); - return results.Select(x => VectorKnowledgeViewModel.From(x)).ToList(); + var results = await orchestrator.Search(request?.Text ?? string.Empty, collection, options); + return results.Select(x => KnowledgeKnowledgeViewModel.From(x)).ToList(); } - [HttpPost("/knowledge/vector/{collection}/page")] - public async Task> GetPagedCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) + [HttpPost("/knowledge/collection/{collection}/page")] + public async Task> GetPagedCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromBody] KnowledgeFilter filter) { - var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter); - var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? []; + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - return new StringIdPagedItems + if (orchestrator == null) + { + return new StringIdPagedItems(); + } + + var data = await orchestrator.GetPagedCollectionData(collection, filter); + var items = data.Items?.Select(x => KnowledgeKnowledgeViewModel.From(x))?.ToList() ?? []; + + return new StringIdPagedItems { Count = data.Count, NextId = data.NextId, @@ -97,62 +133,102 @@ public async Task> GetPagedCollecti }; } - [HttpPost("/knowledge/vector/{collection}/create")] - public async Task CreateVectorKnowledge([FromRoute] string collection, [FromBody] KnowledgeCreateRequest request) + [HttpPost("/knowledge/collection/{collection}/data")] + public async Task CreateCollectionData([FromRoute] string collection, [FromBody] KnowledgeCreateRequest request) { - var create = new VectorCreateModel + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) + { + return false; + } + + var create = new KnowledgeCreateModel { Text = request.Text, Payload = request.Payload }; - var created = await _knowledgeService.CreateVectorCollectionData(collection, create); + var created = await orchestrator.CreateCollectionData(collection, create); return created; } - [HttpGet("/knowledge/vector/{collection}/points")] - public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] QueryVectorDataRequest request) + [HttpGet("/knowledge/collection/{collection}/data")] + public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] QueryVectorDataRequest request) { - var options = new VectorQueryOptions + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return []; + } + + var options = new KnowledgeQueryOptions { WithPayload = request.WithPayload, WithVector = request.WithVector }; - var points = await _knowledgeService.GetVectorCollectionData(collection, request.Ids, options); - return points.Select(x => VectorKnowledgeViewModel.From(x)); + var points = await orchestrator.GetCollectionData(collection, request.Ids, options); + return points.Select(x => KnowledgeKnowledgeViewModel.From(x)); } - [HttpPut("/knowledge/vector/{collection}/update")] + [HttpPut("/knowledge/collection/{collection}/data")] public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeUpdateRequest request) { - var update = new VectorUpdateModel + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (orchestrator == null) + { + return false; + } + + var update = new KnowledgeUpdateModel { Id = request.Id, Text = request.Text, Payload = request.Payload }; - var updated = await _knowledgeService.UpdateVectorCollectionData(collection, update); + var updated = await orchestrator.UpdateCollectionData(collection, update); return updated; } - [HttpDelete("/knowledge/vector/{collection}/data/{id}")] - public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id) + [HttpDelete("/knowledge/collection/{collection}/data/{id}")] + public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id, [FromQuery] string knowledgeType) { - return await _knowledgeService.DeleteVectorCollectionData(collection, id); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return false; + } + + return await orchestrator.DeleteCollectionData(collection, id, new KnowledgeCollectionOptions()); } - [HttpDelete("/knowledge/vector/{collection}/data")] - public async Task DeleteCollectionAllData([FromRoute] string collection) + [HttpDelete("/knowledge/collection/{collection}/data")] + public async Task DeleteCollectionAllData([FromRoute] string collection, [FromQuery] string knowledgeType) { - return await _knowledgeService.DeleteVectorCollectionAllData(collection); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return false; + } + + return await orchestrator.DeleteCollectionData(collection, new KnowledgeCollectionOptions()); } #endregion #region Index - [HttpPost("/knowledge/{collection}/indexes")] + [HttpPost("/knowledge/collection/{collection}/indexes")] public async Task> CreateCollectionIndexes([FromRoute] string collection, [FromBody] CreateCollectionIndexRequest request) { var orchestrator = _services.GetServices() @@ -170,7 +246,7 @@ public async Task> CreateCollectionIndexes([FromRout return await orchestrator.CreateIndexes(collection, options); } - [HttpDelete("/knowledge/{collection}/indexes")] + [HttpDelete("/knowledge/collection/{collection}/indexes")] public async Task> DeleteCollectionIndexes([FromRoute] string collection, [FromBody] DeleteCollectionIndexRequest request) { var orchestrator = _services.GetServices() @@ -191,8 +267,8 @@ public async Task> DeleteCollectionIndexes([FromRout #region Snapshot - [HttpGet("/knowledge/{collection}/snapshots")] - public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType) + [HttpGet("/knowledge/collection/{collection}/snapshots")] + public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -203,11 +279,11 @@ public async Task> GetCollectionS } var snapshots = await orchestrator.GetCollectionSnapshots(collection); - return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x)!); + return snapshots.Select(x => KnowledgeCollectionSnapshotViewModel.From(x)!); } - [HttpPost("/knowledge/{collection}/snapshot")] - public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromQuery] string knowledgeType) + [HttpPost("/knowledge/collection/{collection}/snapshot")] + public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromQuery] string knowledgeType) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -218,10 +294,10 @@ public async Task> GetCollectionS } var snapshot = await orchestrator.CreateCollectionSnapshot(collection); - return VectorCollectionSnapshotViewModel.From(snapshot); + return KnowledgeCollectionSnapshotViewModel.From(snapshot); } - [HttpGet("/knowledge/{collection}/snapshot")] + [HttpGet("/knowledge/collection/{collection}/snapshot")] public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType) { var orchestrator = _services.GetServices() @@ -236,7 +312,7 @@ public async Task GetCollectionSnapshot([FromRoute] string collec return BuildFileResult(snapshotFileName, snapshot); } - [HttpPost("/knowledge/{collection}/snapshot/recover")] + [HttpPost("/knowledge/collection/{collection}/snapshot/recover")] public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromQuery] string knowledgeType) { var orchestrator = _services.GetServices() @@ -253,7 +329,7 @@ public async Task RecoverCollectionFromSnapshot([FromRoute] string collect return done; } - [HttpDelete("/knowledge/{collection}/snapshot")] + [HttpDelete("/knowledgecollection/{collection}/snapshot")] public async Task DeleteCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteCollectionSnapshotRequest request) { var orchestrator = _services.GetServices() diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs index 83f32fdfa..f6e955af1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs @@ -7,8 +7,8 @@ public class CreateCollectionRequest [JsonPropertyName("collection_name")] public string CollectionName { get; set; } - [JsonPropertyName("collection_type")] - public string CollectionType { get; set; } + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } [JsonPropertyName("provider")] public string Provider { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs index 1afeb17e0..15aa9fcff 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs @@ -10,4 +10,7 @@ public class KnowledgeCreateRequest [JsonPropertyName("payload")] public Dictionary? Payload { get; set; } + + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } = null!; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 93b48dc2a..db96251c2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -25,4 +25,7 @@ public class SearchVectorKnowledgeRequest [JsonPropertyName("search_param")] public VectorSearchParamModel? SearchParam { get; set; } + + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } = null!; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionConfigViewModel.cs similarity index 57% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionConfigViewModel.cs index e19fda39f..795369128 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionConfigViewModel.cs @@ -2,11 +2,11 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorCollectionConfigViewModel : VectorCollectionConfig +public class KnowledgeCollectionConfigViewModel : VectorCollectionConfig { - public static VectorCollectionConfigViewModel From(VectorCollectionConfig model) + public static KnowledgeCollectionConfigViewModel From(VectorCollectionConfig model) { - return new VectorCollectionConfigViewModel + return new KnowledgeCollectionConfigViewModel { Name = model.Name, Type = model.Type, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionDetailsViewModel.cs similarity index 73% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionDetailsViewModel.cs index b55981b59..e7a088c59 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionDetailsViewModel.cs @@ -2,13 +2,13 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorCollectionDetailsViewModel : VectorCollectionDetails +public class KnowledgeCollectionDetailsViewModel : VectorCollectionDetails { - public static VectorCollectionDetailsViewModel? From(VectorCollectionDetails? model) + public static KnowledgeCollectionDetailsViewModel? From(VectorCollectionDetails? model) { if (model == null) return null; - return new VectorCollectionDetailsViewModel + return new KnowledgeCollectionDetailsViewModel { Status = model.Status, OptimizerStatus = model.OptimizerStatus, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionSnapshotViewModel.cs similarity index 74% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionSnapshotViewModel.cs index 949507a40..2bb12a81d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionSnapshotViewModel.cs @@ -4,7 +4,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorCollectionSnapshotViewModel +public class KnowledgeCollectionSnapshotViewModel { [JsonPropertyName("name")] public string Name { get; set; } = default!; @@ -18,14 +18,14 @@ public class VectorCollectionSnapshotViewModel [JsonPropertyName("check_sum")] public string? CheckSum { get; set; } - public static VectorCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model) + public static KnowledgeCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model) { if (model == null) { return null; } - return new VectorCollectionSnapshotViewModel + return new KnowledgeCollectionSnapshotViewModel { Name = model.Name, Size = model.Size, @@ -34,14 +34,14 @@ public class VectorCollectionSnapshotViewModel }; } - public static VectorCollectionSnapshotViewModel? From(KnowledgeCollectionSnapshot? model) + public static KnowledgeCollectionSnapshotViewModel? From(KnowledgeCollectionSnapshot? model) { if (model == null) { return null; } - return new VectorCollectionSnapshotViewModel + return new KnowledgeCollectionSnapshotViewModel { Name = model.Name, Size = model.Size, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs similarity index 52% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs index e4c805be5..136f6908c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs @@ -1,9 +1,10 @@ +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeViewModel +public class KnowledgeKnowledgeViewModel { [JsonPropertyName("id")] public string Id { get; set; } @@ -23,9 +24,9 @@ public class VectorKnowledgeViewModel public int? VectorDimension { get; set; } - public static VectorKnowledgeViewModel From(VectorSearchResult result) + public static KnowledgeKnowledgeViewModel From(VectorSearchResult result) { - return new VectorKnowledgeViewModel + return new KnowledgeKnowledgeViewModel { Id = result.Id, Data = result.Data, @@ -35,9 +36,32 @@ public static VectorKnowledgeViewModel From(VectorSearchResult result) }; } - public static VectorKnowledgeViewModel From(VectorCollectionData data) + public static KnowledgeKnowledgeViewModel From(VectorCollectionData data) { - return new VectorKnowledgeViewModel + return new KnowledgeKnowledgeViewModel + { + Id = data.Id, + Data = data.Data, + Payload = data.Payload, + VectorDimension = data.Vector?.Length + }; + } + + public static KnowledgeKnowledgeViewModel From(KnowledgeSearchResult result) + { + return new KnowledgeKnowledgeViewModel + { + Id = result.Id, + Data = result.Data, + Payload = result.Payload, + Score = result.Score, + VectorDimension = result.Vector?.Length + }; + } + + public static KnowledgeKnowledgeViewModel From(KnowledgeCollectionData data) + { + return new KnowledgeKnowledgeViewModel { Id = data.Id, Data = data.Data, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index ab5c29bbe..6e62aa1ae 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -19,11 +19,20 @@ public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var collectionName = !string.IsNullOrEmpty(args.RefinedCollection) - ? args.RefinedCollection + var collectionName = !string.IsNullOrEmpty(args.RefinedCollection) + ? args.RefinedCollection : _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var knowledgeService = _services.GetRequiredService(); - var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); + + if (orchestrator == null) + { + message.Content = "I forgot it"; + return true; + } + + var result = await orchestrator.CreateCollectionData(collectionName, new KnowledgeCreateModel { Text = args.Question, Payload = new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index 00ec683b5..5091954e4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -1,29 +1,19 @@ using BotSharp.Abstraction.Graph.Options; -using BotSharp.Abstraction.VectorStorage.Options; namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeHook : IKnowledgeHook { - private readonly IKnowledgeService _knowledgeService; - private readonly IGraphKnowledgeService _graphKnowledgeService; private readonly IServiceProvider _services; + private readonly IGraphKnowledgeService _graphKnowledgeService; + public KnowledgeHook( - IKnowledgeService knowledgeService, - IGraphKnowledgeService graphKnowledgeService, - IServiceProvider services) + IServiceProvider services, + IGraphKnowledgeService graphKnowledgeService) { - _knowledgeService = knowledgeService; - _graphKnowledgeService = graphKnowledgeService; _services = services; - } - - private async Task> GetKnowledgeBaseNameByAgentIdAsync(string agentId) - { - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(agentId); - return agent.KnowledgeBases; + _graphKnowledgeService = graphKnowledgeService; } public async Task> GetDomainKnowledges(RoleDialogModel message, string text) @@ -46,32 +36,33 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str } else if (knowledgeBase.Type == "document") { - var options = new VectorSearchOptions + var orchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); + var options = new KnowledgeSearchOptions { Fields = null, Limit = 5, Confidence = 0.25f, WithVector = true }; - var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + var result = await orchestrator.Search(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) .Select(x => x.Data["text"].ToString()) .Where(x => x != null)!); } else { - var options = new VectorSearchOptions + var orchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); + var options = new KnowledgeSearchOptions { Fields = null, Limit = 5, Confidence = 0.5f, WithVector = true }; - var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + var result = await orchestrator.Search(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); - } } @@ -84,7 +75,8 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) var results = new List(); // Get all knowledge bases - var knowledgeBases = await _knowledgeService.GetVectorCollections(); + var orchestrator = GetKnowledgeOrchestrator(); + var knowledgeBases = await orchestrator.GetCollections(new KnowledgeCollectionOptions { IncludeAllTypes = true }); foreach (var knowledgeBase in knowledgeBases) { @@ -100,14 +92,15 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) } else { - var options = new VectorSearchOptions + var searchOrchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); + var options = new KnowledgeSearchOptions { Fields = null, Limit = 5, Confidence = 0.5f, WithVector = true }; - var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + var result = await searchOrchestrator.Search(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); @@ -121,4 +114,22 @@ public async Task> CollectChunkedKnowledge() { return new List(); } + + + private async Task> GetKnowledgeBaseNameByAgentIdAsync(string agentId) + { + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(agentId); + return agent.KnowledgeBases; + } + + private IKnowledgeOrchestrator GetKnowledgeOrchestrator(string? type = null) + { + var orchestrators = _services.GetServices(); + if (!string.IsNullOrWhiteSpace(type)) + { + return orchestrators.First(x => x.KnowledgeType == type); + } + return orchestrators.First(); + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs index 3f50166f6..af1ce4e8d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs @@ -88,7 +88,7 @@ public virtual async Task> GetCollections var db = _services.GetRequiredService(); var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { - CollectionTypes = !string.IsNullOrEmpty(KnowledgeType) ? [KnowledgeType] : null, + CollectionTypes = options?.IncludeAllTypes == true ? null : [KnowledgeType], VectorStorageProviders = [vectorDb.Provider] }); diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs index 47b607a3e..384921ec6 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -1,11 +1,7 @@ using static Dapper.SqlMapper; -using Microsoft.Extensions.Logging; -using BotSharp.Core.Infrastructures; using MySqlConnector; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Plugin.SqlDriver.Models; using BotSharp.Abstraction.Models; namespace BotSharp.Plugin.SqlDriver.Services; @@ -26,8 +22,10 @@ public DbKnowledgeService( public async Task Import(ImportDbKnowledgeRequest request) { var sqlDriverSettings = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); var settingService = _services.GetRequiredService(); + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); + var provider = request.Provider ?? "openai"; var model = request.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o); var schema = request.Schema; @@ -71,7 +69,9 @@ public async Task Import(ImportDbKnowledgeRequest request) foreach (var item in knowledges) { - await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + if (orchestrator == null) continue; + + await orchestrator.CreateCollectionData(collectionName, new KnowledgeCreateModel { Text = item.Question, Payload = new Dictionary From 5e6663adae09563f34ec288002b1d5d187de2fc0 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 21 Apr 2026 17:27:15 -0500 Subject: [PATCH 07/23] clean code --- .../SqlGeneration/Functions/SqlSecondaryStageFn.cs | 1 - .../TwoStaging/Functions/SecondaryStagePlanFn.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs index a13c093ee..bab3c2cfc 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs @@ -21,7 +21,6 @@ public SqlSecondaryStageFn( public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); var knowledgeSettings = _services.GetRequiredService(); var states = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs index ab4b01f2d..6a3ec36ae 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs @@ -21,7 +21,6 @@ public SecondaryStagePlanFn( public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); var knowledgeSettings = _services.GetRequiredService(); var states = _services.GetRequiredService(); From 4571a94085c32a96b9994d639a9e03bf4768fee5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 27 Apr 2026 00:22:32 -0500 Subject: [PATCH 08/23] add Taxonomy --- .../Entity/Models/EntityAnalysisOptions.cs | 2 +- .../Knowledges/IKnowledgeOrchestrator.cs | 57 +++++--- .../Models/KnowledgeSearchOptions.cs | 4 +- .../Models/VectorSearchParamModel.cs | 7 - .../Options/VectorSearchOptions.cs | 2 +- .../KnowledgeBaseController.Entity.cs | 20 --- .../KnowledgeBase/KnowledgeBaseController.cs | 4 +- .../Request/SearchVectorKnowledgeRequest.cs | 12 +- .../FuzzySharpPlugin.cs | 2 + .../Services/TaxonomyKnowledgeOrchestrator.cs | 132 ++++++++++++++++++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 7 +- 11 files changed, 194 insertions(+), 55 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs create mode 100644 src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs index b0e08dc63..53f3e4b23 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs @@ -7,7 +7,7 @@ public class EntityAnalysisOptions /// [JsonPropertyName("data_providers")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? DataProviders { get; set; } + public IEnumerable? DataProviders { get; set; } /// /// Maximum n-gram size diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs index 600566d08..8abbbad7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs @@ -8,33 +8,52 @@ public interface IKnowledgeOrchestrator string KnowledgeType { get; } #region Collection - Task ExistCollection(string collectionName, KnowledgeCollectionOptions options); - Task CreateCollection(string collectionName, CollectionCreateOptions options); - Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options); - Task> GetCollections(KnowledgeCollectionOptions options); + Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(false); + Task CreateCollection(string collectionName, CollectionCreateOptions options) + => Task.FromResult(false); + Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(false); + Task> GetCollections(KnowledgeCollectionOptions options) + => Task.FromResult(Enumerable.Empty()); #endregion #region Data - Task> Search(string query, string collectionName, KnowledgeSearchOptions options); - Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter); - Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null); - Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options); - Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options); - Task CreateCollectionData(string collectionName, KnowledgeCreateModel create); - Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update); - Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update); + Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + => Task.FromResult(Enumerable.Empty()); + Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + => Task.FromResult(new StringIdPagedItems()); + Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) + => Task.FromResult(Enumerable.Empty()); + Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) + => Task.FromResult(false); + Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) + => Task.FromResult(false); + Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) + => Task.FromResult(false); + Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) + => Task.FromResult(false); + Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) + => Task.FromResult(false); #endregion #region Index - Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options); - Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options); + Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) + => Task.FromResult(new SuccessFailResponse()); + Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) + => Task.FromResult(new SuccessFailResponse()); #endregion #region Snapshot - Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null); - Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null); - Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null); - Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null); - Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null); + Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(Enumerable.Empty()); + Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(null); + Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(new BinaryData(Array.Empty())); + Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(false); + Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(false); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs index 10b073dbf..70bf6aefc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs @@ -8,10 +8,12 @@ public class KnowledgeSearchOptions public string? DbProvider { get; set; } public IEnumerable? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer]; public IEnumerable? FilterGroups { get; set; } + public IEnumerable? DataProviders { get; set; } + public Dictionary? SearchParam { get; set; } + public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } - public VectorSearchParamModel? SearchParam { get; set; } public static KnowledgeSearchOptions Default() { diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs deleted file mode 100644 index 8e14b05ca..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.VectorStorage.Models; - -public class VectorSearchParamModel -{ - [JsonPropertyName("exact_search")] - public bool? ExactSearch { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs index 4ba0c81c8..4a4358319 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs @@ -10,7 +10,7 @@ public class VectorSearchOptions public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } - public VectorSearchParamModel? SearchParam { get; set; } + public Dictionary? SearchParam { get; set; } public static VectorSearchOptions Default() { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs index 266155539..163570b90 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs @@ -1,27 +1,7 @@ -using BotSharp.OpenAPI.ViewModels.Knowledges; - namespace BotSharp.OpenAPI.Controllers; public partial class KnowledgeBaseController { - /// - /// Entity analyis with options - /// - /// - /// - [HttpPost("knowledge/entity/analyze")] - public async Task EntityAnalyze([FromBody] EntityAnalysisRequest request) - { - var analyzer = _services.GetServices() - .FirstOrDefault(x => x.Provider.IsEqualTo(request.Provider)); - - if (analyzer == null) - { - return null; - } - return await analyzer.AnalyzeAsync(request.Text, request.Options); - } - /// /// Get entity analyzers /// diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index aaf0d8d7c..d1dc936f1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -87,7 +87,7 @@ public async Task DeleteCollection([FromRoute] string collection, [FromQue } [HttpPost("/knowledge/collection/{collection}/search")] - public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); @@ -99,11 +99,13 @@ public async Task> SearchKnowledge([Fro var options = new KnowledgeSearchOptions { + DbProvider = request?.DbProvider, Fields = request?.Fields, FilterGroups = request?.FilterGroups, Limit = request?.Limit ?? 5, Confidence = request?.Confidence ?? 0.5f, WithVector = request?.WithVector ?? false, + DataProviders = request?.DataProviders, SearchParam = request?.SearchParam }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index db96251c2..0f6972dbd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -3,7 +3,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class SearchVectorKnowledgeRequest +public class SearchKnowledgeRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; @@ -24,7 +24,15 @@ public class SearchVectorKnowledgeRequest public bool WithVector { get; set; } [JsonPropertyName("search_param")] - public VectorSearchParamModel? SearchParam { get; set; } + public Dictionary? SearchParam { get; set; } + + [JsonPropertyName("data_providers")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? DataProviders { get; set; } + + [JsonPropertyName("db_provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs index cc6f3e8f1..58d56ea55 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.Plugins; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -21,6 +22,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs new file mode 100644 index 000000000..813a604ff --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs @@ -0,0 +1,132 @@ +using BotSharp.Abstraction.Knowledges; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; +using BotSharp.Abstraction.VectorStorage.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace BotSharp.Plugin.FuzzySharp.Services; + +public class TaxonomyKnowledgeOrchestrator : IKnowledgeOrchestrator +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TaxonomyKnowledgeOrchestrator( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string KnowledgeType => KnowledgeBaseType.Taxonomy; + + public async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + { + var results = new List(); + + try + { + var analyzer = _services.GetServices() + .FirstOrDefault(x => x.Provider.IsEqualTo(options.DbProvider)); + + if (analyzer == null) + { + _logger.LogWarning($"Cannot find entity analyzer with provider '{options.DbProvider}'."); + return results; + } + + var analysisOptions = new EntityAnalysisOptions + { + DataProviders = options.DataProviders, + Cutoff = options.Confidence.HasValue ? (double)options.Confidence.Value : null, + TopK = options.Limit, + MaxNgram = options.SearchParam != null + && options.SearchParam.TryGetValue("max_ngram", out var maxNgramStr) + && int.TryParse(maxNgramStr, out var maxNgram) + ? maxNgram : null + }; + + var response = await analyzer.AnalyzeAsync(query, analysisOptions); + + if (!response.Success || response.Results == null) + { + return results; + } + + foreach (var result in response.Results) + { + var payload = new Dictionary + { + ["token"] = VectorPayloadValue.BuildStringValue(result.Token) + }; + + if (!string.IsNullOrEmpty(result.CanonicalText)) + { + payload["canonical_text"] = VectorPayloadValue.BuildStringValue(result.CanonicalText); + } + + foreach (var kvp in result.Data) + { + if (!payload.ContainsKey(kvp.Key)) + { + payload[kvp.Key] = BuildPayloadValue(kvp.Value); + } + } + + double confidence = 0d; + if (result.Data.TryGetValue("confidence", out var conf)) + { + if (conf is double d) + { + confidence = d; + } + else if (conf is float f) + { + confidence = f; + } + } + + results.Add(new KnowledgeSearchResult + { + Id = Guid.NewGuid().ToString(), + Payload = payload, + Score = confidence + }); + } + + if (options.Limit.HasValue) + { + results = results + .OrderByDescending(x => x.Score) + .Take(options.Limit.Value) + .ToList(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when searching taxonomy knowledge ({query})."); + } + + return results; + } + + private VectorPayloadValue BuildPayloadValue(object value) + { + return value switch + { + string s => VectorPayloadValue.BuildStringValue(s), + double d => VectorPayloadValue.BuildDoubleValue(d), + float f => VectorPayloadValue.BuildDoubleValue(f), + long l => VectorPayloadValue.BuildIntegerValue(l), + int i => VectorPayloadValue.BuildIntegerValue(i), + short sh => VectorPayloadValue.BuildIntegerValue(sh), + byte b => VectorPayloadValue.BuildIntegerValue(b), + bool bl => VectorPayloadValue.BuildBooleanValue(bl), + DateTime dt => VectorPayloadValue.BuildDatetimeValue(dt), + _ => VectorPayloadValue.BuildUnkownValue(value) + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index fab3b9cb2..c315dfe31 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -832,17 +832,18 @@ public async Task DeleteCollectionShapshot(string collectionName, string s }; } - private SearchParams? BuildSearchParam(VectorSearchParamModel? param) + private SearchParams? BuildSearchParam(Dictionary? param) { if (param == null - || param.ExactSearch == null) + || !param.TryGetValue("exact_search", out var exactSearchStr) + || !bool.TryParse(exactSearchStr, out var exactSearch)) { return null; } var search = new SearchParams { - Exact = param.ExactSearch.Value + Exact = exactSearch }; return search; } From 2397e05755431a7facd11acb952563d8726ea4b6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 27 Apr 2026 01:12:36 -0500 Subject: [PATCH 09/23] add graph knowledge orchestrator --- .../Knowledges/Enums/KnowledgeBaseType.cs | 1 + .../Models/KnowledgeSearchOptions.cs | 29 -------- .../CollectionCreateOptions.cs | 4 +- .../Options/GraphKnowledgeSearchOptions.cs | 6 ++ .../KnowledgeQueryOptions.cs | 2 +- .../Options/KnowledgeSearchOptions.cs | 16 +++++ .../Options/TaxonomyKnowledgeSearchOptions.cs | 7 ++ .../BotSharp.Abstraction/Using.cs | 1 + src/Infrastructure/BotSharp.Core/Using.cs | 1 + .../KnowledgeBase/KnowledgeBaseController.cs | 55 ++++++++++---- src/Infrastructure/BotSharp.OpenAPI/Using.cs | 1 + .../Request/SearchVectorKnowledgeRequest.cs | 13 ++++ .../FuzzySharpPlugin.cs | 1 - .../BotSharp.Plugin.KnowledgeBase.csproj | 5 ++ .../Hooks/KnowledgeHook.cs | 35 ++------- .../KnowledgeBasePlugin.cs | 2 + .../Graph/GraphKnowledgeOrchestrator.cs | 71 +++++++++++++++++++ .../TaxonomyKnowledgeOrchestrator.cs | 19 ++--- src/Plugins/BotSharp.Plugin.Qdrant/Using.cs | 3 +- .../BotSharp.Plugin.SqlDriver/Using.cs | 1 + 20 files changed, 182 insertions(+), 91 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs rename src/Infrastructure/BotSharp.Abstraction/Knowledges/{Models => Options}/CollectionCreateOptions.cs (68%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs rename src/Infrastructure/BotSharp.Abstraction/Knowledges/{Models => Options}/KnowledgeQueryOptions.cs (80%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs rename src/Plugins/{BotSharp.Plugin.FuzzySharp/Services => BotSharp.Plugin.KnowledgeBase/Services/Taxonomy}/TaxonomyKnowledgeOrchestrator.cs (85%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs index c2aa125f3..756737f73 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs @@ -5,4 +5,5 @@ public static class KnowledgeBaseType public static string QuestionAnswer = "question-answer"; public static string Document = "document"; public static string Taxonomy = "taxonomy"; + public static string SemanticGraph = "semantic_graph"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs deleted file mode 100644 index 70bf6aefc..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs +++ /dev/null @@ -1,29 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Enums; -using BotSharp.Abstraction.VectorStorage.Models; - -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeSearchOptions -{ - public string? DbProvider { get; set; } - public IEnumerable? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer]; - public IEnumerable? FilterGroups { get; set; } - public IEnumerable? DataProviders { get; set; } - public Dictionary? SearchParam { get; set; } - - public int? Limit { get; set; } = 5; - public float? Confidence { get; set; } = 0.5f; - public bool WithVector { get; set; } - - public static KnowledgeSearchOptions Default() - { - return new() - { - Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer], - FilterGroups = null, - Limit = 5, - Confidence = 0.5f, - WithVector = false - }; - } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/CollectionCreateOptions.cs similarity index 68% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/CollectionCreateOptions.cs index 6fada86dd..dec83e2ad 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/CollectionCreateOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/CollectionCreateOptions.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Options; - -namespace BotSharp.Abstraction.Knowledges.Models; +namespace BotSharp.Abstraction.Knowledges.Options; public class CollectionCreateOptions : KnowledgeOptionBase { diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs new file mode 100644 index 000000000..47dc1a184 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class GraphKnowledgeSearchOptions : KnowledgeSearchOptions +{ + public string? GraphId { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs similarity index 80% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs index 413e6b11a..18cece169 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeQueryOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Abstraction.Knowledges.Models; +namespace BotSharp.Abstraction.Knowledges.Options; public class KnowledgeQueryOptions { diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs new file mode 100644 index 000000000..7d5dcbf25 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs @@ -0,0 +1,16 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeSearchOptions +{ + public string? DbProvider { get; set; } + public IEnumerable? Fields { get; set; } + public IEnumerable? FilterGroups { get; set; } + public Dictionary? SearchParam { get; set; } + public Dictionary? SearchArguments { get; set; } + + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs new file mode 100644 index 000000000..bbe5d8114 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class TaxonomyKnowledgeSearchOptions : KnowledgeSearchOptions +{ + public IEnumerable? DataProviders { get; set; } + public int? MaxNgram { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index ca0cbe7f7..acd2f0bd7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -19,6 +19,7 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Crontab.Models; global using BotSharp.Abstraction.MCP.Models; global using BotSharp.Abstraction.Settings; diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 39ac00cc8..f4a7cae20 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -15,6 +15,7 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Infrastructures.Events; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Loggers.Services; global using BotSharp.Abstraction.MCP.Models; global using BotSharp.Abstraction.MCP.Services; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index d1dc936f1..e48b567a6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -1,7 +1,7 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Graph; using BotSharp.Abstraction.Graph.Options; -using BotSharp.Abstraction.Knowledges.Options; +using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Repositories; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -97,18 +97,7 @@ public async Task> SearchKnowledge([Fro return []; } - var options = new KnowledgeSearchOptions - { - DbProvider = request?.DbProvider, - Fields = request?.Fields, - FilterGroups = request?.FilterGroups, - Limit = request?.Limit ?? 5, - Confidence = request?.Confidence ?? 0.5f, - WithVector = request?.WithVector ?? false, - DataProviders = request?.DataProviders, - SearchParam = request?.SearchParam - }; - + var options = BuildSearchOptions(orchestrator, request); var results = await orchestrator.Search(request?.Text ?? string.Empty, collection, options); return results.Select(x => KnowledgeKnowledgeViewModel.From(x)).ToList(); } @@ -391,5 +380,45 @@ private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) stream.Position = 0; return File(stream, "application/octet-stream", Path.GetFileName(fileName)); } + + private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator orchestrator, SearchKnowledgeRequest? request) + { + if (orchestrator.KnowledgeType.IsEqualTo(KnowledgeBaseType.SemanticGraph)) + { + return new GraphKnowledgeSearchOptions + { + DbProvider = request?.DbProvider, + SearchParam = request?.SearchParam, + SearchArguments = request?.SearchArguments, + GraphId = request?.GraphId + }; + } + + if (orchestrator.KnowledgeType.IsEqualTo(KnowledgeBaseType.Taxonomy)) + { + return new TaxonomyKnowledgeSearchOptions + { + DbProvider = request?.DbProvider, + Limit = request?.Limit ?? 5, + Confidence = request?.Confidence ?? 0.5f, + SearchParam = request?.SearchParam, + SearchArguments = request?.SearchArguments, + DataProviders = request?.DataProviders, + MaxNgram = request?.MaxNgram + }; + } + + return new KnowledgeSearchOptions + { + DbProvider = request?.DbProvider, + Fields = request?.Fields, + FilterGroups = request?.FilterGroups, + Limit = request?.Limit ?? 5, + Confidence = request?.Confidence ?? 0.5f, + WithVector = request?.WithVector ?? false, + SearchParam = request?.SearchParam, + SearchArguments = request?.SearchArguments + }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Using.cs b/src/Infrastructure/BotSharp.OpenAPI/Using.cs index d811452ee..614d01f54 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Using.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Using.cs @@ -28,6 +28,7 @@ global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.VectorStorage.Enums; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Chart.Models; global using BotSharp.Abstraction.Entity; global using BotSharp.Abstraction.Entity.Models; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 0f6972dbd..2ede2f89a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -24,16 +24,29 @@ public class SearchKnowledgeRequest public bool WithVector { get; set; } [JsonPropertyName("search_param")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? SearchParam { get; set; } + [JsonPropertyName("search_arguments")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? SearchArguments { get; set; } + [JsonPropertyName("data_providers")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? DataProviders { get; set; } + [JsonPropertyName("max_ngram")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxNgram { get; set; } + [JsonPropertyName("db_provider")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? DbProvider { get; set; } [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; + + [JsonPropertyName("graph_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? GraphId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs index 58d56ea55..ea3bc048f 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs @@ -22,7 +22,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index e15529d2a..9a8f20dd7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -61,4 +61,9 @@ + + + + + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index 5091954e4..728d2538f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -1,19 +1,13 @@ -using BotSharp.Abstraction.Graph.Options; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeHook : IKnowledgeHook { private readonly IServiceProvider _services; - private readonly IGraphKnowledgeService _graphKnowledgeService; - public KnowledgeHook( - IServiceProvider services, - IGraphKnowledgeService graphKnowledgeService) + IServiceProvider services) { _services = services; - _graphKnowledgeService = graphKnowledgeService; } public async Task> GetDomainKnowledges(RoleDialogModel message, string text) @@ -24,17 +18,7 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str foreach (var knowledgeBase in knowledgeBases) { - if (knowledgeBase.Type == "relationships") - { - var options = new GraphQueryOptions - { - Provider = "Remote", - Method = "local" - }; - var result = await _graphKnowledgeService.ExecuteQueryAsync(text, options); - results.Add(result.Result); - } - else if (knowledgeBase.Type == "document") + if (knowledgeBase.Type == KnowledgeBaseType.Document) { var orchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); var options = new KnowledgeSearchOptions @@ -49,7 +33,7 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str .Select(x => x.Data["text"].ToString()) .Where(x => x != null)!); } - else + else if (knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { var orchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); var options = new KnowledgeSearchOptions @@ -80,17 +64,8 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) foreach (var knowledgeBase in knowledgeBases) { - if (knowledgeBase.Type == "relationships") - { - var options = new GraphQueryOptions - { - Provider = "Remote", - Method = "local" - }; - var result = await _graphKnowledgeService.ExecuteQueryAsync(text, options); - results.Add(result.Result); - } - else + if (knowledgeBase.Type == KnowledgeBaseType.Document + || knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { var searchOrchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); var options = new KnowledgeSearchOptions diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index a8eb90fdf..d3000c807 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -26,6 +26,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs new file mode 100644 index 000000000..757730983 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs @@ -0,0 +1,71 @@ +using BotSharp.Abstraction.Graph.Options; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public class GraphKnowledgeOrchestrator : IKnowledgeOrchestrator +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public GraphKnowledgeOrchestrator( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string KnowledgeType => KnowledgeBaseType.SemanticGraph; + + public async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + { + var results = new List(); + + try + { + var graphDb = GetGraphDb(options?.DbProvider); + if (graphDb == null) + { + _logger.LogWarning($"Cannot find graph db provider '{options?.DbProvider}'."); + return results; + } + + var graphSearchOptions = options as GraphKnowledgeSearchOptions; + var graphOptions = new GraphQueryExecuteOptions + { + GraphId = graphSearchOptions?.GraphId, + Arguments = options?.SearchArguments + }; + + var graphResult = await graphDb.ExecuteQueryAsync(query, graphOptions); + + results = graphResult?.Values?.Select(value => + { + var payload = value.ToDictionary( + kvp => kvp.Key, + kvp => VectorPayloadValue.BuildStringValue(kvp.Value?.ToString() ?? string.Empty) + ); + + return new KnowledgeSearchResult + { + Id = Guid.NewGuid().ToString(), + Payload = payload + }; + })?.ToList() ?? []; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when searching graph knowledge. (Query: {query}, Collection: {collectionName})"); + } + + return results; + } + + #region Private methods + private IGraphDb? GetGraphDb(string? provider = null) + { + var db = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(provider)); + return db; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs similarity index 85% rename from src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs index 813a604ff..b442e51cb 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/TaxonomyKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs @@ -1,12 +1,7 @@ -using BotSharp.Abstraction.Knowledges; -using BotSharp.Abstraction.Knowledges.Enums; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.Utilities; -using BotSharp.Abstraction.VectorStorage.Models; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; +using BotSharp.Abstraction.Entity; +using BotSharp.Abstraction.Entity.Models; -namespace BotSharp.Plugin.FuzzySharp.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public class TaxonomyKnowledgeOrchestrator : IKnowledgeOrchestrator { @@ -38,15 +33,13 @@ public async Task> Search(string query, strin return results; } + var taxonomyOptions = options as TaxonomyKnowledgeSearchOptions; var analysisOptions = new EntityAnalysisOptions { - DataProviders = options.DataProviders, + DataProviders = taxonomyOptions?.DataProviders, Cutoff = options.Confidence.HasValue ? (double)options.Confidence.Value : null, TopK = options.Limit, - MaxNgram = options.SearchParam != null - && options.SearchParam.TryGetValue("max_ngram", out var maxNgramStr) - && int.TryParse(maxNgramStr, out var maxNgram) - ? maxNgram : null + MaxNgram = taxonomyOptions?.MaxNgram }; var response = await analyzer.AnalyzeAsync(query, analysisOptions); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs index b7955f653..2d23665c9 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -6,4 +6,5 @@ global using BotSharp.Abstraction.VectorStorage.Enums; global using BotSharp.Abstraction.VectorStorage.Models; global using BotSharp.Abstraction.Knowledges.Enums; -global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file +global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs index 319abb901..d780e2055 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs @@ -20,6 +20,7 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Knowledges; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Settings; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Instructs; From a3c106c289eecb5efabb4ca7b9c6d7a64c76cf44 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 27 Apr 2026 01:16:36 -0500 Subject: [PATCH 10/23] minor change --- .../Services/Graph/GraphKnowledgeOrchestrator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs index 757730983..3f20e927f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs @@ -43,7 +43,7 @@ public async Task> Search(string query, strin { var payload = value.ToDictionary( kvp => kvp.Key, - kvp => VectorPayloadValue.BuildStringValue(kvp.Value?.ToString() ?? string.Empty) + kvp => new VectorPayloadValue(kvp.Value, VectorPayloadDataType.Unknown) ); return new KnowledgeSearchResult @@ -55,7 +55,7 @@ public async Task> Search(string query, strin } catch (Exception ex) { - _logger.LogError(ex, $"Error when searching graph knowledge. (Query: {query}, Collection: {collectionName})"); + _logger.LogError(ex, $"Error when searching graph knowledge. (Query: {query})"); } return results; From b2294f844b5b407f53d4609d3bc3fff69f8dec97 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 27 Apr 2026 11:06:22 -0500 Subject: [PATCH 11/23] minor change --- .../VectorStorage/Models/VectorPayloadValue.cs | 2 +- .../Services/Graph/GraphKnowledgeOrchestrator.cs | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs index aaa520358..e18843c1a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs @@ -15,7 +15,7 @@ public VectorPayloadValue() } - public VectorPayloadValue(object data, VectorPayloadDataType dataType) + public VectorPayloadValue(object data, VectorPayloadDataType dataType = VectorPayloadDataType.Unknown) { DataValue = data; DataType = dataType; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs index 3f20e927f..23e0ee735 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs @@ -39,18 +39,10 @@ public async Task> Search(string query, strin var graphResult = await graphDb.ExecuteQueryAsync(query, graphOptions); - results = graphResult?.Values?.Select(value => + results = graphResult?.Values?.Select(value => new KnowledgeSearchResult { - var payload = value.ToDictionary( - kvp => kvp.Key, - kvp => new VectorPayloadValue(kvp.Value, VectorPayloadDataType.Unknown) - ); - - return new KnowledgeSearchResult - { - Id = Guid.NewGuid().ToString(), - Payload = payload - }; + Id = Guid.NewGuid().ToString(), + Payload = value.ToDictionary(kvp => kvp.Key, kvp => new VectorPayloadValue(kvp.Value)) })?.ToList() ?? []; } catch (Exception ex) From 32273c10485444d35d3c31c2784b2af383565b99 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 27 Apr 2026 11:07:38 -0500 Subject: [PATCH 12/23] minor change --- .../Request/SearchVectorKnowledgeRequest.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 2ede2f89a..3de4351a7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -31,6 +31,13 @@ public class SearchKnowledgeRequest [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? SearchArguments { get; set; } + [JsonPropertyName("knowledge_type")] + public string KnowledgeType { get; set; } = null!; + + [JsonPropertyName("db_provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } + [JsonPropertyName("data_providers")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? DataProviders { get; set; } @@ -39,13 +46,6 @@ public class SearchKnowledgeRequest [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? MaxNgram { get; set; } - [JsonPropertyName("db_provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } - - [JsonPropertyName("knowledge_type")] - public string KnowledgeType { get; set; } = null!; - [JsonPropertyName("graph_id")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? GraphId { get; set; } From c6577b0e2d17acc2dbe7f51a328c3efc3c89aa7c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 28 Apr 2026 17:34:24 -0500 Subject: [PATCH 13/23] change url --- .../KnowledgeBaseController.File.cs | 13 +++++------ .../KnowledgeBase/KnowledgeBaseController.cs | 4 ++-- .../Request/KnowledgeUploadRequest.cs | 1 - .../Hooks/KnowledgeHook.cs | 2 +- .../Services/KnowledgeService.cs | 2 +- .../Taxonomy/TaxonomyKnowledgeOrchestrator.cs | 23 +++---------------- 6 files changed, 13 insertions(+), 32 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs index 3c114f670..d67a641fd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Knowledges.Options; using BotSharp.Abstraction.Knowledges.Processors; using BotSharp.Abstraction.Knowledges.Responses; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -15,7 +14,7 @@ public IEnumerable GetKnowledgeProcessors() return _services.GetServices().Select(x => x.Provider); } - [HttpPost("/knowledge/file/{collection}/upload")] + [HttpPost("/knowledge/collection/{collection}/file/upload")] public async Task UploadKnowledgeFiles([FromRoute] string collection, [FromBody] KnowledgeUploadRequest request) { var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); @@ -23,7 +22,7 @@ public async Task UploadKnowledgeFiles([FromRoute] stri return response; } - [HttpPost("/knowledge/file/{collection}/form")] + [HttpPost("/knowledge/collection/{collection}/file/form")] public async Task UploadKnowledgeFiles( [FromRoute] string collection, [FromForm] IEnumerable files, @@ -51,7 +50,7 @@ public async Task UploadKnowledgeFiles( return response; } - [HttpDelete("/knowledge/file/{collection}/delete/{fileId}")] + [HttpDelete("/knowledge/collection/{collection}/file/{fileId}")] public async Task DeleteKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) { var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.Orchestrator); @@ -60,7 +59,7 @@ public async Task DeleteKnowledgeFile([FromRoute] string collection, [From return response; } - [HttpDelete("/knowledge/file/{collection}/delete")] + [HttpDelete("/knowledge/collection/{collection}/file")] public async Task DeleteKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) { var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); @@ -68,7 +67,7 @@ public async Task DeleteKnowledgeFiles([FromRoute] string collection, [Fro return response; } - [HttpPost("/knowledge/file/{collection}/page")] + [HttpPost("/knowledge/collection/{collection}/file/page")] public async Task> GetPagedKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) { var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); @@ -81,7 +80,7 @@ public async Task> GetPagedKnowledgeFiles([Fr }; } - [HttpGet("/knowledge/file/{collection}/{fileId}")] + [HttpGet("/knowledge/collection/{collection}/file/{fileId}")] public async Task GetKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) { var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.Orchestrator); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index e48b567a6..a6d106726 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -102,7 +102,7 @@ public async Task> SearchKnowledge([Fro return results.Select(x => KnowledgeKnowledgeViewModel.From(x)).ToList(); } - [HttpPost("/knowledge/collection/{collection}/page")] + [HttpPost("/knowledge/collection/{collection}/data/page")] public async Task> GetPagedCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromBody] KnowledgeFilter filter) { var orchestrator = _services.GetServices() @@ -320,7 +320,7 @@ public async Task RecoverCollectionFromSnapshot([FromRoute] string collect return done; } - [HttpDelete("/knowledgecollection/{collection}/snapshot")] + [HttpDelete("/knowledge/collection/{collection}/snapshot")] public async Task DeleteCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteCollectionSnapshotRequest request) { var orchestrator = _services.GetServices() diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs index 9b2f27b7b..11d29f335 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Options; using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index 728d2538f..c78b07d11 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -60,7 +60,7 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) // Get all knowledge bases var orchestrator = GetKnowledgeOrchestrator(); - var knowledgeBases = await orchestrator.GetCollections(new KnowledgeCollectionOptions { IncludeAllTypes = true }); + var knowledgeBases = await orchestrator.GetCollections(new() { IncludeAllTypes = true }); foreach (var knowledgeBase in knowledgeBases) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 4a288e651..e6df204fd 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -19,7 +19,7 @@ public KnowledgeService( _logger = logger; } - private IVectorDb GetVectorDb() + private IVectorDb? GetVectorDb() { var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); return db; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs index b442e51cb..c5a440335 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs @@ -53,19 +53,19 @@ public async Task> Search(string query, strin { var payload = new Dictionary { - ["token"] = VectorPayloadValue.BuildStringValue(result.Token) + ["token"] = new VectorPayloadValue(result.Token) }; if (!string.IsNullOrEmpty(result.CanonicalText)) { - payload["canonical_text"] = VectorPayloadValue.BuildStringValue(result.CanonicalText); + payload["canonical_text"] = new VectorPayloadValue(result.CanonicalText); } foreach (var kvp in result.Data) { if (!payload.ContainsKey(kvp.Key)) { - payload[kvp.Key] = BuildPayloadValue(kvp.Value); + payload[kvp.Key] = new VectorPayloadValue(kvp.Value); } } @@ -105,21 +105,4 @@ public async Task> Search(string query, strin return results; } - - private VectorPayloadValue BuildPayloadValue(object value) - { - return value switch - { - string s => VectorPayloadValue.BuildStringValue(s), - double d => VectorPayloadValue.BuildDoubleValue(d), - float f => VectorPayloadValue.BuildDoubleValue(f), - long l => VectorPayloadValue.BuildIntegerValue(l), - int i => VectorPayloadValue.BuildIntegerValue(i), - short sh => VectorPayloadValue.BuildIntegerValue(sh), - byte b => VectorPayloadValue.BuildIntegerValue(b), - bool bl => VectorPayloadValue.BuildBooleanValue(bl), - DateTime dt => VectorPayloadValue.BuildDatetimeValue(dt), - _ => VectorPayloadValue.BuildUnkownValue(value) - }; - } } From 26ecb6d40452244582a58e9392901336f80274f6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 28 Apr 2026 17:36:06 -0500 Subject: [PATCH 14/23] rename --- .../Controllers/KnowledgeBase/KnowledgeBaseController.File.cs | 2 +- .../Services/File/KnowledgeFileOrchestrator.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs index d67a641fd..a9a6d42c4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -94,7 +94,7 @@ public async Task GetKnowledgeFile([FromRoute] string collection, private IKnowledgeFileOrchestrator? GetKnowledgeFileOrchestrator(string? provider) { - provider ??= "botsharp-knowledge-doc"; + provider ??= "botsharp-knowledge-file"; var found = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(provider)); return found; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index e680b1c9c..21826c691 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -9,7 +9,7 @@ public class KnowledgeFileOrchestrator : IKnowledgeFileOrchestrator private readonly KnowledgeBaseSettings _settings; private readonly ILogger _logger; - public string Provider => "botsharp-knowledge-doc"; + public string Provider => "botsharp-knowledge-file"; public KnowledgeFileOrchestrator( IServiceProvider services, From 513d98c7d6cee28063607585849a09634e295069 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 29 Apr 2026 16:10:44 -0500 Subject: [PATCH 15/23] add knowledge collection details --- .../Knowledges/IKnowledgeOrchestrator.cs | 2 ++ .../Models/KnowledgeCollectionDetails.cs | 12 +++++++++ .../KnowledgeBase/KnowledgeBaseController.cs | 14 ++++++++++ .../VectorOrchestratorBase.Collection.cs | 26 +++++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs index 8abbbad7e..8490baacf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs @@ -16,6 +16,8 @@ Task DeleteCollection(string collectionName, KnowledgeCollectionOptions op => Task.FromResult(false); Task> GetCollections(KnowledgeCollectionOptions options) => Task.FromResult(Enumerable.Empty()); + Task GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(null); #endregion #region Data diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs new file mode 100644 index 000000000..7adf3b25c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs @@ -0,0 +1,12 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionDetails +{ + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("payload_schema")] + public List PayloadSchema { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index a6d106726..fcd9495c3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -52,6 +52,20 @@ public async Task> GetCollection return collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)); } + [HttpGet("knowledge/collection/{collection}/details")] + public async Task GetCollectionDetails([FromRoute] string collection, [FromQuery] string knowledgeType) + { + var orchestrator = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (orchestrator == null) + { + return null; + } + + return await orchestrator.GetCollectionDetails(collection, new KnowledgeCollectionOptions()); + } + [HttpPost("knowledge/collection")] public async Task CreateCollection([FromBody] CreateCollectionRequest request) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs index af1ce4e8d..92857f208 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs @@ -102,6 +102,32 @@ public virtual async Task> GetCollections }); } + public virtual async Task GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return null; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return null; + } + + var details = await vectorDb.GetCollectionDetails(collectionName); + if (details == null) + { + return null; + } + + return new KnowledgeCollectionDetails + { + Status = details.Status, + PayloadSchema = details.PayloadSchema + }; + } + public virtual async Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) { if (string.IsNullOrWhiteSpace(collectionName)) From 3e53cd8cae3a8985f1fb7b195b44e3b1fc91be2e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 29 Apr 2026 17:39:38 -0500 Subject: [PATCH 16/23] clean code --- .../Knowledges/Enums/KnowledgeBaseType.cs | 2 +- .../Knowledges/Models/KnowledgeFilter.cs | 5 -- .../Options/KnowledgeQueryOptions.cs | 2 +- .../Models/LlmConfigBase.cs | 5 -- .../KnowledgeBaseController.File.cs | 8 +-- .../KnowledgeBase/KnowledgeBaseController.cs | 66 ++++++++++--------- .../Request/CollectionIndexRequest.cs | 8 +++ .../Request/CollectionSnapshotRequest.cs | 11 ++++ .../Request/CreateCollectionRequest.cs | 12 +--- .../Request/DeleteCollectionDataRequest.cs | 11 ++++ .../DeleteCollectionSnapshotRequest.cs | 5 +- .../Request/EntityAnalysisRequest.cs | 8 --- .../Request/GetKnowledgeFilesRequest.cs | 2 +- .../Request/GetPagedCollectionDataRequest.cs | 6 ++ .../KnowledgeCollectionConfigsRequest.cs | 2 - ...quest.cs => KnowledgeDataCreateRequest.cs} | 8 +-- .../Request/KnowledgeDataUpdateRequest.cs | 6 ++ .../Request/KnowledgeFileRequest.cs | 2 +- .../Request/KnowledgeUpdateRequest.cs | 9 --- .../Request/KnowledgeUploadRequest.cs | 5 -- ...quest.cs => QueryCollectionDataRequest.cs} | 3 +- .../Request/SearchVectorKnowledgeRequest.cs | 14 ---- .../Vector/VectorOrchestratorBase.Data.cs | 2 +- 23 files changed, 98 insertions(+), 104 deletions(-) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{KnowledgeCreateRequest.cs => KnowledgeDataCreateRequest.cs} (60%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{QueryVectorDataRequest.cs => QueryCollectionDataRequest.cs} (68%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs index 756737f73..a425188ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs @@ -5,5 +5,5 @@ public static class KnowledgeBaseType public static string QuestionAnswer = "question-answer"; public static string Document = "document"; public static string Taxonomy = "taxonomy"; - public static string SemanticGraph = "semantic_graph"; + public static string SemanticGraph = "semantic-graph"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs index 430d2ae12..76bd019e6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -4,31 +4,26 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFilter : StringIdPagination { - [JsonPropertyName("db_provider")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? DbProvider { get; set; } - [JsonPropertyName("with_vector")] public bool WithVector { get; set; } /// /// Filter group: each item contains a logical operator and a list of key-value pairs /// - [JsonPropertyName("filter_groups")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable? FilterGroups { get; set; } /// /// Order by a specific field /// - [JsonPropertyName("order_by")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public VectorSort? OrderBy { get; set; } /// /// Included payload fields /// - [JsonPropertyName("fields")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable? Fields { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs index 18cece169..85faa2e62 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Options; -public class KnowledgeQueryOptions +public class KnowledgeQueryOptions : KnowledgeOptionBase { public bool WithPayload { get; set; } public bool WithVector { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs index 899f6e7ab..9759f08d1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -15,11 +15,6 @@ public class LlmConfigBase : LlmProviderModel public class LlmProviderModel { - [JsonPropertyName("provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Provider { get; set; } - - [JsonPropertyName("model")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Model { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs index a9a6d42c4..72958ef09 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -53,7 +53,7 @@ public async Task UploadKnowledgeFiles( [HttpDelete("/knowledge/collection/{collection}/file/{fileId}")] public async Task DeleteKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) { - var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.Orchestrator); + var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.FileOrchestrator); var options = !string.IsNullOrWhiteSpace(request?.DbProvider) ? new KnowledgeFileOptions { DbProvider = request.DbProvider } : null; var response = await fileOrchestrator.DeleteKnowledgeFile(collection, fileId, options); return response; @@ -62,7 +62,7 @@ public async Task DeleteKnowledgeFile([FromRoute] string collection, [From [HttpDelete("/knowledge/collection/{collection}/file")] public async Task DeleteKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) { - var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.FileOrchestrator); var response = await fileOrchestrator.DeleteKnowledgeFiles(collection, request); return response; } @@ -70,7 +70,7 @@ public async Task DeleteKnowledgeFiles([FromRoute] string collection, [Fro [HttpPost("/knowledge/collection/{collection}/file/page")] public async Task> GetPagedKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) { - var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.FileOrchestrator); var data = await fileOrchestrator.GetPagedKnowledgeFiles(collection, request); return new PagedItems @@ -83,7 +83,7 @@ public async Task> GetPagedKnowledgeFiles([Fr [HttpGet("/knowledge/collection/{collection}/file/{fileId}")] public async Task GetKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) { - var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.Orchestrator); + var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.FileOrchestrator); var options = !string.IsNullOrWhiteSpace(request?.DbProvider) ? new KnowledgeFileOptions { DbProvider = request.DbProvider } : null; var file = await fileOrchestrator.GetKnowledgeFileBinaryData(collection, fileId, options); var stream = file.FileBinaryData.ToStream(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index fcd9495c3..365c5bdc4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -24,7 +24,7 @@ public KnowledgeBaseController( #region Collection [HttpGet("knowledge/collection/{collection}/exist")] - public async Task ExistCollection([FromRoute] string collection, [FromQuery] string knowledgeType) + public async Task ExistCollection([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -34,11 +34,11 @@ public async Task ExistCollection([FromRoute] string collection, [FromQuer return false; } - return await orchestrator.ExistCollection(collection, new KnowledgeCollectionOptions()); + return await orchestrator.ExistCollection(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } [HttpGet("knowledge/collections")] - public async Task> GetCollections([FromQuery] string knowledgeType) + public async Task> GetCollections([FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -48,12 +48,12 @@ public async Task> GetCollection return []; } - var collections = await orchestrator.GetCollections(new KnowledgeCollectionOptions()); + var collections = await orchestrator.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); return collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)); } [HttpGet("knowledge/collection/{collection}/details")] - public async Task GetCollectionDetails([FromRoute] string collection, [FromQuery] string knowledgeType) + public async Task GetCollectionDetails([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -63,7 +63,7 @@ public async Task> GetCollection return null; } - return await orchestrator.GetCollectionDetails(collection, new KnowledgeCollectionOptions()); + return await orchestrator.GetCollectionDetails(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } [HttpPost("knowledge/collection")] @@ -79,6 +79,7 @@ public async Task CreateCollection([FromBody] CreateCollectionRequest requ var options = new CollectionCreateOptions { + DbProvider = request.DbProvider, LlmProvider = request.Provider, LlmModel = request.Model, EmbeddingDimension = request.Dimension @@ -87,7 +88,7 @@ public async Task CreateCollection([FromBody] CreateCollectionRequest requ } [HttpDelete("knowledge/collection/{collection}")] - public async Task DeleteCollection([FromRoute] string collection, [FromQuery] string knowledgeType) + public async Task DeleteCollection([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -97,7 +98,7 @@ public async Task DeleteCollection([FromRoute] string collection, [FromQue return false; } - return await orchestrator.DeleteCollection(collection, new KnowledgeCollectionOptions()); + return await orchestrator.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } [HttpPost("/knowledge/collection/{collection}/search")] @@ -117,17 +118,17 @@ public async Task> SearchKnowledge([Fro } [HttpPost("/knowledge/collection/{collection}/data/page")] - public async Task> GetPagedCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromBody] KnowledgeFilter filter) + public async Task> GetPagedCollectionData([FromRoute] string collection, [FromBody] GetPagedCollectionDataRequest request) { var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (orchestrator == null) { return new StringIdPagedItems(); } - var data = await orchestrator.GetPagedCollectionData(collection, filter); + var data = await orchestrator.GetPagedCollectionData(collection, request); var items = data.Items?.Select(x => KnowledgeKnowledgeViewModel.From(x))?.ToList() ?? []; return new StringIdPagedItems @@ -139,7 +140,7 @@ public async Task> GetPagedColle } [HttpPost("/knowledge/collection/{collection}/data")] - public async Task CreateCollectionData([FromRoute] string collection, [FromBody] KnowledgeCreateRequest request) + public async Task CreateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataCreateRequest request) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); @@ -151,6 +152,7 @@ public async Task CreateCollectionData([FromRoute] string collection, [Fro var create = new KnowledgeCreateModel { + DbProvider = request.DbProvider, Text = request.Text, Payload = request.Payload }; @@ -160,7 +162,7 @@ public async Task CreateCollectionData([FromRoute] string collection, [Fro } [HttpGet("/knowledge/collection/{collection}/data")] - public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] QueryVectorDataRequest request) + public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] QueryCollectionDataRequest request) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -172,6 +174,7 @@ public async Task> GetCollectionData([F var options = new KnowledgeQueryOptions { + DbProvider = request.DbProvider, WithPayload = request.WithPayload, WithVector = request.WithVector }; @@ -181,7 +184,7 @@ public async Task> GetCollectionData([F } [HttpPut("/knowledge/collection/{collection}/data")] - public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeUpdateRequest request) + public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataUpdateRequest request) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); @@ -193,6 +196,7 @@ public async Task UpdateCollectionData([FromRoute] string collection, [Fro var update = new KnowledgeUpdateModel { + DbProvider = request.DbProvider, Id = request.Id, Text = request.Text, Payload = request.Payload @@ -203,31 +207,31 @@ public async Task UpdateCollectionData([FromRoute] string collection, [Fro } [HttpDelete("/knowledge/collection/{collection}/data/{id}")] - public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id, [FromQuery] string knowledgeType) + public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id, [FromBody] DeleteCollectionDataRequest request) { var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (orchestrator == null) { return false; } - return await orchestrator.DeleteCollectionData(collection, id, new KnowledgeCollectionOptions()); + return await orchestrator.DeleteCollectionData(collection, id, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } [HttpDelete("/knowledge/collection/{collection}/data")] - public async Task DeleteCollectionAllData([FromRoute] string collection, [FromQuery] string knowledgeType) + public async Task DeleteCollectionAllData([FromRoute] string collection, [FromBody] DeleteCollectionDataRequest request) { var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (orchestrator == null) { return false; } - return await orchestrator.DeleteCollectionData(collection, new KnowledgeCollectionOptions()); + return await orchestrator.DeleteCollectionData(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } #endregion @@ -246,6 +250,7 @@ public async Task> CreateCollectionIndexes([FromRout var options = new KnowledgeIndexOptions { + DbProvider = request.DbProvider, Items = request.Options }; return await orchestrator.CreateIndexes(collection, options); @@ -264,6 +269,7 @@ public async Task> DeleteCollectionIndexes([FromRout var options = new KnowledgeIndexOptions { + DbProvider = request.DbProvider, Items = request.Options }; return await orchestrator.DeleteIndexes(collection, options); @@ -273,7 +279,7 @@ public async Task> DeleteCollectionIndexes([FromRout #region Snapshot [HttpGet("/knowledge/collection/{collection}/snapshots")] - public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType) + public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -283,27 +289,27 @@ public async Task> GetCollecti return []; } - var snapshots = await orchestrator.GetCollectionSnapshots(collection); + var snapshots = await orchestrator.GetCollectionSnapshots(collection, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return snapshots.Select(x => KnowledgeCollectionSnapshotViewModel.From(x)!); } [HttpPost("/knowledge/collection/{collection}/snapshot")] - public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromQuery] string knowledgeType) + public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromBody] CollectionSnapshotRequest request) { var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (orchestrator == null) { return null; } - var snapshot = await orchestrator.CreateCollectionSnapshot(collection); + var snapshot = await orchestrator.CreateCollectionSnapshot(collection, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); return KnowledgeCollectionSnapshotViewModel.From(snapshot); } [HttpGet("/knowledge/collection/{collection}/snapshot")] - public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType) + public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -313,12 +319,12 @@ public async Task GetCollectionSnapshot([FromRoute] string collec return BuildFileResult(snapshotFileName, BinaryData.Empty); } - var snapshot = await orchestrator.DownloadCollectionSnapshot(collection, snapshotFileName); + var snapshot = await orchestrator.DownloadCollectionSnapshot(collection, snapshotFileName, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return BuildFileResult(snapshotFileName, snapshot); } [HttpPost("/knowledge/collection/{collection}/snapshot/recover")] - public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromQuery] string knowledgeType) + public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromForm] string knowledgeType, [FromForm] string? dbProvider = null) { var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); @@ -330,7 +336,7 @@ public async Task RecoverCollectionFromSnapshot([FromRoute] string collect var fileName = snapshotFile.FileName; var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile); - var done = await orchestrator.RecoverCollectionFromSnapshot(collection, fileName, binary); + var done = await orchestrator.RecoverCollectionFromSnapshot(collection, fileName, binary, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return done; } @@ -345,7 +351,7 @@ public async Task DeleteCollectionSnapshots([FromRoute] string collection, return false; } - var done = await orchestrator.DeleteCollectionSnapshot(collection, request.SnapshotName); + var done = await orchestrator.DeleteCollectionSnapshot(collection, request.SnapshotName, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); return done; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs index aefa77089..18fd2a3b6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs @@ -8,6 +8,10 @@ public class CreateCollectionIndexRequest [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; + [JsonPropertyName("db_provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } + public IEnumerable Options { get; set; } = []; } @@ -16,5 +20,9 @@ public class DeleteCollectionIndexRequest [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; + [JsonPropertyName("db_provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } + public IEnumerable Options { get; set; } = []; } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs new file mode 100644 index 000000000..d9af5f7f0 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class CollectionSnapshotRequest +{ + public string KnowledgeType { get; set; } = null!; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs index f6e955af1..3d5435227 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs @@ -4,18 +4,12 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class CreateCollectionRequest { - [JsonPropertyName("collection_name")] public string CollectionName { get; set; } - - [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } - - [JsonPropertyName("provider")] public string Provider { get; set; } - - [JsonPropertyName("model")] public string Model { get; set; } - - [JsonPropertyName("dimension")] public int Dimension { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs new file mode 100644 index 000000000..289c8d6a6 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteCollectionDataRequest +{ + public string KnowledgeType { get; set; } = null!; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs index 6d171dd71..3a93944a4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs @@ -4,9 +4,10 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class DeleteCollectionSnapshotRequest { - [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; - [JsonPropertyName("snapshot_name")] public string SnapshotName { get; set; } = default!; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs deleted file mode 100644 index 890113448..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class EntityAnalysisRequest -{ - public string Text { get; set; } = string.Empty; - public string? Provider { get; set; } - public EntityAnalysisOptions? Options { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs index 328a2357a..1ddd163f4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs @@ -4,5 +4,5 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class GetKnowledgeFilesRequest : KnowledgeFileFilter { - public string? Orchestrator { get; set; } + public string? FileOrchestrator { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs new file mode 100644 index 000000000..b311ac88b --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GetPagedCollectionDataRequest : KnowledgeFilter +{ + public string KnowledgeType { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs index ed1541129..e674737b4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs @@ -1,10 +1,8 @@ using BotSharp.Abstraction.VectorStorage.Models; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class KnowledgeCollectionConfigsRequest { - [JsonPropertyName("collections")] public List Collections { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs similarity index 60% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs index 15aa9fcff..29a1da856 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs @@ -1,16 +1,14 @@ using BotSharp.Abstraction.VectorStorage.Models; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class KnowledgeCreateRequest +public class KnowledgeDataCreateRequest { - [JsonPropertyName("text")] public string Text { get; set; } - [JsonPropertyName("payload")] public Dictionary? Payload { get; set; } - [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; + + public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs new file mode 100644 index 000000000..ba829cce2 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeDataUpdateRequest : KnowledgeDataCreateRequest +{ + public string Id { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs index 62fc16c60..2e95d0e9c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs @@ -2,6 +2,6 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class KnowledgeFileRequest { - public string? Orchestrator { get; set; } + public string? FileOrchestrator { get; set; } public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs deleted file mode 100644 index fdff63182..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUpdateRequest.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class KnowledgeUpdateRequest : KnowledgeCreateRequest -{ - [JsonPropertyName("id")] - public string Id { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs index 11d29f335..e7505d6c9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class KnowledgeUploadRequest @@ -7,12 +5,9 @@ public class KnowledgeUploadRequest /// /// Provider for knowledge file orchestrator /// - [JsonPropertyName("orchestrator")] public string? Orchestrator { get; set; } - [JsonPropertyName("files")] public IEnumerable Files { get; set; } = new List(); - [JsonPropertyName("options")] public KnowledgeFileHandleOptions? Options { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryCollectionDataRequest.cs similarity index 68% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryCollectionDataRequest.cs index 331bb01f9..8eccb1500 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryCollectionDataRequest.cs @@ -1,8 +1,9 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class QueryVectorDataRequest +public class QueryCollectionDataRequest { public List Ids { get; set; } = []; public bool WithVector { get; set; } public bool WithPayload { get; set; } + public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 3de4351a7..703ad63ec 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -5,48 +5,34 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class SearchKnowledgeRequest { - [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; - [JsonPropertyName("fields")] public IEnumerable? Fields { get; set; } - [JsonPropertyName("filter_groups")] public IEnumerable? FilterGroups { get; set; } - [JsonPropertyName("limit")] public int? Limit { get; set; } = 5; - [JsonPropertyName("confidence")] public float? Confidence { get; set; } = 0.5f; - [JsonPropertyName("with_vector")] public bool WithVector { get; set; } - [JsonPropertyName("search_param")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? SearchParam { get; set; } - [JsonPropertyName("search_arguments")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? SearchArguments { get; set; } - - [JsonPropertyName("knowledge_type")] public string KnowledgeType { get; set; } = null!; - [JsonPropertyName("db_provider")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? DbProvider { get; set; } - [JsonPropertyName("data_providers")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? DataProviders { get; set; } - [JsonPropertyName("max_ngram")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? MaxNgram { get; set; } - [JsonPropertyName("graph_id")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? GraphId { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs index aaba36290..4cd2936f0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs @@ -71,7 +71,7 @@ public virtual async Task> GetCollectionDat .Select(x => x.ParseResult) .ToList(); - var vectorDb = GetVectorDb(); + var vectorDb = GetVectorDb(options?.DbProvider); if (vectorDb == null) { return []; From 5d270ab9f200e53875bee2d2ed70850c869440f3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 1 May 2026 10:55:41 -0500 Subject: [PATCH 17/23] temp save --- .../KnowledgeBaseController.File.cs | 2 +- .../KnowledgeBase/KnowledgeBaseController.cs | 35 ++++++++++++++----- .../Request/CollectionIndexRequest.cs | 19 ++-------- .../Request/CollectionSnapshotRequest.cs | 8 +---- .../Request/CreateCollectionRequest.cs | 8 +---- .../Request/DeleteCollectionDataRequest.cs | 8 +---- .../Request/DeleteCollectionRequest.cs | 5 +++ .../DeleteCollectionSnapshotRequest.cs | 9 +---- .../Request/KnowledgeBaseRequestBase.cs | 11 ++++++ .../Request/KnowledgeDataCreateRequest.cs | 6 +--- .../Request/KnowledgeUploadRequest.cs | 2 +- .../Request/SearchVectorKnowledgeRequest.cs | 6 +--- .../File/KnowledgeFileOrchestrator.cs | 12 +++---- 13 files changed, 58 insertions(+), 73 deletions(-) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs index 72958ef09..6eb942cb6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -17,7 +17,7 @@ public IEnumerable GetKnowledgeProcessors() [HttpPost("/knowledge/collection/{collection}/file/upload")] public async Task UploadKnowledgeFiles([FromRoute] string collection, [FromBody] KnowledgeUploadRequest request) { - var fileOrchestrator = GetKnowledgeFileOrchestrator(request.Orchestrator); + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.FileOrchestrator); var response = await fileOrchestrator.UploadFilesToKnowledge(collection, request.Files, request.Options); return response; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 365c5bdc4..5bb639975 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -38,18 +38,35 @@ public async Task ExistCollection([FromRoute] string collection, [FromQuer } [HttpGet("knowledge/collections")] - public async Task> GetCollections([FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) + public async Task> GetCollections([FromQuery] string? knowledgeType, [FromQuery] string? dbProvider = null) { - var orchestrator = _services.GetServices() + var results = new List(); + + if (!string.IsNullOrWhiteSpace(knowledgeType)) + { + var orchestrator = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (orchestrator == null) + { + return []; + } + + var collections = await orchestrator.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); + results = collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)).ToList(); + } + else { - return []; + var kgs = _services.GetServices(); + foreach (var kg in kgs) + { + var collections = await kg.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); + var res = collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)); + results.AddRange(res); + } } - var collections = await orchestrator.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); - return collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)); + return results; } [HttpGet("knowledge/collection/{collection}/details")] @@ -88,17 +105,17 @@ public async Task CreateCollection([FromBody] CreateCollectionRequest requ } [HttpDelete("knowledge/collection/{collection}")] - public async Task DeleteCollection([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) + public async Task DeleteCollection([FromRoute] string collection, [FromBody] DeleteCollectionRequest request) { var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (orchestrator == null) { return false; } - return await orchestrator.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); + return await orchestrator.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } [HttpPost("/knowledge/collection/{collection}/search")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs index 18fd2a3b6..bdc6a3d84 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs @@ -1,28 +1,13 @@ using BotSharp.Abstraction.VectorStorage.Options; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class CreateCollectionIndexRequest +public class CreateCollectionIndexRequest : KnowledgeBaseRequestBase { - [JsonPropertyName("knowledge_type")] - public string KnowledgeType { get; set; } = null!; - - [JsonPropertyName("db_provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } - public IEnumerable Options { get; set; } = []; } -public class DeleteCollectionIndexRequest +public class DeleteCollectionIndexRequest : KnowledgeBaseRequestBase { - [JsonPropertyName("knowledge_type")] - public string KnowledgeType { get; set; } = null!; - - [JsonPropertyName("db_provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } - public IEnumerable Options { get; set; } = []; } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs index d9af5f7f0..62e719978 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs @@ -1,11 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class CollectionSnapshotRequest +public class CollectionSnapshotRequest : KnowledgeBaseRequestBase { - public string KnowledgeType { get; set; } = null!; - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs index 3d5435227..ea9b1ffee 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs @@ -1,15 +1,9 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class CreateCollectionRequest +public class CreateCollectionRequest : KnowledgeBaseRequestBase { public string CollectionName { get; set; } - public string KnowledgeType { get; set; } public string Provider { get; set; } public string Model { get; set; } public int Dimension { get; set; } - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs index 289c8d6a6..4b511f711 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs @@ -1,11 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class DeleteCollectionDataRequest +public class DeleteCollectionDataRequest : KnowledgeBaseRequestBase { - public string KnowledgeType { get; set; } = null!; - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs new file mode 100644 index 000000000..694f49eb9 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteCollectionRequest : KnowledgeBaseRequestBase +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs index 3a93944a4..74466d46c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs @@ -1,13 +1,6 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class DeleteCollectionSnapshotRequest +public class DeleteCollectionSnapshotRequest : KnowledgeBaseRequestBase { - public string KnowledgeType { get; set; } = null!; - public string SnapshotName { get; set; } = default!; - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs new file mode 100644 index 000000000..7607eabb5 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeBaseRequestBase +{ + public string KnowledgeType { get; set; } = null!; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs index 29a1da856..7d6e8a436 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs @@ -2,13 +2,9 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class KnowledgeDataCreateRequest +public class KnowledgeDataCreateRequest : KnowledgeBaseRequestBase { public string Text { get; set; } public Dictionary? Payload { get; set; } - - public string KnowledgeType { get; set; } = null!; - - public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs index e7505d6c9..539f466b6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs @@ -5,7 +5,7 @@ public class KnowledgeUploadRequest /// /// Provider for knowledge file orchestrator /// - public string? Orchestrator { get; set; } + public string? FileOrchestrator { get; set; } public IEnumerable Files { get; set; } = new List(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index 703ad63ec..d7a131409 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -3,7 +3,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class SearchKnowledgeRequest +public class SearchKnowledgeRequest : KnowledgeBaseRequestBase { public string Text { get; set; } = string.Empty; @@ -22,10 +22,6 @@ public class SearchKnowledgeRequest [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? SearchArguments { get; set; } - public string KnowledgeType { get; set; } = null!; - - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? DbProvider { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? DataProviders { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index 21826c691..3a1d9732c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -225,10 +225,10 @@ public async Task> GetPagedKnowledgeFiles(string var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var knowledgebaseProvider = filter?.DbProvider ?? _settings.VectorDb.Provider; + var kgDbProvider = filter?.DbProvider ?? _settings.VectorDb.Provider; // Get doc meta data - var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, knowledgebaseProvider, filter); + var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, kgDbProvider, filter); var files = pagedData.Items?.Select(x => new KnowledgeFileModel { @@ -237,7 +237,7 @@ public async Task> GetPagedKnowledgeFiles(string FileSource = x.FileSource, FileExtension = Path.GetExtension(x.FileName), ContentType = x.ContentType, - FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, knowledgebaseProvider, x.FileId, x.FileName), + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, kgDbProvider, x.FileId, x.FileName), RefData = x.RefData })?.ToList() ?? []; @@ -252,10 +252,10 @@ public async Task GetKnowledgeFileBinaryData(string collect { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var knwoledgebaseProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; + var kgDbProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; // Get doc binary data - var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, knwoledgebaseProvider, new KnowledgeFileFilter + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, kgDbProvider, new KnowledgeFileFilter { Size = 1, FileIds = [fileId] @@ -272,7 +272,7 @@ public async Task GetKnowledgeFileBinaryData(string collect }; }; - var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, knwoledgebaseProvider, fileId, metaData.FileName); + var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, kgDbProvider, fileId, metaData.FileName); return new FileBinaryDataModel { FileName = metaData.FileName, From a2f7d7caa84a3070cfce2350f39a450fae8e766e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 1 May 2026 14:55:17 -0500 Subject: [PATCH 18/23] refine --- .../Knowledges/Options/KnowledgeSearchOptions.cs | 4 ++-- .../VectorStorage/Options/VectorSearchOptions.cs | 2 +- .../Controllers/KnowledgeBase/KnowledgeBaseController.cs | 8 +++++--- .../Knowledges/Request/SearchVectorKnowledgeRequest.cs | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs index 7d5dcbf25..89a50f561 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs @@ -7,8 +7,8 @@ public class KnowledgeSearchOptions public string? DbProvider { get; set; } public IEnumerable? Fields { get; set; } public IEnumerable? FilterGroups { get; set; } - public Dictionary? SearchParam { get; set; } - public Dictionary? SearchArguments { get; set; } + public Dictionary? SearchParam { get; set; } + public Dictionary? SearchArguments { get; set; } public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs index 4a4358319..c9ef91f7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs @@ -10,7 +10,7 @@ public class VectorSearchOptions public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } - public Dictionary? SearchParam { get; set; } + public Dictionary? SearchParam { get; set; } public static VectorSearchOptions Default() { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 5bb639975..2954b23d6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -420,12 +420,14 @@ private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator orchestrator, SearchKnowledgeRequest? request) { + var searchParam = request?.SearchParam?.ToDictionary(x => x.Key, x => x.Value?.ConvertToString()); + if (orchestrator.KnowledgeType.IsEqualTo(KnowledgeBaseType.SemanticGraph)) { return new GraphKnowledgeSearchOptions { DbProvider = request?.DbProvider, - SearchParam = request?.SearchParam, + SearchParam = searchParam, SearchArguments = request?.SearchArguments, GraphId = request?.GraphId }; @@ -438,7 +440,7 @@ private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator orchest DbProvider = request?.DbProvider, Limit = request?.Limit ?? 5, Confidence = request?.Confidence ?? 0.5f, - SearchParam = request?.SearchParam, + SearchParam = searchParam, SearchArguments = request?.SearchArguments, DataProviders = request?.DataProviders, MaxNgram = request?.MaxNgram @@ -453,7 +455,7 @@ private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator orchest Limit = request?.Limit ?? 5, Confidence = request?.Confidence ?? 0.5f, WithVector = request?.WithVector ?? false, - SearchParam = request?.SearchParam, + SearchParam = searchParam, SearchArguments = request?.SearchArguments }; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs index d7a131409..4790b5327 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs @@ -18,10 +18,10 @@ public class SearchKnowledgeRequest : KnowledgeBaseRequestBase public bool WithVector { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public Dictionary? SearchParam { get; set; } + public Dictionary? SearchParam { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public Dictionary? SearchArguments { get; set; } + public Dictionary? SearchArguments { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? DataProviders { get; set; } From 3f0617e1da1ae0ab5934796766af786e76738802 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 1 May 2026 15:36:15 -0500 Subject: [PATCH 19/23] rename --- .../KnowledgeBaseController.File.cs | 4 +- .../KnowledgeBase/KnowledgeBaseController.cs | 152 +++++++++--------- .../Functions/MemorizeKnowledgeFn.cs | 8 +- .../Hooks/KnowledgeHook.cs | 24 +-- 4 files changed, 94 insertions(+), 94 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs index 6eb942cb6..2451cc800 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -92,10 +92,10 @@ public async Task GetKnowledgeFile([FromRoute] string collection, return new FileStreamResult(stream, file.ContentType) { FileDownloadName = file.FileName }; } - private IKnowledgeFileOrchestrator? GetKnowledgeFileOrchestrator(string? provider) + private IKnowledgeFileOrchestrator GetKnowledgeFileOrchestrator(string? provider) { provider ??= "botsharp-knowledge-file"; - var found = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(provider)); + var found = _services.GetServices().First(x => x.Provider.IsEqualTo(provider)); return found; } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 2954b23d6..b4461f274 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -26,15 +26,15 @@ public KnowledgeBaseController( [HttpGet("knowledge/collection/{collection}/exist")] public async Task ExistCollection([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } - return await orchestrator.ExistCollection(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); + return await kg.ExistCollection(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } [HttpGet("knowledge/collections")] @@ -44,15 +44,15 @@ public async Task> GetCollection if (!string.IsNullOrWhiteSpace(knowledgeType)) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return []; } - var collections = await orchestrator.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); + var collections = await kg.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); results = collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)).ToList(); } else @@ -72,24 +72,24 @@ public async Task> GetCollection [HttpGet("knowledge/collection/{collection}/details")] public async Task GetCollectionDetails([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return null; } - return await orchestrator.GetCollectionDetails(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); + return await kg.GetCollectionDetails(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } [HttpPost("knowledge/collection")] public async Task CreateCollection([FromBody] CreateCollectionRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } @@ -101,36 +101,36 @@ public async Task CreateCollection([FromBody] CreateCollectionRequest requ LlmModel = request.Model, EmbeddingDimension = request.Dimension }; - return await orchestrator.CreateCollection(request.CollectionName, options); + return await kg.CreateCollection(request.CollectionName, options); } [HttpDelete("knowledge/collection/{collection}")] public async Task DeleteCollection([FromRoute] string collection, [FromBody] DeleteCollectionRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } - return await orchestrator.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); + return await kg.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } [HttpPost("/knowledge/collection/{collection}/search")] public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return []; } - var options = BuildSearchOptions(orchestrator, request); - var results = await orchestrator.Search(request?.Text ?? string.Empty, collection, options); + var options = BuildSearchOptions(kg, request); + var results = await kg.Search(request?.Text ?? string.Empty, collection, options); return results.Select(x => KnowledgeKnowledgeViewModel.From(x)).ToList(); } @@ -159,10 +159,10 @@ public async Task> GetPagedColle [HttpPost("/knowledge/collection/{collection}/data")] public async Task CreateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataCreateRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } @@ -174,17 +174,17 @@ public async Task CreateCollectionData([FromRoute] string collection, [Fro Payload = request.Payload }; - var created = await orchestrator.CreateCollectionData(collection, create); + var created = await kg.CreateCollectionData(collection, create); return created; } [HttpGet("/knowledge/collection/{collection}/data")] public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] QueryCollectionDataRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return []; } @@ -196,17 +196,17 @@ public async Task> GetCollectionData([F WithVector = request.WithVector }; - var points = await orchestrator.GetCollectionData(collection, request.Ids, options); + var points = await kg.GetCollectionData(collection, request.Ids, options); return points.Select(x => KnowledgeKnowledgeViewModel.From(x)); } [HttpPut("/knowledge/collection/{collection}/data")] public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataUpdateRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } @@ -219,36 +219,36 @@ public async Task UpdateCollectionData([FromRoute] string collection, [Fro Payload = request.Payload }; - var updated = await orchestrator.UpdateCollectionData(collection, update); + var updated = await kg.UpdateCollectionData(collection, update); return updated; } [HttpDelete("/knowledge/collection/{collection}/data/{id}")] public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id, [FromBody] DeleteCollectionDataRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } - return await orchestrator.DeleteCollectionData(collection, id, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); + return await kg.DeleteCollectionData(collection, id, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } [HttpDelete("/knowledge/collection/{collection}/data")] public async Task DeleteCollectionAllData([FromRoute] string collection, [FromBody] DeleteCollectionDataRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } - return await orchestrator.DeleteCollectionData(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); + return await kg.DeleteCollectionData(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } #endregion @@ -257,10 +257,10 @@ public async Task DeleteCollectionAllData([FromRoute] string collection, [ [HttpPost("/knowledge/collection/{collection}/indexes")] public async Task> CreateCollectionIndexes([FromRoute] string collection, [FromBody] CreateCollectionIndexRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return new(); } @@ -270,16 +270,16 @@ public async Task> CreateCollectionIndexes([FromRout DbProvider = request.DbProvider, Items = request.Options }; - return await orchestrator.CreateIndexes(collection, options); + return await kg.CreateIndexes(collection, options); } [HttpDelete("/knowledge/collection/{collection}/indexes")] public async Task> DeleteCollectionIndexes([FromRoute] string collection, [FromBody] DeleteCollectionIndexRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return new(); } @@ -289,7 +289,7 @@ public async Task> DeleteCollectionIndexes([FromRout DbProvider = request.DbProvider, Items = request.Options }; - return await orchestrator.DeleteIndexes(collection, options); + return await kg.DeleteIndexes(collection, options); } #endregion @@ -298,77 +298,77 @@ public async Task> DeleteCollectionIndexes([FromRout [HttpGet("/knowledge/collection/{collection}/snapshots")] public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return []; } - var snapshots = await orchestrator.GetCollectionSnapshots(collection, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); + var snapshots = await kg.GetCollectionSnapshots(collection, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return snapshots.Select(x => KnowledgeCollectionSnapshotViewModel.From(x)!); } [HttpPost("/knowledge/collection/{collection}/snapshot")] public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromBody] CollectionSnapshotRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return null; } - var snapshot = await orchestrator.CreateCollectionSnapshot(collection, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); + var snapshot = await kg.CreateCollectionSnapshot(collection, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); return KnowledgeCollectionSnapshotViewModel.From(snapshot); } [HttpGet("/knowledge/collection/{collection}/snapshot")] public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return BuildFileResult(snapshotFileName, BinaryData.Empty); } - var snapshot = await orchestrator.DownloadCollectionSnapshot(collection, snapshotFileName, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); + var snapshot = await kg.DownloadCollectionSnapshot(collection, snapshotFileName, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return BuildFileResult(snapshotFileName, snapshot); } [HttpPost("/knowledge/collection/{collection}/snapshot/recover")] public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromForm] string knowledgeType, [FromForm] string? dbProvider = null) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } var fileName = snapshotFile.FileName; var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile); - var done = await orchestrator.RecoverCollectionFromSnapshot(collection, fileName, binary, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); + var done = await kg.RecoverCollectionFromSnapshot(collection, fileName, binary, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return done; } [HttpDelete("/knowledge/collection/{collection}/snapshot")] public async Task DeleteCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteCollectionSnapshotRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return false; } - var done = await orchestrator.DeleteCollectionSnapshot(collection, request.SnapshotName, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); + var done = await kg.DeleteCollectionSnapshot(collection, request.SnapshotName, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); return done; } #endregion @@ -418,11 +418,11 @@ private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) return File(stream, "application/octet-stream", Path.GetFileName(fileName)); } - private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator orchestrator, SearchKnowledgeRequest? request) + private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator kg, SearchKnowledgeRequest? request) { var searchParam = request?.SearchParam?.ToDictionary(x => x.Key, x => x.Value?.ConvertToString()); - if (orchestrator.KnowledgeType.IsEqualTo(KnowledgeBaseType.SemanticGraph)) + if (kg.KnowledgeType.IsEqualTo(KnowledgeBaseType.SemanticGraph)) { return new GraphKnowledgeSearchOptions { @@ -433,7 +433,7 @@ private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator orchest }; } - if (orchestrator.KnowledgeType.IsEqualTo(KnowledgeBaseType.Taxonomy)) + if (kg.KnowledgeType.IsEqualTo(KnowledgeBaseType.Taxonomy)) { return new TaxonomyKnowledgeSearchOptions { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 6e62aa1ae..d45925b23 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -23,16 +23,16 @@ public async Task Execute(RoleDialogModel message) ? args.RefinedCollection : _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); - if (orchestrator == null) + if (kg == null) { message.Content = "I forgot it"; return true; } - var result = await orchestrator.CreateCollectionData(collectionName, new KnowledgeCreateModel + var result = await kg.CreateCollectionData(collectionName, new KnowledgeCreateModel { Text = args.Question, Payload = new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index c78b07d11..36e82ba16 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -20,7 +20,7 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str { if (knowledgeBase.Type == KnowledgeBaseType.Document) { - var orchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); + var kg = GetKnowledgeService(knowledgeBase.Type); var options = new KnowledgeSearchOptions { Fields = null, @@ -28,14 +28,14 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str Confidence = 0.25f, WithVector = true }; - var result = await orchestrator.Search(text, knowledgeBase.Name, options); + var result = await kg.Search(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) .Select(x => x.Data["text"].ToString()) .Where(x => x != null)!); } else if (knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { - var orchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); + var kg = GetKnowledgeService(knowledgeBase.Type); var options = new KnowledgeSearchOptions { Fields = null, @@ -43,7 +43,7 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str Confidence = 0.5f, WithVector = true }; - var result = await orchestrator.Search(text, knowledgeBase.Name, options); + var result = await kg.Search(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); @@ -59,15 +59,15 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) var results = new List(); // Get all knowledge bases - var orchestrator = GetKnowledgeOrchestrator(); - var knowledgeBases = await orchestrator.GetCollections(new() { IncludeAllTypes = true }); + var kg = GetKnowledgeService(); + var knowledgeBases = await kg.GetCollections(new() { IncludeAllTypes = true }); foreach (var knowledgeBase in knowledgeBases) { if (knowledgeBase.Type == KnowledgeBaseType.Document || knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { - var searchOrchestrator = GetKnowledgeOrchestrator(knowledgeBase.Type); + var kgSearcher = GetKnowledgeService(knowledgeBase.Type); var options = new KnowledgeSearchOptions { Fields = null, @@ -75,7 +75,7 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) Confidence = 0.5f, WithVector = true }; - var result = await searchOrchestrator.Search(text, knowledgeBase.Name, options); + var result = await kgSearcher.Search(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); @@ -98,13 +98,13 @@ private async Task> GetKnowledgeBaseNameByAgentIdAsync( return agent.KnowledgeBases; } - private IKnowledgeOrchestrator GetKnowledgeOrchestrator(string? type = null) + private IKnowledgeOrchestrator GetKnowledgeService(string? type = null) { - var orchestrators = _services.GetServices(); + var kgs = _services.GetServices(); if (!string.IsNullOrWhiteSpace(type)) { - return orchestrators.First(x => x.KnowledgeType == type); + return kgs.First(x => x.KnowledgeType == type); } - return orchestrators.First(); + return kgs.First(); } } \ No newline at end of file From 27b17586a38a64206412d554564c774676640d59 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 1 May 2026 16:58:42 -0500 Subject: [PATCH 20/23] rename --- .../Knowledges/IKnowledgeFileOrchestrator.cs | 18 ++++- .../Knowledges/IKnowledgeOrchestrator.cs | 8 +-- .../Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeCreateModel.cs | 1 - ...rchResult.cs => KnowledgeExecuteResult.cs} | 8 +-- .../Options/GraphKnowledgeSearchOptions.cs | 2 +- .../Options/ImportKnowledgeFileOptions.cs | 9 +++ ...hOptions.cs => KnowledgeExecuteOptions.cs} | 2 +- .../Options/TaxonomyKnowledgeSearchOptions.cs | 2 +- .../Extensions/VectorStorageExtension.cs | 2 +- .../KnowledgeBase/KnowledgeBaseController.cs | 6 +- .../View/KnowledgeKnowledgeViewModel.cs | 2 +- .../Hooks/KnowledgeHook.cs | 12 ++-- .../Vector/VectorOrchestratorBase.Data.cs | 14 ++-- .../File/KnowledgeFileOrchestrator.cs | 70 ++++++++++++++++++- .../Graph/GraphKnowledgeOrchestrator.cs | 6 +- .../Services/KnowledgeService.Document.cs | 10 +-- .../Taxonomy/TaxonomyKnowledgeOrchestrator.cs | 6 +- 18 files changed, 133 insertions(+), 47 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/{KnowledgeSearchResult.cs => KnowledgeExecuteResult.cs} (51%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/{KnowledgeSearchOptions.cs => KnowledgeExecuteOptions.cs} (93%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs index beb40e320..b6d552cac 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Knowledges.Filters; -using BotSharp.Abstraction.Knowledges.Options; using BotSharp.Abstraction.Knowledges.Responses; namespace BotSharp.Abstraction.Knowledges; @@ -15,8 +14,21 @@ public interface IKnowledgeFileOrchestrator /// /// /// - Task UploadFilesToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); - + Task UploadFilesToKnowledge(string collectionName, + IEnumerable files, KnowledgeFileHandleOptions? options = null); + + /// + /// Import file content to knowledgebase without saving the file + /// + /// + /// + /// + /// + /// + /// + Task ImportFileContentToKnowledge(string collectionName, string fileName, string fileSource, + IEnumerable contents, ImportKnowledgeFileOptions? options = null); + /// /// Delete one file and its related knowledge in the collection /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs index 8490baacf..4b9530e11 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs @@ -21,10 +21,10 @@ Task> GetCollections(KnowledgeCollectionO #endregion #region Data - Task> Search(string query, string collectionName, KnowledgeSearchOptions options) - => Task.FromResult(Enumerable.Empty()); - Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) - => Task.FromResult(new StringIdPagedItems()); + Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) + => Task.FromResult(Enumerable.Empty()); + Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + => Task.FromResult(new StringIdPagedItems()); Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) => Task.FromResult(Enumerable.Empty()); Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 35053c629..1c5425dc8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -45,7 +45,7 @@ public interface IKnowledgeService ///// ///// //Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, - // DocMetaRefData? refData = null, Dictionary? payload = null); + // ImportKnowledgeFileOptions? options = null); ///// ///// Delete one document and its related knowledge in the collection ///// diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs index 5e9008585..5020d3605 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Options; using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Abstraction.Knowledges.Models; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs similarity index 51% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs index 65592cd3d..b026fee1a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs @@ -1,15 +1,15 @@ namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeSearchResult : KnowledgeCollectionData +public class KnowledgeExecuteResult : KnowledgeCollectionData { - public KnowledgeSearchResult() + public KnowledgeExecuteResult() { } - public static KnowledgeSearchResult CopyFrom(KnowledgeCollectionData data) + public static KnowledgeExecuteResult CopyFrom(KnowledgeCollectionData data) { - return new KnowledgeSearchResult + return new KnowledgeExecuteResult { Id = data.Id, Payload = data.Payload, diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs index 47dc1a184..caa2d5037 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Options; -public class GraphKnowledgeSearchOptions : KnowledgeSearchOptions +public class GraphKnowledgeSearchOptions : KnowledgeExecuteOptions { public string? GraphId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs new file mode 100644 index 000000000..60610ae5f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs @@ -0,0 +1,9 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Options; + +public class ImportKnowledgeFileOptions : KnowledgeOptionBase +{ + public DocMetaRefData? FileRefData { get; set; } + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs similarity index 93% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs index 89a50f561..90a98d7a0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges.Options; -public class KnowledgeSearchOptions +public class KnowledgeExecuteOptions { public string? DbProvider { get; set; } public IEnumerable? Fields { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs index bbe5d8114..6520e7ff5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Options; -public class TaxonomyKnowledgeSearchOptions : KnowledgeSearchOptions +public class TaxonomyKnowledgeSearchOptions : KnowledgeExecuteOptions { public IEnumerable? DataProviders { get; set; } public int? MaxNgram { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs index 8c977cfbd..00ed10fc0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs @@ -5,7 +5,7 @@ namespace BotSharp.Abstraction.VectorStorage.Extensions; public static class VectorStorageExtension { - public static string ToQuestionAnswer(this VectorSearchResult data) + public static string ToQuestionAnswer(this KnowledgeExecuteResult data) { if (data?.Data == null) return string.Empty; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index b4461f274..060201143 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -130,7 +130,7 @@ public async Task> SearchKnowledge([Fro } var options = BuildSearchOptions(kg, request); - var results = await kg.Search(request?.Text ?? string.Empty, collection, options); + var results = await kg.ExecuteQuery(request?.Text ?? string.Empty, collection, options); return results.Select(x => KnowledgeKnowledgeViewModel.From(x)).ToList(); } @@ -418,7 +418,7 @@ private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) return File(stream, "application/octet-stream", Path.GetFileName(fileName)); } - private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator kg, SearchKnowledgeRequest? request) + private KnowledgeExecuteOptions BuildSearchOptions(IKnowledgeOrchestrator kg, SearchKnowledgeRequest? request) { var searchParam = request?.SearchParam?.ToDictionary(x => x.Key, x => x.Value?.ConvertToString()); @@ -447,7 +447,7 @@ private KnowledgeSearchOptions BuildSearchOptions(IKnowledgeOrchestrator kg, Sea }; } - return new KnowledgeSearchOptions + return new KnowledgeExecuteOptions { DbProvider = request?.DbProvider, Fields = request?.Fields, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs index 136f6908c..ccb1f77e9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs @@ -47,7 +47,7 @@ public static KnowledgeKnowledgeViewModel From(VectorCollectionData data) }; } - public static KnowledgeKnowledgeViewModel From(KnowledgeSearchResult result) + public static KnowledgeKnowledgeViewModel From(KnowledgeExecuteResult result) { return new KnowledgeKnowledgeViewModel { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index 36e82ba16..d6be84b75 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -21,14 +21,14 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str if (knowledgeBase.Type == KnowledgeBaseType.Document) { var kg = GetKnowledgeService(knowledgeBase.Type); - var options = new KnowledgeSearchOptions + var options = new KnowledgeExecuteOptions { Fields = null, Limit = 5, Confidence = 0.25f, WithVector = true }; - var result = await kg.Search(text, knowledgeBase.Name, options); + var result = await kg.ExecuteQuery(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) .Select(x => x.Data["text"].ToString()) .Where(x => x != null)!); @@ -36,14 +36,14 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str else if (knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { var kg = GetKnowledgeService(knowledgeBase.Type); - var options = new KnowledgeSearchOptions + var options = new KnowledgeExecuteOptions { Fields = null, Limit = 5, Confidence = 0.5f, WithVector = true }; - var result = await kg.Search(text, knowledgeBase.Name, options); + var result = await kg.ExecuteQuery(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); @@ -68,14 +68,14 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) || knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { var kgSearcher = GetKnowledgeService(knowledgeBase.Type); - var options = new KnowledgeSearchOptions + var options = new KnowledgeExecuteOptions { Fields = null, Limit = 5, Confidence = 0.5f, WithVector = true }; - var result = await kgSearcher.Search(text, knowledgeBase.Name, options); + var result = await kgSearcher.ExecuteQuery(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs index 4cd2936f0..3251b1083 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs @@ -93,12 +93,12 @@ public virtual async Task> GetCollectionDat }); } - public virtual async Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + public virtual async Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) { var vectorDb = GetVectorDb(filter.DbProvider); if (vectorDb == null) { - return new StringIdPagedItems(); + return new StringIdPagedItems(); } var vectorFilter = new VectorFilter @@ -112,16 +112,16 @@ public virtual async Task> GetPagedCol }; var pagedResult = await vectorDb.GetPagedCollectionData(collectionName, vectorFilter); - return new StringIdPagedItems + return new StringIdPagedItems { Count = pagedResult.Count, - Items = pagedResult.Items.Select(x => KnowledgeSearchResult.CopyFrom(new KnowledgeCollectionData + Items = pagedResult.Items.Select(x => new KnowledgeCollectionData { Id = x.Id, Payload = x.Payload, Score = x.Score, Vector = x.Vector - })), + }), NextId = pagedResult.NextId, }; } @@ -197,7 +197,7 @@ public virtual async Task UpsertCollectionData(string collectionName, Know } #endregion - public virtual async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + public virtual async Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) { var vectorDb = GetVectorDb(options.DbProvider); if (vectorDb == null) @@ -219,7 +219,7 @@ public virtual async Task> Search(string quer }; var found = await vectorDb.Search(collectionName, vector, searchOptions); - return found.Select(x => KnowledgeSearchResult.CopyFrom(new KnowledgeCollectionData + return found.Select(x => KnowledgeExecuteResult.CopyFrom(new KnowledgeCollectionData { Id = x.Id, Payload = x.Payload, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index 3a1d9732c..02209531f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -121,6 +121,72 @@ public async Task UploadFilesToKnowledge( }; } + public async Task ImportFileContentToKnowledge( + string collectionName, + string fileName, + string fileSource, + IEnumerable contents, + ImportKnowledgeFileOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(fileName) + || contents.IsNullOrEmpty()) + { + return false; + } + + try + { + var knowledgebaseProvider = options?.DbProvider ?? _settings.VectorDb.Provider; + var exist = await ExistCollection(collectionName, knowledgebaseProvider); + if (!exist) return false; + + var db = _services.GetRequiredService(); + var userId = await GetUserId(); + + var fileId = Guid.NewGuid(); + var contentType = FileUtility.GetFileContentType(fileName); + + var innerPayload = new Dictionary(options?.Payload ?? []); + innerPayload[KnowledgePayloadName.DataSource] = (VectorPayloadValue)VectorDataSource.File; + innerPayload[KnowledgePayloadName.FileId] = (VectorPayloadValue)fileId.ToString(); + innerPayload[KnowledgePayloadName.FileName] = (VectorPayloadValue)fileName; + innerPayload[KnowledgePayloadName.FileSource] = (VectorPayloadValue)fileSource; + + if (!string.IsNullOrWhiteSpace(options?.FileRefData?.Url)) + { + innerPayload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)options.FileRefData.Url; + } + + var kgFile = new FileKnowledgeWrapper + { + FileId = fileId, + FileSource = fileSource, + FileData = new() + { + FileName = fileName, + ContentType = contentType, + FileBinaryData = BinaryData.Empty + }, + FileKnowledges = new List + { + new() + { + Contents = contents, + Payload = innerPayload + } + } + }; + await HandleKnowledgeFiles(collectionName, knowledgebaseProvider, [kgFile], saveFile: false); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})"); + return false; + } + } + public async Task DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName)) @@ -360,7 +426,7 @@ private async Task> GetFileKnowledge(FileBinaryD return response?.Success == true ? response.Knowledges ?? [] : []; } - private bool SaveDocument(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData binary) + private bool SaveFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData binary) { var fileStorage = _services.GetRequiredService(); var saved = fileStorage.SaveKnowledgeBaseFile(collectionName, knowledgebaseProvider, fileId, fileName, binary); @@ -427,7 +493,7 @@ private async Task HandleKnowledgeFiles( // Save document if (saveFile) { - var saved = SaveDocument(collectionName, knowledgebaseProvider, item.FileId, file.FileName, file.FileBinaryData); + var saved = SaveFile(collectionName, knowledgebaseProvider, item.FileId, file.FileName, file.FileBinaryData); if (!saved) { _logger.LogWarning($"Failed to save knowledge file: {file.FileName} to collection {collectionName}."); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs index 23e0ee735..9ec66dba6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs @@ -17,9 +17,9 @@ public GraphKnowledgeOrchestrator( public string KnowledgeType => KnowledgeBaseType.SemanticGraph; - public async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + public async Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) { - var results = new List(); + var results = new List(); try { @@ -39,7 +39,7 @@ public async Task> Search(string query, strin var graphResult = await graphDb.ExecuteQueryAsync(query, graphOptions); - results = graphResult?.Values?.Select(value => new KnowledgeSearchResult + results = graphResult?.Values?.Select(value => new KnowledgeExecuteResult { Id = Guid.NewGuid().ToString(), Payload = value.ToDictionary(kvp => kvp.Key, kvp => new VectorPayloadValue(kvp.Value)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 31b45a476..8ddf45d84 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -107,7 +107,7 @@ public async Task UploadDocumentsToKnowledge( public async Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, - IEnumerable contents, DocMetaRefData? refData = null, Dictionary? payload = null) + IEnumerable contents, ImportKnowledgeFileOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileName) @@ -123,19 +123,19 @@ public async Task ImportDocumentContentToKnowledge(string collectionName, var db = _services.GetRequiredService(); var userId = await GetUserId(); - var vectorStoreProvider = _settings.VectorDb.Provider; + var vectorStoreProvider = options?.DbProvider ?? _settings.VectorDb.Provider; var fileId = Guid.NewGuid(); var contentType = FileUtility.GetFileContentType(fileName); - var innerPayload = new Dictionary(payload ?? []); + var innerPayload = new Dictionary(options?.Payload ?? []); innerPayload[KnowledgePayloadName.DataSource] = (VectorPayloadValue)VectorDataSource.File; innerPayload[KnowledgePayloadName.FileId] = (VectorPayloadValue)fileId.ToString(); innerPayload[KnowledgePayloadName.FileName] = (VectorPayloadValue)fileName; innerPayload[KnowledgePayloadName.FileSource] = (VectorPayloadValue)fileSource; - if (!string.IsNullOrWhiteSpace(refData?.Url)) + if (!string.IsNullOrWhiteSpace(options?.FileRefData?.Url)) { - innerPayload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)refData.Url; + innerPayload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)options.FileRefData.Url; } var kgFile = new FileKnowledgeWrapper diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs index c5a440335..b86db3d9b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs @@ -18,9 +18,9 @@ public TaxonomyKnowledgeOrchestrator( public string KnowledgeType => KnowledgeBaseType.Taxonomy; - public async Task> Search(string query, string collectionName, KnowledgeSearchOptions options) + public async Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) { - var results = new List(); + var results = new List(); try { @@ -82,7 +82,7 @@ public async Task> Search(string query, strin } } - results.Add(new KnowledgeSearchResult + results.Add(new KnowledgeExecuteResult { Id = Guid.NewGuid().ToString(), Payload = payload, From 3fed7bf50828b1d941ee0c46d18e448a0148ca7d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 1 May 2026 17:01:06 -0500 Subject: [PATCH 21/23] clean code --- .../KnowledgeBase/KnowledgeBaseController.cs | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 060201143..2a5038d19 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Graph; -using BotSharp.Abstraction.Graph.Options; using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Repositories; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -11,14 +9,11 @@ namespace BotSharp.OpenAPI.Controllers; [ApiController] public partial class KnowledgeBaseController : ControllerBase { - private readonly IGraphKnowledgeService _graphKnowledgeService; private readonly IServiceProvider _services; public KnowledgeBaseController( - IGraphKnowledgeService graphKnowledgeService, IServiceProvider services) { - _graphKnowledgeService = graphKnowledgeService; _services = services; } @@ -374,27 +369,6 @@ public async Task DeleteCollectionSnapshots([FromRoute] string collection, #endregion - #region Graph - [HttpPost("/knowledge/graph/search")] - public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) - { - var options = new GraphQueryOptions - { - Provider = request.Provider, - GraphId = request.GraphId, - Arguments = request.Arguments, - Method = request.Method - }; - - var result = await _graphKnowledgeService.ExecuteQueryAsync(request.Query, options); - return new GraphKnowledgeViewModel - { - Result = result.Result - }; - } - #endregion - - #region Common [HttpPost("/knowledge/refresh-configs")] public async Task RefreshCollectionConfigs([FromBody] KnowledgeCollectionConfigsRequest request) From dba3f90a3e761cfef305613b21bbf89c7bd2753c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 1 May 2026 19:26:08 -0500 Subject: [PATCH 22/23] clean code --- .../Knowledges/IKnowledgeOrchestrator.cs | 61 --- .../Knowledges/IKnowledgeService.cs | 119 ++--- .../VectorStorage/Models/VectorCreateModel.cs | 9 - .../VectorStorage/Models/VectorUpdateModel.cs | 6 - ...exOptions.cs => CollectionIndexOptions.cs} | 0 .../KnowledgeBase/KnowledgeBaseController.cs | 52 +- ...eRequest.cs => KnowledgeExecuteRequest.cs} | 2 +- .../Request/SearchGraphKnowledgeRequest.cs | 25 - .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../Hooks/KnowledgeHook.cs | 4 +- .../KnowledgeBasePlugin.cs | 9 +- ...n.cs => VectorKnowledgeBase.Collection.cs} | 2 +- ...se.Data.cs => VectorKnowledgeBase.Data.cs} | 2 +- ....Index.cs => VectorKnowledgeBase.Index.cs} | 2 +- ...hot.cs => VectorKnowledgeBase.Snapshot.cs} | 2 +- ...estratorBase.cs => VectorKnowledgeBase.cs} | 4 +- ...chestrator.cs => DocumentKnowledgeBase.cs} | 6 +- .../File/KnowledgeFileOrchestrator.cs | 2 +- ...rator.cs => SementicGraphKnowledgeBase.cs} | 8 +- .../Services/KnowledgeService.Document.cs | 488 ------------------ .../Services/KnowledgeService.Index.cs | 77 --- .../Services/KnowledgeService.Snapshot.cs | 64 --- .../Services/KnowledgeService.Vector.cs | 364 ------------- .../Services/KnowledgeService.cs | 39 -- ...ator.cs => QuestionAnswerKnowledgeBase.cs} | 6 +- ...chestrator.cs => TaxonomyKnowledgeBase.cs} | 8 +- .../Services/DbKnowledgeService.cs | 8 +- 27 files changed, 109 insertions(+), 1262 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs rename src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/{VectorCollectionIndexOptions.cs => CollectionIndexOptions.cs} (100%) rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/{SearchVectorKnowledgeRequest.cs => KnowledgeExecuteRequest.cs} (94%) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/{VectorOrchestratorBase.Collection.cs => VectorKnowledgeBase.Collection.cs} (98%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/{VectorOrchestratorBase.Data.cs => VectorKnowledgeBase.Data.cs} (99%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/{VectorOrchestratorBase.Index.cs => VectorKnowledgeBase.Index.cs} (97%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/{VectorOrchestratorBase.Snapshot.cs => VectorKnowledgeBase.Snapshot.cs} (98%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/{VectorOrchestratorBase.cs => VectorKnowledgeBase.cs} (91%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/{DocumentKnowledgeOrchestrator.cs => DocumentKnowledgeBase.cs} (56%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/{GraphKnowledgeOrchestrator.cs => SementicGraphKnowledgeBase.cs} (89%) delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/{QuestionAnswerKnowledgeOrchestrator.cs => QuestionAnswerKnowledgeBase.cs} (55%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/{TaxonomyKnowledgeOrchestrator.cs => TaxonomyKnowledgeBase.cs} (93%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs deleted file mode 100644 index 4b9530e11..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeOrchestrator.cs +++ /dev/null @@ -1,61 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.Knowledges.Options; - -namespace BotSharp.Abstraction.Knowledges; - -public interface IKnowledgeOrchestrator -{ - string KnowledgeType { get; } - - #region Collection - Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) - => Task.FromResult(false); - Task CreateCollection(string collectionName, CollectionCreateOptions options) - => Task.FromResult(false); - Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) - => Task.FromResult(false); - Task> GetCollections(KnowledgeCollectionOptions options) - => Task.FromResult(Enumerable.Empty()); - Task GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options) - => Task.FromResult(null); - #endregion - - #region Data - Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) - => Task.FromResult(Enumerable.Empty()); - Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) - => Task.FromResult(new StringIdPagedItems()); - Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) - => Task.FromResult(Enumerable.Empty()); - Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) - => Task.FromResult(false); - Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) - => Task.FromResult(false); - Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) - => Task.FromResult(false); - Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) - => Task.FromResult(false); - Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) - => Task.FromResult(false); - #endregion - - #region Index - Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) - => Task.FromResult(new SuccessFailResponse()); - Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) - => Task.FromResult(new SuccessFailResponse()); - #endregion - - #region Snapshot - Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) - => Task.FromResult(Enumerable.Empty()); - Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) - => Task.FromResult(null); - Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) - => Task.FromResult(new BinaryData(Array.Empty())); - Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) - => Task.FromResult(false); - Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) - => Task.FromResult(false); - #endregion -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 1c5425dc8..e3a6fe3ee 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -1,80 +1,61 @@ -using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Graph.Options; -using BotSharp.Abstraction.Knowledges.Filters; +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Options; -using BotSharp.Abstraction.Knowledges.Responses; -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Abstraction.VectorStorage.Options; namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - #region Vector - //Task ExistVectorCollection(string collectionName); - //Task CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options); - //Task DeleteVectorCollection(string collectionName); - //Task> GetVectorCollections(string? type = null); - //Task GetVectorCollectionDetails(string collectionName); - //Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); - //Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); - //Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null); - //Task DeleteVectorCollectionData(string collectionName, string id); - //Task DeleteVectorCollectionAllData(string collectionName); - //Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); - //Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); - //Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update); - #endregion + string KnowledgeType { get; } - //#region Document - ///// - ///// Save documents and their contents to knowledgebase - ///// - ///// - ///// - ///// - ///// - //Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeFileHandleOptions? options = null); - ///// - ///// Save document content to knowledgebase without saving the document - ///// - ///// - ///// - ///// - ///// - ///// - ///// - //Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, - // ImportKnowledgeFileOptions? options = null); - ///// - ///// Delete one document and its related knowledge in the collection - ///// - ///// - ///// - ///// - //Task DeleteKnowledgeDocument(string collectionName, Guid fileId); - ///// - ///// Delete all documents and their related knowledge in the collection - ///// - ///// - ///// - ///// - //Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - //Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - //Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); - //#endregion + #region Collection + Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(false); + Task CreateCollection(string collectionName, CollectionCreateOptions options) + => Task.FromResult(false); + Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(false); + Task> GetCollections(KnowledgeCollectionOptions options) + => Task.FromResult(Enumerable.Empty()); + Task GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(null); + #endregion - //#region Snapshot - //Task> GetVectorCollectionSnapshots(string collectionName); - //Task CreateVectorCollectionSnapshot(string collectionName); - //Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName); - //Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData); - //Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); - //#endregion + #region Data + Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) + => Task.FromResult(Enumerable.Empty()); + Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + => Task.FromResult(new StringIdPagedItems()); + Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) + => Task.FromResult(Enumerable.Empty()); + Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) + => Task.FromResult(false); + Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) + => Task.FromResult(false); + Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) + => Task.FromResult(false); + Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) + => Task.FromResult(false); + Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) + => Task.FromResult(false); + #endregion - //#region Index - //Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); - //Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); - //#endregion + #region Index + Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) + => Task.FromResult(new SuccessFailResponse()); + Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) + => Task.FromResult(new SuccessFailResponse()); + #endregion + #region Snapshot + Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(Enumerable.Empty()); + Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(null); + Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(new BinaryData(Array.Empty())); + Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(false); + Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(false); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs deleted file mode 100644 index 09d472da2..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs +++ /dev/null @@ -1,9 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Enums; - -namespace BotSharp.Abstraction.VectorStorage.Models; - -public class VectorCreateModel -{ - public string Text { get; set; } - public Dictionary? Payload { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs deleted file mode 100644 index 45600219e..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Abstraction.VectorStorage.Models; - -public class VectorUpdateModel : VectorCreateModel -{ - public string Id { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/CollectionIndexOptions.cs similarity index 100% rename from src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/CollectionIndexOptions.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 2a5038d19..69752d227 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -21,7 +21,7 @@ public KnowledgeBaseController( [HttpGet("knowledge/collection/{collection}/exist")] public async Task ExistCollection([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -39,7 +39,7 @@ public async Task> GetCollection if (!string.IsNullOrWhiteSpace(knowledgeType)) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -52,7 +52,7 @@ public async Task> GetCollection } else { - var kgs = _services.GetServices(); + var kgs = _services.GetServices(); foreach (var kg in kgs) { var collections = await kg.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); @@ -67,7 +67,7 @@ public async Task> GetCollection [HttpGet("knowledge/collection/{collection}/details")] public async Task GetCollectionDetails([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -81,7 +81,7 @@ public async Task> GetCollection [HttpPost("knowledge/collection")] public async Task CreateCollection([FromBody] CreateCollectionRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -102,7 +102,7 @@ public async Task CreateCollection([FromBody] CreateCollectionRequest requ [HttpDelete("knowledge/collection/{collection}")] public async Task DeleteCollection([FromRoute] string collection, [FromBody] DeleteCollectionRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -113,10 +113,10 @@ public async Task DeleteCollection([FromRoute] string collection, [FromBod return await kg.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } - [HttpPost("/knowledge/collection/{collection}/search")] - public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request) + [HttpPost("/knowledge/collection/{collection}/query")] + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] KnowledgeExecuteRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -132,15 +132,15 @@ public async Task> SearchKnowledge([Fro [HttpPost("/knowledge/collection/{collection}/data/page")] public async Task> GetPagedCollectionData([FromRoute] string collection, [FromBody] GetPagedCollectionDataRequest request) { - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - if (orchestrator == null) + if (kg == null) { return new StringIdPagedItems(); } - var data = await orchestrator.GetPagedCollectionData(collection, request); + var data = await kg.GetPagedCollectionData(collection, request); var items = data.Items?.Select(x => KnowledgeKnowledgeViewModel.From(x))?.ToList() ?? []; return new StringIdPagedItems @@ -154,7 +154,7 @@ public async Task> GetPagedColle [HttpPost("/knowledge/collection/{collection}/data")] public async Task CreateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataCreateRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -176,7 +176,7 @@ public async Task CreateCollectionData([FromRoute] string collection, [Fro [HttpGet("/knowledge/collection/{collection}/data")] public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] QueryCollectionDataRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -198,7 +198,7 @@ public async Task> GetCollectionData([F [HttpPut("/knowledge/collection/{collection}/data")] public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataUpdateRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -221,7 +221,7 @@ public async Task UpdateCollectionData([FromRoute] string collection, [Fro [HttpDelete("/knowledge/collection/{collection}/data/{id}")] public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id, [FromBody] DeleteCollectionDataRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -235,7 +235,7 @@ public async Task DeleteCollectionData([FromRoute] string collection, [Fro [HttpDelete("/knowledge/collection/{collection}/data")] public async Task DeleteCollectionAllData([FromRoute] string collection, [FromBody] DeleteCollectionDataRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -252,7 +252,7 @@ public async Task DeleteCollectionAllData([FromRoute] string collection, [ [HttpPost("/knowledge/collection/{collection}/indexes")] public async Task> CreateCollectionIndexes([FromRoute] string collection, [FromBody] CreateCollectionIndexRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -271,7 +271,7 @@ public async Task> CreateCollectionIndexes([FromRout [HttpDelete("/knowledge/collection/{collection}/indexes")] public async Task> DeleteCollectionIndexes([FromRoute] string collection, [FromBody] DeleteCollectionIndexRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -293,7 +293,7 @@ public async Task> DeleteCollectionIndexes([FromRout [HttpGet("/knowledge/collection/{collection}/snapshots")] public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -308,7 +308,7 @@ public async Task> GetCollecti [HttpPost("/knowledge/collection/{collection}/snapshot")] public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromBody] CollectionSnapshotRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -323,7 +323,7 @@ public async Task> GetCollecti [HttpGet("/knowledge/collection/{collection}/snapshot")] public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -338,7 +338,7 @@ public async Task GetCollectionSnapshot([FromRoute] string collec [HttpPost("/knowledge/collection/{collection}/snapshot/recover")] public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromForm] string knowledgeType, [FromForm] string? dbProvider = null) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); if (kg == null) @@ -355,7 +355,7 @@ public async Task RecoverCollectionFromSnapshot([FromRoute] string collect [HttpDelete("/knowledge/collection/{collection}/snapshot")] public async Task DeleteCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteCollectionSnapshotRequest request) { - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); if (kg == null) @@ -392,7 +392,7 @@ private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) return File(stream, "application/octet-stream", Path.GetFileName(fileName)); } - private KnowledgeExecuteOptions BuildSearchOptions(IKnowledgeOrchestrator kg, SearchKnowledgeRequest? request) + private KnowledgeExecuteOptions BuildSearchOptions(IKnowledgeService kg, KnowledgeExecuteRequest? request) { var searchParam = request?.SearchParam?.ToDictionary(x => x.Key, x => x.Value?.ConvertToString()); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeExecuteRequest.cs similarity index 94% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeExecuteRequest.cs index 4790b5327..26aff6651 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeExecuteRequest.cs @@ -3,7 +3,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class SearchKnowledgeRequest : KnowledgeBaseRequestBase +public class KnowledgeExecuteRequest : KnowledgeBaseRequestBase { public string Text { get; set; } = string.Empty; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs deleted file mode 100644 index de3aa2b68..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class SearchGraphKnowledgeRequest -{ - [JsonPropertyName("query")] - public string Query { get; set; } = string.Empty; - - [JsonPropertyName("provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Provider { get; set; } - - [JsonPropertyName("graph_id")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? GraphId { get; set; } - - [JsonPropertyName("arguments")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public Dictionary? Arguments { get; set; } - - [JsonPropertyName("method")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Method { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index d45925b23..be233e2f6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -23,7 +23,7 @@ public async Task Execute(RoleDialogModel message) ? args.RefinedCollection : _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var kg = _services.GetServices() + var kg = _services.GetServices() .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); if (kg == null) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index d6be84b75..645c92373 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -98,9 +98,9 @@ private async Task> GetKnowledgeBaseNameByAgentIdAsync( return agent.KnowledgeBases; } - private IKnowledgeOrchestrator GetKnowledgeService(string? type = null) + private IKnowledgeService GetKnowledgeService(string? type = null) { - var kgs = _services.GetServices(); + var kgs = _services.GetServices(); if (!string.IsNullOrWhiteSpace(type)) { return kgs.First(x => x.KnowledgeType == type); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index d3000c807..3de21e658 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -22,12 +22,11 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddSingleton(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Collection.cs similarity index 98% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Collection.cs index 92857f208..7eba4f12a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Collection.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Collection.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public abstract partial class VectorOrchestratorBase +public abstract partial class VectorKnowledgeBase { #region Collection public virtual async Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Data.cs similarity index 99% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Data.cs index 3251b1083..05a507884 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Data.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Data.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public abstract partial class VectorOrchestratorBase +public abstract partial class VectorKnowledgeBase { #region Collection data public virtual async Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Index.cs similarity index 97% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Index.cs index 43a5c3465..6099cf6b3 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Index.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Index.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public abstract partial class VectorOrchestratorBase +public abstract partial class VectorKnowledgeBase { #region Index public virtual async Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Snapshot.cs similarity index 98% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Snapshot.cs index e7eb717c8..74c4feeb0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.Snapshot.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Snapshot.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public abstract partial class VectorOrchestratorBase +public abstract partial class VectorKnowledgeBase { #region Snapshot public virtual async Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.cs index 482112980..d902e0786 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorOrchestratorBase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public abstract partial class VectorOrchestratorBase : IKnowledgeOrchestrator +public abstract partial class VectorKnowledgeBase : IKnowledgeService { protected readonly IServiceProvider _services; protected readonly ILogger _logger; @@ -8,7 +8,7 @@ public abstract partial class VectorOrchestratorBase : IKnowledgeOrchestrator public abstract string KnowledgeType { get; } - protected VectorOrchestratorBase( + protected VectorKnowledgeBase( IServiceProvider services, ILogger logger, KnowledgeBaseSettings settings) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeBase.cs similarity index 56% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeBase.cs index e88196b6a..28498ca3e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeBase.cs @@ -1,12 +1,12 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public partial class DocumentKnowledgeOrchestrator : VectorOrchestratorBase, IKnowledgeOrchestrator +public partial class DocumentKnowledgeBase : VectorKnowledgeBase, IKnowledgeService { public override string KnowledgeType => KnowledgeBaseType.Document; - public DocumentKnowledgeOrchestrator( + public DocumentKnowledgeBase( IServiceProvider services, - ILogger logger, + ILogger logger, KnowledgeBaseSettings settings) : base(services, logger, settings) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index 02209531f..1094ec20c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -389,7 +389,7 @@ private async Task ExistCollection(string collectionName, string? dbProvid return !configs.IsNullOrEmpty(); } - + private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file) { if (file == null) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs index 9ec66dba6..3373dc44f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/GraphKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs @@ -2,14 +2,14 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public class GraphKnowledgeOrchestrator : IKnowledgeOrchestrator +public class SementicGraphKnowledgeBase : IKnowledgeService { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; - public GraphKnowledgeOrchestrator( + public SementicGraphKnowledgeBase( IServiceProvider services, - ILogger logger) + ILogger logger) { _services = services; _logger = logger; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs deleted file mode 100644 index 8ddf45d84..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ /dev/null @@ -1,488 +0,0 @@ -using System.Net.Http; - -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task UploadDocumentsToKnowledge( - string collectionName, - IEnumerable files, - KnowledgeFileHandleOptions? options = null) - { - var res = new UploadKnowledgeResponse - { - Success = [], - Failed = files?.Select(x => x.FileName) ?? [] - }; - - if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty()) - { - return res; - } - - var exist = await ExistVectorCollection(collectionName); - if (!exist) - { - return res; - } - - var fileStoreage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; - var knowledgeFiles = new List(); - var successFiles = new List(); - var failedFiles = new List(); - - foreach (var file in files!) - { - if (string.IsNullOrWhiteSpace(file.FileData) - && string.IsNullOrWhiteSpace(file.FileUrl)) - { - continue; - } - - try - { - // Get document info - var (contentType, binary) = await GetFileInfo(file); - var fileData = new FileBinaryDataModel - { - FileName = file.FileName, - ContentType = contentType, - FileBinaryData = binary - }; - var knowledges = await GetFileKnowledge(fileData, options); - if (knowledges.IsNullOrEmpty()) - { - failedFiles.Add(file.FileName); - continue; - } - - var fileId = Guid.NewGuid(); - var payload = new Dictionary() - { - { KnowledgePayloadName.DataSource, (VectorPayloadValue)VectorDataSource.File }, - { KnowledgePayloadName.FileId, (VectorPayloadValue)fileId.ToString() }, - { KnowledgePayloadName.FileName, (VectorPayloadValue)file.FileName }, - { KnowledgePayloadName.FileSource, (VectorPayloadValue)file.FileSource } - }; - - if (!string.IsNullOrWhiteSpace(file.FileUrl)) - { - payload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)file.FileUrl; - } - - foreach (var kg in knowledges) - { - var kgPayload = new Dictionary(kg.Payload ?? new Dictionary()); - foreach (var pair in payload) - { - kgPayload[pair.Key] = pair.Value; - } - kg.Payload = kgPayload; - } - - knowledgeFiles.Add(new() - { - FileId = fileId, - FileData = fileData, - FileSource = VectorDataSource.File, - FileKnowledges = knowledges - }); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when processing knowledge file ({file.FileName})."); - failedFiles.Add(file.FileName); - continue; - } - } - - var response = await HandleKnowledgeFiles(collectionName, vectorStoreProvider, knowledgeFiles, saveFile: true); - return new UploadKnowledgeResponse - { - Success = successFiles.Concat(response.Success).Distinct(), - Failed = failedFiles.Concat(response.Failed).Distinct() - }; - } - - - public async Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, - IEnumerable contents, ImportKnowledgeFileOptions? options = null) - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(fileName) - || contents.IsNullOrEmpty()) - { - return false; - } - - try - { - var exist = await ExistVectorCollection(collectionName); - if (!exist) return false; - - var db = _services.GetRequiredService(); - var userId = await GetUserId(); - var vectorStoreProvider = options?.DbProvider ?? _settings.VectorDb.Provider; - var fileId = Guid.NewGuid(); - var contentType = FileUtility.GetFileContentType(fileName); - - var innerPayload = new Dictionary(options?.Payload ?? []); - innerPayload[KnowledgePayloadName.DataSource] = (VectorPayloadValue)VectorDataSource.File; - innerPayload[KnowledgePayloadName.FileId] = (VectorPayloadValue)fileId.ToString(); - innerPayload[KnowledgePayloadName.FileName] = (VectorPayloadValue)fileName; - innerPayload[KnowledgePayloadName.FileSource] = (VectorPayloadValue)fileSource; - - if (!string.IsNullOrWhiteSpace(options?.FileRefData?.Url)) - { - innerPayload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)options.FileRefData.Url; - } - - var kgFile = new FileKnowledgeWrapper - { - FileId = fileId, - FileSource = fileSource, - FileData = new() - { - FileName = fileName, - ContentType = contentType, - FileBinaryData = BinaryData.Empty - }, - FileKnowledges = new List - { - new() - { - Contents = contents, - Payload = innerPayload - } - } - }; - await HandleKnowledgeFiles(collectionName, vectorStoreProvider, [kgFile], saveFile: false); - return true; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})"); - return false; - } - } - - - public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - try - { - var db = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var vectorDb = GetVectorDb(); - var vectorStoreProvider = _settings.VectorDb.Provider; - - // Get doc meta data - var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter - { - Size = 1, - FileIds = [ fileId ] - }); - - // Delete doc - fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId); - - var found = pageData?.Items?.FirstOrDefault(); - if (found != null && !found.VectorDataIds.IsNullOrEmpty()) - { - var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList(); - await vectorDb.DeleteCollectionData(collectionName, guids); - } - - await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId); - return true; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting knowledge document " + - $"(Collection: {collectionName}, File id: {fileId})"); - return false; - } - } - - public async Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) - { - if (string.IsNullOrWhiteSpace(collectionName)) return false; - - - var pageSize = filter.Size; - var innerFilter = new KnowledgeFileFilter - { - Page = 1, - Size = pageSize, - FileIds = filter.FileIds, - FileNames = filter.FileNames, - FileSources = filter.FileSources, - ContentTypes = filter.ContentTypes - }; - - var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); - - var total = pageData.Count; - if (total == 0) return false; - - var page = 1; - var totalPages = total % pageSize == 0 ? total / pageSize : total / pageSize + 1; - - while (page <= totalPages) - { - if (page > 1) - { - pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); - } - - var fileIds = pageData.Items.Select(x => x.FileId).ToList(); - foreach (var fileId in fileIds) - { - try - { - await DeleteKnowledgeDocument(collectionName, fileId); - } - catch - { - continue; - } - } - - page++; - } - - return true; - } - - - public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return new PagedItems(); - } - - var db = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; - - // Get doc meta data - var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter); - - var files = pagedData.Items?.Select(x => new KnowledgeFileModel - { - FileId = x.FileId, - FileName = x.FileName, - FileSource = x.FileSource, - FileExtension = Path.GetExtension(x.FileName), - ContentType = x.ContentType, - FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName), - RefData = x.RefData - })?.ToList() ?? new List(); - - return new PagedItems - { - Items = files, - Count = pagedData.Count - }; - } - - public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) - { - var db = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; - - // Get doc binary data - var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter - { - Size = 1, - FileIds = [ fileId ] - }); - - var metaData = pageData?.Items?.FirstOrDefault(); - if (metaData == null) - { - return new FileBinaryDataModel - { - FileName = "error.txt", - ContentType = "text/plain", - FileBinaryData = BinaryData.Empty - }; - }; - - var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); - return new FileBinaryDataModel - { - FileName = metaData.FileName, - ContentType = metaData.ContentType, - FileBinaryData = binaryData - }; - } - - - #region Private methods - /// - /// Get file content type and file bytes - /// - /// - /// - private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file) - { - if (file == null) - { - return (string.Empty, BinaryData.Empty); - } - - if (!string.IsNullOrWhiteSpace(file.FileUrl)) - { - var http = _services.GetRequiredService(); - var contentType = FileUtility.GetFileContentType(file.FileName); - using var client = http.CreateClient(); - var bytes = await client.GetByteArrayAsync(file.FileUrl); - return (contentType, BinaryData.FromBytes(bytes)); - } - else if (!string.IsNullOrWhiteSpace(file.FileData)) - { - var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - return (contentType, binary); - } - - return (string.Empty, BinaryData.Empty); - } - - #region Read doc content - private async Task> GetFileKnowledge(FileBinaryDataModel file, KnowledgeFileHandleOptions? options) - { - var processor = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(options?.Processor)); - if (processor == null) - { - return Enumerable.Empty(); - } - - var response = await processor.GetFileKnowledgeAsync(file, options: options); - return response?.Success == true ? response.Knowledges ?? [] : []; - } - #endregion - - - private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData binary) - { - var fileStoreage = _services.GetRequiredService(); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, binary); - return saved; - } - - private async Task> SaveToVectorDb(string collectionName, IEnumerable contents, Dictionary? payload = null) - { - if (contents.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var dataIds = new List(); - var vectorDb = GetVectorDb(); - var textEmbedding = await GetTextEmbedding(collectionName); - - for (int i = 0; i < contents.Count(); i++) - { - var content = contents.ElementAt(i); - - try - { - var vector = await textEmbedding.GetVectorAsync(content); - var dataId = Guid.NewGuid(); - var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload ?? []); - - if (!saved) - { - continue; - } - - dataIds.Add(dataId.ToString()); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when saving file knowledge to vector db collection {collectionName}. (Content: {content.SubstringMax(20)})"); - } - } - - return dataIds; - } - - private async Task HandleKnowledgeFiles( - string collectionName, - string vectorStore, - IEnumerable knowledgeFiles, - bool saveFile = false) - { - if (knowledgeFiles.IsNullOrEmpty()) - { - return new(); - } - - var successFiles = new List(); - var failedFiles = new List(); - var db = _services.GetRequiredService(); - - var userId = await GetUserId(); - foreach (var item in knowledgeFiles) - { - var file = item.FileData; - - // Save document - if (saveFile) - { - var saved = SaveDocument(collectionName, vectorStore, item.FileId, file.FileName, file.FileBinaryData); - if (!saved) - { - _logger.LogWarning($"Failed to save knowledge file: {file.FileName} to collection {collectionName}."); - failedFiles.Add(file.FileName); - continue; - } - } - - // Save to vector db - var dataIds = new List(); - foreach (var kg in item.FileKnowledges) - { - var ids = await SaveToVectorDb(collectionName, kg.Contents, kg.Payload?.ToDictionary()); - dataIds.AddRange(ids); - } - - if (!dataIds.IsNullOrEmpty()) - { - await db.SaveKnolwedgeBaseFileMeta(new KnowledgeFileMetaData - { - Collection = collectionName, - FileId = item.FileId, - FileName = file.FileName, - FileSource = item.FileSource ?? VectorDataSource.File, - ContentType = file.ContentType, - VectorStoreProvider = vectorStore, - VectorDataIds = dataIds, - CreateDate = DateTime.UtcNow, - CreateUserId = userId - }); - successFiles.Add(file.FileName); - } - else - { - failedFiles.Add(file.FileName); - } - } - - return new UploadKnowledgeResponse - { - Success = successFiles, - Failed = failedFiles - }; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs deleted file mode 100644 index d55198d9c..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs +++ /dev/null @@ -1,77 +0,0 @@ -using BotSharp.Abstraction.Models; -using BotSharp.Abstraction.VectorStorage.Options; - -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty()) - { - return new(); - } - - var response = new SuccessFailResponse(); - var innerOptions = options.DistinctBy(x => x.FieldName).ToList(); - var vectorDb = GetVectorDb(); - foreach (var option in innerOptions) - { - var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option); - var field = $"{option.FieldName} ({option.FieldSchemaType})"; - if (created) - { - response.Success.Add(field); - } - else - { - _logger.LogError($"Failed to create vector collection payload index ({collectionName} => {field})."); - response.Fail.Add(field); - } - } - return response; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when creating vector collection payload index ({collectionName})."); - return new(); - } - } - - public async Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty()) - { - return new(); - } - - var response = new SuccessFailResponse(); - var innerOptions = options.DistinctBy(x => x.FieldName).ToList(); - var vectorDb = GetVectorDb(); - foreach (var option in innerOptions) - { - var deleted = await vectorDb.DeleteCollectionPayloadIndex(collectionName, option); - var field = $"{option.FieldName}"; - if (deleted) - { - response.Success.Add(field); - } - else - { - _logger.LogError($"Failed to deleting vector collection payload index ({collectionName}-{field})."); - response.Fail.Add(field); - } - } - return response; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting vector collection payload index ({collectionName})."); - return new(); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs deleted file mode 100644 index 4b8393321..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs +++ /dev/null @@ -1,64 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> GetVectorCollectionSnapshots(string collectionName) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return Enumerable.Empty(); - } - - var db = GetVectorDb(); - var snapshots = await db.GetCollectionSnapshots(collectionName); - return snapshots; - } - - public async Task CreateVectorCollectionSnapshot(string collectionName) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return null; - } - - var db = GetVectorDb(); - var snapshot = await db.CreateCollectionShapshot(collectionName); - return snapshot; - } - - public async Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName) - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) - { - return BinaryData.Empty; - } - - var db = GetVectorDb(); - var snapshot = await db.DownloadCollectionSnapshot(collectionName, snapshotFileName); - return snapshot; - } - - public async Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var db = GetVectorDb(); - var done = await db.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData); - return done; - } - - public async Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName) - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) - { - return false; - } - - var db = GetVectorDb(); - var done = await db.DeleteCollectionShapshot(collectionName, snapshotName); - return done; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs deleted file mode 100644 index f26436340..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ /dev/null @@ -1,364 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Filters; -using BotSharp.Abstraction.VectorStorage.Options; - -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - #region Collection - public async Task ExistVectorCollection(string collectionName) - { - var db = _services.GetRequiredService(); - var vectorDb = GetVectorDb(); - - var exist = await vectorDb.DoesCollectionExist(collectionName); - if (exist) return true; - - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionNames = [collectionName], - VectorStorageProviders = [_settings.VectorDb.Provider] - }); - - return !configs.IsNullOrEmpty(); - } - - public async Task CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var db = _services.GetRequiredService(); - var created = await db.AddKnowledgeCollectionConfigs(new List - { - new VectorCollectionConfig - { - Name = collectionName, - Type = collectionType, - VectorStore = new VectorStoreConfig - { - Provider = _settings.VectorDb.Provider - }, - TextEmbedding = new KnowledgeEmbeddingConfig - { - Provider = options.Provider, - Model = options.Model, - Dimension = options.Dimension - } - } - }); - - if (created) - { - var vectorDb = GetVectorDb(); - created = await vectorDb.CreateCollection(collectionName, options); - } - - return created; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when creating a vector collection ({collectionName})."); - return false; - } - } - - public async Task> GetVectorCollections(string? type = null) - { - try - { - var db = _services.GetRequiredService(); - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionTypes = !string.IsNullOrEmpty(type) ? [type] : null, - VectorStorageProviders = [_settings.VectorDb.Provider] - }); - - var vectorDb = GetVectorDb(); - if (vectorDb == null) - { - return []; - } - var dbCollections = await vectorDb.GetCollections(); - return configs.Where(x => dbCollections.Contains(x.Name)); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when getting vector db collections."); - return []; - } - } - - public async Task GetVectorCollectionDetails(string collectionName) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) return null; - - var db = _services.GetRequiredService(); - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionNames = [collectionName] - }); - - var vectorDb = GetVectorDb(); - var details = await vectorDb.GetCollectionDetails(collectionName); - if (details != null) - { - details.BasicInfo = configs.FirstOrDefault(); - } - return details; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when getting vector db collection details."); - return null; - } - } - - public async Task DeleteVectorCollection(string collectionName) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var vectorDb = GetVectorDb(); - var deleted = await vectorDb.DeleteCollection(collectionName); - - if (deleted) - { - var db = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; - - await db.DeleteKnowledgeCollectionConfig(collectionName); - fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider); - await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider); - } - - return deleted; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting collection ({collectionName})."); - return false; - } - } - #endregion - - #region Collection data - public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) - { - return false; - } - - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(create.Text); - - var db = GetVectorDb(); - var guid = Guid.NewGuid(); - var payload = create.Payload ?? new(); - - if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) - { - payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); - } - - return await db.Upsert(collectionName, guid, vector, create.Text, payload); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when creating vector collection data."); - return false; - } - } - - public async Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(update.Text) - || !Guid.TryParse(update.Id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, [guid]); - if (found.IsNullOrEmpty()) - { - return false; - } - - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(update.Text); - var payload = update.Payload ?? new(); - - if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) - { - - payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); - } - - return await db.Upsert(collectionName, guid, vector, update.Text, payload); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when updating vector collection data."); - return false; - } - } - - public async Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(update.Text) - || !Guid.TryParse(update.Id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, [guid], options: new() { WithVector = true, WithPayload = true }); - if (!found.IsNullOrEmpty()) - { - if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text) - { - // Only update payload - return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload); - } - } - - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(update.Text); - var payload = update.Payload ?? new(); - - if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) - { - payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); - } - - return await db.Upsert(collectionName, guid, vector, update.Text, payload); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when updating vector collection data."); - return false; - } - } - - public async Task DeleteVectorCollectionData(string collectionName, string id) - { - try - { - if (!Guid.TryParse(id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - return await db.DeleteCollectionData(collectionName, [guid]); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName}-{id})."); - return false; - } - } - - - public async Task DeleteVectorCollectionAllData(string collectionName) - { - try - { - var db = GetVectorDb(); - return await db.DeleteCollectionAllData(collectionName); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName})."); - return false; - } - } - - public async Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter) - { - try - { - var db = GetVectorDb(); - var pagedResult = await db.GetPagedCollectionData(collectionName, filter); - return new StringIdPagedItems - { - Count = pagedResult.Count, - Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)), - NextId = pagedResult.NextId, - }; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when getting vector knowledge collection data ({collectionName})."); - return new StringIdPagedItems(); - } - } - - public async Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty()) - { - return []; - } - - var pointIds = ids.Select(x => new { Id = x, IsValid = Guid.TryParse(x, out var guid), ParseResult = guid }) - .Where(x => x.IsValid) - .Select(x => x.ParseResult) - .ToList(); - - var db = GetVectorDb(); - var points = await db.GetCollectionData(collectionName, pointIds, options); - return points; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when querying vector collection {collectionName} points."); - return []; - } - } - - public async Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options) - { - try - { - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(query); - - // Vector search - var db = GetVectorDb(); - var found = await db.Search(collectionName, vector, options); - - var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); - return results; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when searching vector knowledge ({collectionName})."); - return []; - } - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs deleted file mode 100644 index e6df204fd..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ /dev/null @@ -1,39 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService : IKnowledgeService -{ - private readonly IServiceProvider _services; - private readonly IUserIdentity _user; - private readonly KnowledgeBaseSettings _settings; - private readonly ILogger _logger; - - public KnowledgeService( - IServiceProvider services, - IUserIdentity user, - KnowledgeBaseSettings settings, - ILogger logger) - { - _services = services; - _user = user; - _settings = settings; - _logger = logger; - } - - private IVectorDb? GetVectorDb() - { - var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); - return db; - } - - private async Task GetTextEmbedding(string collectionName) - { - return await KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); - } - - private async Task GetUserId() - { - var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); - return user.Id; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeBase.cs similarity index 55% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeBase.cs index 6b9f4a7d0..584c508b5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeBase.cs @@ -1,12 +1,12 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public class QuestionAnswerKnowledgeOrchestrator : VectorOrchestratorBase, IKnowledgeOrchestrator +public class QuestionAnswerKnowledgeBase : VectorKnowledgeBase, IKnowledgeService { public override string KnowledgeType => KnowledgeBaseType.QuestionAnswer; - public QuestionAnswerKnowledgeOrchestrator( + public QuestionAnswerKnowledgeBase( IServiceProvider services, - ILogger logger, + ILogger logger, KnowledgeBaseSettings settings) : base(services, logger, settings) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs similarity index 93% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs index b86db3d9b..c165be3c7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeOrchestrator.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs @@ -3,14 +3,14 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public class TaxonomyKnowledgeOrchestrator : IKnowledgeOrchestrator +public class TaxonomyKnowledgeBase : IKnowledgeService { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; - public TaxonomyKnowledgeOrchestrator( + public TaxonomyKnowledgeBase( IServiceProvider services, - ILogger logger) + ILogger logger) { _services = services; _logger = logger; diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs index 384921ec6..aa5186f60 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -23,8 +23,8 @@ public async Task Import(ImportDbKnowledgeRequest request) { var sqlDriverSettings = _services.GetRequiredService(); var settingService = _services.GetRequiredService(); - var orchestrator = _services.GetServices() - .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); var provider = request.Provider ?? "openai"; var model = request.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o); @@ -69,9 +69,9 @@ public async Task Import(ImportDbKnowledgeRequest request) foreach (var item in knowledges) { - if (orchestrator == null) continue; + if (kg == null) continue; - await orchestrator.CreateCollectionData(collectionName, new KnowledgeCreateModel + await kg.CreateCollectionData(collectionName, new KnowledgeCreateModel { Text = item.Question, Payload = new Dictionary From 0333d0489b1e91e1c75bb03617dbf99f401d4ed9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 4 May 2026 12:11:54 -0500 Subject: [PATCH 23/23] clean graph kg service --- .../Graph/IGraphKnowledgeService.cs | 9 ----- ...ons.cs => GraphKnowledgeExecuteOptions.cs} | 2 +- .../Options/KnowledgeExecuteOptions.cs | 2 +- ....cs => TaxonomyKnowledgeExecuteOptions.cs} | 2 +- .../KnowledgeBase/KnowledgeBaseController.cs | 4 +- .../View/KnowledgeKnowledgeViewModel.cs | 25 ------------- .../Graph/GraphKnowledgeService.cs | 37 ------------------- .../KnowledgeBasePlugin.cs | 2 - .../Graph/SementicGraphKnowledgeBase.cs | 4 +- .../Taxonomy/TaxonomyKnowledgeBase.cs | 2 +- 10 files changed, 8 insertions(+), 81 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/{GraphKnowledgeSearchOptions.cs => GraphKnowledgeExecuteOptions.cs} (58%) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/{TaxonomyKnowledgeSearchOptions.cs => TaxonomyKnowledgeExecuteOptions.cs} (68%) delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs deleted file mode 100644 index 6f612cf1c..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs +++ /dev/null @@ -1,9 +0,0 @@ -using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Graph.Options; - -namespace BotSharp.Abstraction.Graph; - -public interface IGraphKnowledgeService -{ - Task ExecuteQueryAsync(string query, GraphQueryOptions? options = null); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeExecuteOptions.cs similarity index 58% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeExecuteOptions.cs index caa2d5037..24d1a6932 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeExecuteOptions.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Options; -public class GraphKnowledgeSearchOptions : KnowledgeExecuteOptions +public class GraphKnowledgeExecuteOptions : KnowledgeExecuteOptions { public string? GraphId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs index 90a98d7a0..540381749 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs @@ -8,7 +8,7 @@ public class KnowledgeExecuteOptions public IEnumerable? Fields { get; set; } public IEnumerable? FilterGroups { get; set; } public Dictionary? SearchParam { get; set; } - public Dictionary? SearchArguments { get; set; } + public Dictionary? SearchArguments { get; set; } public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeExecuteOptions.cs similarity index 68% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeExecuteOptions.cs index 6520e7ff5..ac04fdafe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeExecuteOptions.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Options; -public class TaxonomyKnowledgeSearchOptions : KnowledgeExecuteOptions +public class TaxonomyKnowledgeExecuteOptions : KnowledgeExecuteOptions { public IEnumerable? DataProviders { get; set; } public int? MaxNgram { get; set; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 69752d227..4d949ec10 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -398,7 +398,7 @@ private KnowledgeExecuteOptions BuildSearchOptions(IKnowledgeService kg, Knowled if (kg.KnowledgeType.IsEqualTo(KnowledgeBaseType.SemanticGraph)) { - return new GraphKnowledgeSearchOptions + return new GraphKnowledgeExecuteOptions { DbProvider = request?.DbProvider, SearchParam = searchParam, @@ -409,7 +409,7 @@ private KnowledgeExecuteOptions BuildSearchOptions(IKnowledgeService kg, Knowled if (kg.KnowledgeType.IsEqualTo(KnowledgeBaseType.Taxonomy)) { - return new TaxonomyKnowledgeSearchOptions + return new TaxonomyKnowledgeExecuteOptions { DbProvider = request?.DbProvider, Limit = request?.Limit ?? 5, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs index ccb1f77e9..6f1cf6a81 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; using System.Text.Json.Serialization; @@ -23,30 +22,6 @@ public class KnowledgeKnowledgeViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? VectorDimension { get; set; } - - public static KnowledgeKnowledgeViewModel From(VectorSearchResult result) - { - return new KnowledgeKnowledgeViewModel - { - Id = result.Id, - Data = result.Data, - Payload = result.Payload, - Score = result.Score, - VectorDimension = result.Vector?.Length - }; - } - - public static KnowledgeKnowledgeViewModel From(VectorCollectionData data) - { - return new KnowledgeKnowledgeViewModel - { - Id = data.Id, - Data = data.Data, - Payload = data.Payload, - VectorDimension = data.Vector?.Length - }; - } - public static KnowledgeKnowledgeViewModel From(KnowledgeExecuteResult result) { return new KnowledgeKnowledgeViewModel diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs deleted file mode 100644 index 45b439bea..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs +++ /dev/null @@ -1,37 +0,0 @@ -using BotSharp.Abstraction.Graph.Options; - -namespace BotSharp.Plugin.KnowledgeBase.Graph; - -public partial class GraphKnowledgeService : IGraphKnowledgeService -{ - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly KnowledgeBaseSettings _settings; - - public GraphKnowledgeService( - IServiceProvider services, - ILogger logger, - KnowledgeBaseSettings settings) - { - _services = services; - _logger = logger; - _settings = settings; - } - - public async Task ExecuteQueryAsync(string query, GraphQueryOptions? options = null) - { - var db = GetGraphDb(options?.Provider); - var result = await db.ExecuteQueryAsync(query, options); - return result; - } - - - #region Private methods - private IGraphDb GetGraphDb(string? provider = null) - { - var graphProvider = provider ?? _settings.GraphDb.Provider; - var db = _services.GetServices().FirstOrDefault(x => x.Provider == graphProvider); - return db; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 3de21e658..b7ec32131 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; -using BotSharp.Plugin.KnowledgeBase.Graph; using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.KnowledgeBase; @@ -29,7 +28,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs index 3373dc44f..a95119b48 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs @@ -30,10 +30,10 @@ public async Task> ExecuteQuery(string query return results; } - var graphSearchOptions = options as GraphKnowledgeSearchOptions; + var graphExecuteOptions = options as GraphKnowledgeExecuteOptions; var graphOptions = new GraphQueryExecuteOptions { - GraphId = graphSearchOptions?.GraphId, + GraphId = graphExecuteOptions?.GraphId, Arguments = options?.SearchArguments }; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs index c165be3c7..fc37a4f81 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs @@ -33,7 +33,7 @@ public async Task> ExecuteQuery(string query return results; } - var taxonomyOptions = options as TaxonomyKnowledgeSearchOptions; + var taxonomyOptions = options as TaxonomyKnowledgeExecuteOptions; var analysisOptions = new EntityAnalysisOptions { DataProviders = taxonomyOptions?.DataProviders,