diff --git a/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs index b0e08dc63..53f3e4b23 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs @@ -7,7 +7,7 @@ public class EntityAnalysisOptions /// [JsonPropertyName("data_providers")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public List? DataProviders { get; set; } + public IEnumerable? DataProviders { get; set; } /// /// Maximum n-gram size diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 72e8fddb4..7f785d188 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -62,17 +62,9 @@ public interface IFileStorageService #endregion #region Knowledge - bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData); - - /// - /// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection. - /// - /// - /// - /// - /// - bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null); - string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); - BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName); + bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData); + bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null); + string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName); + BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs deleted file mode 100644 index 6f612cf1c..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs +++ /dev/null @@ -1,9 +0,0 @@ -using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Graph.Options; - -namespace BotSharp.Abstraction.Graph; - -public interface IGraphKnowledgeService -{ - Task ExecuteQueryAsync(string query, GraphQueryOptions? options = null); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs similarity index 52% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs index b63488964..a425188ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs @@ -1,7 +1,9 @@ namespace BotSharp.Abstraction.Knowledges.Enums; -public static class KnowledgeCollectionType +public static class KnowledgeBaseType { public static string QuestionAnswer = "question-answer"; public static string Document = "document"; + public static string Taxonomy = "taxonomy"; + public static string SemanticGraph = "semantic-graph"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs index ec2504181..676a74d21 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs @@ -2,12 +2,10 @@ namespace BotSharp.Abstraction.Knowledges.Filters; public class KnowledgeFileFilter : Pagination { + public string? DbProvider { get; set; } public IEnumerable? FileIds { get; set; } - public IEnumerable? FileNames { get; set; } - public IEnumerable? ContentTypes { get; set; } - public IEnumerable? FileSources { get; set; } public KnowledgeFileFilter() diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs new file mode 100644 index 000000000..b6d552cac --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs @@ -0,0 +1,65 @@ +using BotSharp.Abstraction.Knowledges.Filters; +using BotSharp.Abstraction.Knowledges.Responses; + +namespace BotSharp.Abstraction.Knowledges; + +public interface IKnowledgeFileOrchestrator +{ + string Provider { get; } + + /// + /// Save files and their contents to knowledgebase + /// + /// + /// + /// + /// + Task UploadFilesToKnowledge(string collectionName, + IEnumerable files, KnowledgeFileHandleOptions? options = null); + + /// + /// Import file content to knowledgebase without saving the file + /// + /// + /// + /// + /// + /// + /// + Task ImportFileContentToKnowledge(string collectionName, string fileName, string fileSource, + IEnumerable contents, ImportKnowledgeFileOptions? options = null); + + /// + /// Delete one file and its related knowledge in the collection + /// + /// + /// + /// + /// + Task DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); + + /// + /// Delete all files and their related knowledge in the collection + /// + /// + /// + /// + Task DeleteKnowledgeFiles(string collectionName, KnowledgeFileFilter filter); + + /// + /// Get knowledge files by pagination + /// + /// + /// + /// + Task> GetPagedKnowledgeFiles(string collectionName, KnowledgeFileFilter filter); + + /// + /// Get knowledge file binary data + /// + /// + /// + /// + /// + Task GetKnowledgeFileBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 4af6d7e6c..e3a6fe3ee 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -1,83 +1,61 @@ -using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Graph.Options; -using BotSharp.Abstraction.Knowledges.Filters; +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Options; -using BotSharp.Abstraction.Knowledges.Responses; -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Abstraction.VectorStorage.Options; namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - #region Vector - Task ExistVectorCollection(string collectionName); - Task CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options); - Task DeleteVectorCollection(string collectionName); - Task> GetVectorCollections(string? type = null); - Task GetVectorCollectionDetails(string collectionName); - Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); - Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter); - Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null); - Task DeleteVectorCollectionData(string collectionName, string id); - Task DeleteVectorCollectionAllData(string collectionName); - Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); - Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); - Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update); - #endregion + string KnowledgeType { get; } - #region Document - /// - /// Save documents and their contents to knowledgebase - /// - /// - /// - /// - /// - Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files, KnowledgeDocOptions? options = null); - /// - /// Save document content to knowledgebase without saving the document - /// - /// - /// - /// - /// - /// - /// - Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, - DocMetaRefData? refData = null, Dictionary? payload = null); - /// - /// Delete one document and its related knowledge in the collection - /// - /// - /// - /// - Task DeleteKnowledgeDocument(string collectionName, Guid fileId); - /// - /// Delete all documents and their related knowledge in the collection - /// - /// - /// - /// - Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); - Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); + #region Collection + Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(false); + Task CreateCollection(string collectionName, CollectionCreateOptions options) + => Task.FromResult(false); + Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(false); + Task> GetCollections(KnowledgeCollectionOptions options) + => Task.FromResult(Enumerable.Empty()); + Task GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options) + => Task.FromResult(null); #endregion - #region Snapshot - Task> GetVectorCollectionSnapshots(string collectionName); - Task CreateVectorCollectionSnapshot(string collectionName); - Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName); - Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData); - Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); + #region Data + Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) + => Task.FromResult(Enumerable.Empty()); + Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + => Task.FromResult(new StringIdPagedItems()); + Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) + => Task.FromResult(Enumerable.Empty()); + Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) + => Task.FromResult(false); + Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) + => Task.FromResult(false); + Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) + => Task.FromResult(false); + Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) + => Task.FromResult(false); + Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) + => Task.FromResult(false); #endregion #region Index - Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); - Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) + => Task.FromResult(new SuccessFailResponse()); + Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) + => Task.FromResult(new SuccessFailResponse()); #endregion - #region Common - Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs); + #region Snapshot + Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(Enumerable.Empty()); + Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(null); + Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(new BinaryData(Array.Empty())); + Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(false); + Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) + => Task.FromResult(false); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs new file mode 100644 index 000000000..633e1bc3d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs @@ -0,0 +1,7 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionConfig : VectorCollectionConfig +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs new file mode 100644 index 000000000..59ff18fac --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -0,0 +1,14 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionData +{ + public string Id { get; set; } + public Dictionary Payload { get; set; } = new(); + public Dictionary Data => Payload?.ToDictionary(x => x.Key, x => x.Value.DataValue) ?? []; + public double? Score { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs new file mode 100644 index 000000000..7adf3b25c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs @@ -0,0 +1,12 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionDetails +{ + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("payload_schema")] + public List PayloadSchema { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs similarity index 53% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs index 95b8a3251..ba0b340b8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs @@ -1,30 +1,22 @@ using BotSharp.Abstraction.VectorStorage.Models; -using System.Text.Json.Serialization; -namespace BotSharp.OpenAPI.ViewModels.Knowledges; +namespace BotSharp.Abstraction.Knowledges.Models; -public class VectorCollectionSnapshotViewModel +public class KnowledgeCollectionSnapshot { - [JsonPropertyName("name")] public string Name { get; set; } = default!; - - [JsonPropertyName("size")] public long Size { get; set; } - - [JsonPropertyName("created_time")] public DateTime CreatedTime { get; set; } - - [JsonPropertyName("check_sum")] public string? CheckSum { get; set; } - public static VectorCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model) + public static KnowledgeCollectionSnapshot? CopyFrom(VectorCollectionSnapshot? model) { if (model == null) { return null; } - return new VectorCollectionSnapshotViewModel + return new KnowledgeCollectionSnapshot { Name = model.Name, Size = model.Size, diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs new file mode 100644 index 000000000..5020d3605 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs @@ -0,0 +1,9 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCreateModel : KnowledgeOptionBase +{ + public string Text { get; set; } + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs new file mode 100644 index 000000000..b026fee1a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeExecuteResult : KnowledgeCollectionData +{ + public KnowledgeExecuteResult() + { + + } + + public static KnowledgeExecuteResult CopyFrom(KnowledgeCollectionData data) + { + return new KnowledgeExecuteResult + { + Id = data.Id, + Payload = data.Payload, + Score = data.Score, + Vector = data.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs similarity index 97% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs index a1139c901..810e0ef96 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeDocMetaData +public class KnowledgeFileMetaData { [JsonPropertyName("collection")] public string Collection { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs new file mode 100644 index 000000000..76bd019e6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFilter : StringIdPagination +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } + + public bool WithVector { get; set; } + + /// + /// Filter group: each item contains a logical operator and a list of key-value pairs + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? FilterGroups { get; set; } + + /// + /// Order by a specific field + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public VectorSort? OrderBy { get; set; } + + /// + /// Included payload fields + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Fields { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs new file mode 100644 index 000000000..214ab9c01 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeUpdateModel.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeUpdateModel : KnowledgeCreateModel +{ + public string Id { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/CollectionCreateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/CollectionCreateOptions.cs new file mode 100644 index 000000000..dec83e2ad --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/CollectionCreateOptions.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class CollectionCreateOptions : KnowledgeOptionBase +{ + public int EmbeddingDimension { get; set; } + public string LlmProvider { get; set; } = null!; + public string LlmModel { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeExecuteOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeExecuteOptions.cs new file mode 100644 index 000000000..24d1a6932 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/GraphKnowledgeExecuteOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class GraphKnowledgeExecuteOptions : KnowledgeExecuteOptions +{ + public string? GraphId { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs new file mode 100644 index 000000000..60610ae5f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/ImportKnowledgeFileOptions.cs @@ -0,0 +1,9 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Options; + +public class ImportKnowledgeFileOptions : KnowledgeOptionBase +{ + public DocMetaRefData? FileRefData { get; set; } + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs new file mode 100644 index 000000000..69bdb9260 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeCollectionOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeCollectionOptions : KnowledgeOptionBase +{ + public bool IncludeAllTypes { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs deleted file mode 100644 index 52c123426..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeDocOptions.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Options; - -public class KnowledgeDocOptions : FileKnowledgeHandleOptions -{ - public string? Processor { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs new file mode 100644 index 000000000..540381749 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeExecuteOptions.cs @@ -0,0 +1,16 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeExecuteOptions +{ + public string? DbProvider { get; set; } + public IEnumerable? Fields { get; set; } + public IEnumerable? FilterGroups { get; set; } + public Dictionary? SearchParam { get; set; } + public Dictionary? SearchArguments { get; set; } + + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs new file mode 100644 index 000000000..48c505344 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileHandleOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeFileHandleOptions : FileKnowledgeHandleOptions +{ + public string? DbProvider { get; set; } + public string? Processor { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs new file mode 100644 index 000000000..b66e37431 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeFileOptions.cs @@ -0,0 +1,5 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeFileOptions : KnowledgeOptionBase +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs new file mode 100644 index 000000000..cbf69b697 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeIndexOptions.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.VectorStorage.Options; + +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeIndexOptions : KnowledgeOptionBase +{ + public IEnumerable Items { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs new file mode 100644 index 000000000..29a5fb8a2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeOptionBase.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeOptionBase +{ + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs new file mode 100644 index 000000000..85faa2e62 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeQueryOptions.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeQueryOptions : KnowledgeOptionBase +{ + public bool WithPayload { get; set; } + public bool WithVector { get; set; } + + public static KnowledgeQueryOptions Default() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs new file mode 100644 index 000000000..9c02cb2fa --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/KnowledgeSnapshotOptions.cs @@ -0,0 +1,5 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class KnowledgeSnapshotOptions : KnowledgeOptionBase +{ +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeExecuteOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeExecuteOptions.cs new file mode 100644 index 000000000..ac04fdafe --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Options/TaxonomyKnowledgeExecuteOptions.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Options; + +public class TaxonomyKnowledgeExecuteOptions : KnowledgeExecuteOptions +{ + public IEnumerable? DataProviders { get; set; } + public int? MaxNgram { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs index 899f6e7ab..9759f08d1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -15,11 +15,6 @@ public class LlmConfigBase : LlmProviderModel public class LlmProviderModel { - [JsonPropertyName("provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Provider { get; set; } - - [JsonPropertyName("model")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Model { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 437e6cdd7..fef9c9684 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -239,21 +239,14 @@ Task DeleteKnowledgeCollectionConfig(string collectionName) => throw new NotImplementedException(); Task> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) => throw new NotImplementedException(); - Task GetKnowledgeCollectionConfig(string collectionName, string vectorStroageProvider) + Task GetKnowledgeCollectionConfig(string collectionName, string knowledgebaseProvider) => throw new NotImplementedException(); - Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + Task SaveKnolwedgeBaseFileMeta(KnowledgeFileMetaData metaData) => throw new NotImplementedException(); - /// - /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection. - /// - /// - /// - /// - /// - Task DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + Task DeleteKnolwedgeBaseFileMeta(string collectionName, string knowledgebaseProvider, Guid? fileId = null) => throw new NotImplementedException(); - Task> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + Task> GetKnowledgeBaseFileMeta(string collectionName, string knowledgebaseProvider, KnowledgeFileFilter filter) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index ca0cbe7f7..acd2f0bd7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -19,6 +19,7 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Crontab.Models; global using BotSharp.Abstraction.MCP.Models; global using BotSharp.Abstraction.Settings; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs index 8c977cfbd..00ed10fc0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs @@ -5,7 +5,7 @@ namespace BotSharp.Abstraction.VectorStorage.Extensions; public static class VectorStorageExtension { - public static string ToQuestionAnswer(this VectorSearchResult data) + public static string ToQuestionAnswer(this KnowledgeExecuteResult data) { if (data?.Data == null) return string.Empty; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs index 14d5126d4..10b27e203 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Filters/VectorCollectionConfigFilter.cs @@ -4,7 +4,7 @@ public class VectorCollectionConfigFilter { public IEnumerable? CollectionNames { get; set; } public IEnumerable? CollectionTypes { get; set; } - public IEnumerable? VectorStroageProviders { get; set; } + public IEnumerable? VectorStorageProviders { get; set; } public static VectorCollectionConfigFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index c862785a1..bd32e7d22 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -40,8 +40,8 @@ Task RecoverCollectionFromShapshot(string collectionName, string snapshotF Task DeleteCollectionShapshot(string collectionName, string snapshotName) => throw new NotImplementedException(); - Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) + Task CreateCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) => throw new NotImplementedException(); - Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) + Task DeleteCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs index 01ffcaf19..7a9b2e0cf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs @@ -1,11 +1,5 @@ namespace BotSharp.Abstraction.VectorStorage.Models; -public class VectorCollectionConfigsModel -{ - [JsonPropertyName("collections")] - public List Collections { get; set; } = new(); -} - public class VectorCollectionConfig { /// diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs deleted file mode 100644 index 09d472da2..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs +++ /dev/null @@ -1,9 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Enums; - -namespace BotSharp.Abstraction.VectorStorage.Models; - -public class VectorCreateModel -{ - public string Text { get; set; } - public Dictionary? Payload { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs index aaa520358..e18843c1a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorPayloadValue.cs @@ -15,7 +15,7 @@ public VectorPayloadValue() } - public VectorPayloadValue(object data, VectorPayloadDataType dataType) + public VectorPayloadValue(object data, VectorPayloadDataType dataType = VectorPayloadDataType.Unknown) { DataValue = data; DataType = dataType; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs deleted file mode 100644 index 8e14b05ca..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchParamModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.VectorStorage.Models; - -public class VectorSearchParamModel -{ - [JsonPropertyName("exact_search")] - public bool? ExactSearch { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs deleted file mode 100644 index 45600219e..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Abstraction.VectorStorage.Models; - -public class VectorUpdateModel : VectorCreateModel -{ - public string Id { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/CollectionIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/CollectionIndexOptions.cs new file mode 100644 index 000000000..47d5a5df7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/CollectionIndexOptions.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.VectorStorage.Options; + +public class CollectionIndexOptions +{ + [JsonPropertyName("field_name")] + public string FieldName { get; set; } = null!; + + [JsonPropertyName("field_schema_type")] + public string? FieldSchemaType { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs deleted file mode 100644 index 1f49a0cff..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorCollectionIndexOptions.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace BotSharp.Abstraction.VectorStorage.Options; - -public class VectorCollectionIndexOptions -{ - [JsonPropertyName("field_name")] - public string FieldName { get; set; } = null!; -} - -public class CreateVectorCollectionIndexOptions : VectorCollectionIndexOptions -{ - [JsonPropertyName("field_schema_type")] - public string FieldSchemaType { get; set; } = null!; -} - -public class DeleteVectorCollectionIndexOptions : VectorCollectionIndexOptions -{ - -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs index 4ba0c81c8..c9ef91f7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Options/VectorSearchOptions.cs @@ -10,7 +10,7 @@ public class VectorSearchOptions public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; public bool WithVector { get; set; } - public VectorSearchParamModel? SearchParam { get; set; } + public Dictionary? SearchParam { get; set; } public static VectorSearchOptions Default() { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 72170fea0..e860c9ce1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -4,17 +4,17 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } try { - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var dir = Path.Combine(docDir, fileId.ToString()); if (ExistDirectory(dir)) { @@ -32,20 +32,20 @@ public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvi catch (Exception ex) { _logger.LogWarning(ex, $"Error when saving knowledge file " + - $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})."); + $"(Vector store provider: {knowledgebaseProvider}, Collection: {collectionName}, File name: {fileName})."); return false; } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!ExistDirectory(dir)) { return false; @@ -67,16 +67,16 @@ public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvide return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return string.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var file = Path.Combine(docDir, fileId.ToString(), fileName); if (!File.Exists(file)) { @@ -86,16 +86,16 @@ public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreP return $"/knowledge/document/{collectionName}/file/{fileId}"; } - public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return BinaryData.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var file = Path.Combine(docDir, fileId.ToString(), fileName); if (!File.Exists(file)) @@ -111,9 +111,9 @@ public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string v #region Private methods - private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider) + private string BuildKnowledgeCollectionFileDir(string collectionName, string knowledgebaseProvider) { - return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr()); + return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, knowledgebaseProvider.CleanStr(), collectionName.CleanStr()); } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index 94b5aa0c5..1a4a7f853 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -103,9 +103,9 @@ public async Task> GetKnowledgeCollectionCon configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList(); } - if (!filter.VectorStroageProviders.IsNullOrEmpty()) + if (!filter.VectorStorageProviders.IsNullOrEmpty()) { - configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStore?.Provider)).ToList(); + configs = configs.Where(x => filter.VectorStorageProviders.Contains(x.VectorStore?.Provider)).ToList(); } return configs; @@ -114,12 +114,12 @@ public async Task> GetKnowledgeCollectionCon #if !DEBUG [SharpCache(10)] #endif - public async Task GetKnowledgeCollectionConfig(string collectionName, string vectorStroageProvider) + public async Task GetKnowledgeCollectionConfig(string collectionName, string knowledgebaseProvider) { var configs = await GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStroageProviders = [vectorStroageProvider] + VectorStorageProviders = [knowledgebaseProvider] }); return configs?.FirstOrDefault(); } @@ -127,7 +127,7 @@ public async Task GetKnowledgeCollectionConfig(string co #region Documents - public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + public async Task SaveKnolwedgeBaseFileMeta(KnowledgeFileMetaData metaData) { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) @@ -149,15 +149,15 @@ public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) return true; } - public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!Directory.Exists(dir)) { return false; @@ -179,21 +179,21 @@ public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, strin return await Task.FromResult(true); } - public async Task> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + public async Task> GetKnowledgeBaseFileMeta(string collectionName, string knowledgebaseProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { - return new PagedItems(); + return new PagedItems(); } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!Directory.Exists(dir)) { - return new PagedItems(); + return new PagedItems(); } - var records = new List(); + var records = new List(); foreach (var folder in Directory.EnumerateDirectories(dir)) { var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE); @@ -203,7 +203,7 @@ public async Task> GetKnowledgeBaseFileMeta(str } var content = await File.ReadAllTextAsync(metaFile); - var metaData = JsonSerializer.Deserialize(content, _options); + var metaData = JsonSerializer.Deserialize(content, _options); if (metaData == null) { continue; @@ -244,7 +244,7 @@ public async Task> GetKnowledgeBaseFileMeta(str records.Add(metaData); } - return new PagedItems + return new PagedItems { Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size), Count = records.Count diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 39ac00cc8..f4a7cae20 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -15,6 +15,7 @@ global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Infrastructures.Events; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Loggers.Services; global using BotSharp.Abstraction.MCP.Models; global using BotSharp.Abstraction.MCP.Services; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs deleted file mode 100644 index cdd91c2dd..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs +++ /dev/null @@ -1,87 +0,0 @@ -using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Knowledges.Options; -using BotSharp.Abstraction.Knowledges.Processors; -using BotSharp.Abstraction.Knowledges.Responses; -using BotSharp.OpenAPI.ViewModels.Knowledges; - -namespace BotSharp.OpenAPI.Controllers; - -public partial class KnowledgeBaseController -{ - #region Document - [HttpGet("/knowledge/processors")] - public IEnumerable GetKnowledgeProcessors() - { - return _services.GetServices().Select(x => x.Provider); - } - - [HttpPost("/knowledge/document/{collection}/upload")] - public async Task UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) - { - var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, request.Files, request.Options); - return response; - } - - [HttpPost("/knowledge/document/{collection}/form")] - public async Task UploadKnowledgeDocuments( - [FromRoute] string collection, - [FromForm] IEnumerable files, - [FromForm] KnowledgeDocOptions? options = null) - { - if (files.IsNullOrEmpty()) - { - return new UploadKnowledgeResponse(); - } - - var docs = new List(); - foreach (var file in files) - { - var data = FileUtility.BuildFileDataFromFile(file); - docs.Add(new ExternalFileModel - { - FileName = file.FileName, - FileData = data - }); - } - - var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, docs, options); - return response; - } - - [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")] - public async Task DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) - { - var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId); - return response; - } - - [HttpDelete("/knowledge/document/{collection}/delete")] - public async Task DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) - { - var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request); - return response; - } - - [HttpPost("/knowledge/document/{collection}/page")] - public async Task> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) - { - var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request); - - return new PagedItems - { - Items = data.Items.Select(x => KnowledgeFileViewModel.From(x)), - Count = data.Count - }; - } - - [HttpGet("/knowledge/document/{collection}/file/{fileId}")] - public async Task GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId) - { - var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId); - var stream = file.FileBinaryData.ToStream(); - stream.Position = 0; - - return new FileStreamResult(stream, file.ContentType) { FileDownloadName = file.FileName }; - } - #endregion -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs index 266155539..163570b90 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs @@ -1,27 +1,7 @@ -using BotSharp.OpenAPI.ViewModels.Knowledges; - namespace BotSharp.OpenAPI.Controllers; public partial class KnowledgeBaseController { - /// - /// Entity analyis with options - /// - /// - /// - [HttpPost("knowledge/entity/analyze")] - public async Task EntityAnalyze([FromBody] EntityAnalysisRequest request) - { - var analyzer = _services.GetServices() - .FirstOrDefault(x => x.Provider.IsEqualTo(request.Provider)); - - if (analyzer == null) - { - return null; - } - return await analyzer.AnalyzeAsync(request.Text, request.Options); - } - /// /// Get entity analyzers /// diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs new file mode 100644 index 000000000..2451cc800 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.File.cs @@ -0,0 +1,102 @@ +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Knowledges.Processors; +using BotSharp.Abstraction.Knowledges.Responses; +using BotSharp.OpenAPI.ViewModels.Knowledges; + +namespace BotSharp.OpenAPI.Controllers; + +public partial class KnowledgeBaseController +{ + #region File + [HttpGet("/knowledge/processors")] + public IEnumerable GetKnowledgeProcessors() + { + return _services.GetServices().Select(x => x.Provider); + } + + [HttpPost("/knowledge/collection/{collection}/file/upload")] + public async Task UploadKnowledgeFiles([FromRoute] string collection, [FromBody] KnowledgeUploadRequest request) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.FileOrchestrator); + var response = await fileOrchestrator.UploadFilesToKnowledge(collection, request.Files, request.Options); + return response; + } + + [HttpPost("/knowledge/collection/{collection}/file/form")] + public async Task UploadKnowledgeFiles( + [FromRoute] string collection, + [FromForm] IEnumerable files, + [FromForm] string? orchestrator = null, + [FromForm] KnowledgeFileHandleOptions? options = null) + { + if (files.IsNullOrEmpty()) + { + return new UploadKnowledgeResponse(); + } + + var docs = new List(); + foreach (var file in files) + { + var data = FileUtility.BuildFileDataFromFile(file); + docs.Add(new ExternalFileModel + { + FileName = file.FileName, + FileData = data + }); + } + + var fileOrchestrator = GetKnowledgeFileOrchestrator(orchestrator); + var response = await fileOrchestrator.UploadFilesToKnowledge(collection, docs, options); + return response; + } + + [HttpDelete("/knowledge/collection/{collection}/file/{fileId}")] + public async Task DeleteKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.FileOrchestrator); + var options = !string.IsNullOrWhiteSpace(request?.DbProvider) ? new KnowledgeFileOptions { DbProvider = request.DbProvider } : null; + var response = await fileOrchestrator.DeleteKnowledgeFile(collection, fileId, options); + return response; + } + + [HttpDelete("/knowledge/collection/{collection}/file")] + public async Task DeleteKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.FileOrchestrator); + var response = await fileOrchestrator.DeleteKnowledgeFiles(collection, request); + return response; + } + + [HttpPost("/knowledge/collection/{collection}/file/page")] + public async Task> GetPagedKnowledgeFiles([FromRoute] string collection, [FromBody] GetKnowledgeFilesRequest request) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request.FileOrchestrator); + var data = await fileOrchestrator.GetPagedKnowledgeFiles(collection, request); + + return new PagedItems + { + Items = data.Items.Select(x => KnowledgeFileViewModel.From(x)), + Count = data.Count + }; + } + + [HttpGet("/knowledge/collection/{collection}/file/{fileId}")] + public async Task GetKnowledgeFile([FromRoute] string collection, [FromRoute] Guid fileId, [FromQuery] KnowledgeFileRequest? request = null) + { + var fileOrchestrator = GetKnowledgeFileOrchestrator(request?.FileOrchestrator); + var options = !string.IsNullOrWhiteSpace(request?.DbProvider) ? new KnowledgeFileOptions { DbProvider = request.DbProvider } : null; + var file = await fileOrchestrator.GetKnowledgeFileBinaryData(collection, fileId, options); + var stream = file.FileBinaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, file.ContentType) { FileDownloadName = file.FileName }; + } + + private IKnowledgeFileOrchestrator GetKnowledgeFileOrchestrator(string? provider) + { + provider ??= "botsharp-knowledge-file"; + var found = _services.GetServices().First(x => x.Provider.IsEqualTo(provider)); + return found; + } + #endregion +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs index 1ab9278b9..4d949ec10 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Graph; -using BotSharp.Abstraction.Graph.Options; -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Abstraction.VectorStorage.Options; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.Repositories; using BotSharp.OpenAPI.ViewModels.Knowledges; namespace BotSharp.OpenAPI.Controllers; @@ -11,83 +9,141 @@ namespace BotSharp.OpenAPI.Controllers; [ApiController] public partial class KnowledgeBaseController : ControllerBase { - private readonly IKnowledgeService _knowledgeService; - private readonly IGraphKnowledgeService _graphKnowledgeService; private readonly IServiceProvider _services; public KnowledgeBaseController( - IKnowledgeService knowledgeService, - IGraphKnowledgeService graphKnowledgeService, IServiceProvider services) { - _knowledgeService = knowledgeService; - _graphKnowledgeService = graphKnowledgeService; _services = services; } - #region Vector - [HttpGet("knowledge/vector/{collection}/exist")] - public async Task ExistVectorCollection([FromRoute] string collection) + #region Collection + [HttpGet("knowledge/collection/{collection}/exist")] + public async Task ExistCollection([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - return await _knowledgeService.ExistVectorCollection(collection); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return false; + } + + return await kg.ExistCollection(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } - [HttpGet("knowledge/vector/collections")] - public async Task> GetVectorCollections([FromQuery] string? type = null) + [HttpGet("knowledge/collections")] + public async Task> GetCollections([FromQuery] string? knowledgeType, [FromQuery] string? dbProvider = null) { - var collections = await _knowledgeService.GetVectorCollections(type); - return collections.Select(x => VectorCollectionConfigViewModel.From(x)); + var results = new List(); + + if (!string.IsNullOrWhiteSpace(knowledgeType)) + { + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return []; + } + + var collections = await kg.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); + results = collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)).ToList(); + } + else + { + var kgs = _services.GetServices(); + foreach (var kg in kgs) + { + var collections = await kg.GetCollections(new KnowledgeCollectionOptions { DbProvider = dbProvider }); + var res = collections.Select(x => KnowledgeCollectionConfigViewModel.From(x)); + results.AddRange(res); + } + } + + return results; } - [HttpGet("knowledge/vector/{collection}/details")] - public async Task GetVectorCollectionDetails([FromRoute] string collection) + [HttpGet("knowledge/collection/{collection}/details")] + public async Task GetCollectionDetails([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var details = await _knowledgeService.GetVectorCollectionDetails(collection); - return VectorCollectionDetailsViewModel.From(details); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return null; + } + + return await kg.GetCollectionDetails(collection, new KnowledgeCollectionOptions { DbProvider = dbProvider }); } - [HttpPost("knowledge/vector/create-collection")] - public async Task CreateVectorCollection([FromBody] CreateVectorCollectionRequest request) + [HttpPost("knowledge/collection")] + public async Task CreateCollection([FromBody] CreateCollectionRequest request) { - var options = new VectorCollectionCreateOptions + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return false; + } + + var options = new CollectionCreateOptions { - Provider = request.Provider, - Model = request.Model, - Dimension = request.Dimension + DbProvider = request.DbProvider, + LlmProvider = request.Provider, + LlmModel = request.Model, + EmbeddingDimension = request.Dimension }; - return await _knowledgeService.CreateVectorCollection(request.CollectionName, request.CollectionType, options); + return await kg.CreateCollection(request.CollectionName, options); } - [HttpDelete("knowledge/vector/{collection}/delete-collection")] - public async Task DeleteVectorCollection([FromRoute] string collection) + [HttpDelete("knowledge/collection/{collection}")] + public async Task DeleteCollection([FromRoute] string collection, [FromBody] DeleteCollectionRequest request) { - return await _knowledgeService.DeleteVectorCollection(collection); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return false; + } + + return await kg.DeleteCollection(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } - [HttpPost("/knowledge/vector/{collection}/search")] - public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) + [HttpPost("/knowledge/collection/{collection}/query")] + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] KnowledgeExecuteRequest request) { - var options = new VectorSearchOptions + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) { - Fields = request?.Fields, - FilterGroups = request?.FilterGroups, - Limit = request?.Limit ?? 5, - Confidence = request?.Confidence ?? 0.5f, - WithVector = request?.WithVector ?? false, - SearchParam = request?.SearchParam - }; + return []; + } - var results = await _knowledgeService.SearchVectorKnowledge(request?.Text ?? string.Empty, collection, options); - return results.Select(x => VectorKnowledgeViewModel.From(x)).ToList(); + var options = BuildSearchOptions(kg, request); + var results = await kg.ExecuteQuery(request?.Text ?? string.Empty, collection, options); + return results.Select(x => KnowledgeKnowledgeViewModel.From(x)).ToList(); } - [HttpPost("/knowledge/vector/{collection}/page")] - public async Task> GetPagedVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) + [HttpPost("/knowledge/collection/{collection}/data/page")] + public async Task> GetPagedCollectionData([FromRoute] string collection, [FromBody] GetPagedCollectionDataRequest request) { - var data = await _knowledgeService.GetPagedVectorCollectionData(collection, filter); - var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))?.ToList() ?? []; + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - return new StringIdPagedItems + if (kg == null) + { + return new StringIdPagedItems(); + } + + var data = await kg.GetPagedCollectionData(collection, request); + var items = data.Items?.Select(x => KnowledgeKnowledgeViewModel.From(x))?.ToList() ?? []; + + return new StringIdPagedItems { Count = data.Count, NextId = data.NextId, @@ -95,142 +151,235 @@ public async Task> GetPagedVectorCo }; } - [HttpPost("/knowledge/vector/{collection}/create")] - public async Task CreateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeCreateRequest request) + [HttpPost("/knowledge/collection/{collection}/data")] + public async Task CreateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataCreateRequest request) { - var create = new VectorCreateModel + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return false; + } + + var create = new KnowledgeCreateModel { + DbProvider = request.DbProvider, Text = request.Text, Payload = request.Payload }; - var created = await _knowledgeService.CreateVectorCollectionData(collection, create); + var created = await kg.CreateCollectionData(collection, create); return created; } - [HttpGet("/knowledge/vector/{collection}/points")] - public async Task> GetVectorCollectionData([FromRoute] string collection, [FromQuery] QueryVectorDataRequest request) + [HttpGet("/knowledge/collection/{collection}/data")] + public async Task> GetCollectionData([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] QueryCollectionDataRequest request) { - var options = new VectorQueryOptions + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return []; + } + + var options = new KnowledgeQueryOptions { + DbProvider = request.DbProvider, WithPayload = request.WithPayload, WithVector = request.WithVector }; - var points = await _knowledgeService.GetVectorCollectionData(collection, request.Ids, options); - return points.Select(x => VectorKnowledgeViewModel.From(x)); + var points = await kg.GetCollectionData(collection, request.Ids, options); + return points.Select(x => KnowledgeKnowledgeViewModel.From(x)); } - [HttpPut("/knowledge/vector/{collection}/update")] - public async Task UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request) + [HttpPut("/knowledge/collection/{collection}/data")] + public async Task UpdateCollectionData([FromRoute] string collection, [FromBody] KnowledgeDataUpdateRequest request) { - var update = new VectorUpdateModel + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) { + return false; + } + + var update = new KnowledgeUpdateModel + { + DbProvider = request.DbProvider, Id = request.Id, Text = request.Text, Payload = request.Payload }; - var updated = await _knowledgeService.UpdateVectorCollectionData(collection, update); + var updated = await kg.UpdateCollectionData(collection, update); return updated; } - [HttpDelete("/knowledge/vector/{collection}/data/{id}")] - public async Task DeleteVectorCollectionData([FromRoute] string collection, [FromRoute] string id) + [HttpDelete("/knowledge/collection/{collection}/data/{id}")] + public async Task DeleteCollectionData([FromRoute] string collection, [FromRoute] string id, [FromBody] DeleteCollectionDataRequest request) { - return await _knowledgeService.DeleteVectorCollectionData(collection, id); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return false; + } + + return await kg.DeleteCollectionData(collection, id, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } - [HttpDelete("/knowledge/vector/{collection}/data")] - public async Task DeleteVectorCollectionAllData([FromRoute] string collection) + [HttpDelete("/knowledge/collection/{collection}/data")] + public async Task DeleteCollectionAllData([FromRoute] string collection, [FromBody] DeleteCollectionDataRequest request) { - return await _knowledgeService.DeleteVectorCollectionAllData(collection); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return false; + } + + return await kg.DeleteCollectionData(collection, new KnowledgeCollectionOptions { DbProvider = request.DbProvider }); } #endregion #region Index - [HttpPost("/knowledge/vector/{collection}/payload/indexes")] - public async Task> CreateCollectionPayloadIndexes([FromRoute] string collection, [FromBody] CreateVectorCollectionIndexRequest request) + [HttpPost("/knowledge/collection/{collection}/indexes")] + public async Task> CreateCollectionIndexes([FromRoute] string collection, [FromBody] CreateCollectionIndexRequest request) { - return await _knowledgeService.CreateVectorCollectionPayloadIndexes(collection, request.Options); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return new(); + } + + var options = new KnowledgeIndexOptions + { + DbProvider = request.DbProvider, + Items = request.Options + }; + return await kg.CreateIndexes(collection, options); } - [HttpDelete("/knowledge/vector/{collection}/payload/indexes")] - public async Task> DeleteCollectionPayloadIndexes([FromRoute] string collection, [FromBody] DeleteVectorCollectionIndexRequest request) + [HttpDelete("/knowledge/collection/{collection}/indexes")] + public async Task> DeleteCollectionIndexes([FromRoute] string collection, [FromBody] DeleteCollectionIndexRequest request) { - return await _knowledgeService.DeleteVectorCollectionPayloadIndexes(collection, request.Options); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return new(); + } + + var options = new KnowledgeIndexOptions + { + DbProvider = request.DbProvider, + Items = request.Options + }; + return await kg.DeleteIndexes(collection, options); } #endregion #region Snapshot - [HttpGet("/knowledge/vector/{collection}/snapshots")] - public async Task> GetVectorCollectionSnapshots([FromRoute] string collection) + [HttpGet("/knowledge/collection/{collection}/snapshots")] + public async Task> GetCollectionSnapshots([FromRoute] string collection, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var snapshots = await _knowledgeService.GetVectorCollectionSnapshots(collection); - return snapshots.Select(x => VectorCollectionSnapshotViewModel.From(x)); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return []; + } + + var snapshots = await kg.GetCollectionSnapshots(collection, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); + return snapshots.Select(x => KnowledgeCollectionSnapshotViewModel.From(x)!); } - [HttpPost("/knowledge/vector/{collection}/snapshot")] - public async Task CreateVectorCollectionSnapshot([FromRoute] string collection) + [HttpPost("/knowledge/collection/{collection}/snapshot")] + public async Task CreateCollectionSnapshot([FromRoute] string collection, [FromBody] CollectionSnapshotRequest request) { - var snapshot = await _knowledgeService.CreateVectorCollectionSnapshot(collection); - return VectorCollectionSnapshotViewModel.From(snapshot); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); + + if (kg == null) + { + return null; + } + + var snapshot = await kg.CreateCollectionSnapshot(collection, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); + return KnowledgeCollectionSnapshotViewModel.From(snapshot); } - [HttpGet("/knowledge/vector/{collection}/snapshot")] - public async Task GetVectorCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName) + [HttpGet("/knowledge/collection/{collection}/snapshot")] + public async Task GetCollectionSnapshot([FromRoute] string collection, [FromQuery] string snapshotFileName, [FromQuery] string knowledgeType, [FromQuery] string? dbProvider = null) { - var snapshot = await _knowledgeService.DownloadVectorCollectionSnapshot(collection, snapshotFileName); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return BuildFileResult(snapshotFileName, BinaryData.Empty); + } + + var snapshot = await kg.DownloadCollectionSnapshot(collection, snapshotFileName, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return BuildFileResult(snapshotFileName, snapshot); } - [HttpPost("/knowledge/vector/{collection}/snapshot/recover")] - public async Task RecoverVectorCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile) + [HttpPost("/knowledge/collection/{collection}/snapshot/recover")] + public async Task RecoverCollectionFromSnapshot([FromRoute] string collection, IFormFile snapshotFile, [FromForm] string knowledgeType, [FromForm] string? dbProvider = null) { + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(knowledgeType)); + + if (kg == null) + { + return false; + } + var fileName = snapshotFile.FileName; var binary = FileUtility.BuildBinaryDataFromFile(snapshotFile); - var done = await _knowledgeService.RecoverVectorCollectionFromSnapshot(collection, fileName, binary); + var done = await kg.RecoverCollectionFromSnapshot(collection, fileName, binary, new KnowledgeSnapshotOptions { DbProvider = dbProvider }); return done; } - [HttpDelete("/knowledge/vector/{collection}/snapshot")] - public async Task DeleteVectorCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteVectorCollectionSnapshotRequest request) + [HttpDelete("/knowledge/collection/{collection}/snapshot")] + public async Task DeleteCollectionSnapshots([FromRoute] string collection, [FromBody] DeleteCollectionSnapshotRequest request) { - var done = await _knowledgeService.DeleteVectorCollectionSnapshot(collection, request.SnapshotName); - return done; - } - #endregion - + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(request.KnowledgeType)); - #region Graph - [HttpPost("/knowledge/graph/search")] - public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) - { - var options = new GraphQueryOptions + if (kg == null) { - Provider = request.Provider, - GraphId = request.GraphId, - Arguments = request.Arguments, - Method = request.Method - }; + return false; + } - var result = await _graphKnowledgeService.ExecuteQueryAsync(request.Query, options); - return new GraphKnowledgeViewModel - { - Result = result.Result - }; + var done = await kg.DeleteCollectionSnapshot(collection, request.SnapshotName, new KnowledgeSnapshotOptions { DbProvider = request.DbProvider }); + return done; } #endregion #region Common - [HttpPost("/knowledge/vector/refresh-configs")] - public async Task RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request) + [HttpPost("/knowledge/refresh-configs")] + public async Task RefreshCollectionConfigs([FromBody] KnowledgeCollectionConfigsRequest request) { - var saved = await _knowledgeService.RefreshVectorKnowledgeConfigs(request); - return saved ? "Success" : "Fail"; + var db = _services.GetRequiredService(); + if (request.Collections.IsNullOrEmpty()) + { + return false; + } + var saved = await db.AddKnowledgeCollectionConfigs(request.Collections, reset: true); + return saved; } #endregion @@ -242,5 +391,47 @@ private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) stream.Position = 0; return File(stream, "application/octet-stream", Path.GetFileName(fileName)); } + + private KnowledgeExecuteOptions BuildSearchOptions(IKnowledgeService kg, KnowledgeExecuteRequest? request) + { + var searchParam = request?.SearchParam?.ToDictionary(x => x.Key, x => x.Value?.ConvertToString()); + + if (kg.KnowledgeType.IsEqualTo(KnowledgeBaseType.SemanticGraph)) + { + return new GraphKnowledgeExecuteOptions + { + DbProvider = request?.DbProvider, + SearchParam = searchParam, + SearchArguments = request?.SearchArguments, + GraphId = request?.GraphId + }; + } + + if (kg.KnowledgeType.IsEqualTo(KnowledgeBaseType.Taxonomy)) + { + return new TaxonomyKnowledgeExecuteOptions + { + DbProvider = request?.DbProvider, + Limit = request?.Limit ?? 5, + Confidence = request?.Confidence ?? 0.5f, + SearchParam = searchParam, + SearchArguments = request?.SearchArguments, + DataProviders = request?.DataProviders, + MaxNgram = request?.MaxNgram + }; + } + + return new KnowledgeExecuteOptions + { + DbProvider = request?.DbProvider, + Fields = request?.Fields, + FilterGroups = request?.FilterGroups, + Limit = request?.Limit ?? 5, + Confidence = request?.Confidence ?? 0.5f, + WithVector = request?.WithVector ?? false, + SearchParam = searchParam, + SearchArguments = request?.SearchArguments + }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Using.cs b/src/Infrastructure/BotSharp.OpenAPI/Using.cs index d811452ee..614d01f54 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Using.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Using.cs @@ -28,6 +28,7 @@ global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.VectorStorage.Enums; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Chart.Models; global using BotSharp.Abstraction.Entity; global using BotSharp.Abstraction.Entity.Models; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs new file mode 100644 index 000000000..bdc6a3d84 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionIndexRequest.cs @@ -0,0 +1,13 @@ +using BotSharp.Abstraction.VectorStorage.Options; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class CreateCollectionIndexRequest : KnowledgeBaseRequestBase +{ + public IEnumerable Options { get; set; } = []; +} + +public class DeleteCollectionIndexRequest : KnowledgeBaseRequestBase +{ + public IEnumerable Options { get; set; } = []; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs new file mode 100644 index 000000000..62e719978 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CollectionSnapshotRequest.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class CollectionSnapshotRequest : KnowledgeBaseRequestBase +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs new file mode 100644 index 000000000..ea9b1ffee --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateCollectionRequest.cs @@ -0,0 +1,9 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class CreateCollectionRequest : KnowledgeBaseRequestBase +{ + public string CollectionName { get; set; } + public string Provider { get; set; } + public string Model { get; set; } + public int Dimension { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs deleted file mode 100644 index 7df04941f..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/CreateVectorCollectionRequest.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class CreateVectorCollectionRequest -{ - [JsonPropertyName("collection_name")] - public string CollectionName { get; set; } - - [JsonPropertyName("collection_type")] - public string CollectionType { get; set; } - - [JsonPropertyName("provider")] - public string Provider { get; set; } - - [JsonPropertyName("model")] - public string Model { get; set; } - - [JsonPropertyName("dimension")] - public int Dimension { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs new file mode 100644 index 000000000..4b511f711 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionDataRequest.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteCollectionDataRequest : KnowledgeBaseRequestBase +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs new file mode 100644 index 000000000..694f49eb9 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionRequest.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteCollectionRequest : KnowledgeBaseRequestBase +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs new file mode 100644 index 000000000..74466d46c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteCollectionSnapshotRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class DeleteCollectionSnapshotRequest : KnowledgeBaseRequestBase +{ + public string SnapshotName { get; set; } = default!; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs deleted file mode 100644 index 30398b2c0..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/DeleteVectorCollectionSnapshotRequest.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class DeleteVectorCollectionSnapshotRequest -{ - [JsonPropertyName("snapshot_name")] - public string SnapshotName { get; set; } = default!; -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs deleted file mode 100644 index 890113448..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class EntityAnalysisRequest -{ - public string Text { get; set; } = string.Empty; - public string? Provider { get; set; } - public EntityAnalysisOptions? Options { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs deleted file mode 100644 index 52022b70b..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeDocsRequest.cs +++ /dev/null @@ -1,7 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Filters; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class GetKnowledgeDocsRequest : KnowledgeFileFilter -{ -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs new file mode 100644 index 000000000..1ddd163f4 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetKnowledgeFilesRequest.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Knowledges.Filters; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GetKnowledgeFilesRequest : KnowledgeFileFilter +{ + public string? FileOrchestrator { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs new file mode 100644 index 000000000..b311ac88b --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/GetPagedCollectionDataRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GetPagedCollectionDataRequest : KnowledgeFilter +{ + public string KnowledgeType { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs new file mode 100644 index 000000000..7607eabb5 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeBaseRequestBase.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeBaseRequestBase +{ + public string KnowledgeType { get; set; } = null!; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs new file mode 100644 index 000000000..e674737b4 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeCollectionConfigsRequest.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionConfigsRequest +{ + public List Collections { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs similarity index 59% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs index c2117b69f..7d6e8a436 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataCreateRequest.cs @@ -1,13 +1,10 @@ using BotSharp.Abstraction.VectorStorage.Models; -using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeCreateRequest +public class KnowledgeDataCreateRequest : KnowledgeBaseRequestBase { - [JsonPropertyName("text")] public string Text { get; set; } - [JsonPropertyName("payload")] public Dictionary? Payload { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs new file mode 100644 index 000000000..ba829cce2 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeDataUpdateRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeDataUpdateRequest : KnowledgeDataCreateRequest +{ + public string Id { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeExecuteRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeExecuteRequest.cs new file mode 100644 index 000000000..26aff6651 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeExecuteRequest.cs @@ -0,0 +1,34 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeExecuteRequest : KnowledgeBaseRequestBase +{ + public string Text { get; set; } = string.Empty; + + public IEnumerable? Fields { get; set; } + + public IEnumerable? FilterGroups { get; set; } + + public int? Limit { get; set; } = 5; + + public float? Confidence { get; set; } = 0.5f; + + public bool WithVector { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? SearchParam { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? SearchArguments { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? DataProviders { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxNgram { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? GraphId { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs new file mode 100644 index 000000000..2e95d0e9c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeFileRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeFileRequest +{ + public string? FileOrchestrator { get; set; } + public string? DbProvider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs new file mode 100644 index 000000000..539f466b6 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/KnowledgeUploadRequest.cs @@ -0,0 +1,13 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeUploadRequest +{ + /// + /// Provider for knowledge file orchestrator + /// + public string? FileOrchestrator { get; set; } + + public IEnumerable Files { get; set; } = new List(); + + public KnowledgeFileHandleOptions? Options { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryCollectionDataRequest.cs similarity index 68% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryCollectionDataRequest.cs index 331bb01f9..8eccb1500 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryCollectionDataRequest.cs @@ -1,8 +1,9 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class QueryVectorDataRequest +public class QueryCollectionDataRequest { public List Ids { get; set; } = []; public bool WithVector { get; set; } public bool WithPayload { get; set; } + public string? DbProvider { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs deleted file mode 100644 index de3aa2b68..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchGraphKnowledgeRequest.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class SearchGraphKnowledgeRequest -{ - [JsonPropertyName("query")] - public string Query { get; set; } = string.Empty; - - [JsonPropertyName("provider")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Provider { get; set; } - - [JsonPropertyName("graph_id")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? GraphId { get; set; } - - [JsonPropertyName("arguments")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public Dictionary? Arguments { get; set; } - - [JsonPropertyName("method")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Method { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs deleted file mode 100644 index 93b48dc2a..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/SearchVectorKnowledgeRequest.cs +++ /dev/null @@ -1,28 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Models; -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class SearchVectorKnowledgeRequest -{ - [JsonPropertyName("text")] - public string Text { get; set; } = string.Empty; - - [JsonPropertyName("fields")] - public IEnumerable? Fields { get; set; } - - [JsonPropertyName("filter_groups")] - public IEnumerable? FilterGroups { get; set; } - - [JsonPropertyName("limit")] - public int? Limit { get; set; } = 5; - - [JsonPropertyName("confidence")] - public float? Confidence { get; set; } = 0.5f; - - [JsonPropertyName("with_vector")] - public bool WithVector { get; set; } - - [JsonPropertyName("search_param")] - public VectorSearchParamModel? SearchParam { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs deleted file mode 100644 index 5ff787fa6..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs +++ /dev/null @@ -1,13 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Options; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class CreateVectorCollectionIndexRequest -{ - public IEnumerable Options { get; set; } = []; -} - -public class DeleteVectorCollectionIndexRequest -{ - public IEnumerable Options { get; set; } = []; -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs deleted file mode 100644 index 355ae7760..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUpdateRequest.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class VectorKnowledgeUpdateRequest : VectorKnowledgeCreateRequest -{ - [JsonPropertyName("id")] - public string Id { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs deleted file mode 100644 index 2a97733e4..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorKnowledgeUploadRequest.cs +++ /dev/null @@ -1,13 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Options; -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class VectorKnowledgeUploadRequest -{ - [JsonPropertyName("files")] - public IEnumerable Files { get; set; } = new List(); - - [JsonPropertyName("options")] - public KnowledgeDocOptions? Options { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionConfigViewModel.cs similarity index 57% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionConfigViewModel.cs index e19fda39f..795369128 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionConfigViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionConfigViewModel.cs @@ -2,11 +2,11 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorCollectionConfigViewModel : VectorCollectionConfig +public class KnowledgeCollectionConfigViewModel : VectorCollectionConfig { - public static VectorCollectionConfigViewModel From(VectorCollectionConfig model) + public static KnowledgeCollectionConfigViewModel From(VectorCollectionConfig model) { - return new VectorCollectionConfigViewModel + return new KnowledgeCollectionConfigViewModel { Name = model.Name, Type = model.Type, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionDetailsViewModel.cs similarity index 73% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionDetailsViewModel.cs index b55981b59..e7a088c59 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionDetailsViewModel.cs @@ -2,13 +2,13 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorCollectionDetailsViewModel : VectorCollectionDetails +public class KnowledgeCollectionDetailsViewModel : VectorCollectionDetails { - public static VectorCollectionDetailsViewModel? From(VectorCollectionDetails? model) + public static KnowledgeCollectionDetailsViewModel? From(VectorCollectionDetails? model) { if (model == null) return null; - return new VectorCollectionDetailsViewModel + return new KnowledgeCollectionDetailsViewModel { Status = model.Status, OptimizerStatus = model.OptimizerStatus, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionSnapshotViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionSnapshotViewModel.cs new file mode 100644 index 000000000..2bb12a81d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeCollectionSnapshotViewModel.cs @@ -0,0 +1,52 @@ +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionSnapshotViewModel +{ + [JsonPropertyName("name")] + public string Name { get; set; } = default!; + + [JsonPropertyName("size")] + public long Size { get; set; } + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } + + [JsonPropertyName("check_sum")] + public string? CheckSum { get; set; } + + public static KnowledgeCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model) + { + if (model == null) + { + return null; + } + + return new KnowledgeCollectionSnapshotViewModel + { + Name = model.Name, + Size = model.Size, + CreatedTime = model.CreatedTime, + CheckSum = model.CheckSum + }; + } + + public static KnowledgeCollectionSnapshotViewModel? From(KnowledgeCollectionSnapshot? model) + { + if (model == null) + { + return null; + } + + return new KnowledgeCollectionSnapshotViewModel + { + Name = model.Name, + Size = model.Size, + CreatedTime = model.CreatedTime, + CheckSum = model.CheckSum + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs similarity index 78% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs index e4c805be5..6f1cf6a81 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/KnowledgeKnowledgeViewModel.cs @@ -3,7 +3,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class VectorKnowledgeViewModel +public class KnowledgeKnowledgeViewModel { [JsonPropertyName("id")] public string Id { get; set; } @@ -22,10 +22,9 @@ public class VectorKnowledgeViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? VectorDimension { get; set; } - - public static VectorKnowledgeViewModel From(VectorSearchResult result) + public static KnowledgeKnowledgeViewModel From(KnowledgeExecuteResult result) { - return new VectorKnowledgeViewModel + return new KnowledgeKnowledgeViewModel { Id = result.Id, Data = result.Data, @@ -35,9 +34,9 @@ public static VectorKnowledgeViewModel From(VectorSearchResult result) }; } - public static VectorKnowledgeViewModel From(VectorCollectionData data) + public static KnowledgeKnowledgeViewModel From(KnowledgeCollectionData data) { - return new VectorKnowledgeViewModel + return new KnowledgeKnowledgeViewModel { Id = data.Id, Data = data.Data, diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs index cc6f3e8f1..ea3bc048f 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.Plugins; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index e15529d2a..9a8f20dd7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -61,4 +61,9 @@ + + + + + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index ab5c29bbe..be233e2f6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -19,11 +19,20 @@ public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var collectionName = !string.IsNullOrEmpty(args.RefinedCollection) - ? args.RefinedCollection + var collectionName = !string.IsNullOrEmpty(args.RefinedCollection) + ? args.RefinedCollection : _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var knowledgeService = _services.GetRequiredService(); - var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); + + if (kg == null) + { + message.Content = "I forgot it"; + return true; + } + + var result = await kg.CreateCollectionData(collectionName, new KnowledgeCreateModel { Text = args.Question, Payload = new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs deleted file mode 100644 index 45b439bea..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Graph/GraphKnowledgeService.cs +++ /dev/null @@ -1,37 +0,0 @@ -using BotSharp.Abstraction.Graph.Options; - -namespace BotSharp.Plugin.KnowledgeBase.Graph; - -public partial class GraphKnowledgeService : IGraphKnowledgeService -{ - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly KnowledgeBaseSettings _settings; - - public GraphKnowledgeService( - IServiceProvider services, - ILogger logger, - KnowledgeBaseSettings settings) - { - _services = services; - _logger = logger; - _settings = settings; - } - - public async Task ExecuteQueryAsync(string query, GraphQueryOptions? options = null) - { - var db = GetGraphDb(options?.Provider); - var result = await db.ExecuteQueryAsync(query, options); - return result; - } - - - #region Private methods - private IGraphDb GetGraphDb(string? provider = null) - { - var graphProvider = provider ?? _settings.GraphDb.Provider; - var db = _services.GetServices().FirstOrDefault(x => x.Provider == graphProvider); - return db; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index 00ec683b5..645c92373 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -1,31 +1,15 @@ -using BotSharp.Abstraction.Graph.Options; -using BotSharp.Abstraction.VectorStorage.Options; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeHook : IKnowledgeHook { - private readonly IKnowledgeService _knowledgeService; - private readonly IGraphKnowledgeService _graphKnowledgeService; private readonly IServiceProvider _services; public KnowledgeHook( - IKnowledgeService knowledgeService, - IGraphKnowledgeService graphKnowledgeService, IServiceProvider services) { - _knowledgeService = knowledgeService; - _graphKnowledgeService = graphKnowledgeService; _services = services; } - private async Task> GetKnowledgeBaseNameByAgentIdAsync(string agentId) - { - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(agentId); - return agent.KnowledgeBases; - } - public async Task> GetDomainKnowledges(RoleDialogModel message, string text) { // Get agent Id by knowledge base name @@ -34,44 +18,35 @@ public async Task> GetDomainKnowledges(RoleDialogModel message, str foreach (var knowledgeBase in knowledgeBases) { - if (knowledgeBase.Type == "relationships") + if (knowledgeBase.Type == KnowledgeBaseType.Document) { - var options = new GraphQueryOptions - { - Provider = "Remote", - Method = "local" - }; - var result = await _graphKnowledgeService.ExecuteQueryAsync(text, options); - results.Add(result.Result); - } - else if (knowledgeBase.Type == "document") - { - var options = new VectorSearchOptions + var kg = GetKnowledgeService(knowledgeBase.Type); + var options = new KnowledgeExecuteOptions { Fields = null, Limit = 5, Confidence = 0.25f, WithVector = true }; - var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + var result = await kg.ExecuteQuery(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) .Select(x => x.Data["text"].ToString()) .Where(x => x != null)!); } - else + else if (knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { - var options = new VectorSearchOptions + var kg = GetKnowledgeService(knowledgeBase.Type); + var options = new KnowledgeExecuteOptions { Fields = null, Limit = 5, Confidence = 0.5f, WithVector = true }; - var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + var result = await kg.ExecuteQuery(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); - } } @@ -84,30 +59,23 @@ public async Task> GetGlobalKnowledges(RoleDialogModel message) var results = new List(); // Get all knowledge bases - var knowledgeBases = await _knowledgeService.GetVectorCollections(); + var kg = GetKnowledgeService(); + var knowledgeBases = await kg.GetCollections(new() { IncludeAllTypes = true }); foreach (var knowledgeBase in knowledgeBases) { - if (knowledgeBase.Type == "relationships") - { - var options = new GraphQueryOptions - { - Provider = "Remote", - Method = "local" - }; - var result = await _graphKnowledgeService.ExecuteQueryAsync(text, options); - results.Add(result.Result); - } - else + if (knowledgeBase.Type == KnowledgeBaseType.Document + || knowledgeBase.Type == KnowledgeBaseType.QuestionAnswer) { - var options = new VectorSearchOptions + var kgSearcher = GetKnowledgeService(knowledgeBase.Type); + var options = new KnowledgeExecuteOptions { Fields = null, Limit = 5, Confidence = 0.5f, WithVector = true }; - var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + var result = await kgSearcher.ExecuteQuery(text, knowledgeBase.Name, options); results.AddRange(result.Where(x => x.Data != null && (x.Data.ContainsKey("text") || x.Data.ContainsKey("answer"))) .Select(x => x.Data.ContainsKey("answer") ? x.Data["text"].ToString() + "\r\n\r\n" + x.Data["answer"].ToString() : x.Data["text"].ToString()) .Where(x => x != null)!); @@ -121,4 +89,22 @@ public async Task> CollectChunkedKnowledge() { return new List(); } + + + private async Task> GetKnowledgeBaseNameByAgentIdAsync(string agentId) + { + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(agentId); + return agent.KnowledgeBases; + } + + private IKnowledgeService GetKnowledgeService(string? type = null) + { + var kgs = _services.GetServices(); + if (!string.IsNullOrWhiteSpace(type)) + { + return kgs.First(x => x.KnowledgeType == type); + } + return kgs.First(); + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index dd974d6d3..b7ec32131 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; -using BotSharp.Plugin.KnowledgeBase.Graph; using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.KnowledgeBase; @@ -22,8 +21,13 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddSingleton(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Collection.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Collection.cs new file mode 100644 index 000000000..7eba4f12a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Collection.cs @@ -0,0 +1,159 @@ +using BotSharp.Abstraction.VectorStorage.Filters; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorKnowledgeBase +{ + #region Collection + public virtual async Task ExistCollection(string collectionName, KnowledgeCollectionOptions options) + { + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var exist = await vectorDb.DoesCollectionExist(collectionName); + if (exist) + { + return true; + } + + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = [collectionName], + VectorStorageProviders = [vectorDb.Provider] + }); + + return !configs.IsNullOrEmpty(); + } + + public virtual async Task CreateCollection(string collectionName, CollectionCreateOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var vectorDb = GetVectorDb(options.DbProvider); + if (vectorDb == null) + { + return false; + } + + var db = _services.GetRequiredService(); + var created = await db.AddKnowledgeCollectionConfigs(new List + { + new VectorCollectionConfig + { + Name = collectionName, + Type = KnowledgeType, + VectorStore = new VectorStoreConfig + { + Provider = vectorDb.Provider + }, + TextEmbedding = new KnowledgeEmbeddingConfig + { + Provider = options.LlmProvider, + Model = options.LlmModel, + Dimension = options.EmbeddingDimension + } + } + }); + + if (!created) + { + return false; + } + + created = await vectorDb.CreateCollection(collectionName, options: new() + { + Provider = options.LlmProvider, + Model = options.LlmProvider, + Dimension = options.EmbeddingDimension + }); + + return created; + } + + public virtual async Task> GetCollections(KnowledgeCollectionOptions options) + { + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return []; + } + + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionTypes = options?.IncludeAllTypes == true ? null : [KnowledgeType], + VectorStorageProviders = [vectorDb.Provider] + }); + + var dbCollections = await vectorDb.GetCollections(); + return configs.Where(x => dbCollections.Contains(x.Name)).Select(x => new KnowledgeCollectionConfig + { + Name = x.Name, + Type = x.Type, + VectorStore = x.VectorStore, + TextEmbedding = x.TextEmbedding + }); + } + + public virtual async Task GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return null; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return null; + } + + var details = await vectorDb.GetCollectionDetails(collectionName); + if (details == null) + { + return null; + } + + return new KnowledgeCollectionDetails + { + Status = details.Status, + PayloadSchema = details.PayloadSchema + }; + } + + public virtual async Task DeleteCollection(string collectionName, KnowledgeCollectionOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var deleted = await vectorDb.DeleteCollection(collectionName); + + if (deleted) + { + var db = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + + await db.DeleteKnowledgeCollectionConfig(collectionName); + fileStorage.DeleteKnowledgeFile(collectionName, vectorDb.Provider); + await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorDb.Provider); + } + + return deleted; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Data.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Data.cs new file mode 100644 index 000000000..05a507884 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Data.cs @@ -0,0 +1,230 @@ +using BotSharp.Abstraction.VectorStorage.Options; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorKnowledgeBase +{ + #region Collection data + public virtual async Task CreateCollectionData(string collectionName, KnowledgeCreateModel create) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) + { + return false; + } + + var vectorDb = GetVectorDb(create.DbProvider); + if (vectorDb == null) + { + return false; + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(create.Text); + + var guid = Guid.NewGuid(); + var payload = create.Payload ?? new(); + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); + } + + return await vectorDb.Upsert(collectionName, guid, vector, create.Text, payload); + } + + public virtual async Task DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options) + { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + return await vectorDb.DeleteCollectionData(collectionName, [guid]); + } + + public virtual async Task DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options) + { + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + return await vectorDb.DeleteCollectionAllData(collectionName); + } + + public virtual async Task> GetCollectionData(string collectionName, IEnumerable ids, KnowledgeQueryOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty()) + { + return []; + } + + var pointIds = ids.Select(x => new { Id = x, IsValid = Guid.TryParse(x, out var guid), ParseResult = guid }) + .Where(x => x.IsValid) + .Select(x => x.ParseResult) + .ToList(); + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return []; + } + + var queryOptions = options != null ? new VectorQueryOptions + { + WithPayload = options.WithPayload, + WithVector = options.WithVector + } : null; + + var points = await vectorDb.GetCollectionData(collectionName, pointIds, queryOptions); + return points.Select(x => new KnowledgeCollectionData + { + Id = x.Id, + Payload = x.Payload, + Score = x.Score, + Vector = x.Vector + }); + } + + public virtual async Task> GetPagedCollectionData(string collectionName, KnowledgeFilter filter) + { + var vectorDb = GetVectorDb(filter.DbProvider); + if (vectorDb == null) + { + return new StringIdPagedItems(); + } + + var vectorFilter = new VectorFilter + { + WithVector = filter.WithVector, + FilterGroups = filter.FilterGroups, + OrderBy = filter.OrderBy, + Fields = filter.Fields, + Size = filter.Size, + StartId = filter.StartId + }; + + var pagedResult = await vectorDb.GetPagedCollectionData(collectionName, vectorFilter); + return new StringIdPagedItems + { + Count = pagedResult.Count, + Items = pagedResult.Items.Select(x => new KnowledgeCollectionData + { + Id = x.Id, + Payload = x.Payload, + Score = x.Score, + Vector = x.Vector + }), + NextId = pagedResult.NextId, + }; + } + + public virtual async Task UpdateCollectionData(string collectionName, KnowledgeUpdateModel update) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(update.Text) + || !Guid.TryParse(update.Id, out var guid)) + { + return false; + } + + var vectorDb = GetVectorDb(update.DbProvider); + if (vectorDb == null) + { + return false; + } + + var found = await vectorDb.GetCollectionData(collectionName, [guid]); + if (found.IsNullOrEmpty()) + { + return false; + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(update.Text); + var payload = update.Payload ?? new(); + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); + } + + return await vectorDb.Upsert(collectionName, guid, vector, update.Text, payload); + } + + public virtual async Task UpsertCollectionData(string collectionName, KnowledgeUpdateModel update) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(update.Text) + || !Guid.TryParse(update.Id, out var guid)) + { + return false; + } + + var vectorDb = GetVectorDb(update.DbProvider); + if (vectorDb == null) + { + return false; + } + + var found = await vectorDb.GetCollectionData(collectionName, [guid], options: new() { WithVector = true, WithPayload = true }); + if (!found.IsNullOrEmpty()) + { + if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text) + { + // Only update payload + return await vectorDb.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload); + } + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(update.Text); + var payload = update.Payload ?? new(); + + if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) + { + payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); + } + + return await vectorDb.Upsert(collectionName, guid, vector, update.Text, payload); + } + #endregion + + public virtual async Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) + { + var vectorDb = GetVectorDb(options.DbProvider); + if (vectorDb == null) + { + return []; + } + + var textEmbedding = await GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(query); + + var searchOptions = new VectorSearchOptions + { + Fields = options.Fields, + FilterGroups = options.FilterGroups, + Limit = options.Limit, + Confidence = options.Confidence, + WithVector = options.WithVector, + SearchParam = options.SearchParam + }; + + var found = await vectorDb.Search(collectionName, vector, searchOptions); + return found.Select(x => KnowledgeExecuteResult.CopyFrom(new KnowledgeCollectionData + { + Id = x.Id, + Payload = x.Payload, + Score = x.Score, + Vector = x.Vector + })).ToList(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Index.cs new file mode 100644 index 000000000..6099cf6b3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Index.cs @@ -0,0 +1,82 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorKnowledgeBase +{ + #region Index + public virtual async Task> CreateIndexes(string collectionName, KnowledgeIndexOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return new(); + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return new(); + } + + var response = new SuccessFailResponse(); + var innerOptions = options?.Items?.DistinctBy(x => x.FieldName)?.ToList(); + if (innerOptions.IsNullOrEmpty()) + { + return new(); + } + + foreach (var option in innerOptions!) + { + var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option); + var field = $"{option.FieldName} ({option.FieldSchemaType})"; + if (created) + { + response.Success.Add(field); + } + else + { + response.Fail.Add(field); + } + } + + return response; + } + + public virtual async Task> DeleteIndexes(string collectionName, KnowledgeIndexOptions options) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return new(); + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return new(); + } + + var response = new SuccessFailResponse(); + var innerOptions = options?.Items?.DistinctBy(x => x.FieldName)?.ToList(); + if (innerOptions.IsNullOrEmpty()) + { + return new(); + } + + foreach (var option in innerOptions!) + { + var deleted = await vectorDb.DeleteCollectionPayloadIndex(collectionName, option); + var field = $"{option.FieldName}"; + if (deleted) + { + response.Success.Add(field); + } + else + { + response.Fail.Add(field); + } + } + + return response; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Snapshot.cs new file mode 100644 index 000000000..74c4feeb0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.Snapshot.cs @@ -0,0 +1,91 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorKnowledgeBase +{ + #region Snapshot + public virtual async Task> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return []; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return []; + } + + var snapshots = await vectorDb.GetCollectionSnapshots(collectionName); + return snapshots.Select(x => KnowledgeCollectionSnapshot.CopyFrom(x)!); + } + + public virtual async Task CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return null; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return null; + } + + var snapshot = await vectorDb.CreateCollectionShapshot(collectionName); + return KnowledgeCollectionSnapshot.CopyFrom(snapshot); + } + + public virtual async Task DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) + { + return BinaryData.Empty; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return BinaryData.Empty; + } + + var snapshot = await vectorDb.DownloadCollectionSnapshot(collectionName, snapshotFileName); + return snapshot; + } + + public virtual async Task RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var done = await vectorDb.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData); + return done; + } + + public virtual async Task DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) + { + return false; + } + + var vectorDb = GetVectorDb(options?.DbProvider); + if (vectorDb == null) + { + return false; + } + + var done = await vectorDb.DeleteCollectionShapshot(collectionName, snapshotName); + return done; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.cs new file mode 100644 index 000000000..d902e0786 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Base/Vector/VectorKnowledgeBase.cs @@ -0,0 +1,40 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public abstract partial class VectorKnowledgeBase : IKnowledgeService +{ + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + protected readonly KnowledgeBaseSettings _settings; + + public abstract string KnowledgeType { get; } + + protected VectorKnowledgeBase( + IServiceProvider services, + ILogger logger, + KnowledgeBaseSettings settings) + { + _services = services; + _logger = logger; + _settings = settings; + } + + protected IVectorDb? GetVectorDb(string? dbProvider = null) + { + var provider = dbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + return db; + } + + protected async Task GetTextEmbedding(string collectionName) + { + return await KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); + } + + protected async Task GetUserId() + { + var userIdentity = _services.GetRequiredService(); + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(userIdentity.Id); + return user.Id; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeBase.cs new file mode 100644 index 000000000..28498ca3e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Document/DocumentKnowledgeBase.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class DocumentKnowledgeBase : VectorKnowledgeBase, IKnowledgeService +{ + public override string KnowledgeType => KnowledgeBaseType.Document; + + public DocumentKnowledgeBase( + IServiceProvider services, + ILogger logger, + KnowledgeBaseSettings settings) + : base(services, logger, settings) + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs similarity index 71% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs index 5f2222eb9..1094ec20c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/File/KnowledgeFileOrchestrator.cs @@ -1,13 +1,30 @@ using System.Net.Http; +using BotSharp.Abstraction.VectorStorage.Filters; namespace BotSharp.Plugin.KnowledgeBase.Services; -public partial class KnowledgeService +public class KnowledgeFileOrchestrator : IKnowledgeFileOrchestrator { - public async Task UploadDocumentsToKnowledge( + private readonly IServiceProvider _services; + private readonly KnowledgeBaseSettings _settings; + private readonly ILogger _logger; + + public string Provider => "botsharp-knowledge-file"; + + public KnowledgeFileOrchestrator( + IServiceProvider services, + KnowledgeBaseSettings settings, + ILogger logger) + { + _services = services; + _settings = settings; + _logger = logger; + } + + public async Task UploadFilesToKnowledge( string collectionName, IEnumerable files, - KnowledgeDocOptions? options = null) + KnowledgeFileHandleOptions? options = null) { var res = new UploadKnowledgeResponse { @@ -20,14 +37,13 @@ public async Task UploadDocumentsToKnowledge( return res; } - var exist = await ExistVectorCollection(collectionName); + var knowledgebaseProvider = options?.DbProvider ?? _settings.VectorDb.Provider; + var exist = await ExistCollection(collectionName, knowledgebaseProvider); if (!exist) { return res; } - var fileStoreage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; var knowledgeFiles = new List(); var successFiles = new List(); var failedFiles = new List(); @@ -97,7 +113,7 @@ public async Task UploadDocumentsToKnowledge( } } - var response = await HandleKnowledgeFiles(collectionName, vectorStoreProvider, knowledgeFiles, saveFile: true); + var response = await HandleKnowledgeFiles(collectionName, knowledgebaseProvider, knowledgeFiles, saveFile: true); return new UploadKnowledgeResponse { Success = successFiles.Concat(response.Success).Distinct(), @@ -105,9 +121,12 @@ public async Task UploadDocumentsToKnowledge( }; } - - public async Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, - IEnumerable contents, DocMetaRefData? refData = null, Dictionary? payload = null) + public async Task ImportFileContentToKnowledge( + string collectionName, + string fileName, + string fileSource, + IEnumerable contents, + ImportKnowledgeFileOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileName) @@ -118,24 +137,25 @@ public async Task ImportDocumentContentToKnowledge(string collectionName, try { - var exist = await ExistVectorCollection(collectionName); + var knowledgebaseProvider = options?.DbProvider ?? _settings.VectorDb.Provider; + var exist = await ExistCollection(collectionName, knowledgebaseProvider); if (!exist) return false; var db = _services.GetRequiredService(); var userId = await GetUserId(); - var vectorStoreProvider = _settings.VectorDb.Provider; + var fileId = Guid.NewGuid(); var contentType = FileUtility.GetFileContentType(fileName); - var innerPayload = new Dictionary(payload ?? []); + var innerPayload = new Dictionary(options?.Payload ?? []); innerPayload[KnowledgePayloadName.DataSource] = (VectorPayloadValue)VectorDataSource.File; innerPayload[KnowledgePayloadName.FileId] = (VectorPayloadValue)fileId.ToString(); innerPayload[KnowledgePayloadName.FileName] = (VectorPayloadValue)fileName; innerPayload[KnowledgePayloadName.FileSource] = (VectorPayloadValue)fileSource; - if (!string.IsNullOrWhiteSpace(refData?.Url)) + if (!string.IsNullOrWhiteSpace(options?.FileRefData?.Url)) { - innerPayload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)refData.Url; + innerPayload[KnowledgePayloadName.FileUrl] = (VectorPayloadValue)options.FileRefData.Url; } var kgFile = new FileKnowledgeWrapper @@ -157,7 +177,7 @@ public async Task ImportDocumentContentToKnowledge(string collectionName, } } }; - await HandleKnowledgeFiles(collectionName, vectorStoreProvider, [kgFile], saveFile: false); + await HandleKnowledgeFiles(collectionName, knowledgebaseProvider, [kgFile], saveFile: false); return true; } catch (Exception ex) @@ -167,8 +187,7 @@ public async Task ImportDocumentContentToKnowledge(string collectionName, } } - - public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId) + public async Task DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -179,19 +198,23 @@ public async Task DeleteKnowledgeDocument(string collectionName, Guid file { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var vectorDb = GetVectorDb(); - var vectorStoreProvider = _settings.VectorDb.Provider; + var knowledgebaseProvider = options?.DbProvider ?? _settings.VectorDb.Provider; + var vectorDb = GetVectorDb(knowledgebaseProvider); + if (vectorDb == null) + { + return false; + } // Get doc meta data - var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, knowledgebaseProvider, new KnowledgeFileFilter { Size = 1, - FileIds = [ fileId ] + FileIds = [fileId] }); // Delete doc - fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId); - + fileStorage.DeleteKnowledgeFile(collectionName, knowledgebaseProvider, fileId); + var found = pageData?.Items?.FirstOrDefault(); if (found != null && !found.VectorDataIds.IsNullOrEmpty()) { @@ -199,7 +222,7 @@ public async Task DeleteKnowledgeDocument(string collectionName, Guid file await vectorDb.DeleteCollectionData(collectionName, guids); } - await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId); + await db.DeleteKnolwedgeBaseFileMeta(collectionName, knowledgebaseProvider, fileId); return true; } catch (Exception ex) @@ -210,11 +233,10 @@ public async Task DeleteKnowledgeDocument(string collectionName, Guid file } } - public async Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + public async Task DeleteKnowledgeFiles(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) return false; - var pageSize = filter.Size; var innerFilter = new KnowledgeFileFilter { @@ -226,7 +248,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg ContentTypes = filter.ContentTypes }; - var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + var pageData = await GetPagedKnowledgeFiles(collectionName, innerFilter); var total = pageData.Count; if (total == 0) return false; @@ -238,7 +260,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg { if (page > 1) { - pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + pageData = await GetPagedKnowledgeFiles(collectionName, innerFilter); } var fileIds = pageData.Items.Select(x => x.FileId).ToList(); @@ -246,7 +268,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg { try { - await DeleteKnowledgeDocument(collectionName, fileId); + await DeleteKnowledgeFile(collectionName, fileId); } catch { @@ -260,8 +282,7 @@ public async Task DeleteKnowledgeDocuments(string collectionName, Knowledg return true; } - - public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + public async Task> GetPagedKnowledgeFiles(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) { @@ -270,10 +291,10 @@ public async Task> GetPagedKnowledgeDocuments(str var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; + var kgDbProvider = filter?.DbProvider ?? _settings.VectorDb.Provider; // Get doc meta data - var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter); + var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, kgDbProvider, filter); var files = pagedData.Items?.Select(x => new KnowledgeFileModel { @@ -282,9 +303,9 @@ public async Task> GetPagedKnowledgeDocuments(str FileSource = x.FileSource, FileExtension = Path.GetExtension(x.FileName), ContentType = x.ContentType, - FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName), + FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, kgDbProvider, x.FileId, x.FileName), RefData = x.RefData - })?.ToList() ?? new List(); + })?.ToList() ?? []; return new PagedItems { @@ -293,17 +314,17 @@ public async Task> GetPagedKnowledgeDocuments(str }; } - public async Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId) + public async Task GetKnowledgeFileBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null) { var db = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; + var kgDbProvider = options?.DbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider) ?? _settings.VectorDb.Provider; // Get doc binary data - var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, kgDbProvider, new KnowledgeFileFilter { Size = 1, - FileIds = [ fileId ] + FileIds = [fileId] }); var metaData = pageData?.Items?.FirstOrDefault(); @@ -317,7 +338,7 @@ public async Task GetKnowledgeDocumentBinaryData(string col }; }; - var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName); + var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, kgDbProvider, fileId, metaData.FileName); return new FileBinaryDataModel { FileName = metaData.FileName, @@ -328,11 +349,47 @@ public async Task GetKnowledgeDocumentBinaryData(string col #region Private methods - /// - /// Get file content type and file bytes - /// - /// - /// + private IVectorDb? GetVectorDb(string? dbProvider = null) + { + var provider = dbProvider.IfNullOrEmptyAs(_settings.VectorDb.Provider); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == provider); + return db; + } + + private async Task GetTextEmbedding(string collectionName) + { + return await KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); + } + + private async Task GetUserId() + { + var userIdentity = _services.GetRequiredService(); + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(userIdentity.Id); + return user.Id; + } + + private async Task ExistCollection(string collectionName, string? dbProvider) + { + var vectorDb = GetVectorDb(dbProvider); + if (vectorDb == null) + { + return false; + } + + var exist = await vectorDb.DoesCollectionExist(collectionName); + if (exist) return true; + + var db = _services.GetRequiredService(); + var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = [collectionName], + VectorStorageProviders = [vectorDb.Provider] + }); + + return !configs.IsNullOrEmpty(); + } + private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file) { if (file == null) @@ -357,8 +414,7 @@ public async Task GetKnowledgeDocumentBinaryData(string col return (string.Empty, BinaryData.Empty); } - #region Read doc content - private async Task> GetFileKnowledge(FileBinaryDataModel file, KnowledgeDocOptions? options) + private async Task> GetFileKnowledge(FileBinaryDataModel file, KnowledgeFileHandleOptions? options) { var processor = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(options?.Processor)); if (processor == null) @@ -369,17 +425,15 @@ private async Task> GetFileKnowledge(FileBinaryD var response = await processor.GetFileKnowledgeAsync(file, options: options); return response?.Success == true ? response.Knowledges ?? [] : []; } - #endregion - - private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData binary) + private bool SaveFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData binary) { - var fileStoreage = _services.GetRequiredService(); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, binary); + var fileStorage = _services.GetRequiredService(); + var saved = fileStorage.SaveKnowledgeBaseFile(collectionName, knowledgebaseProvider, fileId, fileName, binary); return saved; } - private async Task> SaveToVectorDb(string collectionName, IEnumerable contents, Dictionary? payload = null) + private async Task> SaveToKnowledgebase(string collectionName, string knowledgebaseProvider, IEnumerable contents, Dictionary? payload = null) { if (contents.IsNullOrEmpty()) { @@ -387,7 +441,7 @@ private async Task> SaveToVectorDb(string collectionName, IE } var dataIds = new List(); - var vectorDb = GetVectorDb(); + var vectorDb = GetVectorDb(knowledgebaseProvider); var textEmbedding = await GetTextEmbedding(collectionName); for (int i = 0; i < contents.Count(); i++) @@ -418,7 +472,7 @@ private async Task> SaveToVectorDb(string collectionName, IE private async Task HandleKnowledgeFiles( string collectionName, - string vectorStore, + string knowledgebaseProvider, IEnumerable knowledgeFiles, bool saveFile = false) { @@ -439,7 +493,7 @@ private async Task HandleKnowledgeFiles( // Save document if (saveFile) { - var saved = SaveDocument(collectionName, vectorStore, item.FileId, file.FileName, file.FileBinaryData); + var saved = SaveFile(collectionName, knowledgebaseProvider, item.FileId, file.FileName, file.FileBinaryData); if (!saved) { _logger.LogWarning($"Failed to save knowledge file: {file.FileName} to collection {collectionName}."); @@ -452,20 +506,20 @@ private async Task HandleKnowledgeFiles( var dataIds = new List(); foreach (var kg in item.FileKnowledges) { - var ids = await SaveToVectorDb(collectionName, kg.Contents, kg.Payload?.ToDictionary()); + var ids = await SaveToKnowledgebase(collectionName, knowledgebaseProvider, kg.Contents, kg.Payload?.ToDictionary()); dataIds.AddRange(ids); } if (!dataIds.IsNullOrEmpty()) { - await db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData + await db.SaveKnolwedgeBaseFileMeta(new KnowledgeFileMetaData { Collection = collectionName, FileId = item.FileId, FileName = file.FileName, FileSource = item.FileSource ?? VectorDataSource.File, ContentType = file.ContentType, - VectorStoreProvider = vectorStore, + VectorStoreProvider = knowledgebaseProvider, VectorDataIds = dataIds, CreateDate = DateTime.UtcNow, CreateUserId = userId diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs new file mode 100644 index 000000000..a95119b48 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Graph/SementicGraphKnowledgeBase.cs @@ -0,0 +1,63 @@ +using BotSharp.Abstraction.Graph.Options; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public class SementicGraphKnowledgeBase : IKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public SementicGraphKnowledgeBase( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string KnowledgeType => KnowledgeBaseType.SemanticGraph; + + public async Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) + { + var results = new List(); + + try + { + var graphDb = GetGraphDb(options?.DbProvider); + if (graphDb == null) + { + _logger.LogWarning($"Cannot find graph db provider '{options?.DbProvider}'."); + return results; + } + + var graphExecuteOptions = options as GraphKnowledgeExecuteOptions; + var graphOptions = new GraphQueryExecuteOptions + { + GraphId = graphExecuteOptions?.GraphId, + Arguments = options?.SearchArguments + }; + + var graphResult = await graphDb.ExecuteQueryAsync(query, graphOptions); + + results = graphResult?.Values?.Select(value => new KnowledgeExecuteResult + { + Id = Guid.NewGuid().ToString(), + Payload = value.ToDictionary(kvp => kvp.Key, kvp => new VectorPayloadValue(kvp.Value)) + })?.ToList() ?? []; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when searching graph knowledge. (Query: {query})"); + } + + return results; + } + + #region Private methods + private IGraphDb? GetGraphDb(string? provider = null) + { + var db = _services.GetServices().FirstOrDefault(x => x.Provider.IsEqualTo(provider)); + return db; + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs deleted file mode 100644 index aa758d796..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs) - { - var db = _services.GetRequiredService(); - var collections = configs.Collections ?? new(); - var saved = await db.AddKnowledgeCollectionConfigs(collections, reset: true); - return saved; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs deleted file mode 100644 index 4ea034487..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs +++ /dev/null @@ -1,77 +0,0 @@ -using BotSharp.Abstraction.Models; -using BotSharp.Abstraction.VectorStorage.Options; - -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty()) - { - return new(); - } - - var response = new SuccessFailResponse(); - var innerOptions = options.DistinctBy(x => x.FieldName).ToList(); - var vectorDb = GetVectorDb(); - foreach (var option in innerOptions) - { - var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option); - var field = $"{option.FieldName} ({option.FieldSchemaType})"; - if (created) - { - response.Success.Add(field); - } - else - { - _logger.LogError($"Failed to create vector collection payload index ({collectionName} => {field})."); - response.Fail.Add(field); - } - } - return response; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when creating vector collection payload index ({collectionName})."); - return new(); - } - } - - public async Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty()) - { - return new(); - } - - var response = new SuccessFailResponse(); - var innerOptions = options.DistinctBy(x => x.FieldName).ToList(); - var vectorDb = GetVectorDb(); - foreach (var option in innerOptions) - { - var deleted = await vectorDb.DeleteCollectionPayloadIndex(collectionName, option); - var field = $"{option.FieldName}"; - if (deleted) - { - response.Success.Add(field); - } - else - { - _logger.LogError($"Failed to deleting vector collection payload index ({collectionName}-{field})."); - response.Fail.Add(field); - } - } - return response; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting vector collection payload index ({collectionName})."); - return new(); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs deleted file mode 100644 index 4b8393321..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Snapshot.cs +++ /dev/null @@ -1,64 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> GetVectorCollectionSnapshots(string collectionName) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return Enumerable.Empty(); - } - - var db = GetVectorDb(); - var snapshots = await db.GetCollectionSnapshots(collectionName); - return snapshots; - } - - public async Task CreateVectorCollectionSnapshot(string collectionName) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return null; - } - - var db = GetVectorDb(); - var snapshot = await db.CreateCollectionShapshot(collectionName); - return snapshot; - } - - public async Task DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName) - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotFileName)) - { - return BinaryData.Empty; - } - - var db = GetVectorDb(); - var snapshot = await db.DownloadCollectionSnapshot(collectionName, snapshotFileName); - return snapshot; - } - - public async Task RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData) - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var db = GetVectorDb(); - var done = await db.RecoverCollectionFromShapshot(collectionName, snapshotFileName, snapshotData); - return done; - } - - public async Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName) - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(snapshotName)) - { - return false; - } - - var db = GetVectorDb(); - var done = await db.DeleteCollectionShapshot(collectionName, snapshotName); - return done; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs deleted file mode 100644 index 85f81a1e1..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ /dev/null @@ -1,364 +0,0 @@ -using BotSharp.Abstraction.VectorStorage.Filters; -using BotSharp.Abstraction.VectorStorage.Options; - -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - #region Collection - public async Task ExistVectorCollection(string collectionName) - { - var db = _services.GetRequiredService(); - var vectorDb = GetVectorDb(); - - var exist = await vectorDb.DoesCollectionExist(collectionName); - if (exist) return true; - - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionNames = [collectionName], - VectorStroageProviders = [_settings.VectorDb.Provider] - }); - - return !configs.IsNullOrEmpty(); - } - - public async Task CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var db = _services.GetRequiredService(); - var created = await db.AddKnowledgeCollectionConfigs(new List - { - new VectorCollectionConfig - { - Name = collectionName, - Type = collectionType, - VectorStore = new VectorStoreConfig - { - Provider = _settings.VectorDb.Provider - }, - TextEmbedding = new KnowledgeEmbeddingConfig - { - Provider = options.Provider, - Model = options.Model, - Dimension = options.Dimension - } - } - }); - - if (created) - { - var vectorDb = GetVectorDb(); - created = await vectorDb.CreateCollection(collectionName, options); - } - - return created; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when creating a vector collection ({collectionName})."); - return false; - } - } - - public async Task> GetVectorCollections(string? type = null) - { - try - { - var db = _services.GetRequiredService(); - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionTypes = !string.IsNullOrEmpty(type) ? [type] : null, - VectorStroageProviders = [_settings.VectorDb.Provider] - }); - - var vectorDb = GetVectorDb(); - if (vectorDb == null) - { - return []; - } - var dbCollections = await vectorDb.GetCollections(); - return configs.Where(x => dbCollections.Contains(x.Name)); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when getting vector db collections."); - return []; - } - } - - public async Task GetVectorCollectionDetails(string collectionName) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) return null; - - var db = _services.GetRequiredService(); - var configs = await db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter - { - CollectionNames = [collectionName] - }); - - var vectorDb = GetVectorDb(); - var details = await vectorDb.GetCollectionDetails(collectionName); - if (details != null) - { - details.BasicInfo = configs.FirstOrDefault(); - } - return details; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when getting vector db collection details."); - return null; - } - } - - public async Task DeleteVectorCollection(string collectionName) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var vectorDb = GetVectorDb(); - var deleted = await vectorDb.DeleteCollection(collectionName); - - if (deleted) - { - var db = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var vectorStoreProvider = _settings.VectorDb.Provider; - - await db.DeleteKnowledgeCollectionConfig(collectionName); - fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider); - await db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider); - } - - return deleted; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting collection ({collectionName})."); - return false; - } - } - #endregion - - #region Collection data - public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) - { - return false; - } - - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(create.Text); - - var db = GetVectorDb(); - var guid = Guid.NewGuid(); - var payload = create.Payload ?? new(); - - if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) - { - payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); - } - - return await db.Upsert(collectionName, guid, vector, create.Text, payload); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when creating vector collection data."); - return false; - } - } - - public async Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(update.Text) - || !Guid.TryParse(update.Id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, [guid]); - if (found.IsNullOrEmpty()) - { - return false; - } - - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(update.Text); - var payload = update.Payload ?? new(); - - if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) - { - - payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); - } - - return await db.Upsert(collectionName, guid, vector, update.Text, payload); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when updating vector collection data."); - return false; - } - } - - public async Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(update.Text) - || !Guid.TryParse(update.Id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, [guid], options: new() { WithVector = true, WithPayload = true }); - if (!found.IsNullOrEmpty()) - { - if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text) - { - // Only update payload - return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload); - } - } - - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(update.Text); - var payload = update.Payload ?? new(); - - if (!payload.TryGetValue(KnowledgePayloadName.DataSource, out _)) - { - payload[KnowledgePayloadName.DataSource] = VectorPayloadValue.BuildStringValue(VectorDataSource.Api); - } - - return await db.Upsert(collectionName, guid, vector, update.Text, payload); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when updating vector collection data."); - return false; - } - } - - public async Task DeleteVectorCollectionData(string collectionName, string id) - { - try - { - if (!Guid.TryParse(id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - return await db.DeleteCollectionData(collectionName, [guid]); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName}-{id})."); - return false; - } - } - - - public async Task DeleteVectorCollectionAllData(string collectionName) - { - try - { - var db = GetVectorDb(); - return await db.DeleteCollectionAllData(collectionName); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName})."); - return false; - } - } - - public async Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter) - { - try - { - var db = GetVectorDb(); - var pagedResult = await db.GetPagedCollectionData(collectionName, filter); - return new StringIdPagedItems - { - Count = pagedResult.Count, - Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)), - NextId = pagedResult.NextId, - }; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when getting vector knowledge collection data ({collectionName})."); - return new StringIdPagedItems(); - } - } - - public async Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty()) - { - return []; - } - - var pointIds = ids.Select(x => new { Id = x, IsValid = Guid.TryParse(x, out var guid), ParseResult = guid }) - .Where(x => x.IsValid) - .Select(x => x.ParseResult) - .ToList(); - - var db = GetVectorDb(); - var points = await db.GetCollectionData(collectionName, pointIds, options); - return points; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when querying vector collection {collectionName} points."); - return []; - } - } - - public async Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options) - { - try - { - var textEmbedding = await GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(query); - - // Vector search - var db = GetVectorDb(); - var found = await db.Search(collectionName, vector, options); - - var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); - return results; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when searching vector knowledge ({collectionName})."); - return []; - } - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs deleted file mode 100644 index 4a288e651..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ /dev/null @@ -1,39 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService : IKnowledgeService -{ - private readonly IServiceProvider _services; - private readonly IUserIdentity _user; - private readonly KnowledgeBaseSettings _settings; - private readonly ILogger _logger; - - public KnowledgeService( - IServiceProvider services, - IUserIdentity user, - KnowledgeBaseSettings settings, - ILogger logger) - { - _services = services; - _user = user; - _settings = settings; - _logger = logger; - } - - private IVectorDb GetVectorDb() - { - var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); - return db; - } - - private async Task GetTextEmbedding(string collectionName) - { - return await KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); - } - - private async Task GetUserId() - { - var userService = _services.GetRequiredService(); - var user = await userService.GetUser(_user.Id); - return user.Id; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeBase.cs new file mode 100644 index 000000000..584c508b5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/QuestionAnswer/QuestionAnswerKnowledgeBase.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public class QuestionAnswerKnowledgeBase : VectorKnowledgeBase, IKnowledgeService +{ + public override string KnowledgeType => KnowledgeBaseType.QuestionAnswer; + + public QuestionAnswerKnowledgeBase( + IServiceProvider services, + ILogger logger, + KnowledgeBaseSettings settings) + : base(services, logger, settings) + { + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs new file mode 100644 index 000000000..fc37a4f81 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/Taxonomy/TaxonomyKnowledgeBase.cs @@ -0,0 +1,108 @@ +using BotSharp.Abstraction.Entity; +using BotSharp.Abstraction.Entity.Models; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public class TaxonomyKnowledgeBase : IKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TaxonomyKnowledgeBase( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public string KnowledgeType => KnowledgeBaseType.Taxonomy; + + public async Task> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options) + { + var results = new List(); + + try + { + var analyzer = _services.GetServices() + .FirstOrDefault(x => x.Provider.IsEqualTo(options.DbProvider)); + + if (analyzer == null) + { + _logger.LogWarning($"Cannot find entity analyzer with provider '{options.DbProvider}'."); + return results; + } + + var taxonomyOptions = options as TaxonomyKnowledgeExecuteOptions; + var analysisOptions = new EntityAnalysisOptions + { + DataProviders = taxonomyOptions?.DataProviders, + Cutoff = options.Confidence.HasValue ? (double)options.Confidence.Value : null, + TopK = options.Limit, + MaxNgram = taxonomyOptions?.MaxNgram + }; + + var response = await analyzer.AnalyzeAsync(query, analysisOptions); + + if (!response.Success || response.Results == null) + { + return results; + } + + foreach (var result in response.Results) + { + var payload = new Dictionary + { + ["token"] = new VectorPayloadValue(result.Token) + }; + + if (!string.IsNullOrEmpty(result.CanonicalText)) + { + payload["canonical_text"] = new VectorPayloadValue(result.CanonicalText); + } + + foreach (var kvp in result.Data) + { + if (!payload.ContainsKey(kvp.Key)) + { + payload[kvp.Key] = new VectorPayloadValue(kvp.Value); + } + } + + double confidence = 0d; + if (result.Data.TryGetValue("confidence", out var conf)) + { + if (conf is double d) + { + confidence = d; + } + else if (conf is float f) + { + confidence = f; + } + } + + results.Add(new KnowledgeExecuteResult + { + Id = Guid.NewGuid().ToString(), + Payload = payload, + Score = confidence + }); + } + + if (options.Limit.HasValue) + { + results = results + .OrderByDescending(x => x.Score) + .Take(options.Limit.Value) + .ToList(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when searching taxonomy knowledge ({query})."); + } + + return results; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index e4aa6797f..1318824fd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -99,9 +99,9 @@ public async Task> GetKnowledgeCollectionCon filters.Add(builder.In(x => x.Type, filter.CollectionTypes)); } - if (!filter.VectorStroageProviders.IsNullOrEmpty()) + if (!filter.VectorStorageProviders.IsNullOrEmpty()) { - filters.Add(builder.In(x => x.VectorStore.Provider, filter.VectorStroageProviders)); + filters.Add(builder.In(x => x.VectorStore.Provider, filter.VectorStorageProviders)); } // Get data @@ -119,19 +119,19 @@ public async Task> GetKnowledgeCollectionCon #if !DEBUG [SharpCache(10)] #endif - public async Task GetKnowledgeCollectionConfig(string collectionName, string vectorStroageProvider) + public async Task GetKnowledgeCollectionConfig(string collectionName, string knowledgebaseProvider) { var configs = await GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter { CollectionNames = [collectionName], - VectorStroageProviders = [vectorStroageProvider] + VectorStorageProviders = [knowledgebaseProvider] }); return configs?.FirstOrDefault(); } #endregion #region Documents - public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) + public async Task SaveKnolwedgeBaseFileMeta(KnowledgeFileMetaData metaData) { if (metaData == null || string.IsNullOrWhiteSpace(metaData.Collection) @@ -159,10 +159,10 @@ public async Task SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) return true; } - public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } @@ -171,7 +171,7 @@ public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, strin var filters = new List>() { builder.Eq(x => x.Collection, collectionName), - builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + builder.Eq(x => x.VectorStoreProvider, knowledgebaseProvider) }; if (fileId != null) @@ -183,19 +183,19 @@ public async Task DeleteKnolwedgeBaseFileMeta(string collectionName, strin return res.DeletedCount > 0; } - public async Task> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + public async Task> GetKnowledgeBaseFileMeta(string collectionName, string knowledgebaseProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { - return new PagedItems(); + return new PagedItems(); } var builder = Builders.Filter; var docFilters = new List>() { builder.Eq(x => x.Collection, collectionName), - builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider) + builder.Eq(x => x.VectorStoreProvider, knowledgebaseProvider) }; // Apply filters @@ -237,7 +237,7 @@ public async Task> GetKnowledgeBaseFileMeta(str var docs = docsTask.Result.ToList(); var count = countTask.Result; - var files = docs?.Select(x => new KnowledgeDocMetaData + var files = docs?.Select(x => new KnowledgeFileMetaData { Collection = x.Collection, FileId = x.FileId, @@ -251,7 +251,7 @@ public async Task> GetKnowledgeBaseFileMeta(str CreateUserId = x.CreateUserId })?.ToList() ?? []; - return new PagedItems + return new PagedItems { Items = files, Count = count diff --git a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs index a13c093ee..bab3c2cfc 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs @@ -21,7 +21,6 @@ public SqlSecondaryStageFn( public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); var knowledgeSettings = _services.GetRequiredService(); var states = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs index ab4b01f2d..6a3ec36ae 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs @@ -21,7 +21,6 @@ public SecondaryStagePlanFn( public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); var knowledgeSettings = _services.GetRequiredService(); var states = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index cbca32ae5..c315dfe31 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -346,7 +346,7 @@ public async Task DeleteCollectionAllData(string collectionName) #endregion #region Payload index - public async Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) + public async Task CreateCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) { var exist = await DoesCollectionExist(collectionName); if (!exist) @@ -360,7 +360,7 @@ public async Task CreateCollectionPayloadIndex(string collectionName, Crea return result.Status == UpdateStatus.Completed; } - public async Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) + public async Task DeleteCollectionPayloadIndex(string collectionName, CollectionIndexOptions options) { var exist = await DoesCollectionExist(collectionName); if (!exist) @@ -832,17 +832,18 @@ public async Task DeleteCollectionShapshot(string collectionName, string s }; } - private SearchParams? BuildSearchParam(VectorSearchParamModel? param) + private SearchParams? BuildSearchParam(Dictionary? param) { if (param == null - || param.ExactSearch == null) + || !param.TryGetValue("exact_search", out var exactSearchStr) + || !bool.TryParse(exactSearchStr, out var exactSearch)) { return null; } var search = new SearchParams { - Exact = param.ExactSearch.Value + Exact = exactSearch }; return search; } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs index b7955f653..2d23665c9 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -6,4 +6,5 @@ global using BotSharp.Abstraction.VectorStorage.Enums; global using BotSharp.Abstraction.VectorStorage.Models; global using BotSharp.Abstraction.Knowledges.Enums; -global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file +global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs index 47b607a3e..aa5186f60 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -1,11 +1,7 @@ using static Dapper.SqlMapper; -using Microsoft.Extensions.Logging; -using BotSharp.Core.Infrastructures; using MySqlConnector; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Plugin.SqlDriver.Models; using BotSharp.Abstraction.Models; namespace BotSharp.Plugin.SqlDriver.Services; @@ -26,8 +22,10 @@ public DbKnowledgeService( public async Task Import(ImportDbKnowledgeRequest request) { var sqlDriverSettings = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); var settingService = _services.GetRequiredService(); + var kg = _services.GetServices() + .FirstOrDefault(x => x.KnowledgeType.IsEqualTo(KnowledgeBaseType.QuestionAnswer)); + var provider = request.Provider ?? "openai"; var model = request.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o); var schema = request.Schema; @@ -71,7 +69,9 @@ public async Task Import(ImportDbKnowledgeRequest request) foreach (var item in knowledges) { - await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + if (kg == null) continue; + + await kg.CreateCollectionData(collectionName, new KnowledgeCreateModel { Text = item.Question, Payload = new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs index 319abb901..d780e2055 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs @@ -20,6 +20,7 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Knowledges; global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Knowledges.Options; global using BotSharp.Abstraction.Settings; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Instructs; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs index 4ae82e9e2..bc81f9af9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs @@ -2,17 +2,17 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData) + public bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } try { - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var dir = $"{docDir}/{fileId}"; if (ExistDirectory(dir)) { @@ -26,20 +26,20 @@ public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvi catch (Exception ex) { _logger.LogWarning(ex, $"Error when saving knowledge file " + - $"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})."); + $"(Vector store provider: {knowledgebaseProvider}, Collection: {collectionName}, File name: {fileName})."); return false; } } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider)) + || string.IsNullOrWhiteSpace(knowledgebaseProvider)) { return false; } - var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var dir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); if (!ExistDirectory(dir)) return false; if (fileId == null) @@ -58,16 +58,16 @@ public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvide return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return string.Empty; } - var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName); + var docDir = BuildKnowledgeCollectionFileDir(knowledgebaseProvider, collectionName); var fileDir = $"{docDir}/{fileId}"; if (!ExistDirectory(fileDir)) { @@ -77,10 +77,10 @@ public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreP return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ; } - public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { if (string.IsNullOrWhiteSpace(collectionName) - || string.IsNullOrWhiteSpace(vectorStoreProvider) + || string.IsNullOrWhiteSpace(knowledgebaseProvider) || string.IsNullOrWhiteSpace(fileName)) { return BinaryData.Empty; @@ -88,7 +88,7 @@ public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string v try { - var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider); + var docDir = BuildKnowledgeCollectionFileDir(collectionName, knowledgebaseProvider); var fileDir = $"{docDir}/{fileId}"; if (!ExistDirectory(fileDir)) { @@ -106,7 +106,7 @@ public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string v } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})"); + _logger.LogWarning(ex, $"Error when downloading collection file ({collectionName}-{knowledgebaseProvider}-{fileId}-{fileName})"); return BinaryData.Empty; } } diff --git a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs index 10236d0c5..4c87ff2e5 100644 --- a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs +++ b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs @@ -114,23 +114,23 @@ public BinaryData GetSpeechFile(string conversationId, string fileName) return BinaryData.FromBytes(new byte[] { 0x03, 0x04, 0x05 }); } - public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, + public bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData) { return true; } - public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null) + public bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null) { return true; } - public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName) + public string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { return $"https://fakeurl.com/{fileName}"; } - public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, + public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName) { return BinaryData.FromBytes(new byte[] { 0x06, 0x07, 0x08 });