Features/refine knowledge service

src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs

@@ -7,7 +7,7 @@ public class EntityAnalysisOptions
     /// </summary>
     [JsonPropertyName("data_providers")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public List<string>? DataProviders { get; set; }
+    public IEnumerable<string>? DataProviders { get; set; }
 
     /// <summary>
     /// Maximum n-gram size

src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs

@@ -62,17 +62,9 @@ public interface IFileStorageService
     #endregion
 
     #region Knowledge
-    bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData);
-
-    /// <summary>
-    /// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection.
-    /// </summary>
-    /// <param name="collectionName"></param>
-    /// <param name="vectorStoreProvider"></param>
-    /// <param name="fileId"></param>
-    /// <returns></returns>
-    bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
-    string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
-    BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
+    bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData);
+    bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null);
+    string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName);
+    BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs → src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs

@@ -1,7 +1,9 @@
 namespace BotSharp.Abstraction.Knowledges.Enums;
 
-public static class KnowledgeCollectionType
+public static class KnowledgeBaseType
 {
     public static string QuestionAnswer = "question-answer";
     public static string Document = "document";
+    public static string Taxonomy = "taxonomy";
+    public static string SemanticGraph = "semantic-graph";
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs

@@ -2,12 +2,10 @@ namespace BotSharp.Abstraction.Knowledges.Filters;
 
 public class KnowledgeFileFilter : Pagination
 {
+    public string? DbProvider { get; set; }
     public IEnumerable<Guid>? FileIds { get; set; }
-
     public IEnumerable<string>? FileNames { get; set; }
-
     public IEnumerable<string>? ContentTypes { get; set; }
-
     public IEnumerable<string>? FileSources { get; set; }
 
     public KnowledgeFileFilter()

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs

@@ -0,0 +1,65 @@
+using BotSharp.Abstraction.Knowledges.Filters;
+using BotSharp.Abstraction.Knowledges.Responses;
+
+namespace BotSharp.Abstraction.Knowledges;
+
+public interface IKnowledgeFileOrchestrator
+{
+    string Provider { get; }
+
+    /// <summary>
+    /// Save files and their contents to knowledgebase
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="files"></param>
+    /// <param name="options"></param>
+    /// <returns></returns>
+    Task<UploadKnowledgeResponse> UploadFilesToKnowledge(string collectionName,
+        IEnumerable<ExternalFileModel> files, KnowledgeFileHandleOptions? options = null);
+
+    /// <summary>
+    /// Import file content to knowledgebase without saving the file
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileName"></param>
+    /// <param name="fileSource"></param>
+    /// <param name="contents"></param>
+    /// <param name="options"></param>
+    /// <returns></returns>
+    Task<bool> ImportFileContentToKnowledge(string collectionName, string fileName, string fileSource,
+        IEnumerable<string> contents, ImportKnowledgeFileOptions? options = null);
+
+    /// <summary>
+    /// Delete one file and its related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileId"></param>
+    /// <param name="options"></param>
+    /// <returns></returns>
+    Task<bool> DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null);
+
+    /// <summary>
+    /// Delete all files and their related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="filter"></param>
+    /// <returns></returns>
+    Task<bool> DeleteKnowledgeFiles(string collectionName, KnowledgeFileFilter filter);
+
+    /// <summary>
+    /// Get knowledge files by pagination
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="filter"></param>
+    /// <returns></returns>
+    Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeFiles(string collectionName, KnowledgeFileFilter filter);
+
+    /// <summary>
+    /// Get knowledge file binary data
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileId"></param>
+    /// <param name="options"></param>
+    /// <returns></returns>
+    Task<FileBinaryDataModel> GetKnowledgeFileBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null);
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -1,83 +1,61 @@
-using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.Graph.Options;
-using BotSharp.Abstraction.Knowledges.Filters;
+using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.Knowledges.Options;
-using BotSharp.Abstraction.Knowledges.Responses;
-using BotSharp.Abstraction.VectorStorage.Models;
-using BotSharp.Abstraction.VectorStorage.Options;
 
 namespace BotSharp.Abstraction.Knowledges;
 
 public interface IKnowledgeService
 {
-    #region Vector
-    Task<bool> ExistVectorCollection(string collectionName);
-    Task<bool> CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options);
-    Task<bool> DeleteVectorCollection(string collectionName);
-    Task<IEnumerable<VectorCollectionConfig>> GetVectorCollections(string? type = null);
-    Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
-    Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
-    Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
-    Task<IEnumerable<VectorCollectionData>> GetVectorCollectionData(string collectionName, IEnumerable<string> ids, VectorQueryOptions? options = null);
-    Task<bool> DeleteVectorCollectionData(string collectionName, string id);
-    Task<bool> DeleteVectorCollectionAllData(string collectionName);
-    Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
-    Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
-    Task<bool> UpsertVectorCollectionData(string collectionName, VectorUpdateModel update);
-    #endregion
+    string KnowledgeType { get; }
 
-    #region Document
-    /// <summary>
-    /// Save documents and their contents to knowledgebase
-    /// </summary>
-    /// <param name="collectionName"></param>
-    /// <param name="files"></param>
-    /// <param name="option"></param>
-    /// <returns></returns>
-    Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files, KnowledgeDocOptions? options = null);
-    /// <summary>
-    /// Save document content to knowledgebase without saving the document
-    /// </summary>
-    /// <param name="collectionName"></param>
-    /// <param name="fileName"></param>
-    /// <param name="fileSource"></param>
-    /// <param name="contents"></param>
-    /// <param name="refData"></param>
-    /// <returns></returns>
-    Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable<string> contents,
-        DocMetaRefData? refData = null, Dictionary<string, VectorPayloadValue>? payload = null);
-    /// <summary>
-    /// Delete one document and its related knowledge in the collection
-    /// </summary>
-    /// <param name="collectionName"></param>
-    /// <param name="fileId"></param>
-    /// <returns></returns>
-    Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
-    /// <summary>
-    /// Delete all documents and their related knowledge in the collection
-    /// </summary>
-    /// <param name="collectionName"></param>
-    /// <param name="filter"></param>
-    /// <returns></returns>
-    Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
-    Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
-    Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
+    #region Collection
+    Task<bool> ExistCollection(string collectionName, KnowledgeCollectionOptions options)
+        => Task.FromResult(false);
+    Task<bool> CreateCollection(string collectionName, CollectionCreateOptions options)
+        => Task.FromResult(false);
+    Task<bool> DeleteCollection(string collectionName, KnowledgeCollectionOptions options)
+        => Task.FromResult(false);
+    Task<IEnumerable<KnowledgeCollectionConfig>> GetCollections(KnowledgeCollectionOptions options)
+        => Task.FromResult(Enumerable.Empty<KnowledgeCollectionConfig>());
+    Task<KnowledgeCollectionDetails?> GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options)
+        => Task.FromResult<KnowledgeCollectionDetails?>(null);
     #endregion
 
-    #region Snapshot
-    Task<IEnumerable<VectorCollectionSnapshot>> GetVectorCollectionSnapshots(string collectionName);
-    Task<VectorCollectionSnapshot?> CreateVectorCollectionSnapshot(string collectionName);
-    Task<BinaryData> DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName);
-    Task<bool> RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData);
-    Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
+    #region Data
+    Task<IEnumerable<KnowledgeExecuteResult>> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options)
+        => Task.FromResult(Enumerable.Empty<KnowledgeExecuteResult>());
+    Task<StringIdPagedItems<KnowledgeCollectionData>> GetPagedCollectionData(string collectionName, KnowledgeFilter filter)
+        => Task.FromResult(new StringIdPagedItems<KnowledgeCollectionData>());
+    Task<IEnumerable<KnowledgeCollectionData>> GetCollectionData(string collectionName, IEnumerable<string> ids, KnowledgeQueryOptions? options = null)
+        => Task.FromResult(Enumerable.Empty<KnowledgeCollectionData>());
+    Task<bool> DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options)
+        => Task.FromResult(false);
+    Task<bool> DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options)
+        => Task.FromResult(false);
+    Task<bool> CreateCollectionData(string collectionName, KnowledgeCreateModel create)
+        => Task.FromResult(false);
+    Task<bool> UpdateCollectionData(string collectionName, KnowledgeUpdateModel update)
+        => Task.FromResult(false);
+    Task<bool> UpsertCollectionData(string collectionName, KnowledgeUpdateModel update)
+        => Task.FromResult(false);
     #endregion
 
     #region Index
-    Task<SuccessFailResponse<string>> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable<CreateVectorCollectionIndexOptions> options);
-    Task<SuccessFailResponse<string>> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable<DeleteVectorCollectionIndexOptions> options);
+    Task<SuccessFailResponse<string>> CreateIndexes(string collectionName, KnowledgeIndexOptions options)
+        => Task.FromResult(new SuccessFailResponse<string>());
+    Task<SuccessFailResponse<string>> DeleteIndexes(string collectionName, KnowledgeIndexOptions options)
+        => Task.FromResult(new SuccessFailResponse<string>());
     #endregion
 
-    #region Common
-    Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
+    #region Snapshot
+    Task<IEnumerable<KnowledgeCollectionSnapshot>> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null)
+        => Task.FromResult(Enumerable.Empty<KnowledgeCollectionSnapshot>());
+    Task<KnowledgeCollectionSnapshot?> CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null)
+        => Task.FromResult<KnowledgeCollectionSnapshot?>(null);
+    Task<BinaryData> DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null)
+        => Task.FromResult(new BinaryData(Array.Empty<byte>()));
+    Task<bool> RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null)
+        => Task.FromResult(false);
+    Task<bool> DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null)
+        => Task.FromResult(false);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs

@@ -0,0 +1,7 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeCollectionConfig : VectorCollectionConfig
+{
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs

@@ -0,0 +1,14 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeCollectionData
+{
+    public string Id { get; set; }
+    public Dictionary<string, VectorPayloadValue> Payload { get; set; } = new();
+    public Dictionary<string, object> Data => Payload?.ToDictionary(x => x.Key, x => x.Value.DataValue) ?? [];
+    public double? Score { get; set; }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public float[]? Vector { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeCollectionDetails
+{
+    [JsonPropertyName("status")]
+    public string Status { get; set; }
+
+    [JsonPropertyName("payload_schema")]
+    public List<PayloadSchemaDetail> PayloadSchema { get; set; } = [];
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionSnapshotViewModel.cs → src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionSnapshot.cs

@@ -1,30 +1,22 @@
 using BotSharp.Abstraction.VectorStorage.Models;
-using System.Text.Json.Serialization;
 
-namespace BotSharp.OpenAPI.ViewModels.Knowledges;
+namespace BotSharp.Abstraction.Knowledges.Models;
 
-public class VectorCollectionSnapshotViewModel
+public class KnowledgeCollectionSnapshot
 {
-    [JsonPropertyName("name")]
     public string Name { get; set; } = default!;
-
-    [JsonPropertyName("size")]
     public long Size { get; set; }
-
-    [JsonPropertyName("created_time")]
     public DateTime CreatedTime { get; set; }
-
-    [JsonPropertyName("check_sum")]
     public string? CheckSum { get; set; }
 
-    public static VectorCollectionSnapshotViewModel? From(VectorCollectionSnapshot? model)
+    public static KnowledgeCollectionSnapshot? CopyFrom(VectorCollectionSnapshot? model)
     {
         if (model == null)
         {
             return null;
         }
 
-        return new VectorCollectionSnapshotViewModel
+        return new KnowledgeCollectionSnapshot
         {
             Name = model.Name,
             Size = model.Size,

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreateModel.cs

@@ -0,0 +1,9 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeCreateModel : KnowledgeOptionBase
+{
+    public string Text { get; set; }
+    public Dictionary<string, VectorPayloadValue>? Payload { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeExecuteResult.cs

@@ -0,0 +1,20 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeExecuteResult : KnowledgeCollectionData
+{
+    public KnowledgeExecuteResult()
+    {
+
+    }
+
+    public static KnowledgeExecuteResult CopyFrom(KnowledgeCollectionData data)
+    {
+        return new KnowledgeExecuteResult
+        {
+            Id = data.Id,
+            Payload = data.Payload,
+            Score = data.Score,
+            Vector = data.Vector
+        };
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs → src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileMetaData.cs

@@ -1,6 +1,6 @@
 namespace BotSharp.Abstraction.Knowledges.Models;
 
-public class KnowledgeDocMetaData
+public class KnowledgeFileMetaData
 {
     [JsonPropertyName("collection")]
     public string Collection { get; set; }