From efa2395f654cc2979e5df621760b8fdf34b36775 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 19 May 2026 20:57:03 +0800 Subject: [PATCH 01/11] async worker --- CMakeLists.txt | 1 + src/binding/async_workers.cc | 56 ++++++++++++++++++++++++++ src/binding/async_workers.h | 48 ++++++++++++++++++++++ src/binding/collection.cc | 52 ++++++++++++++++++++++++ src/binding/collection.h | 4 ++ src/binding/types.h | 78 ++++++++++++++++++------------------ src/index.d.ts | 15 +++++++ 7 files changed, 216 insertions(+), 38 deletions(-) create mode 100644 src/binding/async_workers.cc create mode 100644 src/binding/async_workers.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 65cecf0..b9cc5a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -270,6 +270,7 @@ endif() add_library(${PROJECT_NAME} SHARED ${CMAKE_JS_SRC} src/binding/addon.cc + src/binding/async_workers.cc src/binding/collection.cc src/binding/config.cc src/binding/doc.cc diff --git a/src/binding/async_workers.cc b/src/binding/async_workers.cc new file mode 100644 index 0000000..cbea0a4 --- /dev/null +++ b/src/binding/async_workers.cc @@ -0,0 +1,56 @@ +#include "async_workers.h" +#include "types.h" + + +namespace binding { + + +DeleteByFilterWorker::DeleteByFilterWorker(Napi::Env env, + zvec::Collection::Ptr collection, + std::string filter, + Napi::Promise::Deferred deferred) + : Napi::AsyncWorker(env), + collection_(collection), + filter_(std::move(filter)), + deferred_(deferred) {} + +void DeleteByFilterWorker::Execute() { + status_ = collection_->DeleteByFilter(filter_); +} + +void DeleteByFilterWorker::OnOK() { + Napi::Env env = Env(); + deferred_.Resolve(CreateStatusObject(env, status_)); +} + +void DeleteByFilterWorker::OnError(const Napi::Error &error) { + deferred_.Reject(error.Value()); +} + + +OptimizeWorker::OptimizeWorker(Napi::Env env, zvec::Collection::Ptr collection, + zvec::OptimizeOptions options, + Napi::Promise::Deferred deferred) + : Napi::AsyncWorker(env), + collection_(collection), + options_(options), + deferred_(deferred) {} + +void OptimizeWorker::Execute() { + status_ = collection_->Optimize(options_); +} + +void OptimizeWorker::OnOK() { + Napi::Env env = Env(); + if (status_.ok()) { + deferred_.Resolve(env.Undefined()); + } else { + RejectIfNotOk(env, status_, deferred_); + } +} +void OptimizeWorker::OnError(const Napi::Error &error) { + deferred_.Reject(error.Value()); +} + + +} // namespace binding diff --git a/src/binding/async_workers.h b/src/binding/async_workers.h new file mode 100644 index 0000000..4e5e067 --- /dev/null +++ b/src/binding/async_workers.h @@ -0,0 +1,48 @@ +#pragma once + + +#include +#include +#include "zvec/db/collection.h" +#include "zvec/db/status.h" + + +namespace binding { + + +class DeleteByFilterWorker : public Napi::AsyncWorker { + public: + DeleteByFilterWorker(Napi::Env env, zvec::Collection::Ptr collection, + std::string filter, Napi::Promise::Deferred deferred); + + void Execute() override; + void OnOK() override; + void OnError(const Napi::Error &error) override; + + private: + zvec::Collection::Ptr collection_; + std::string filter_; + zvec::Status status_; + Napi::Promise::Deferred deferred_; +}; + + +class OptimizeWorker : public Napi::AsyncWorker { + public: + OptimizeWorker(Napi::Env env, zvec::Collection::Ptr collection, + zvec::OptimizeOptions options, + Napi::Promise::Deferred deferred); + + void Execute() override; + void OnOK() override; + void OnError(const Napi::Error &error) override; + + private: + zvec::Collection::Ptr collection_; + zvec::OptimizeOptions options_; + zvec::Status status_; + Napi::Promise::Deferred deferred_; +}; + + +} // namespace binding diff --git a/src/binding/collection.cc b/src/binding/collection.cc index d1b80b6..3713444 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -1,4 +1,5 @@ #include "collection.h" +#include "async_workers.h" #include "doc.h" #include "params.h" #include "schema.h" @@ -153,9 +154,11 @@ Napi::Object Collection::Init(Napi::Env env, Napi::Object exports, InstanceMethod("updateSync", &Collection::Update), InstanceMethod("deleteSync", &Collection::Delete), InstanceMethod("deleteByFilterSync", &Collection::DeleteByFilter), + InstanceMethod("deleteByFilter", &Collection::DeleteByFilterAsync), InstanceMethod("_internalQuery", &Collection::Query), InstanceMethod("fetchSync", &Collection::Fetch), InstanceMethod("optimizeSync", &Collection::Optimize), + InstanceMethod("optimize", &Collection::OptimizeAsync), InstanceMethod("closeSync", &Collection::Close), InstanceMethod("destroySync", &Collection::Destroy), InstanceMethod("addColumnSync", &Collection::AddColumn), @@ -516,6 +519,24 @@ Napi::Value Collection::DeleteByFilter(const Napi::CallbackInfo &info) { } +Napi::Value Collection::DeleteByFilterAsync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1 || !info[0].IsString()) { + ThrowIfNotOk( + env, zvec::Status::InvalidArgument( + "Collection.deleteByFilter(): Expected exactly 1 argument. " + "Argument must be a string")); + return env.Undefined(); + } + std::string filter = info[0].As().Utf8Value(); + auto deferred = Napi::Promise::Deferred::New(env); + auto *worker = + new DeleteByFilterWorker(env, collection_, std::move(filter), deferred); + worker->Queue(); + return deferred.Promise(); +} + + Napi::Value Collection::Query(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (info.Length() != 1) { @@ -630,6 +651,37 @@ Napi::Value Collection::Optimize(const Napi::CallbackInfo &info) { } +Napi::Value Collection::OptimizeAsync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() > 1) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.optimize(): Expected 0 to 1 argument. " + "Argument must be an OptimizeOptions object.")); + return env.Undefined(); + } + auto options{zvec::OptimizeOptions{}}; + if (info.Length() == 1) { + if (!info[0].IsObject()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.optimize(): Expected 0 to 1 argument. " + "Argument must be an OptimizeOptions object.")); + return env.Undefined(); + } + auto parsed_options = ParseOptimizeOptions(info[0].As()); + if (parsed_options) { + options = parsed_options.value(); + } else { + ThrowIfNotOk(env, parsed_options.error()); + return env.Undefined(); + } + } + auto deferred = Napi::Promise::Deferred::New(env); + auto *worker = new OptimizeWorker(env, collection_, options, deferred); + worker->Queue(); + return deferred.Promise(); +} + + Napi::Value Collection::Close(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (info.Length() != 0) { diff --git a/src/binding/collection.h b/src/binding/collection.h index 5ccfd28..de7b8fc 100644 --- a/src/binding/collection.h +++ b/src/binding/collection.h @@ -53,12 +53,16 @@ class Collection : public Napi::ObjectWrap { Napi::Value DeleteByFilter(const Napi::CallbackInfo &info); + Napi::Value DeleteByFilterAsync(const Napi::CallbackInfo &info); + Napi::Value Query(const Napi::CallbackInfo &info); Napi::Value Fetch(const Napi::CallbackInfo &info); Napi::Value Optimize(const Napi::CallbackInfo &info); + Napi::Value OptimizeAsync(const Napi::CallbackInfo &info); + Napi::Value Close(const Napi::CallbackInfo &info); Napi::Value Destroy(const Napi::CallbackInfo &info); diff --git a/src/binding/types.h b/src/binding/types.h index 598fa50..b1dfde9 100644 --- a/src/binding/types.h +++ b/src/binding/types.h @@ -9,68 +9,70 @@ namespace binding { -inline void ThrowIfNotOk(const Napi::Env &env, const zvec::Status &status) { - if (status.ok()) return; - - Napi::Error error; +inline Napi::Error CreateZVecError(const Napi::Env &env, + const zvec::Status &status) { + auto error = Napi::Error::New(env, status.message()); + auto errorValue = error.Value(); switch (status.code()) { case zvec::StatusCode::NOT_FOUND: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "NotFoundError"; - error.Value()["code"] = "ZVEC_NOT_FOUND"; + errorValue["name"] = "NotFoundError"; + errorValue["code"] = "ZVEC_NOT_FOUND"; break; case zvec::StatusCode::ALREADY_EXISTS: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "AlreadyExistsError"; - error.Value()["code"] = "ZVEC_ALREADY_EXISTS"; + errorValue["name"] = "AlreadyExistsError"; + errorValue["code"] = "ZVEC_ALREADY_EXISTS"; break; case zvec::StatusCode::INVALID_ARGUMENT: - error = Napi::TypeError::New(env, status.message()); - error.Value()["name"] = "InvalidArgumentError"; - error.Value()["code"] = "ZVEC_INVALID_ARGUMENT"; + errorValue["name"] = "InvalidArgumentError"; + errorValue["code"] = "ZVEC_INVALID_ARGUMENT"; break; case zvec::StatusCode::PERMISSION_DENIED: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "PermissionDeniedError"; - error.Value()["code"] = "ZVEC_PERMISSION_DENIED"; + errorValue["name"] = "PermissionDeniedError"; + errorValue["code"] = "ZVEC_PERMISSION_DENIED"; break; case zvec::StatusCode::FAILED_PRECONDITION: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "FailedPreconditionError"; - error.Value()["code"] = "ZVEC_FAILED_PRECONDITION"; + errorValue["name"] = "FailedPreconditionError"; + errorValue["code"] = "ZVEC_FAILED_PRECONDITION"; break; case zvec::StatusCode::RESOURCE_EXHAUSTED: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "ResourceExhaustedError"; - error.Value()["code"] = "ZVEC_RESOURCE_EXHAUSTED"; + errorValue["name"] = "ResourceExhaustedError"; + errorValue["code"] = "ZVEC_RESOURCE_EXHAUSTED"; break; case zvec::StatusCode::UNAVAILABLE: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "UnavailableError"; - error.Value()["code"] = "ZVEC_UNAVAILABLE"; + errorValue["name"] = "UnavailableError"; + errorValue["code"] = "ZVEC_UNAVAILABLE"; break; case zvec::StatusCode::INTERNAL_ERROR: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "InternalError"; - error.Value()["code"] = "ZVEC_INTERNAL_ERROR"; + errorValue["name"] = "InternalError"; + errorValue["code"] = "ZVEC_INTERNAL_ERROR"; break; case zvec::StatusCode::NOT_SUPPORTED: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "NotSupportedError"; - error.Value()["code"] = "ZVEC_NOT_SUPPORTED"; + errorValue["name"] = "NotSupportedError"; + errorValue["code"] = "ZVEC_NOT_SUPPORTED"; break; case zvec::StatusCode::UNKNOWN: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "UnknownError"; - error.Value()["code"] = "ZVEC_UNKNOWN"; + errorValue["name"] = "UnknownError"; + errorValue["code"] = "ZVEC_UNKNOWN"; break; default: - error = Napi::Error::New(env, status.message()); - error.Value()["name"] = "InvalidStatusCodeError"; - error.Value()["code"] = "ZVEC_INVALID_STATUS_CODE"; + errorValue["name"] = "InvalidStatusCodeError"; + errorValue["code"] = "ZVEC_INVALID_STATUS_CODE"; break; } - error.ThrowAsJavaScriptException(); + return error; +} + + +inline void ThrowIfNotOk(const Napi::Env &env, const zvec::Status &status) { + if (status.ok()) return; + CreateZVecError(env, status).ThrowAsJavaScriptException(); +} + + +inline void RejectIfNotOk(const Napi::Env &env, const zvec::Status &status, + Napi::Promise::Deferred &deferred) { + if (status.ok()) return; + deferred.Reject(CreateZVecError(env, status).Value()); } diff --git a/src/index.d.ts b/src/index.d.ts index 7b06946..35dea3a 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -909,6 +909,13 @@ export declare class ZVecCollection { */ deleteByFilterSync(filter: string): ZVecStatus; + /** + * Asynchronously deletes documents based on a filter expression. + * @param filter - A string representing the filter expression. + * @returns A promise that resolves with the status of the operation. + */ + deleteByFilter(filter: string): Promise; + /** * Performs a vector similarity search query. * @param params - The query parameters. @@ -930,6 +937,14 @@ export declare class ZVecCollection { */ optimizeSync(options?: ZVecOptimizeOptions): void; + /** + * Asynchronously optimizes the collection's internal structures for better performance. + * @param options - Optional parameters for the operation. + * @returns A promise that resolves when optimization is complete. + * @rejects {ZVecError} If the operation fails. + */ + optimize(options?: ZVecOptimizeOptions): Promise; + /** * Closes the collection and releases its resources. * After calling this method, the collection object should not be used. From 46b68b398b658874c53f48a596491a7b0d433880 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 19 May 2026 23:36:30 +0800 Subject: [PATCH 02/11] update --- src/binding/async_workers.cc | 38 ++++++++++++++++++++++++++++++++++++ src/binding/async_workers.h | 20 +++++++++++++++++++ src/binding/collection.cc | 26 +++++++++++++++++++++++- src/binding/collection.h | 2 ++ src/index.d.ts | 11 +++++++++-- src/index.js | 20 +++++++++++++------ 6 files changed, 108 insertions(+), 9 deletions(-) diff --git a/src/binding/async_workers.cc b/src/binding/async_workers.cc index cbea0a4..4e0f9d1 100644 --- a/src/binding/async_workers.cc +++ b/src/binding/async_workers.cc @@ -1,4 +1,5 @@ #include "async_workers.h" +#include "doc.h" #include "types.h" @@ -28,6 +29,43 @@ void DeleteByFilterWorker::OnError(const Napi::Error &error) { } +QueryWorker::QueryWorker(Napi::Env env, zvec::Collection::Ptr collection, + zvec::CollectionSchema::Ptr schema, + zvec::VectorQuery query, + Napi::Promise::Deferred deferred) + : Napi::AsyncWorker(env), + collection_(collection), + schema_(schema), + query_(std::move(query)), + deferred_(deferred) {} + +void QueryWorker::Execute() { + auto res = collection_->Query(query_); + if (res) { + results_ = std::move(res.value()); + } else { + status_ = res.error(); + } +} + +void QueryWorker::OnOK() { + Napi::Env env = Env(); + if (status_.ok()) { + Napi::Array array = Napi::Array::New(env); + for (size_t i = 0; i < results_.size(); i++) { + array.Set(i, CreateDoc(env, schema_, results_[i])); + } + deferred_.Resolve(array); + } else { + RejectIfNotOk(env, status_, deferred_); + } +} + +void QueryWorker::OnError(const Napi::Error &error) { + deferred_.Reject(error.Value()); +} + + OptimizeWorker::OptimizeWorker(Napi::Env env, zvec::Collection::Ptr collection, zvec::OptimizeOptions options, Napi::Promise::Deferred deferred) diff --git a/src/binding/async_workers.h b/src/binding/async_workers.h index 4e5e067..71ad3ad 100644 --- a/src/binding/async_workers.h +++ b/src/binding/async_workers.h @@ -27,6 +27,26 @@ class DeleteByFilterWorker : public Napi::AsyncWorker { }; +class QueryWorker : public Napi::AsyncWorker { + public: + QueryWorker(Napi::Env env, zvec::Collection::Ptr collection, + zvec::CollectionSchema::Ptr schema, zvec::VectorQuery query, + Napi::Promise::Deferred deferred); + + void Execute() override; + void OnOK() override; + void OnError(const Napi::Error &error) override; + + private: + zvec::Collection::Ptr collection_; + zvec::CollectionSchema::Ptr schema_; + zvec::VectorQuery query_; + Napi::Promise::Deferred deferred_; + zvec::Status status_; + zvec::DocPtrList results_; +}; + + class OptimizeWorker : public Napi::AsyncWorker { public: OptimizeWorker(Napi::Env env, zvec::Collection::Ptr collection, diff --git a/src/binding/collection.cc b/src/binding/collection.cc index 3713444..4a0bbfd 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -156,6 +156,7 @@ Napi::Object Collection::Init(Napi::Env env, Napi::Object exports, InstanceMethod("deleteByFilterSync", &Collection::DeleteByFilter), InstanceMethod("deleteByFilter", &Collection::DeleteByFilterAsync), InstanceMethod("_internalQuery", &Collection::Query), + InstanceMethod("_internalQueryAsync", &Collection::QueryAsync), InstanceMethod("fetchSync", &Collection::Fetch), InstanceMethod("optimizeSync", &Collection::Optimize), InstanceMethod("optimize", &Collection::OptimizeAsync), @@ -542,7 +543,7 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) { if (info.Length() != 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.query(): Expected exactly 1 argument. " - "Argument must be an Query object")); + "Argument must be a Query object")); return env.Undefined(); } @@ -567,6 +568,29 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) { } +Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.query(): Expected exactly 1 argument. " + "Argument must be a Query object")); + return env.Undefined(); + } + + if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema()); + parsed_query) { + auto deferred = Napi::Promise::Deferred::New(env); + auto *worker = new QueryWorker(env, collection_, get_wrapped_schema(), + std::move(parsed_query.value()), deferred); + worker->Queue(); + return deferred.Promise(); + } else { + ThrowIfNotOk(env, parsed_query.error()); + return env.Undefined(); + } +} + + Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (info.Length() != 1) { diff --git a/src/binding/collection.h b/src/binding/collection.h index de7b8fc..f95566e 100644 --- a/src/binding/collection.h +++ b/src/binding/collection.h @@ -57,6 +57,8 @@ class Collection : public Napi::ObjectWrap { Napi::Value Query(const Napi::CallbackInfo &info); + Napi::Value QueryAsync(const Napi::CallbackInfo &info); + Napi::Value Fetch(const Napi::CallbackInfo &info); Napi::Value Optimize(const Napi::CallbackInfo &info); diff --git a/src/index.d.ts b/src/index.d.ts index 35dea3a..4e0c9e8 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -917,12 +917,19 @@ export declare class ZVecCollection { deleteByFilter(filter: string): Promise; /** - * Performs a vector similarity search query. + * Performs a search query against the collection. * @param params - The query parameters. - * @returns An array of documents ranked by similarity to the query vector. + * @returns An array of documents matching the query. */ querySync(params: ZVecQuery): ZVecDoc[]; + /** + * Asynchronously performs a search query against the collection. + * @param params - The query parameters. + * @returns A promise that resolves with an array of documents matching the query. + */ + query(params: ZVecQuery): Promise; + /** * Fetches documents by their IDs. * @param ids - A single document ID or an array of document IDs to fetch. diff --git a/src/index.js b/src/index.js index e257a5d..be9ba72 100644 --- a/src/index.js +++ b/src/index.js @@ -22,15 +22,15 @@ try { } -binding.Collection.prototype.querySync = function (queryObj) { - if (arguments.length !== 1) { - const err = new Error("Collection.querySync(): Expected exactly 1 argument. Argument must be an Query object"); +function validateQueryArg(methodName, queryObj, argCount) { + if (argCount !== 1) { + const err = new Error(`Collection.${methodName}(): Expected exactly 1 argument. Argument must be an Query object`); err.name = "InvalidArgumentError"; err.code = "ZVEC_INVALID_ARGUMENT"; throw err; } if (queryObj === null || typeof queryObj !== 'object') { - const err = new Error("Collection.querySync(): Expected exactly 1 argument. Argument must be an Query object"); + const err = new Error(`Collection.${methodName}(): Expected exactly 1 argument. Argument must be an Query object`); err.name = "InvalidArgumentError"; err.code = "ZVEC_INVALID_ARGUMENT"; throw err; @@ -40,9 +40,17 @@ binding.Collection.prototype.querySync = function (queryObj) { err.name = "NotSupportedError"; err.code = "ZVEC_NOT_SUPPORTED"; throw err; - } else { - return this._internalQuery(queryObj); } +} + +binding.Collection.prototype.querySync = function (queryObj) { + validateQueryArg('querySync', queryObj, arguments.length); + return this._internalQuery(queryObj); +}; + +binding.Collection.prototype.query = function (queryObj) { + validateQueryArg('query', queryObj, arguments.length); + return this._internalQueryAsync(queryObj); }; From b25abdeca5f92cb766912a60e098150f36e2c047 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Wed, 20 May 2026 15:54:39 +0800 Subject: [PATCH 03/11] update --- src/binding/collection.cc | 50 +++++++++++++++++++++++++++++---------- src/binding/collection.h | 6 ++--- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/binding/collection.cc b/src/binding/collection.cc index 4a0bbfd..44cf5d9 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -199,7 +199,6 @@ zvec::Collection::Ptr Collection::get_wrapped_collection() { zvec::CollectionSchema::Ptr Collection::get_wrapped_schema() { - std::shared_lock lock(schema_lock_); return schema_; } @@ -216,7 +215,6 @@ void Collection::set_wrapped_collection(zvec::Collection::Ptr collection) { void Collection::set_wrapped_schema(Napi::Env &env) { - std::unique_lock lock(schema_lock_); if (auto schema = collection_->Schema(); schema) { schema_ = std::make_shared(schema.value()); } else { @@ -225,18 +223,32 @@ void Collection::set_wrapped_schema(Napi::Env &env) { } +bool Collection::ThrowIfClosed(Napi::Env &env) { + if (collection_) { + return false; + } else { + ThrowIfNotOk(env, zvec::Status(zvec::StatusCode::FAILED_PRECONDITION, + "Collection is closed")); + return true; + } +} + + Napi::Value Collection::Path(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (auto path = collection_->Path(); path) { - return Napi::String::New(info.Env(), path.value()); + return Napi::String::New(env, path.value()); } else { - ThrowIfNotOk(info.Env(), path.error()); - return info.Env().Undefined(); + ThrowIfNotOk(env, path.error()); + return env.Undefined(); } } Napi::Value Collection::Schema(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (auto schema = collection_->Schema(); schema) { auto constructors = get_constructors(env); if (!constructors) { @@ -257,6 +269,7 @@ Napi::Value Collection::Schema(const Napi::CallbackInfo &info) { Napi::Value Collection::Options(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (auto options = collection_->Options(); options) { return CreateCollectionOptions(env, options.value()); } else { @@ -268,6 +281,7 @@ Napi::Value Collection::Options(const Napi::CallbackInfo &info) { Napi::Value Collection::Stats(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (auto stats = collection_->Stats(); stats) { auto obj = Napi::Object::New(env); obj.Set("docCount", stats.value().doc_count); @@ -286,6 +300,7 @@ Napi::Value Collection::Stats(const Napi::CallbackInfo &info) { Napi::Value Collection::Insert(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -341,6 +356,7 @@ Napi::Value Collection::Insert(const Napi::CallbackInfo &info) { Napi::Value Collection::Upsert(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -396,6 +412,7 @@ Napi::Value Collection::Upsert(const Napi::CallbackInfo &info) { Napi::Value Collection::Update(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -451,6 +468,7 @@ Napi::Value Collection::Update(const Napi::CallbackInfo &info) { Napi::Value Collection::Delete(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -508,6 +526,7 @@ Napi::Value Collection::Delete(const Napi::CallbackInfo &info) { Napi::Value Collection::DeleteByFilter(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsString()) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -522,6 +541,7 @@ Napi::Value Collection::DeleteByFilter(const Napi::CallbackInfo &info) { Napi::Value Collection::DeleteByFilterAsync(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsString()) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -540,6 +560,7 @@ Napi::Value Collection::DeleteByFilterAsync(const Napi::CallbackInfo &info) { Napi::Value Collection::Query(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.query(): Expected exactly 1 argument. " @@ -570,6 +591,7 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) { Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.query(): Expected exactly 1 argument. " @@ -593,6 +615,7 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1) { ThrowIfNotOk( env, zvec::Status::InvalidArgument( @@ -648,6 +671,7 @@ Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { Napi::Value Collection::Optimize(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() > 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.optimize(): Expected 0 to 1 argument. " @@ -667,7 +691,7 @@ Napi::Value Collection::Optimize(const Napi::CallbackInfo &info) { options = parsed_options.value(); } else { ThrowIfNotOk(env, parsed_options.error()); - return Napi::Object::New(env); + return env.Undefined(); } } ThrowIfNotOk(env, collection_->Optimize(options)); @@ -677,6 +701,7 @@ Napi::Value Collection::Optimize(const Napi::CallbackInfo &info) { Napi::Value Collection::OptimizeAsync(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() > 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.optimize(): Expected 0 to 1 argument. " @@ -713,7 +738,6 @@ Napi::Value Collection::Close(const Napi::CallbackInfo &info) { "Collection.close(): Expected no argument")); return env.Undefined(); } - std::unique_lock lock(ddl_lock_); collection_ = nullptr; return env.Undefined(); } @@ -721,18 +745,21 @@ Napi::Value Collection::Close(const Napi::CallbackInfo &info) { Napi::Value Collection::Destroy(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 0) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.destroy(): Expected no argument")); return env.Undefined(); } ThrowIfNotOk(env, collection_->Destroy()); + collection_ = nullptr; return env.Undefined(); } Napi::Value Collection::AddColumn(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsObject()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.addColumn(): Expected exactly 1 " @@ -782,7 +809,6 @@ Napi::Value Collection::AddColumn(const Napi::CallbackInfo &info) { return env.Undefined(); } } - std::unique_lock lock(ddl_lock_); ThrowIfNotOk(env, collection_->AddColumn(parsed_field_schema.value(), expression, options)); set_wrapped_schema(env); @@ -792,6 +818,7 @@ Napi::Value Collection::AddColumn(const Napi::CallbackInfo &info) { Napi::Value Collection::DropColumn(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsString()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.dropColumn(): Expected exactly 1 " @@ -799,7 +826,6 @@ Napi::Value Collection::DropColumn(const Napi::CallbackInfo &info) { return env.Undefined(); } auto field_name = info[0].As().Utf8Value(); - std::unique_lock lock(ddl_lock_); ThrowIfNotOk(env, collection_->DropColumn(field_name)); set_wrapped_schema(env); return env.Undefined(); @@ -808,6 +834,7 @@ Napi::Value Collection::DropColumn(const Napi::CallbackInfo &info) { Napi::Value Collection::AlterColumn(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsObject()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.alterColumn(): Expected exactly 1 " @@ -882,7 +909,6 @@ Napi::Value Collection::AlterColumn(const Napi::CallbackInfo &info) { "Collection.alterColumn(): 'newColumnName' and " "'fieldSchema' are mutually exclusive")); } else { - std::unique_lock lock(ddl_lock_); ThrowIfNotOk(env, collection_->AlterColumn(column_name, new_column_name, field_schema, options)); set_wrapped_schema(env); @@ -893,6 +919,7 @@ Napi::Value Collection::AlterColumn(const Napi::CallbackInfo &info) { Napi::Value Collection::CreateIndex(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsObject()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.createIndex(): Expected exactly 1 " @@ -946,7 +973,6 @@ Napi::Value Collection::CreateIndex(const Napi::CallbackInfo &info) { } } - std::unique_lock lock(ddl_lock_); ThrowIfNotOk(env, collection_->CreateIndex( field_name, parsed_index_params.value(), options)); set_wrapped_schema(env); @@ -956,6 +982,7 @@ Napi::Value Collection::CreateIndex(const Napi::CallbackInfo &info) { Napi::Value Collection::DropIndex(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + if (ThrowIfClosed(env)) return env.Undefined(); if (info.Length() != 1 || !info[0].IsString()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.dropIndex(): Expected exactly 1 " @@ -963,7 +990,6 @@ Napi::Value Collection::DropIndex(const Napi::CallbackInfo &info) { return env.Undefined(); } auto field_name = info[0].As().Utf8Value(); - std::unique_lock lock(ddl_lock_); ThrowIfNotOk(env, collection_->DropIndex(field_name)); set_wrapped_schema(env); return env.Undefined(); diff --git a/src/binding/collection.h b/src/binding/collection.h index f95566e..1f48f6a 100644 --- a/src/binding/collection.h +++ b/src/binding/collection.h @@ -2,8 +2,6 @@ #include -#include -#include #include "zvec/db/collection.h" #include "addon.h" @@ -80,10 +78,10 @@ class Collection : public Napi::ObjectWrap { Napi::Value DropIndex(const Napi::CallbackInfo &info); + bool ThrowIfClosed(Napi::Env &env); + zvec::Collection::Ptr collection_{nullptr}; zvec::CollectionSchema::Ptr schema_{nullptr}; - mutable std::shared_mutex schema_lock_; - mutable std::mutex ddl_lock_{}; }; From 669ad869d7d2223455e7473e53e5ff8c3f3757e1 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Wed, 20 May 2026 20:05:02 +0800 Subject: [PATCH 04/11] update code , need to write ut --- src/binding/collection.cc | 56 ++++++++++++++++++++++----------------- src/index.js | 6 ++++- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/binding/collection.cc b/src/binding/collection.cc index 44cf5d9..00fa75f 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -542,15 +542,17 @@ Napi::Value Collection::DeleteByFilter(const Napi::CallbackInfo &info) { Napi::Value Collection::DeleteByFilterAsync(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (ThrowIfClosed(env)) return env.Undefined(); + auto deferred = Napi::Promise::Deferred::New(env); if (info.Length() != 1 || !info[0].IsString()) { - ThrowIfNotOk( - env, zvec::Status::InvalidArgument( - "Collection.deleteByFilter(): Expected exactly 1 argument. " - "Argument must be a string")); - return env.Undefined(); + RejectIfNotOk( + env, + zvec::Status::InvalidArgument( + "Collection.deleteByFilter(): Expected exactly 1 argument. " + "Argument must be a string"), + deferred); + return deferred.Promise(); } std::string filter = info[0].As().Utf8Value(); - auto deferred = Napi::Promise::Deferred::New(env); auto *worker = new DeleteByFilterWorker(env, collection_, std::move(filter), deferred); worker->Queue(); @@ -592,23 +594,25 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) { Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (ThrowIfClosed(env)) return env.Undefined(); + auto deferred = Napi::Promise::Deferred::New(env); if (info.Length() != 1) { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.query(): Expected exactly 1 argument. " - "Argument must be a Query object")); - return env.Undefined(); + RejectIfNotOk(env, + zvec::Status::InvalidArgument( + "Collection.query(): Expected exactly 1 argument. " + "Argument must be a Query object"), + deferred); + return deferred.Promise(); } if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema()); parsed_query) { - auto deferred = Napi::Promise::Deferred::New(env); auto *worker = new QueryWorker(env, collection_, get_wrapped_schema(), std::move(parsed_query.value()), deferred); worker->Queue(); return deferred.Promise(); } else { - ThrowIfNotOk(env, parsed_query.error()); - return env.Undefined(); + RejectIfNotOk(env, parsed_query.error(), deferred); + return deferred.Promise(); } } @@ -702,29 +706,33 @@ Napi::Value Collection::Optimize(const Napi::CallbackInfo &info) { Napi::Value Collection::OptimizeAsync(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (ThrowIfClosed(env)) return env.Undefined(); + auto deferred = Napi::Promise::Deferred::New(env); if (info.Length() > 1) { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.optimize(): Expected 0 to 1 argument. " - "Argument must be an OptimizeOptions object.")); - return env.Undefined(); + RejectIfNotOk(env, + zvec::Status::InvalidArgument( + "Collection.optimize(): Expected 0 to 1 argument. " + "Argument must be an OptimizeOptions object."), + deferred); + return deferred.Promise(); } auto options{zvec::OptimizeOptions{}}; if (info.Length() == 1) { if (!info[0].IsObject()) { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.optimize(): Expected 0 to 1 argument. " - "Argument must be an OptimizeOptions object.")); - return env.Undefined(); + RejectIfNotOk(env, + zvec::Status::InvalidArgument( + "Collection.optimize(): Expected 0 to 1 argument. " + "Argument must be an OptimizeOptions object."), + deferred); + return deferred.Promise(); } auto parsed_options = ParseOptimizeOptions(info[0].As()); if (parsed_options) { options = parsed_options.value(); } else { - ThrowIfNotOk(env, parsed_options.error()); - return env.Undefined(); + RejectIfNotOk(env, parsed_options.error(), deferred); + return deferred.Promise(); } } - auto deferred = Napi::Promise::Deferred::New(env); auto *worker = new OptimizeWorker(env, collection_, options, deferred); worker->Queue(); return deferred.Promise(); diff --git a/src/index.js b/src/index.js index be9ba72..80d54db 100644 --- a/src/index.js +++ b/src/index.js @@ -49,7 +49,11 @@ binding.Collection.prototype.querySync = function (queryObj) { }; binding.Collection.prototype.query = function (queryObj) { - validateQueryArg('query', queryObj, arguments.length); + try { + validateQueryArg('query', queryObj, arguments.length); + } catch (err) { + return Promise.reject(err); + } return this._internalQueryAsync(queryObj); }; From 8614bc09f58d8ac9e60df75a5e4ecf08bb924a4e Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Thu, 21 May 2026 16:25:14 +0800 Subject: [PATCH 05/11] update --- jest.config.json | 3 +- package-lock.json | 6 +- tests/collection/lifecycle.test.ts | 233 +++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 tests/collection/lifecycle.test.ts diff --git a/jest.config.json b/jest.config.json index 96fdb68..611abf0 100644 --- a/jest.config.json +++ b/jest.config.json @@ -2,6 +2,7 @@ "preset": "ts-jest", "testEnvironment": "node", "testMatch": [ - "/tests/*.test.ts" + "/tests/*.test.ts", + "/tests/**/*.test.ts" ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0a52639..447590e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4271,9 +4271,9 @@ } }, "node_modules/typedoc/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", "dev": true, "license": "MIT", "dependencies": { diff --git a/tests/collection/lifecycle.test.ts b/tests/collection/lifecycle.test.ts new file mode 100644 index 0000000..45093dd --- /dev/null +++ b/tests/collection/lifecycle.test.ts @@ -0,0 +1,233 @@ +import * as fs from 'fs'; +import { + isZVecError, + ZVecCollection, + ZVecCollectionSchema, + ZVecCreateAndOpen, + ZVecDataType, + ZVecHnswIndexParams, + ZVecIndexType, + ZVecInitialize, + ZVecLogLevel, + ZVecLogType, + ZVecMetricType, + ZVecOpen, + ZVecQuantizeType, + ZVecStatus +} from '../../src/index'; + + +ZVecInitialize({ + logType: ZVecLogType.CONSOLE, + logLevel: ZVecLogLevel.WARN, +}); + + +describe('Collection Lifecycle', () => { + const collectionPath = './test_lifecycle_collection'; + const collectionName = 'lifecycle_test'; + + const schema = new ZVecCollectionSchema({ + name: collectionName, + vectors: { + name: 'embedding', + dataType: ZVecDataType.VECTOR_FP32, + dimension: 8, + indexParams: { + indexType: ZVecIndexType.HNSW, + metricType: ZVecMetricType.COSINE, + m: 16, + quantizeType: ZVecQuantizeType.FP16 + } + }, + fields: { + name: 'label', + dataType: ZVecDataType.STRING, + nullable: true, + indexParams: { indexType: ZVecIndexType.INVERT } + } + }); + + beforeAll(() => { + if (fs.existsSync(collectionPath)) { + fs.rmSync(collectionPath, { recursive: true, force: true }); + } + }); + + afterAll(() => { + if (fs.existsSync(collectionPath)) { + fs.rmSync(collectionPath, { recursive: true, force: true }); + } + }); + + + describe('create', () => { + it('should create a new collection on disk', () => { + const collection = ZVecCreateAndOpen(collectionPath, schema); + expect(collection).toBeDefined(); + expect(collection.path).toBe(collectionPath); + expect(fs.existsSync(collectionPath)).toBe(true); + collection.closeSync(); + }); + + it('should reflect the schema correctly after creation', () => { + const collection = ZVecOpen(collectionPath); + + expect(collection.schema.name).toBe(collectionName); + + const vectors = collection.schema.vectors(); + expect(vectors.length).toBe(1); + expect(vectors[0].name).toBe('embedding'); + expect(vectors[0].dataType).toBe(ZVecDataType.VECTOR_FP32); + expect(vectors[0].dimension).toBe(8); + expect(vectors[0].indexParams!.indexType).toBe(ZVecIndexType.HNSW); + expect(vectors[0].indexParams!.metricType).toBe(ZVecMetricType.COSINE); + expect((vectors[0].indexParams as ZVecHnswIndexParams).m).toBe(16); + expect((vectors[0].indexParams as ZVecHnswIndexParams).quantizeType).toBe(ZVecQuantizeType.FP16); + + const fields = collection.schema.fields(); + expect(fields.length).toBe(1); + expect(fields[0].name).toBe('label'); + expect(fields[0].dataType).toBe(ZVecDataType.STRING); + expect(fields[0].nullable).toBe(true); + expect(fields[0].indexParams!.indexType).toBe(ZVecIndexType.INVERT); + + expect(collection.stats.docCount).toBe(0); + + collection.closeSync(); + }); + + it('should fail to create over an existing collection path', () => { + try { + ZVecCreateAndOpen(collectionPath, schema); + fail('Expected an error to be thrown'); + } catch (error) { + expect(isZVecError(error)).toBe(true); + } + }); + }); + + + describe('open', () => { + it('should open an existing collection', () => { + const collection = ZVecOpen(collectionPath); + expect(collection).toBeDefined(); + expect(collection.path).toBe(collectionPath); + expect(collection.schema.name).toBe(collectionName); + expect(collection.stats.docCount).toBe(0); + collection.closeSync(); + }); + + it('should fail to open a non-existing path', () => { + try { + ZVecOpen('./non_existing_path'); + fail('Expected an error to be thrown'); + } catch (error) { + expect(isZVecError(error)).toBe(true); + } + }); + }); + + + describe('basic operations after open', () => { + let collection: ZVecCollection; + + beforeAll(() => { + collection = ZVecOpen(collectionPath); + }); + + afterAll(() => { + collection.closeSync(); + }); + + it('should insert documents', () => { + const result = collection.insertSync([ + { + id: 'a1', + vectors: { embedding: [1, 0, 0, 0, 0, 0, 0, 0] }, + fields: { label: 'first' } + }, + { + id: 'a2', + vectors: { embedding: [0, 1, 0, 0, 0, 0, 0, 0] }, + fields: { label: 'second' } + }, + { + id: 'a3', + vectors: { embedding: [0, 0, 1, 0, 0, 0, 0, 0] }, + fields: { label: 'third' } + } + ]); + expect(result.every((r: ZVecStatus) => r.ok)).toBe(true); + expect(collection.stats.docCount).toBe(3); + }); + + it('should fetch inserted documents', () => { + const fetched = collection.fetchSync(['a1', 'a2']); + expect(fetched['a1'].fields.label).toBe('first'); + expect(fetched['a2'].fields.label).toBe('second'); + }); + + it('should query by vector similarity', () => { + const results = collection.querySync({ + fieldName: 'embedding', + vector: [1, 0, 0, 0, 0, 0, 0, 0], + topk: 1, + }); + expect(results[0].id).toBe('a1'); + }); + + it('should delete a document', () => { + const result = collection.deleteSync('a3'); + expect(result.ok).toBe(true); + expect(collection.stats.docCount).toBe(2); + + const fetched = collection.fetchSync('a3'); + expect('a3' in fetched).toBe(false); + }); + }); + + + describe('close', () => { + it('should close without error', () => { + const collection = ZVecOpen(collectionPath); + expect(() => collection.closeSync()).not.toThrow(); + }); + + it('should throw when using collection after close', () => { + const collection = ZVecOpen(collectionPath); + collection.closeSync(); + + const ops: [string, () => void][] = [ + ['insertSync', () => collection.insertSync({ + id: 'should_fail', + vectors: { embedding: [0, 0, 0, 0, 0, 0, 0, 0] }, + fields: { label: 'fail' } + })], + ['fetchSync', () => collection.fetchSync('a1')], + ['querySync', () => collection.querySync({ + fieldName: 'embedding', + vector: [1, 0, 0, 0, 0, 0, 0, 0], + })], + ]; + + for (const [name, op] of ops) { + try { + op(); + fail(`${name} should have thrown`); + } catch (error) { + expect(isZVecError(error)).toBe(true); + } + } + }); + }); + + + describe('destroy', () => { + it('should remove the collection from disk', () => { + const collection = ZVecOpen(collectionPath); + collection.destroySync(); + expect(fs.existsSync(collectionPath)).toBe(false); + }); + }); +}); From 99a4d6c42ede49053146d16f3ad2b668da6b97f0 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Fri, 22 May 2026 11:29:29 +0800 Subject: [PATCH 06/11] update ut --- tests/collection/columns.test.ts | 231 +++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 tests/collection/columns.test.ts diff --git a/tests/collection/columns.test.ts b/tests/collection/columns.test.ts new file mode 100644 index 0000000..0010120 --- /dev/null +++ b/tests/collection/columns.test.ts @@ -0,0 +1,231 @@ +import * as fs from 'fs'; +import { + isZVecError, + ZVecCollection, + ZVecCollectionSchema, + ZVecCreateAndOpen, + ZVecDataType, + ZVecIndexType, + ZVecInitialize, + ZVecLogLevel, + ZVecLogType, + ZVecMetricType, +} from '../../src/index'; + + +ZVecInitialize({ + logType: ZVecLogType.CONSOLE, + logLevel: ZVecLogLevel.WARN, +}); + + +describe('Collection Columns', () => { + const collectionPath = './test_columns_collection'; + const dimension = 17; + let collection: ZVecCollection; + + const makeEmbedding = (hotIndex: number) => + Array.from({ length: dimension }, (_, j) => (j === hotIndex % dimension ? 1 : 0.1)); + + beforeAll(() => { + if (fs.existsSync(collectionPath)) { + fs.rmSync(collectionPath, { recursive: true, force: true }); + } + + const schema = new ZVecCollectionSchema({ + name: 'columns_test', + vectors: { + name: 'embedding', + dataType: ZVecDataType.VECTOR_FP32, + dimension, + indexParams: { + indexType: ZVecIndexType.HNSW, + metricType: ZVecMetricType.COSINE, + } + }, + fields: { + name: 'tag', + dataType: ZVecDataType.ARRAY_STRING, + nullable: true, + indexParams: { indexType: ZVecIndexType.INVERT } + } + }); + + collection = ZVecCreateAndOpen(collectionPath, schema); + + const docs = Array.from({ length: 20 }, (_, i) => ({ + id: `doc${i + 1}`, + vectors: { embedding: makeEmbedding(i) }, + fields: { tag: [`category_${i % 5}`] } + })); + collection.insertSync(docs); + collection.optimizeSync(); + }); + + afterAll(() => { + if (collection) { + collection.destroySync(); + } + if (fs.existsSync(collectionPath)) { + fs.rmSync(collectionPath, { recursive: true, force: true }); + } + }); + + + describe('addColumn', () => { + it('should add a new scalar column', () => { + collection.addColumnSync({ + fieldSchema: { + name: 'price', + dataType: ZVecDataType.INT64, + nullable: true, + indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: true } + } + }); + + const fields = collection.schema.fields(); + expect(fields.length).toBe(2); + + const priceField = collection.schema.field('price'); + expect(priceField.name).toBe('price'); + expect(priceField.dataType).toBe(ZVecDataType.INT64); + expect(priceField.nullable).toBe(true); + expect(priceField.indexParams!.indexType).toBe(ZVecIndexType.INVERT); + + const tagField = collection.schema.field('tag'); + expect(tagField.name).toBe('tag'); + expect(tagField.dataType).toBe(ZVecDataType.ARRAY_STRING); + expect(tagField.nullable).toBe(true); + expect(tagField.indexParams!.indexType).toBe(ZVecIndexType.INVERT); + + const vectors = collection.schema.vectors(); + expect(vectors.length).toBe(1); + expect(vectors[0].name).toBe('embedding'); + expect(vectors[0].dimension).toBe(17); + }); + + it('should support insert, query, and delete with the new column', () => { + const insertResults = collection.insertSync([ + { + id: 'doc21', + vectors: { embedding: makeEmbedding(3) }, + fields: { tag: ['tech'], price: 199 } + }, + { + id: 'doc22', + vectors: { embedding: makeEmbedding(5) }, + fields: { tag: ['art'], price: 50 } + }, + { + id: 'doc23', + vectors: { embedding: makeEmbedding(7) }, + fields: { tag: ['science'], price: 320 } + }, + ]); + expect(insertResults.every(r => r.ok)).toBe(true); + + const fetched = collection.fetchSync(['doc21', 'doc22', 'doc23']); + expect(fetched['doc21'].fields['price']).toBe(199); + expect(fetched['doc22'].fields['price']).toBe(50); + expect(fetched['doc23'].fields['price']).toBe(320); + + const queryResults = collection.querySync({ filter: 'price > 100' }); + const ids = queryResults.map(r => r.id); + expect(ids).toContain('doc21'); + expect(ids).toContain('doc23'); + expect(ids).not.toContain('doc22'); + + expect(collection.deleteSync('doc5').ok).toBe(true); + expect(collection.deleteSync('doc22').ok).toBe(true); + + const afterDelete = collection.fetchSync(['doc5', 'doc22']); + expect('doc5' in afterDelete).toBe(false); + expect('doc22' in afterDelete).toBe(false); + + expect(collection.fetchSync('doc1')['doc1']).toBeDefined(); + expect(collection.fetchSync('doc21')['doc21']).toBeDefined(); + }); + + it('should throw on duplicate column name', () => { + try { + collection.addColumnSync({ + fieldSchema: { name: 'price', dataType: ZVecDataType.UINT32 } + }); + fail('Expected an error to be thrown'); + } catch (error) { + expect(isZVecError(error)).toBe(true); + if (isZVecError(error)) { + expect(error.code).toBe('ZVEC_INVALID_ARGUMENT'); + } + } + }); + }); + + + describe('alterColumn', () => { + it('should rename an existing column', () => { + collection.alterColumnSync({ + columnName: 'price', + newColumnName: 'cost' + }); + + expect(collection.schema.fields().length).toBe(2); + const costField = collection.schema.field('cost'); + expect(costField.name).toBe('cost'); + expect(costField.dataType).toBe(ZVecDataType.INT64); + }); + + it('should not find the old column name after rename', () => { + try { + collection.schema.field('price'); + fail('Expected an error to be thrown'); + } catch (error) { + expect(isZVecError(error)).toBe(true); + if (isZVecError(error)) { + expect(error.code).toBe('ZVEC_NOT_FOUND'); + } + } + }); + + it('should preserve data under the new column name', () => { + const fetched = collection.fetchSync(['doc21', 'doc23']); + expect(fetched['doc21'].fields['cost']).toBe(199); + expect(fetched['doc23'].fields['cost']).toBe(320); + }); + }); + + + describe('dropColumn', () => { + it('should remove an existing column', () => { + collection.dropColumnSync('cost'); + expect(collection.schema.fields().length).toBe(1); + }); + + it('should throw when dropping a non-existing column', () => { + try { + collection.dropColumnSync('cost'); + fail('Expected an error to be thrown'); + } catch (error) { + expect(isZVecError(error)).toBe(true); + if (isZVecError(error)) { + expect(error.code).toBe('ZVEC_INVALID_ARGUMENT'); + } + } + }); + + it('should still allow fetching and querying after column drop', () => { + const fetched = collection.fetchSync(['doc1', 'doc10', 'doc21']); + expect(fetched['doc1'].fields['tag']).toEqual(['category_0']); + expect(fetched['doc10'].fields['tag']).toEqual(['category_4']); + expect(fetched['doc21'].fields['tag']).toEqual(['tech']); + expect('cost' in fetched['doc21'].fields).toBe(false); + + const queryResult = collection.querySync({ + fieldName: 'embedding', + vector: makeEmbedding(0), + topk: 1, + }); + expect(queryResult[0].id).toBe('doc1'); + }); + }); +}); From d1d3c7096d895f5efe38758d74738c9af2e883eb Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Fri, 22 May 2026 11:37:09 +0800 Subject: [PATCH 07/11] update --- src/binding/collection.cc | 2 + tests/collectionDDL.test.ts | 313 ------------------------------------ 2 files changed, 2 insertions(+), 313 deletions(-) delete mode 100644 tests/collectionDDL.test.ts diff --git a/src/binding/collection.cc b/src/binding/collection.cc index 00fa75f..ebef24a 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -747,6 +747,7 @@ Napi::Value Collection::Close(const Napi::CallbackInfo &info) { return env.Undefined(); } collection_ = nullptr; + schema_ = nullptr; return env.Undefined(); } @@ -761,6 +762,7 @@ Napi::Value Collection::Destroy(const Napi::CallbackInfo &info) { } ThrowIfNotOk(env, collection_->Destroy()); collection_ = nullptr; + schema_ = nullptr; return env.Undefined(); } diff --git a/tests/collectionDDL.test.ts b/tests/collectionDDL.test.ts deleted file mode 100644 index 6307d36..0000000 --- a/tests/collectionDDL.test.ts +++ /dev/null @@ -1,313 +0,0 @@ -import * as fs from 'fs'; -import { - isZVecError, - ZVecCollection, - ZVecCollectionSchema, - ZVecCreateAndOpen, - ZVecDataType, - ZVecHnswIndexParams, - ZVecIndexType, - ZVecInitialize, - ZVecLogLevel, - ZVecLogType, - ZVecMetricType, - ZVecOpen, - ZVecQuantizeType -} from '../src/index'; - - -ZVecInitialize({ - logType: ZVecLogType.CONSOLE, - logLevel: ZVecLogLevel.WARN, -}); - - -describe('Collection Data Definition Operations', () => { - const testCollectionName = 'test_ddl_collection'; - const testCollectionPath = './test_ddl_collection'; - - - beforeAll(() => { - if (fs.existsSync(testCollectionPath)) { - fs.rmSync(testCollectionPath, { recursive: true, force: true }); - } - }); - - afterAll(() => { - if (fs.existsSync(testCollectionPath)) { - fs.rmSync(testCollectionPath, { recursive: true, force: true }); - } - }); - - - it('should create a simple collection successfully', () => { - const schema = new ZVecCollectionSchema({ - name: testCollectionName, - vectors: { - name: 'vector', - dataType: ZVecDataType.VECTOR_FP32, - dimension: 10, - indexParams: { - indexType: ZVecIndexType.HNSW, - metricType: ZVecMetricType.COSINE, - m: 77, - quantizeType: ZVecQuantizeType.FP16 - } - }, - fields: { - name: 'tag', - dataType: ZVecDataType.ARRAY_STRING, - nullable: true, - indexParams: { indexType: ZVecIndexType.INVERT } - } - }); - - const collection: ZVecCollection = ZVecCreateAndOpen(testCollectionPath, schema); - expect(collection).toBeDefined(); - - expect(collection.path).toBe(testCollectionPath); - - expect(collection.schema.name).toBe(testCollectionName); - const vectors = collection.schema.vectors(); - expect(vectors.length).toBe(1); - expect(vectors[0].name).toBe('vector'); - expect(vectors[0].dataType).toBe(ZVecDataType.VECTOR_FP32); - expect(vectors[0].dimension).toBe(10); - expect(vectors[0].indexParams!.indexType).toBe(ZVecIndexType.HNSW); - expect(vectors[0].indexParams!.metricType).toBe(ZVecMetricType.COSINE); - expect((vectors[0].indexParams as ZVecHnswIndexParams)["m"]).toBe(77); - expect((vectors[0].indexParams as ZVecHnswIndexParams).quantizeType).toBe(ZVecQuantizeType.FP16); - const fields = collection.schema.fields(); - expect(fields.length).toBe(1); - expect(fields[0].name).toBe('tag'); - expect(fields[0].dataType).toBe(ZVecDataType.ARRAY_STRING); - expect(fields[0].nullable).toBe(true); - expect(fields[0].indexParams!.indexType).toBe(ZVecIndexType.INVERT); - - expect(collection.stats.docCount).toBe(0); - - collection.closeSync(); - }); - - - it('should open an existing collection successfully', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - expect(collection.path).toBe(testCollectionPath); - - expect(collection.schema.name).toBe(testCollectionName); - const vectors = collection.schema.vectors(); - expect(vectors.length).toBe(1); - expect(vectors[0].name).toBe('vector'); - expect(vectors[0].dataType).toBe(ZVecDataType.VECTOR_FP32); - expect(vectors[0].dimension).toBe(10); - expect(vectors[0].indexParams!.indexType).toBe(ZVecIndexType.HNSW); - expect(vectors[0].indexParams!.metricType).toBe(ZVecMetricType.COSINE); - expect((vectors[0].indexParams as ZVecHnswIndexParams)["m"]).toBe(77); - expect((vectors[0].indexParams as ZVecHnswIndexParams).quantizeType).toBe(ZVecQuantizeType.FP16); - const fields = collection.schema.fields(); - expect(fields.length).toBe(1); - expect(fields[0].name).toBe('tag'); - expect(fields[0].dataType).toBe(ZVecDataType.ARRAY_STRING); - expect(fields[0].nullable).toBe(true); - expect(fields[0].indexParams!.indexType).toBe(ZVecIndexType.INVERT); - - expect(collection.stats.docCount).toBe(0); - - collection.closeSync(); - }); - - - it('should insert sample documents to the collection', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - const singleResult = collection.insertSync({ - id: 'doc1', - vectors: { 'vector': Array.from({ length: 10 }, () => Math.random()) }, - fields: { 'tag': ['Music', 'Sports'] } - }); - expect(singleResult.ok).toBe(true); - expect(singleResult.code).toBe('ZVEC_OK'); - - const sampleDocuments = [ - { - id: 'doc2', - vectors: { vector: Array.from({ length: 10 }, () => Math.random()) }, - fields: { tag: ['Music', 'Movie'] } - }, - { - id: 'doc3', - vectors: { vector: Array.from({ length: 10 }, () => Math.random()) }, - fields: { tag: ['Literature', 'Sports'] } - } - ]; - - const multiResults = collection.insertSync(sampleDocuments); - expect(multiResults.length).toBe(2); - expect(multiResults[0].ok).toBe(true); - expect(multiResults[0].code).toBe('ZVEC_OK'); - expect(multiResults[1].ok).toBe(true); - expect(multiResults[1].code).toBe('ZVEC_OK'); - - expect(collection.stats.docCount).toBe(3); - - collection.closeSync(); - }); - - - it('should add more columns successfully', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - collection.addColumnSync({ - fieldSchema: { - name: 'price', - dataType: ZVecDataType.INT64, - nullable: true, - indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: true } - } - }); - - const fields = collection.schema.fields(); - expect(fields.length).toBe(2); - const tagField = collection.schema.field('tag'); - expect(tagField.name).toBe('tag'); - expect(tagField.dataType).toBe(ZVecDataType.ARRAY_STRING); - expect(tagField.nullable).toBe(true); - expect(tagField.indexParams!.indexType).toBe(ZVecIndexType.INVERT); - const priceField = collection.schema.field('price'); - expect(priceField.name).toBe('price'); - expect(priceField.dataType).toBe(ZVecDataType.INT64); - expect(priceField.nullable).toBe(true); - expect(priceField.indexParams!.indexType).toBe(ZVecIndexType.INVERT); - - const insertResult = collection.insertSync({ - id: 'doc4', - vectors: { 'vector': Array.from({ length: 10 }, () => Math.random()) }, - fields: { - 'tag': ['Technology'], - 'price': 199, - } - }); - expect(insertResult.ok).toBe(true); - const fetchResult = collection.fetchSync('doc4'); - expect(fetchResult['doc4'].fields['price']).toBe(199); - - // Test adding a duplicate column - let errorOccurred = false; - let errorCode: string = ''; - try { - collection.addColumnSync({ - fieldSchema: { - name: 'price', - dataType: ZVecDataType.UINT32, - nullable: true - } - }); - } catch (error) { - if (isZVecError(error)) { - errorOccurred = true; - errorCode = error.code; - } - else { - throw (error); - } - } - expect(errorOccurred).toBe(true); - expect(errorCode).toBe('ZVEC_INVALID_ARGUMENT'); - - collection.closeSync(); - }); - - - it('should rename a column successfully', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - collection.alterColumnSync({ - columnName: 'price', - newColumnName: 'price-usd' - }); - - expect(collection.schema.fields().length).toBe(2); - const priceUSD = collection.schema.field('price-usd'); - expect(priceUSD.name).toBe('price-usd'); - expect(priceUSD.dataType).toBe(ZVecDataType.INT64); - - // Test getting a non-existing column - let errorOccurred = false; - let errorCode: string = ''; - try { - const field = collection.schema.field('price'); - console.log(field); - } catch (error) { - if (isZVecError(error)) { - errorOccurred = true; - errorCode = error.code; - } - else { - throw (error); - } - } - expect(errorOccurred).toBe(true); - expect(errorCode).toBe('ZVEC_NOT_FOUND'); - - collection.closeSync(); - }); - - - it('should drop a column successfully', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - let errorOccurred = false; - let errorCode: string = ''; - - collection.dropColumnSync('price-usd'); - - expect(collection.schema.fields().length).toBe(1); - try { - collection.schema.field('price-usd'); - } catch (error) { - if (isZVecError(error)) { - errorOccurred = true; - errorCode = error.code; - } - else { - throw (error); - } - } - expect(errorOccurred).toBe(true); - expect(errorCode).toBe('ZVEC_NOT_FOUND'); - - // Test dropping a non-existing column - errorOccurred = false; - errorCode = ''; - try { - collection.dropColumnSync('price'); - } catch (error) { - if (isZVecError(error)) { - errorOccurred = true; - errorCode = error.code; - } - else { - throw (error); - } - } - expect(errorOccurred).toBe(true); - expect(errorCode).toBe('ZVEC_INVALID_ARGUMENT'); - - - collection.closeSync(); - }); - - - it('should destroy a collection successfully', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - collection.destroySync(); - expect(fs.existsSync(testCollectionPath)).toBe(false); - }); -}); \ No newline at end of file From 9c7d9e2f1075088ec11c2b8e04c8eec0e913bdf0 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Fri, 22 May 2026 13:36:51 +0800 Subject: [PATCH 08/11] fix ut --- jest.config.json | 9 +++++++-- tests/collection/columns.test.ts | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/jest.config.json b/jest.config.json index 611abf0..a957fc5 100644 --- a/jest.config.json +++ b/jest.config.json @@ -1,8 +1,13 @@ { - "preset": "ts-jest", "testEnvironment": "node", "testMatch": [ "/tests/*.test.ts", "/tests/**/*.test.ts" - ] + ], + "modulePathIgnorePatterns": [ + "/build/" + ], + "transform": { + "^.+\\.ts$": ["ts-jest", { "diagnostics": { "ignoreCodes": [151002] } }] + } } \ No newline at end of file diff --git a/tests/collection/columns.test.ts b/tests/collection/columns.test.ts index 0010120..817846d 100644 --- a/tests/collection/columns.test.ts +++ b/tests/collection/columns.test.ts @@ -21,7 +21,7 @@ ZVecInitialize({ describe('Collection Columns', () => { const collectionPath = './test_columns_collection'; - const dimension = 17; + const dimension = 43; let collection: ZVecCollection; const makeEmbedding = (hotIndex: number) => @@ -101,7 +101,7 @@ describe('Collection Columns', () => { const vectors = collection.schema.vectors(); expect(vectors.length).toBe(1); expect(vectors[0].name).toBe('embedding'); - expect(vectors[0].dimension).toBe(17); + expect(vectors[0].dimension).toBe(dimension); }); it('should support insert, query, and delete with the new column', () => { From 2d8f4572157dd3a944e0afe9401bc028ba247e25 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Fri, 22 May 2026 15:32:33 +0800 Subject: [PATCH 09/11] update ut --- tests/data/helpers.ts | 160 ++++++++++++++++++++++++++++++++++ tests/data/operations.test.ts | 72 +++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 tests/data/helpers.ts create mode 100644 tests/data/operations.test.ts diff --git a/tests/data/helpers.ts b/tests/data/helpers.ts new file mode 100644 index 0000000..222ccda --- /dev/null +++ b/tests/data/helpers.ts @@ -0,0 +1,160 @@ +import { + ZVecCollection, + ZVecCollectionSchema, + ZVecDataType, + ZVecDoc, + ZVecDocInput, + ZVecIndexType, + ZVecMetricType, + ZVecQuantizeType +} from '../../src/index'; + + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const DIMENSION = 128; + +const TEST_SCHEMA = { + vectors: [ + { + name: 'dense', + dataType: ZVecDataType.VECTOR_FP32, + dimension: DIMENSION, + indexParams: { + indexType: ZVecIndexType.IVF, + metricType: ZVecMetricType.COSINE, + efConstruction: 200, + quantizeType: ZVecQuantizeType.FP16 + } + }, + { + name: 'sparse', + dataType: ZVecDataType.SPARSE_VECTOR_FP32, + indexParams: { indexType: ZVecIndexType.HNSW } + } + ], + fields: [ + { + name: 'title', + dataType: ZVecDataType.STRING, + indexParams: { indexType: ZVecIndexType.INVERT } + }, + { + name: 'price', + dataType: ZVecDataType.FLOAT, + nullable: true, + indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: true } + } + ] +}; + +export function createTestSchema(name: string): ZVecCollectionSchema { + return new ZVecCollectionSchema({ name, ...TEST_SCHEMA }); +} + + +// ─── Versioned doc generation ──────────────────────────────────────────────── + +function title(k: number, version: number): string { return `Product_${k}_v${version}`; } +function price(k: number, version: number): number { return (k + 0.99) * version; } +function dense(k: number, version: number): number[] { + const s = k * 1000 + version; + return Array.from({ length: DIMENSION }, (_, i) => Math.sin(s * (i + 1))); +} +function sparse(k: number, version: number): Record { + const s = k * 1000 + version; + return Object.fromEntries([[0, 1.0], ...Array.from({ length: 4 }, (_, j) => [s * 10 + j + 1, j === 0 ? 5.0 : 0.5])]); +} + +export function makeDoc(k: number, fieldVersion: number, vectorVersion: number): ZVecDocInput { + return { + id: `doc_${k}`, + vectors: { dense: dense(k, vectorVersion), sparse: sparse(k, vectorVersion) }, + fields: { title: title(k, fieldVersion), price: price(k, fieldVersion) } + }; +} + +export function makeUpdate(k: number, fieldVersion: number): ZVecDocInput { + return { + id: `doc_${k}`, + fields: { title: title(k, fieldVersion), price: price(k, fieldVersion) } + }; +} + + +// ─── Batch operations ──────────────────────────────────────────────────────── + +const BATCH_SIZE = 500; +type Operation = 'insert' | 'upsert' | 'update'; + +export function batch( + collection: ZVecCollection, + operation: Operation, + start: number, + end: number, + fieldVersion: number, + vectorVersion: number +): void { + if (start < 1 || end < start) { + throw new Error(`Invalid range: [${start}, ${end}]`); + } + + const gen = operation === 'update' + ? (k: number) => makeUpdate(k, fieldVersion) + : (k: number) => makeDoc(k, fieldVersion, vectorVersion); + const method = collection[`${operation}Sync`].bind(collection) as (docs: ZVecDocInput[]) => any; + + // Single doc first + const singleResult = collection[`${operation}Sync`](gen(start)); + expect(singleResult.ok).toBe(true); + + // Remaining in batches + for (let k = start + 1; k <= end; k += BATCH_SIZE) { + const to = Math.min(k + BATCH_SIZE - 1, end); + const docs = Array.from({ length: to - k + 1 }, (_, i) => gen(k + i)); + const results = method(docs); + const statuses = Array.isArray(results) ? results : [results]; + for (const status of statuses) { + expect(status.ok).toBe(true); + } + } +} + +export function verifyDocs( + collection: ZVecCollection, + start: number, + end: number, + fieldVersion: number, + vectorVersion: number +): void { + for (let from = start; from <= end; from += BATCH_SIZE) { + const to = Math.min(from + BATCH_SIZE - 1, end); + const ids = Array.from({ length: to - from + 1 }, (_, i) => `doc_${from + i}`); + const fetched = collection.fetchSync(ids); + for (let k = from; k <= to; k++) { + expectDoc(fetched[`doc_${k}`], k, fieldVersion, vectorVersion); + } + } +} + + +// ─── Verification helpers ──────────────────────────────────────────────────── + +export function expectDoc( + doc: ZVecDoc, + k: number, + fieldVersion: number, + vectorVersion: number +): void { + expect(doc.id).toBe(`doc_${k}`); + expect(doc.fields.title).toBe(title(k, fieldVersion)); + expect(doc.fields.price).toBeCloseTo(price(k, fieldVersion), 2); + const expectedDense = dense(k, vectorVersion); + for (let i = 0; i < expectedDense.length; i++) { + expect(doc.vectors.dense[i]).toBeCloseTo(expectedDense[i], 4); + } + const expectedSparse = sparse(k, vectorVersion); + for (const [dim, val] of Object.entries(expectedSparse)) { + expect(doc.vectors.sparse[Number(dim)]).toBeCloseTo(val, 4); + } +} diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts new file mode 100644 index 0000000..75809b1 --- /dev/null +++ b/tests/data/operations.test.ts @@ -0,0 +1,72 @@ +import * as fs from 'fs'; +import { + ZVecCollection, + ZVecCreateAndOpen, + ZVecInitialize, + ZVecLogLevel, + ZVecLogType, +} from '../../src/index'; +import { batch, createTestSchema, expectDoc, makeDoc, verifyDocs } from './helpers'; + + +const COLLECTION_PATH = './test_operations_collection'; + + +ZVecInitialize({ logType: ZVecLogType.CONSOLE, logLevel: ZVecLogLevel.WARN }); + + +describe('Data Operations Pipeline', () => { + let collection: ZVecCollection; + + beforeAll(() => { + if (fs.existsSync(COLLECTION_PATH)) { + fs.rmSync(COLLECTION_PATH, { recursive: true, force: true }); + } + collection = ZVecCreateAndOpen(COLLECTION_PATH, createTestSchema('operations_test')); + }); + + afterAll(() => { + collection?.destroySync(); + if (fs.existsSync(COLLECTION_PATH)) { + fs.rmSync(COLLECTION_PATH, { recursive: true, force: true }); + } + }); + + + describe('insert', () => { + it('should insert 1000 docs and verify all', () => { + batch(collection, 'insert', 1, 1000, 1, 1); + expect(collection.stats.docCount).toBe(1000); + verifyDocs(collection, 1, 1000, 1, 1); + }); + + it('should return correct results from vector query', () => { + const doc = makeDoc(42, 1, 1); + const results = collection.querySync({ + fieldName: 'dense', vector: doc.vectors!.dense, topk: 1, includeVector: true + }); + expectDoc(results[0], 42, 1, 1); + }); + }); + + + describe('optimize', () => { + it('should reach full index completeness', async () => { + await collection.optimize(); + expect(collection.stats.indexCompleteness['dense']).toBeCloseTo(1); + expect(collection.stats.indexCompleteness['sparse']).toBeCloseTo(1); + }); + + it('should still return correct data after optimize', () => { + verifyDocs(collection, 1, 1000, 1, 1); + }); + + it('should return correct results from async query', async () => { + const doc = makeDoc(42, 1, 1); + const results = await collection.query({ + fieldName: 'dense', vector: doc.vectors!.dense, topk: 1, includeVector: true + }); + expectDoc(results[0], 42, 1, 1); + }); + }); +}); From b958488d824253da38f8a571f69eaf035c3fb169 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Fri, 22 May 2026 16:05:27 +0800 Subject: [PATCH 10/11] update ut --- tests/data/operations.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts index 75809b1..1e91687 100644 --- a/tests/data/operations.test.ts +++ b/tests/data/operations.test.ts @@ -51,8 +51,8 @@ describe('Data Operations Pipeline', () => { describe('optimize', () => { - it('should reach full index completeness', async () => { - await collection.optimize(); + it('should reach full index completeness', () => { + collection.optimizeSync(); expect(collection.stats.indexCompleteness['dense']).toBeCloseTo(1); expect(collection.stats.indexCompleteness['sparse']).toBeCloseTo(1); }); @@ -61,9 +61,9 @@ describe('Data Operations Pipeline', () => { verifyDocs(collection, 1, 1000, 1, 1); }); - it('should return correct results from async query', async () => { + it('should return correct results from vector query', () => { const doc = makeDoc(42, 1, 1); - const results = await collection.query({ + const results = collection.querySync({ fieldName: 'dense', vector: doc.vectors!.dense, topk: 1, includeVector: true }); expectDoc(results[0], 42, 1, 1); From d2f62a54c7b150c2201e241c7396bfd4a73e1527 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Fri, 22 May 2026 16:26:11 +0800 Subject: [PATCH 11/11] udpate ut --- tests/collectionDML.test.ts | 295 ---------------------------------- tests/data/operations.test.ts | 24 +++ 2 files changed, 24 insertions(+), 295 deletions(-) delete mode 100644 tests/collectionDML.test.ts diff --git a/tests/collectionDML.test.ts b/tests/collectionDML.test.ts deleted file mode 100644 index 4ac29a6..0000000 --- a/tests/collectionDML.test.ts +++ /dev/null @@ -1,295 +0,0 @@ -import * as fs from 'fs'; -import { - ZVecCollection, - ZVecCollectionSchema, - ZVecCreateAndOpen, - ZVecDataType, - ZVecIndexType, - ZVecInitialize, - ZVecLogLevel, - ZVecLogType, - ZVecMetricType, - ZVecOpen, - ZVecQuantizeType -} from '../src/index'; - - -ZVecInitialize({ - logType: ZVecLogType.CONSOLE, - logLevel: ZVecLogLevel.WARN, -}); - - -describe('Collection Data Modification Operations', () => { - const testCollectionName = 'test_dml_collection'; - const testCollectionPath = './test_dml_collection'; - const dimension = 512; - - const makeDense = (k: number) => - Array.from({ length: dimension }, (_, idx) => (idx === (k - 1) % dimension ? 1 : 0.1)); - - const makeSparse = (k: number) => - Object.fromEntries( - Array.from({ length: dimension }, (_, j) => { - const key = j + 1; - return [key, key === ((k - 1) % dimension) + 1 ? 5 : 0.1]; - }) - ); - - const makeDoc = (k: number) => ({ - id: `doc${k}`, - vectors: { - dense: makeDense(k), - sparse: makeSparse(k), - }, - fields: { - title: `Product_${k}`, - price: k + 0.99, - } - }); - - - beforeAll(() => { - if (fs.existsSync(testCollectionPath)) { - fs.rmSync(testCollectionPath, { recursive: true, force: true }); - } - }); - - afterAll(() => { - if (fs.existsSync(testCollectionPath)) { - fs.rmSync(testCollectionPath, { recursive: true, force: true }); - } - }); - - - it('should handle insert operations correctly', () => { - const schema = new ZVecCollectionSchema({ - name: testCollectionName, - vectors: [ - { - name: 'dense', - dataType: ZVecDataType.VECTOR_FP32, - dimension: dimension, - indexParams: { - indexType: ZVecIndexType.HNSW, - metricType: ZVecMetricType.COSINE, - m: 17, - efConstruction: 200, - quantizeType: ZVecQuantizeType.FP16 - } - }, - { - name: 'sparse', - dataType: ZVecDataType.SPARSE_VECTOR_FP32, - indexParams: { indexType: ZVecIndexType.HNSW } - } - ], - fields: [ - { - name: 'title', - dataType: ZVecDataType.STRING, - indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: false } - }, - { - name: 'price', - dataType: ZVecDataType.FLOAT, - nullable: true, - indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: true } - } - ] - }); - - const collection: ZVecCollection = ZVecCreateAndOpen(testCollectionPath, schema); - expect(collection).toBeDefined(); - - const singleInsertResult = collection.insertSync(makeDoc(1)); - expect(singleInsertResult.ok).toBe(true); - expect(singleInsertResult.code).toBe('ZVEC_OK'); - expect(collection.stats.docCount).toBe(1); - - const multiInsertResult = collection.insertSync([makeDoc(2), makeDoc(3)]); - expect(multiInsertResult.length).toBe(2); - expect(multiInsertResult[0].ok).toBe(true); - expect(multiInsertResult[0].code).toBe('ZVEC_OK'); - expect(multiInsertResult[1].ok).toBe(true); - expect(multiInsertResult[1].code).toBe('ZVEC_OK'); - expect(collection.stats.docCount).toBe(3); - - const fetchResult = collection.fetchSync(['doc1', 'doc2', 'doc3']); - for (let k = 1; k <= 3; k++) { - const doc = fetchResult[`doc${k}`]; - expect(doc).toBeDefined(); - expect(doc.fields.title).toBe(`Product_${k}`); - expect(doc.fields.price).toBeCloseTo(k + 0.99, 2); - expect(doc.vectors.dense[(k - 1) % dimension]).toBeCloseTo(1, 6); - expect(doc.vectors.dense[(k % dimension)]).toBeCloseTo(0.1, 6); // some other dimension - expect(doc.vectors.sparse[((k - 1) % dimension) + 1]).toBeCloseTo(5, 6); - } - - const queryResult = collection.querySync({ - fieldName: 'dense', - vector: makeDense(3), - outputFields: ['title'], - }); - expect(queryResult).toBeDefined(); - expect(queryResult[0].id).toBe('doc3'); - expect('title' in queryResult[0].fields).toBe(true); - expect('price' in queryResult[0].fields).toBe(false); - expect(queryResult[0].fields['title']).toBe('Product_3'); - - collection.closeSync(); - }); - - - it('should handle lots of insert operations correctly', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - for (let k = 4; k <= 500; k++) { - const result = collection.insertSync(makeDoc(k)); - expect(result.ok).toBe(true); - expect(result.code).toBe('ZVEC_OK'); - } - collection.optimizeSync(); - expect(collection.stats.docCount).toBe(500); - expect(collection.stats.indexCompleteness['dense']).toBeCloseTo(1); - expect(collection.stats.indexCompleteness['sparse']).toBeCloseTo(1); - - const denseQuery = collection.querySync({ - fieldName: 'dense', - vector: makeDense(300), - topk: 25, - }); - expect(denseQuery[0].id).toBe('doc300'); - expect('title' in denseQuery[0].fields).toBe(true); - expect('price' in denseQuery[0].fields).toBe(true); - expect(denseQuery[0].fields['title']).toBe('Product_300'); - expect(denseQuery[0].fields['price']).toBeCloseTo(300.99); - - const sparseQuery = collection.querySync({ - fieldName: 'sparse', - vector: makeSparse(200), - topk: 25, - }); - expect(sparseQuery[0].id).toBe('doc200'); - expect('title' in sparseQuery[0].fields).toBe(true); - expect('price' in sparseQuery[0].fields).toBe(true); - expect(sparseQuery[0].fields['title']).toBe('Product_200'); - expect(sparseQuery[0].fields['price']).toBeCloseTo(200.99); - - const filteredQuery = collection.querySync({ - filter: 'price > 100 and price < 105.5' - }); - expect(filteredQuery.length).toBe(5); - - const filteredVectorQuery = collection.querySync({ - fieldName: 'sparse', - vector: makeSparse(200), - filter: 'price > 190 and price < 210' - }); - expect(filteredVectorQuery[0].id).toBe('doc200'); - - collection.closeSync(); - }); - - - it('should handle update operations correctly', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - const updateResult = collection.updateSync({ - id: 'doc1', - fields: { - 'title': 'updated_Product_1', - 'price': 10000 - } - }); - expect(updateResult.ok).toBe(true); - expect(updateResult.code).toBe('ZVEC_OK'); - - const updateResults = collection.updateSync([ - { - id: 'doc2', - fields: { 'title': 'updated_Product_2' } - }, - { - id: 'doc3', - fields: { 'title': 'updated_Product_3' } - } - ]); - expect(updateResults[0].ok).toBe(true); - expect(updateResults[0].code).toBe('ZVEC_OK'); - expect(updateResults[1].ok).toBe(true); - expect(updateResults[1].code).toBe('ZVEC_OK'); - - const fetchResult = collection.fetchSync('doc1'); - expect(fetchResult['doc1']).toBeDefined(); - expect(fetchResult['doc1'].fields.title).toBe('updated_Product_1'); - expect(fetchResult['doc1'].fields.price).toBeCloseTo(10000); - - const fetchResults = collection.fetchSync(['doc2', 'doc3']); - expect(fetchResults['doc2']).toBeDefined(); - expect(fetchResults['doc2'].fields.title).toBe('updated_Product_2'); - expect(fetchResults['doc2'].fields.price).toBeCloseTo(2.99); - expect(fetchResults['doc3']).toBeDefined(); - expect(fetchResults['doc3'].fields.title).toBe('updated_Product_3'); - expect(fetchResults['doc3'].fields.price).toBeCloseTo(3.99); - - collection.closeSync(); - }); - - - it('should handle upsert operations correctly', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - const upsertResult = collection.upsertSync({ - id: 'doc1', - vectors: { - 'dense': Array.from({ length: dimension }, () => (0.7)), - 'sparse': { 9999: 0.5 } - }, - fields: { - 'title': 'upserted_Product_1', - 'price': 99999 - } - }); - expect(upsertResult.ok).toBe(true); - expect(upsertResult.code).toBe('ZVEC_OK'); - - const fetchResult = collection.fetchSync('doc1'); - expect(fetchResult['doc1']).toBeDefined(); - expect(fetchResult['doc1'].vectors.dense[0]).toBeCloseTo(0.7); - expect(fetchResult['doc1'].vectors.sparse[9999]).toBeCloseTo(0.5); - expect(fetchResult['doc1'].fields.title).toBe('upserted_Product_1'); - expect(fetchResult['doc1'].fields.price).toBeCloseTo(99999); - - collection.closeSync(); - }); - - - it('should handle delete operations correctly', () => { - const collection: ZVecCollection = ZVecOpen(testCollectionPath); - expect(collection).toBeDefined(); - - const deleteSingleResult = collection.deleteSync('doc3'); - expect(deleteSingleResult.ok).toBe(true); - expect(deleteSingleResult.code).toBe('ZVEC_OK'); - expect(collection.stats.docCount).toBe(500 - 1); - const fetchResult = collection.fetchSync('doc3'); - expect('doc3' in fetchResult).toBe(false); - - const deleteByFilterResult = collection.deleteByFilterSync('price < 10'); - expect(deleteByFilterResult.ok).toBe(true); - expect(deleteByFilterResult.code).toBe('ZVEC_OK'); - expect(collection.stats.docCount).toBe(500 - 8); - const queryResult = collection.querySync({ - filter: 'price < 10' - }) - expect(queryResult.length).toBe(0); - - collection.closeSync(); - }); - - -}); \ No newline at end of file diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts index 1e91687..3d00e3b 100644 --- a/tests/data/operations.test.ts +++ b/tests/data/operations.test.ts @@ -69,4 +69,28 @@ describe('Data Operations Pipeline', () => { expectDoc(results[0], 42, 1, 1); }); }); + + + describe('upsert', () => { + it('should upsert existing docs with new versions', () => { + batch(collection, 'upsert', 1, 500, 2, 2); + verifyDocs(collection, 1, 500, 2, 2); + }); + + it('should not affect other docs', () => { + verifyDocs(collection, 501, 1000, 1, 1); + }); + + it('should upsert new docs beyond the original range', () => { + batch(collection, 'upsert', 1001, 1500, 1, 1); + expect(collection.stats.docCount).toBe(1500); + verifyDocs(collection, 1001, 1500, 1, 1); + }); + + }); + + // NOTE: update, delete, and re-optimize tests are blocked by an engine bug. + // ReduceVectorIndex in segment_helper.cc uses MakeQuantizeVectorIndexPath + // for the primary index when quantization is enabled, causing "Failed to open index" + // on any re-optimize after new data is written to an already-optimized collection. });