From 95ac50bf2ed072475ae808aac75dd6fa3c552b50 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Thu, 27 Feb 2025 17:58:43 +0800 Subject: [PATCH 01/11] make sure interactive support relative large number of vertex and edge labels Committed-by: xiaolei.zl@alibaba-inc.com from Dev container Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/CMakeLists.txt | 8 + .../graph_db/database/update_transaction.cc | 161 ++++++----- .../graph_db/runtime/common/accessors.h | 59 ++-- .../engines/graph_db/runtime/common/rt_any.cc | 7 +- flex/engines/graph_db/runtime/common/types.cc | 27 -- flex/engines/graph_db/runtime/common/types.h | 4 - .../execute/ops/retrieve/procedure_call.cc | 34 ++- flex/engines/hqps_db/core/operator/sink.h | 17 +- flex/engines/hqps_db/core/utils/hqps_utils.h | 213 +++++++++----- .../hqps_db/database/mutable_csr_interface.h | 2 +- .../http_server/actor/admin_actor.act.cc | 38 ++- .../python/gs_interactive/tests/conftest.py | 122 ++++++++ .../gs_interactive/tests/test_robustness.py | 24 ++ .../loader/basic_fragment_loader.cc | 41 ++- .../loader/basic_fragment_loader.h | 8 +- .../mutable_property_fragment.cc | 265 +++++++----------- .../mutable_property_fragment.h | 16 +- flex/storages/rt_mutable_graph/schema.cc | 244 +++++++++------- flex/storages/rt_mutable_graph/schema.h | 46 ++- flex/storages/rt_mutable_graph/types.h | 4 + flex/utils/id_indexer.h | 2 +- flex/utils/property/types.cc | 22 ++ flex/utils/property/types.h | 5 + 23 files changed, 777 insertions(+), 592 deletions(-) diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index 2c27898a9e8e..2623c93c9256 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -17,6 +17,8 @@ option(USE_PTHASH "Whether to use pthash" OFF) option(OPTIMIZE_FOR_HOST "Whether to optimize on host" ON) # Whether to build optimized code on host option(USE_STATIC_ARROW "Whether to use static arrow" OFF) # Whether to link arrow statically, default is OFF option(BUILD_WITH_OTEL "Whether to build with opentelemetry-cpp" OFF) # Whether to build with opentelemetry-cpp, default is OFF +# option(LABEL_TYPE "The label type of the graph, valid values: uint16, uint8" "uint8") # The type of the label in the graph +SET(LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") #print options message(STATUS "Build test: ${BUILD_TEST}") @@ -58,6 +60,12 @@ if(USE_PTHASH) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/murmurhash) endif() +# check label_type in {uint8, uint16} +if(NOT LABEL_TYPE STREQUAL "uint8_t" AND NOT LABEL_TYPE STREQUAL "uint16_t") + message(FATAL_ERROR "LABEL_TYPE must be uint8 or uint16, but got ${LABEL_TYPE}") +endif() +add_compile_definitions(FLEX_LABEL_TYPE=${LABEL_TYPE}) + set(DEFAULT_BUILD_TYPE "Release") if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' as none was specified.") diff --git a/flex/engines/graph_db/database/update_transaction.cc b/flex/engines/graph_db/database/update_transaction.cc index 7d8a53eefdb9..fc9d832942f1 100644 --- a/flex/engines/graph_db/database/update_transaction.cc +++ b/flex/engines/graph_db/database/update_transaction.cc @@ -93,7 +93,7 @@ UpdateTransaction::UpdateTransaction(const GraphDBSession& session, extra_vertex_properties_[i].resize(4096); } - size_t csr_num = 2 * vertex_label_num_ * vertex_label_num_ * edge_label_num_; + size_t csr_num = 2 * schema().get_edge_triplet_num(); added_edges_.resize(csr_num); updated_edge_data_.resize(csr_num); } @@ -560,7 +560,6 @@ void UpdateTransaction::IngestWal(MutablePropertyFragment& graph, updated_edge_data; size_t vertex_label_num = graph.schema().vertex_label_num(); - size_t edge_label_num = graph.schema().edge_label_num(); for (label_t idx = 0; idx < vertex_label_num; ++idx) { if (graph.lf_indexers_[idx].get_type() == PropertyType::kInt64) { @@ -603,7 +602,7 @@ void UpdateTransaction::IngestWal(MutablePropertyFragment& graph, extra_vertex_properties[i].resize(4096); } - size_t csr_num = 2 * vertex_label_num * vertex_label_num * edge_label_num; + size_t csr_num = 2 * graph.schema().get_edge_triplet_num(); added_edges.resize(csr_num); updated_edge_data.resize(csr_num); @@ -681,16 +680,14 @@ void UpdateTransaction::IngestWal(MutablePropertyFragment& graph, size_t UpdateTransaction::get_in_csr_index(label_t src_label, label_t dst_label, label_t edge_label) const { - return src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + edge_label; + return graph_.schema().get_edge_triplet_id(src_label, dst_label, edge_label); } size_t UpdateTransaction::get_out_csr_index(label_t src_label, label_t dst_label, label_t edge_label) const { - return src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + edge_label + - vertex_label_num_ * vertex_label_num_ * edge_label_num_; + return graph_.schema().get_edge_triplet_id(dst_label, src_label, edge_label) + + graph_.schema().get_edge_triplet_num(); } bool UpdateTransaction::oid_to_lid(label_t label, const Any& oid, @@ -820,89 +817,89 @@ void UpdateTransaction::applyVerticesUpdates() { } void UpdateTransaction::applyEdgesUpdates() { - for (label_t src_label = 0; src_label < vertex_label_num_; ++src_label) { - for (label_t dst_label = 0; dst_label < vertex_label_num_; ++dst_label) { - for (label_t edge_label = 0; edge_label < edge_label_num_; ++edge_label) { - size_t oe_csr_index = - get_out_csr_index(src_label, dst_label, edge_label); - for (auto& pair : updated_edge_data_[oe_csr_index]) { - auto& updates = pair.second; - if (updates.empty()) { - continue; - } + for (size_t index = 0; index < graph_.schema().get_edge_triplet_num(); + ++index) { + auto edge_triplet = graph_.schema().get_edge_triplet(index); + label_t src_label = std::get<0>(edge_triplet); + label_t dst_label = std::get<1>(edge_triplet); + label_t edge_label = std::get<2>(edge_triplet); + size_t oe_csr_index = get_out_csr_index(src_label, dst_label, edge_label); + for (auto& pair : updated_edge_data_[oe_csr_index]) { + auto& updates = pair.second; + if (updates.empty()) { + continue; + } - std::shared_ptr edge_iter = - graph_.get_outgoing_edges_mut(src_label, pair.first, dst_label, - edge_label); - for (auto& edge : updates) { - if (edge.second.second != std::numeric_limits::max()) { - auto& iter = *edge_iter; - iter += edge.second.second; - if (iter.is_valid() && iter.get_neighbor() == edge.first) { - iter.set_data(edge.second.first, timestamp_); - } else if (iter.is_valid() && iter.get_neighbor() != edge.first) { - LOG(FATAL) << "Inconsistent neighbor id:" << iter.get_neighbor() - << " " << edge.first << "\n"; - } else { - LOG(FATAL) << "Illegal offset: " << edge.first << " " - << edge.second.second << "\n"; - } - } + std::shared_ptr edge_iter = + graph_.get_outgoing_edges_mut(src_label, pair.first, dst_label, + edge_label); + for (auto& edge : updates) { + if (edge.second.second != std::numeric_limits::max()) { + auto& iter = *edge_iter; + iter += edge.second.second; + if (iter.is_valid() && iter.get_neighbor() == edge.first) { + iter.set_data(edge.second.first, timestamp_); + } else if (iter.is_valid() && iter.get_neighbor() != edge.first) { + LOG(FATAL) << "Inconsistent neighbor id:" << iter.get_neighbor() + << " " << edge.first << "\n"; + } else { + LOG(FATAL) << "Illegal offset: " << edge.first << " " + << edge.second.second << "\n"; } } + } + } - for (auto& pair : added_edges_[oe_csr_index]) { - vid_t v = pair.first; - auto& add_list = pair.second; + for (auto& pair : added_edges_[oe_csr_index]) { + vid_t v = pair.first; + auto& add_list = pair.second; - if (add_list.empty()) { - continue; - } - std::sort(add_list.begin(), add_list.end()); - auto& edge_data = updated_edge_data_[oe_csr_index].at(v); - for (size_t idx = 0; idx < add_list.size(); ++idx) { - if (idx && add_list[idx] == add_list[idx - 1]) - continue; - auto u = add_list[idx]; - auto value = edge_data.at(u).first; - grape::InArchive iarc; - serialize_field(iarc, value); - grape::OutArchive oarc(std::move(iarc)); - graph_.IngestEdge(src_label, v, dst_label, u, edge_label, - timestamp_, oarc, alloc_); - } - } + if (add_list.empty()) { + continue; + } + std::sort(add_list.begin(), add_list.end()); + auto& edge_data = updated_edge_data_[oe_csr_index].at(v); + for (size_t idx = 0; idx < add_list.size(); ++idx) { + if (idx && add_list[idx] == add_list[idx - 1]) + continue; + auto u = add_list[idx]; + auto value = edge_data.at(u).first; + grape::InArchive iarc; + serialize_field(iarc, value); + grape::OutArchive oarc(std::move(iarc)); + graph_.IngestEdge(src_label, v, dst_label, u, edge_label, timestamp_, + oarc, alloc_); } } } - for (label_t src_label = 0; src_label < vertex_label_num_; ++src_label) { - for (label_t dst_label = 0; dst_label < vertex_label_num_; ++dst_label) { - for (label_t edge_label = 0; edge_label < edge_label_num_; ++edge_label) { - size_t ie_csr_index = - get_in_csr_index(src_label, dst_label, edge_label); - for (auto& pair : updated_edge_data_[ie_csr_index]) { - auto& updates = pair.second; - if (updates.empty()) { - continue; - } - std::shared_ptr edge_iter = - graph_.get_incoming_edges_mut(dst_label, pair.first, src_label, - edge_label); - for (auto& edge : updates) { - if (edge.second.second != std::numeric_limits::max()) { - auto& iter = *edge_iter; - iter += edge.second.second; - if (iter.is_valid() && iter.get_neighbor() == edge.first) { - iter.set_data(edge.second.first, timestamp_); - } else if (iter.is_valid() && iter.get_neighbor() != edge.first) { - LOG(FATAL) << "Inconsistent neighbor id:" << iter.get_neighbor() - << " " << edge.first << "\n"; - } else { - LOG(FATAL) << "Illegal offset: " << edge.first << " " - << edge.second.second << "\n"; - } - } + for (size_t index = 0; index < graph_.schema().get_edge_triplet_num(); + ++index) { + auto edge_triplet = graph_.schema().get_edge_triplet(index); + label_t src_label = std::get<0>(edge_triplet); + label_t dst_label = std::get<1>(edge_triplet); + label_t edge_label = std::get<2>(edge_triplet); + size_t ie_csr_index = get_in_csr_index(src_label, dst_label, edge_label); + for (auto& pair : updated_edge_data_[ie_csr_index]) { + auto& updates = pair.second; + if (updates.empty()) { + continue; + } + std::shared_ptr edge_iter = + graph_.get_incoming_edges_mut(dst_label, pair.first, src_label, + edge_label); + for (auto& edge : updates) { + if (edge.second.second != std::numeric_limits::max()) { + auto& iter = *edge_iter; + iter += edge.second.second; + if (iter.is_valid() && iter.get_neighbor() == edge.first) { + iter.set_data(edge.second.first, timestamp_); + } else if (iter.is_valid() && iter.get_neighbor() != edge.first) { + LOG(FATAL) << "Inconsistent neighbor id:" << iter.get_neighbor() + << " " << edge.first << "\n"; + } else { + LOG(FATAL) << "Illegal offset: " << edge.first << " " + << edge.second.second << "\n"; } } } diff --git a/flex/engines/graph_db/runtime/common/accessors.h b/flex/engines/graph_db/runtime/common/accessors.h index 3cf14e7e5282..eea247e76c90 100644 --- a/flex/engines/graph_db/runtime/common/accessors.h +++ b/flex/engines/graph_db/runtime/common/accessors.h @@ -419,16 +419,16 @@ class MultiPropsEdgePropertyPathAccessor : public IAccessor { MultiPropsEdgePropertyPathAccessor(const GraphInterface& graph, const std::string& prop_name, const Context& ctx, int tag) - : col_(*std::dynamic_pointer_cast(ctx.get(tag))) { + : graph_(graph), + col_(*std::dynamic_pointer_cast(ctx.get(tag))) { const auto& labels = col_.get_labels(); vertex_label_num_ = graph.schema().vertex_label_num(); edge_label_num_ = graph.schema().edge_label_num(); - prop_index_.resize( - 2 * vertex_label_num_ * vertex_label_num_ * edge_label_num_, - std::numeric_limits::max()); + prop_index_.resize(graph.schema().get_edge_triplet_num(), + std::numeric_limits::max()); for (auto& label : labels) { - size_t idx = label.src_label * vertex_label_num_ * edge_label_num_ + - label.dst_label * edge_label_num_ + label.edge_label; + size_t idx = graph.schema().get_edge_triplet_id( + label.src_label, label.dst_label, label.edge_label); const auto& names = graph.schema().get_edge_property_names( label.src_label, label.dst_label, label.edge_label); for (size_t i = 0; i < names.size(); ++i) { @@ -478,8 +478,8 @@ class MultiPropsEdgePropertyPathAccessor : public IAccessor { bool is_optional() const override { return col_.is_optional(); } size_t get_index(const LabelTriplet& label) const { - size_t idx = label.src_label * vertex_label_num_ * edge_label_num_ + - label.dst_label * edge_label_num_ + label.edge_label; + size_t idx = graph_.schema().get_edge_triplet_id( + label.src_label, label.dst_label, label.edge_label); return prop_index_[idx]; } @@ -495,6 +495,7 @@ class MultiPropsEdgePropertyPathAccessor : public IAccessor { } private: + const GraphInterface& graph_; const IEdgeColumn& col_; std::vector prop_index_; size_t vertex_label_num_; @@ -556,32 +557,23 @@ class MultiPropsEdgePropertyEdgeAccessor : public IAccessor { using elem_t = T; MultiPropsEdgePropertyEdgeAccessor(const GraphInterface& graph, - const std::string& name) { + const std::string& name) + : graph_(graph) { edge_label_num_ = graph.schema().edge_label_num(); vertex_label_num_ = graph.schema().vertex_label_num(); - indexs.resize(2 * vertex_label_num_ * vertex_label_num_ * edge_label_num_, + indexs.resize(graph.schema().get_edge_triplet_num(), std::numeric_limits::max()); - for (label_t src_label = 0; src_label < vertex_label_num_; ++src_label) { - auto src = graph.schema().get_vertex_label_name(src_label); - for (label_t dst_label = 0; dst_label < vertex_label_num_; ++dst_label) { - auto dst = graph.schema().get_vertex_label_name(dst_label); - for (label_t edge_label = 0; edge_label < edge_label_num_; - ++edge_label) { - auto edge = graph.schema().get_edge_label_name(edge_label); - if (!graph.schema().exist(src, dst, edge)) { - continue; - } - size_t idx = src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + edge_label; - const std::vector& names = - graph.schema().get_edge_property_names(src_label, dst_label, - edge_label); - for (size_t i = 0; i < names.size(); ++i) { - if (names[i] == name) { - indexs[idx] = i; - break; - } - } + + for (size_t index = 0; index < graph.schema().get_edge_triplet_num(); + ++index) { + auto label = graph.schema().get_edge_triplet(index); + const std::vector& names = + graph.schema().get_edge_property_names( + std::get<0>(label), std::get<1>(label), std::get<2>(label)); + for (size_t i = 0; i < names.size(); ++i) { + if (names[i] == name) { + indexs[index] = i; + break; } } } @@ -613,12 +605,13 @@ class MultiPropsEdgePropertyEdgeAccessor : public IAccessor { } size_t get_index(const LabelTriplet& label) const { - size_t idx = label.src_label * vertex_label_num_ * edge_label_num_ + - label.dst_label * edge_label_num_ + label.edge_label; + size_t idx = graph_.schema().get_edge_triplet_id( + label.src_label, label.dst_label, label.edge_label); return indexs[idx]; } private: + const GraphInterface& graph_; std::vector indexs; size_t vertex_label_num_; size_t edge_label_num_; diff --git a/flex/engines/graph_db/runtime/common/rt_any.cc b/flex/engines/graph_db/runtime/common/rt_any.cc index 7da1d88b0850..c7380272c08c 100644 --- a/flex/engines/graph_db/runtime/common/rt_any.cc +++ b/flex/engines/graph_db/runtime/common/rt_any.cc @@ -826,12 +826,13 @@ void RTAny::sink(const GraphReadInterface& graph, int id, auto [label, src, dst, prop, dir] = this->as_edge(); e->mutable_src_label()->set_id(label.src_label); e->mutable_dst_label()->set_id(label.dst_label); - auto edge_label = generate_edge_label_id(label.src_label, label.dst_label, - label.edge_label); + + auto edge_triplet_id = graph.schema().get_edge_triplet_id( + label.src_label, label.dst_label, label.edge_label); e->mutable_label()->set_id(label.edge_label); e->set_src_id(encode_unique_vertex_id(label.src_label, src)); e->set_dst_id(encode_unique_vertex_id(label.dst_label, dst)); - e->set_id(encode_unique_edge_id(edge_label, src, dst)); + e->set_id(encode_unique_edge_id(edge_triplet_id, src, dst)); auto& prop_names = graph.schema().get_edge_property_names( label.src_label, label.dst_label, label.edge_label); if (prop_names.size() == 1) { diff --git a/flex/engines/graph_db/runtime/common/types.cc b/flex/engines/graph_db/runtime/common/types.cc index f217595b8ba4..4f0a296bb7a5 100644 --- a/flex/engines/graph_db/runtime/common/types.cc +++ b/flex/engines/graph_db/runtime/common/types.cc @@ -29,33 +29,6 @@ std::pair decode_unique_vertex_id(uint64_t unique_id) { GlobalId::get_vid(unique_id)}; } -uint32_t generate_edge_label_id(label_t src_label_id, label_t dst_label_id, - label_t edge_label_id) { - uint32_t unique_edge_label_id = src_label_id; - static constexpr int num_bits = sizeof(label_t) * 8; - static_assert(num_bits * 3 <= sizeof(uint32_t) * 8, - "label_t is too large to be encoded in 32 bits"); - unique_edge_label_id = unique_edge_label_id << num_bits; - unique_edge_label_id = unique_edge_label_id | dst_label_id; - unique_edge_label_id = unique_edge_label_id << num_bits; - unique_edge_label_id = unique_edge_label_id | edge_label_id; - return unique_edge_label_id; -} - -std::tuple decode_edge_label_id( - uint32_t edge_label_id) { - static constexpr int num_bits = sizeof(label_t) * 8; - static_assert(num_bits * 3 <= sizeof(uint32_t) * 8, - "label_t is too large to be encoded in 32 bits"); - auto mask = (1 << num_bits) - 1; - label_t edge_label = edge_label_id & mask; - edge_label_id = edge_label_id >> num_bits; - label_t dst_label = edge_label_id & mask; - edge_label_id = edge_label_id >> num_bits; - label_t src_label = edge_label_id & mask; - return std::make_tuple(src_label, dst_label, edge_label); -} - int64_t encode_unique_edge_id(uint32_t label_id, vid_t src, vid_t dst) { // We assume label_id is only used by 24 bits. int64_t unique_edge_id = label_id; diff --git a/flex/engines/graph_db/runtime/common/types.h b/flex/engines/graph_db/runtime/common/types.h index 1df7fe5ffc74..4088e0a49fc2 100644 --- a/flex/engines/graph_db/runtime/common/types.h +++ b/flex/engines/graph_db/runtime/common/types.h @@ -27,12 +27,8 @@ namespace runtime { uint64_t encode_unique_vertex_id(label_t label_id, vid_t vid); std::pair decode_unique_vertex_id(uint64_t unique_id); -uint32_t generate_edge_label_id(label_t src_label_id, label_t dst_label_id, - label_t edge_label_id); int64_t encode_unique_edge_id(uint32_t label_id, vid_t src, vid_t dst); -std::tuple decode_edge_label_id( - uint32_t edge_label_id); enum class Direction { kOut, kIn, diff --git a/flex/engines/graph_db/runtime/execute/ops/retrieve/procedure_call.cc b/flex/engines/graph_db/runtime/execute/ops/retrieve/procedure_call.cc index ffd646f2fbba..caaa7aacada1 100644 --- a/flex/engines/graph_db/runtime/execute/ops/retrieve/procedure_call.cc +++ b/flex/engines/graph_db/runtime/execute/ops/retrieve/procedure_call.cc @@ -129,11 +129,13 @@ RTAny vertex_to_rt_any(const results::Vertex& vertex) { return RTAny::from_vertex(label_id, label_id_vid.second); } -RTAny edge_to_rt_any(const results::Edge& edge) { +RTAny edge_to_rt_any(const GraphReadInterface& graph, + const results::Edge& edge) { LOG(FATAL) << "Not implemented."; label_t src_label_id = (label_t) edge.src_label().id(); label_t dst_label_id = (label_t) edge.dst_label().id(); - auto edge_triplet_tuple = decode_edge_label_id(edge.label().id()); + auto edge_triplet_tuple = graph.schema().get_edge_triplet(edge.label().id()); + CHECK((src_label_id == std::get<0>(edge_triplet_tuple)) && (dst_label_id == std::get<1>(edge_triplet_tuple))) << "Inconsistent src label id."; @@ -171,11 +173,12 @@ RTAny graph_path_to_rt_any(const results::GraphPath& path) { LOG(FATAL) << "Not implemented."; } -RTAny element_to_rt_any(const results::Element& element) { +RTAny element_to_rt_any(const GraphReadInterface& txn, + const results::Element& element) { if (element.inner_case() == results::Element::kVertex) { return vertex_to_rt_any(element.vertex()); } else if (element.inner_case() == results::Element::kEdge) { - return edge_to_rt_any(element.edge()); + return edge_to_rt_any(txn, element.edge()); } else if (element.inner_case() == results::Element::kObject) { return object_to_rt_any(element.object()); } else if (element.inner_case() == results::Element::kGraphPath) { @@ -185,27 +188,30 @@ RTAny element_to_rt_any(const results::Element& element) { } } -RTAny collection_to_rt_any(const results::Collection& collection) { +RTAny collection_to_rt_any(const GraphReadInterface& txn, + const results::Collection& collection) { std::vector values; for (const auto& element : collection.collection()) { - values.push_back(element_to_rt_any(element)); + values.push_back(element_to_rt_any(txn, element)); } LOG(FATAL) << "Not implemented."; return RTAny(); } -RTAny column_to_rt_any(const results::Column& column) { +RTAny column_to_rt_any(const GraphReadInterface& txn, + const results::Column& column) { auto& entry = column.entry(); if (entry.has_element()) { - return element_to_rt_any(entry.element()); + return element_to_rt_any(txn, entry.element()); } else if (entry.has_collection()) { - return collection_to_rt_any(entry.collection()); + return collection_to_rt_any(txn, entry.collection()); } else { LOG(FATAL) << "Unsupported column entry type: " << entry.inner_case(); } } -std::vector result_to_rt_any(const results::Results& result) { +std::vector result_to_rt_any(const GraphReadInterface& txn, + const results::Results& result) { auto& record = result.record(); if (record.columns_size() == 0) { LOG(WARNING) << "Empty result."; @@ -213,7 +219,7 @@ std::vector result_to_rt_any(const results::Results& result) { } else { std::vector tuple; for (int32_t i = 0; i < record.columns_size(); ++i) { - tuple.push_back(column_to_rt_any(record.columns(i))); + tuple.push_back(column_to_rt_any(txn, record.columns(i))); } return tuple; } @@ -221,7 +227,7 @@ std::vector result_to_rt_any(const results::Results& result) { std::pair>, std::vector> collective_result_vec_to_column( - int32_t expect_col_num, + const GraphReadInterface& txn, int32_t expect_col_num, const std::vector& collective_results_vec) { std::vector offsets; offsets.push_back(0); @@ -233,7 +239,7 @@ collective_result_vec_to_column( std::vector> any_vec(expect_col_num); for (size_t i = 0; i < collective_results_vec.size(); ++i) { for (int32_t j = 0; j < collective_results_vec[i].results_size(); ++j) { - auto tuple = result_to_rt_any(collective_results_vec[i].results(j)); + auto tuple = result_to_rt_any(txn, collective_results_vec[i].results(j)); CHECK(tuple.size() == (size_t) expect_col_num) << "Inconsistent column number."; for (int32_t k = 0; k < expect_col_num; ++k) { @@ -367,7 +373,7 @@ bl::result eval_procedure_call(const std::vector& aliases, } auto column_and_offsets = - collective_result_vec_to_column(aliases.size(), results); + collective_result_vec_to_column(txn, aliases.size(), results); auto& columns = column_and_offsets.first; auto& offsets = column_and_offsets.second; if (columns.size() != aliases.size()) { diff --git a/flex/engines/hqps_db/core/operator/sink.h b/flex/engines/hqps_db/core/operator/sink.h index 9ff64bf309b2..e06d93844e26 100644 --- a/flex/engines/hqps_db/core/operator/sink.h +++ b/flex/engines/hqps_db/core/operator/sink.h @@ -996,7 +996,7 @@ class SinkOp { auto mutable_edge = new_col->mutable_entry()->mutable_element()->mutable_edge(); CHECK(iter != end_iter); - auto unique_edge_label = generate_edge_label_id( + auto unique_edge_label = graph.schema().get_edge_triplet_id( iter.GetSrcLabel(), iter.GetDstLabel(), iter.GetEdgeLabel()); mutable_edge->mutable_label()->set_id(iter.GetEdgeLabel()); mutable_edge->set_id(encode_unique_edge_id( @@ -1033,7 +1033,7 @@ class SinkOp { new_col->mutable_name_or_id()->set_id(tag_id); auto mutable_edge = new_col->mutable_entry()->mutable_element()->mutable_edge(); - auto unique_edge_label = generate_edge_label_id( + auto unique_edge_label = graph.schema().get_edge_triplet_id( iter.GetSrcLabel(), iter.GetDstLabel(), iter.GetEdgeLabel()); mutable_edge->mutable_label()->set_id(iter.GetEdgeLabel()); mutable_edge->set_id(encode_unique_edge_id( @@ -1155,19 +1155,6 @@ class SinkOp { unique_edge_id = unique_edge_id | dst; return unique_edge_id; } - - // actually produce uint24_t. - static uint32_t generate_edge_label_id(label_id_t src_label_id, - label_id_t dst_label_id, - label_id_t edge_label_id) { - uint32_t unique_edge_label_id = src_label_id; - static constexpr int num_bits = sizeof(label_id_t) * 8; - unique_edge_label_id = unique_edge_label_id << num_bits; - unique_edge_label_id = unique_edge_label_id | dst_label_id; - unique_edge_label_id = unique_edge_label_id << num_bits; - unique_edge_label_id = unique_edge_label_id | edge_label_id; - return unique_edge_label_id; - } }; } // namespace gs diff --git a/flex/engines/hqps_db/core/utils/hqps_utils.h b/flex/engines/hqps_db/core/utils/hqps_utils.h index efdfb68dc647..781693e74e49 100644 --- a/flex/engines/hqps_db/core/utils/hqps_utils.h +++ b/flex/engines/hqps_db/core/utils/hqps_utils.h @@ -145,14 +145,16 @@ template struct group_key_on_property : public std::true_type {}; template -struct group_key_on_property> +struct group_key_on_property< + AliasTagProp> : public std::false_type {}; template struct group_key_on_property> : public std::true_type {}; template -struct group_key_on_property> : public std::false_type {}; +struct group_key_on_property> + : public std::false_type {}; // check edge_dir and vopt consistency inline bool check_edge_dir_consist_vopt(const Direction& dir, VOpt vopt) { @@ -167,18 +169,24 @@ inline bool check_edge_dir_consist_vopt(const Direction& dir, VOpt vopt) { return false; } -template ::type* = nullptr> -constexpr auto remove_nth_element_impl(std::index_sequence, std::index_sequence, +template < + std::size_t nth, std::size_t... Head, std::size_t... Tail, + typename... Types, + typename std::enable_if<(nth + 1 != sizeof...(Types))>::type* = nullptr> +constexpr auto remove_nth_element_impl(std::index_sequence, + std::index_sequence, const std::tuple& tup) { return std::tuple{std::get(tup)..., // We +1 to refer one element after the one removed std::get(tup)...}; } -template ::type* = nullptr> -constexpr auto remove_nth_element_impl(std::index_sequence, std::index_sequence, +template < + std::size_t nth, std::size_t... Head, std::size_t... Tail, + typename... Types, + typename std::enable_if<(nth + 1 == sizeof...(Types))>::type* = nullptr> +constexpr auto remove_nth_element_impl(std::index_sequence, + std::index_sequence, const std::tuple& tup) { return std::tuple{std::get(tup)...}; } @@ -186,8 +194,9 @@ constexpr auto remove_nth_element_impl(std::index_sequence, std::index_ template constexpr auto remove_nth_element(const std::tuple& tup) { static_assert(nth < sizeof...(Types)); - return remove_nth_element_impl(std::make_index_sequence(), - std::make_index_sequence(), tup); + return remove_nth_element_impl( + std::make_index_sequence(), + std::make_index_sequence(), tup); } template @@ -214,7 +223,8 @@ template struct remove_ith_type> { typedef decltype(std::tuple_cat( std::declval>(), - std::declval>::type>())) type; + std::declval>::type>())) + type; }; // I != J @@ -222,13 +232,15 @@ template struct remove_ith_jth_type {}; template -struct remove_ith_jth_type, typename std::enable_if<(I < J)>::type> { +struct remove_ith_jth_type, + typename std::enable_if<(I < J)>::type> { using first_type = typename remove_ith_type>::type; using type = typename remove_ith_type::type; }; template -struct remove_ith_jth_type, typename std::enable_if<(I > J)>::type> { +struct remove_ith_jth_type, + typename std::enable_if<(I > J)>::type> { using type = typename remove_ith_jth_type>::type; }; @@ -237,17 +249,20 @@ struct Edge; template < size_t Is, typename... PROP_T, - typename std::enable_if::type* = nullptr> void props_to_string_array( - std::tuple& props, - std::array>>& res) { + typename std::enable_if::type* = nullptr> void + props_to_string_array( + std::tuple& props, + std::array>>& + res) { res[Is] = std::get(props).property_name; props_to_string_array(props, res); } template ::type* = nullptr> -void props_to_string_array(std::tuple& props, - std::array>>& res) { +void props_to_string_array( + std::tuple& props, + std::array>>& res) { res[Is] = std::get(props).property_name; } template @@ -262,8 +277,8 @@ struct tuple_element; // recursive case template -struct tuple_element> : gs::tuple_element> { -}; +struct tuple_element> + : gs::tuple_element> {}; // base case template @@ -281,7 +296,8 @@ struct tuple_element<-1, std::tuple> { template auto unwrap_future_tuple(std::tuple&& tuple) { - return unwrap_future_tuple(std::move(tuple), std::make_index_sequence()); + return unwrap_future_tuple(std::move(tuple), + std::make_index_sequence()); } template auto unwrap_future_tuple(std::tuple&& tuple, std::index_sequence) { @@ -322,7 +338,9 @@ struct FirstElement { // Create a tuple of const references to the elements of a tuple. template auto make_tuple_of_const_refs(const std::tuple& t) { - return std::apply([](const Args&... args) { return std::make_tuple(std::cref(args)...); }, t); + return std::apply( + [](const Args&... args) { return std::make_tuple(std::cref(args)...); }, + t); } template @@ -340,8 +358,9 @@ struct first_n_impl; template struct first_n_impl, Out...> { - typedef typename first_n_impl, Out..., First>::type - type; // move first input to output. + typedef + typename first_n_impl, Out..., First>::type + type; // move first input to output. }; // need First, Other... here to resolve ambiguity on n = 0 @@ -377,34 +396,42 @@ constexpr auto tuple_slice_impl(T&& t, std::index_sequence) { template constexpr auto tuple_slice(T&& t) { static_assert(r >= l, "invalid slice"); - static_assert(std::tuple_size>::value >= r, "slice index out of bounds"); - return tuple_slice_impl(std::forward(t), std::make_index_sequence{}); + static_assert(std::tuple_size>::value >= r, + "slice index out of bounds"); + return tuple_slice_impl(std::forward(t), + std::make_index_sequence{}); } // [l, tuple_size - 1] template constexpr auto tuple_slice(T&& t) { - static_assert(std::tuple_size>::value > l, "slice index out of bounds"); + static_assert(std::tuple_size>::value > l, + "slice index out of bounds"); return tuple_slice_impl( - std::forward(t), std::make_index_sequence>::value - l>{}); + std::forward(t), + std::make_index_sequence>::value - l>{}); } -template = 0)>::type* = nullptr> +template = 0)>::type* = nullptr> inline auto get_from_tuple(std::tuple& tuple) { return std::get(tuple); } -template ::type* = nullptr> +template ::type* = nullptr> inline auto get_from_tuple(std::tuple& tuple) { static constexpr size_t num = sizeof...(T); return std::get(tuple); } -template = 0)>::type* = nullptr> +template = 0)>::type* = nullptr> inline const auto& get_from_tuple(const std::tuple& tuple) { return std::get(tuple); } -template ::type* = nullptr> +template ::type* = nullptr> inline const auto& get_from_tuple(const std::tuple& tuple) { static constexpr size_t num = sizeof...(T); return std::get(tuple); @@ -413,26 +440,30 @@ inline const auto& get_from_tuple(const std::tuple& tuple) { // vertex/edge property associate with type template ::type> -auto transform_array_impl(std::array&& array, FUNC_T&& func, std::index_sequence) { +auto transform_array_impl(std::array&& array, FUNC_T&& func, + std::index_sequence) { return std::array{std::move(func(std::move(array[Is])))...}; } template auto transform_array(std::array&& array, FUNC_T&& func) { - return transform_array_impl(std::move(array), std::move(func), std::make_index_sequence()); + return transform_array_impl(std::move(array), std::move(func), + std::make_index_sequence()); } template ::type> auto transform_tuple_impl(const std::tuple&& tuple, FUNC_T&& func, std::index_sequence) { - return std::make_tuple(std::move(func(Is, std::move(std::get(tuple))))...); + return std::make_tuple( + std::move(func(Is, std::move(std::get(tuple))))...); } template auto transform_tuple(const std::tuple&& tuple, FUNC_T&& func) { static constexpr size_t N = sizeof...(T); - return transform_tuple_impl(std::move(tuple), std::move(func), std::make_index_sequence()); + return transform_tuple_impl(std::move(tuple), std::move(func), + std::make_index_sequence()); } template @@ -443,28 +474,32 @@ bool apply_on_tuple_impl(const FUNC& func, const std::tuple& tuple, template bool apply_on_tuple(const FUNC& func, const std::tuple& tuple) { - return apply_on_tuple_impl(func, tuple, std::make_index_sequence()); + return apply_on_tuple_impl(func, tuple, + std::make_index_sequence()); } template ::type> -auto apply_array_impl(const std::array& array, FUNC_T&& func, std::index_sequence) { +auto apply_array_impl(const std::array& array, FUNC_T&& func, + std::index_sequence) { return std::array{std::move(func(array[Is]))...}; } template auto apply_array(const std::array& array, FUNC_T&& func) { - return apply_array_impl(array, std::move(func), std::make_index_sequence()); + return apply_array_impl(array, std::move(func), + std::make_index_sequence()); } template -void apply_tuple_impl(const std::tuple& tuple, const FUNC_T& func, std::index_sequence, - OTHER_ARGS&... other_args) { +void apply_tuple_impl(const std::tuple& tuple, const FUNC_T& func, + std::index_sequence, OTHER_ARGS&... other_args) { ((func(std::get(tuple), std::forward(other_args)...)), ...); } template -auto apply_tuple(const std::tuple& tuple, const FUNC_T& func, OTHER_ARGS&... other_args) { +auto apply_tuple(const std::tuple& tuple, const FUNC_T& func, + OTHER_ARGS&... other_args) { static constexpr size_t N = sizeof...(T); return apply_tuple_impl(tuple, func, std::make_index_sequence(), std::forward(other_args)...); @@ -473,8 +508,8 @@ auto apply_tuple(const std::tuple& tuple, const FUNC_T& func, OTHER_ARGS&. template constexpr auto make_array(Args&&... args) { if constexpr (std::is_same::value) { - return std::array...>, sizeof...(Args)>{ - {std::forward(args)...}}; + return std::array...>, + sizeof...(Args)>{{std::forward(args)...}}; } else { return std::array{{std::forward(args)...}}; } @@ -522,11 +557,13 @@ struct NumberLarger { }; template -using make_index_range = decltype(add(std::make_index_sequence())); +using make_index_range = + decltype(add(std::make_index_sequence())); template struct TupleCatT { - using tuple_cat_t = decltype(std::tuple_cat(std::declval(), std::declval())); + using tuple_cat_t = + decltype(std::tuple_cat(std::declval(), std::declval())); }; template @@ -540,7 +577,8 @@ struct TupleCatT> { }; template -auto make_getter_tuple(label_t label, std::tuple&& tuple, std::index_sequence) { +auto make_getter_tuple(label_t label, std::tuple&& tuple, + std::index_sequence) { return std::make_tuple(std::get(tuple).CreateGetter(label)...); } @@ -582,8 +620,9 @@ struct ColumnAccessorImpl {}; // Recursive template -struct ColumnAccessorImpl : public SingleColumn, - public ColumnAccessorImpl {}; +struct ColumnAccessorImpl + : public SingleColumn, + public ColumnAccessorImpl {}; // multiple single columns. @@ -641,8 +680,8 @@ std::vector array_to_vec(const std::array& array) { } template -static typename PRIORITY_QUEUE_T::container_type priority_queue_to_vec(PRIORITY_QUEUE_T& pq, - bool reversed = false) { +static typename PRIORITY_QUEUE_T::container_type priority_queue_to_vec( + PRIORITY_QUEUE_T& pq, bool reversed = false) { auto pq_size = pq.size(); typename PRIORITY_QUEUE_T::container_type res; res.reserve(pq_size); @@ -681,7 +720,8 @@ struct to_string_impl> { // map{key:value, ...} ss << "map{"; for (auto& [k, v] : vec) { - ss << to_string_impl::to_string(k) << ":" << to_string_impl::to_string(v) << ","; + ss << to_string_impl::to_string(k) << ":" + << to_string_impl::to_string(v) << ","; } ss << "}"; return ss.str(); @@ -701,7 +741,8 @@ struct to_string_impl> { template struct to_string_impl, M>> { - static inline std::string to_string(const std::array, M>& empty) { + static inline std::string to_string( + const std::array, M>& empty) { std::stringstream ss; ss << "["; for (auto i : empty) { @@ -729,7 +770,9 @@ struct to_string_impl { template <> struct to_string_impl { - static inline std::string to_string(const Dist& empty) { return std::to_string(empty.dist); } + static inline std::string to_string(const Dist& empty) { + return std::to_string(empty.dist); + } }; template <> @@ -741,12 +784,16 @@ struct to_string_impl { template <> struct to_string_impl { - static inline std::string to_string(const std::string_view& empty) { return std::string(empty); } + static inline std::string to_string(const std::string_view& empty) { + return std::string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const grape::EmptyType& empty) { return ""; } + static inline std::string to_string(const grape::EmptyType& empty) { + return ""; + } }; template <> @@ -756,39 +803,60 @@ struct to_string_impl { } }; +template <> +struct to_string_impl { + static inline std::string to_string(const uint16_t& empty) { + return std::to_string((int32_t) empty); + } +}; + template <> struct to_string_impl { - static inline std::string to_string(const int64_t& empty) { return std::to_string(empty); } + static inline std::string to_string(const int64_t& empty) { + return std::to_string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const bool& empty) { return std::to_string(empty); } + static inline std::string to_string(const bool& empty) { + return std::to_string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const unsigned long& empty) { return std::to_string(empty); } + static inline std::string to_string(const unsigned long& empty) { + return std::to_string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const int32_t& empty) { return std::to_string(empty); } + static inline std::string to_string(const int32_t& empty) { + return std::to_string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const uint32_t& empty) { return std::to_string(empty); } + static inline std::string to_string(const uint32_t& empty) { + return std::to_string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const double& empty) { return std::to_string(empty); } + static inline std::string to_string(const double& empty) { + return std::to_string(empty); + } }; template <> struct to_string_impl { - static inline std::string to_string(const std::string& empty) { return empty; } + static inline std::string to_string(const std::string& empty) { + return empty; + } }; template <> @@ -874,8 +942,8 @@ struct to_string_impl> { std::apply( [&result](const auto&... v) { ((result += - (to_string_impl>>::to_string( - v)) + + (to_string_impl>>::to_string(v)) + ","), ...); }, @@ -904,9 +972,11 @@ template struct Edge { VID_T src, dst; const std::tuple& edata; - Edge(VID_T s, VID_T d, const std::tuple& data) : src(s), dst(d), edata(data) {} + Edge(VID_T s, VID_T d, const std::tuple& data) + : src(s), dst(d), edata(data) {} std::string to_string() const { - return std::to_string(src) + "->" + std::to_string(dst) + "(" + gs::to_string(edata) + ")"; + return std::to_string(src) + "->" + std::to_string(dst) + "(" + + gs::to_string(edata) + ")"; } }; @@ -924,7 +994,8 @@ template using DefaultEdge = Edge; template -inline bool operator==(const DefaultEdge& lhs, const DefaultEdge& rhs) { +inline bool operator==(const DefaultEdge& lhs, + const DefaultEdge& rhs) { return lhs.src == rhs.src && lhs.dst == rhs.dst; } @@ -958,9 +1029,9 @@ struct function_traits }; template -static std::tuple get_graph_label_pair(Direction& direction, - label_id_t query_src_label, - label_id_t query_dst_label) { +static std::tuple get_graph_label_pair( + Direction& direction, label_id_t query_src_label, + label_id_t query_dst_label) { label_id_t src_label, dst_label; if (direction == Direction::In) { src_label = query_dst_label; diff --git a/flex/engines/hqps_db/database/mutable_csr_interface.h b/flex/engines/hqps_db/database/mutable_csr_interface.h index 1bb3a237836e..97d599712fce 100644 --- a/flex/engines/hqps_db/database/mutable_csr_interface.h +++ b/flex/engines/hqps_db/database/mutable_csr_interface.h @@ -80,7 +80,7 @@ class MutableCSRInterface { using vertex_id_t = vid_t; using gid_t = uint64_t; - using label_id_t = uint8_t; + using label_id_t = gs::label_t; using nbr_list_array_t = mutable_csr_graph_impl::NbrListArray; diff --git a/flex/engines/http_server/actor/admin_actor.act.cc b/flex/engines/http_server/actor/admin_actor.act.cc index 526b769dd59d..5199d4173f8e 100644 --- a/flex/engines/http_server/actor/admin_actor.act.cc +++ b/flex/engines/http_server/actor/admin_actor.act.cc @@ -51,27 +51,25 @@ gs::GraphStatistics get_graph_statistics(const gs::GraphDBSession& sess) { auto edge_label_name = schema.get_edge_label_name(edge_label_id); std::vector> vertex_type_pair_statistics; - for (auto src_label_id = 0; src_label_id < vertex_label_num; - ++src_label_id) { - auto src_label_name = schema.get_vertex_label_name(src_label_id); - for (auto dst_label_id = 0; dst_label_id < vertex_label_num; - ++dst_label_id) { - auto dst_label_name = schema.get_vertex_label_name(dst_label_id); - if (schema.exist(src_label_id, dst_label_id, edge_label_id)) { - auto oe_csr = - graph.get_oe_csr(src_label_id, dst_label_id, edge_label_id); - auto ie_csr = - graph.get_ie_csr(dst_label_id, src_label_id, edge_label_id); - size_t cur_edge_cnt = 0; - if (oe_csr) { - cur_edge_cnt += oe_csr->edge_num(); - } else if (ie_csr) { - cur_edge_cnt += ie_csr->edge_num(); - } - stat.total_edge_count += cur_edge_cnt; - vertex_type_pair_statistics.emplace_back( - std::tuple{src_label_name, dst_label_name, cur_edge_cnt}); + for (size_t index = 0; index < schema.get_edge_triplet_num(); ++index) { + auto triplet = schema.get_edge_triplet(index); + auto src_label_id = std::get<0>(triplet); + auto dst_label_id = std::get<1>(triplet); + if (schema.exist(src_label_id, dst_label_id, edge_label_id)) { + auto oe_csr = + graph.get_oe_csr(src_label_id, dst_label_id, edge_label_id); + auto ie_csr = + graph.get_ie_csr(dst_label_id, src_label_id, edge_label_id); + size_t cur_edge_cnt = 0; + if (oe_csr) { + cur_edge_cnt += oe_csr->edge_num(); + } else if (ie_csr) { + cur_edge_cnt += ie_csr->edge_num(); } + stat.total_edge_count += cur_edge_cnt; + vertex_type_pair_statistics.emplace_back(std::tuple{ + schema.get_vertex_label_name(src_label_id), + schema.get_vertex_label_name(dst_label_id), cur_edge_cnt}); } } if (!vertex_type_pair_statistics.empty()) { diff --git a/flex/interactive/sdk/python/gs_interactive/tests/conftest.py b/flex/interactive/sdk/python/gs_interactive/tests/conftest.py index 4eba360f3b7b..48ef79061d8f 100644 --- a/flex/interactive/sdk/python/gs_interactive/tests/conftest.py +++ b/flex/interactive/sdk/python/gs_interactive/tests/conftest.py @@ -33,6 +33,8 @@ from gs_interactive.models import StartServiceRequest from gs_interactive.models import UpdateProcedureRequest +MANY_LABEL_GRAPH_LABEL_NUM = 1000 + cur_dir = os.path.dirname(os.path.abspath(__file__)) MODERN_GRAPH_DATA_DIR = os.path.abspath( os.path.join(cur_dir, "../../../../examples/modern_graph") @@ -1115,6 +1117,65 @@ def create_graph_algo_graph_with_x_csr_params(interactive_session): delete_running_graph(interactive_session, graph_id) +@pytest.fixture(scope="function") +def create_many_label_graph(interactive_session): + """ + Create a graph with many labels for testing the performance of the query + """ + many_label_graph = { + "name": "many_label_graph", + "schema": { + "vertex_types": [], + "edge_types": [], + }, + } + for i in range(MANY_LABEL_GRAPH_LABEL_NUM): + many_label_graph["schema"]["vertex_types"].append( + { + "type_name": f"person{i}", + "properties": [ + { + "property_name": "id", + "property_type": {"primitive_type": "DT_SIGNED_INT64"}, + }, + { + "property_name": "name", + "property_type": {"string": {"long_text": ""}}, + }, + ], + "primary_keys": ["id"], + } + ) + + for i in range(MANY_LABEL_GRAPH_LABEL_NUM): + many_label_graph["schema"]["edge_types"].append( + { + "type_name": f"knows{i}", + "vertex_type_pair_relations": [ + { + "source_vertex": f"person{i}", + "destination_vertex": f"person{i}", + "relation": "MANY_TO_MANY", + } + ], + "properties": [ + { + "property_name": "weight", + "property_type": {"primitive_type": "DT_DOUBLE"}, + } + ], + "primary_keys": [], + } + ) + + create_graph_request = CreateGraphRequest.from_dict(many_label_graph) + resp = interactive_session.create_graph(create_graph_request) + assert resp.is_ok() + graph_id = resp.get_value().graph_id + yield graph_id + delete_running_graph(interactive_session, graph_id) + + def wait_job_finish(sess: Session, job_id: str): assert job_id is not None while True: @@ -1237,6 +1298,67 @@ def import_data_to_new_graph_algo_graph(sess: Session, graph_id: str): assert wait_job_finish(sess, job_id) +def import_data_to_many_label_graph(sess: Session, graph_id: str): + # get temp dir + import shutil + import tempfile + + data_dir = os.path.join(tempfile.gettempdir(), "many_label_graph") + if os.path.exists(data_dir): + shutil.rmtree(data_dir) + os.makedirs(data_dir) + many_label_graph_import_config = { + "loading_config": { + "data_source": {"scheme": "file", "location": data_dir}, + "import_option": "init", + "format": { + "type": "csv", + "metadata": { + "delimiter": "|", + }, + }, + }, + "vertex_mappings": [], + "edge_mappings": [], + } + edge_file_name = "person_knows_person.csv" + vertex_file_name = "person.csv" + + for i in range(MANY_LABEL_GRAPH_LABEL_NUM): + many_label_graph_import_config["vertex_mappings"].append( + { + "type_name": f"person{i}", + "inputs": [vertex_file_name], + } + ) + for i in range(MANY_LABEL_GRAPH_LABEL_NUM): + many_label_graph_import_config["edge_mappings"].append( + { + "type_triplet": { + "edge": f"knows{i}", + "source_vertex": f"person{i}", + "destination_vertex": f"person{i}", + }, + "inputs": [edge_file_name], + } + ) + # generate the data + tmp_person_csv = open(data_dir + "/" + vertex_file_name, "w") + tmp_person_csv.write("id|name\n" + "1|marko\n") + tmp_person_csv.close() + tmp_person_knows_person_csv = open(data_dir + "/" + edge_file_name, "w") + tmp_person_knows_person_csv.write("source|target|weight\n" + "1|1|1.0\n") + tmp_person_knows_person_csv.close() + schema_mapping = SchemaMapping.from_dict(many_label_graph_import_config) + resp = sess.bulk_loading(graph_id, schema_mapping) + assert resp.is_ok() + job_id = resp.get_value().job_id + assert wait_job_finish(sess, job_id) + + # remove the temp dir + shutil.rmtree(data_dir) + + def submit_query_via_neo4j_endpoint( neo4j_sess: Neo4jSession, graph_id: str, query: str ): diff --git a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py index ff07fff41553..9d55000fd226 100644 --- a/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py +++ b/flex/interactive/sdk/python/gs_interactive/tests/test_robustness.py @@ -25,12 +25,14 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) +from gs_interactive.tests.conftest import MANY_LABEL_GRAPH_LABEL_NUM from gs_interactive.tests.conftest import call_procedure # noqa: E402 from gs_interactive.tests.conftest import create_procedure from gs_interactive.tests.conftest import delete_procedure from gs_interactive.tests.conftest import ensure_compiler_schema_ready from gs_interactive.tests.conftest import import_data_to_full_graph_algo_graph from gs_interactive.tests.conftest import import_data_to_full_modern_graph +from gs_interactive.tests.conftest import import_data_to_many_label_graph from gs_interactive.tests.conftest import import_data_to_modern_graph_temporal_type from gs_interactive.tests.conftest import import_data_to_new_graph_algo_graph from gs_interactive.tests.conftest import import_data_to_partial_modern_graph @@ -561,3 +563,25 @@ def test_graph_with_long_text_property( assert len(records) == 4 for record in records: assert len(record["name"]) > 4096 + + +def test_many_label_graph(interactive_session, neo4j_session, create_many_label_graph): + print("[Test many label graph]") + import_data_to_many_label_graph(interactive_session, create_many_label_graph) + start_service_on_graph(interactive_session, create_many_label_graph) + ensure_compiler_schema_ready( + interactive_session, neo4j_session, create_many_label_graph + ) + + # Get number vertex labels + result = neo4j_session.run("MATCH (n) return count(distinct(labels(n)));") + record = result.fetch(1) + assert record[0]["$f0"] == MANY_LABEL_GRAPH_LABEL_NUM + + result = neo4j_session.run( + f"MATCH (n: person{MANY_LABEL_GRAPH_LABEL_NUM - 1}) return count(n)" + ) + record = result.fetch(1) + assert ( + record[0]["$f0"] == 1 + ) # only one vertex with label person_{MANY_LABEL_GRAPH_LABEL_NUM - 1} diff --git a/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc index fdef0cb69b6b..731a3b985c43 100644 --- a/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc @@ -54,10 +54,9 @@ BasicFragmentLoader::BasicFragmentLoader(const Schema& schema, vertex_label_num_(schema_.vertex_label_num()), edge_label_num_(schema_.edge_label_num()) { vertex_data_.resize(vertex_label_num_); - ie_.resize(vertex_label_num_ * vertex_label_num_ * edge_label_num_, NULL); - oe_.resize(vertex_label_num_ * vertex_label_num_ * edge_label_num_, NULL); - dual_csr_list_.resize(vertex_label_num_ * vertex_label_num_ * edge_label_num_, - NULL); + ie_.resize(schema_.get_edge_triplet_num(), NULL); + oe_.resize(schema_.get_edge_triplet_num(), NULL); + dual_csr_list_.resize(schema_.get_edge_triplet_num(), NULL); lf_indexers_.resize(vertex_label_num_); std::filesystem::create_directories(runtime_dir(prefix)); std::filesystem::create_directories(snapshot_dir(prefix, 0)); @@ -102,20 +101,16 @@ void BasicFragmentLoader::init_loading_status_file() { append_vertex_loading_progress(label_name, LoadingStatus::kLoading); } VLOG(1) << "Finish init vertex status files"; - for (size_t src_label = 0; src_label < vertex_label_num_; src_label++) { - std::string src_label_name = schema_.get_vertex_label_name(src_label); - for (size_t dst_label = 0; dst_label < vertex_label_num_; dst_label++) { - std::string dst_label_name = schema_.get_vertex_label_name(dst_label); - for (size_t edge_label = 0; edge_label < edge_label_num_; edge_label++) { - std::string edge_label_name = schema_.get_edge_label_name(edge_label); - if (schema_.exist(src_label_name, dst_label_name, edge_label_name)) { - append_edge_loading_progress(src_label_name, dst_label_name, - edge_label_name, - LoadingStatus::kLoading); - } - } - } + for (uint32_t i = 0; i < schema_.get_edge_triplet_num(); ++i) { + auto triplet = schema_.get_edge_triplet(i); + + auto src_label_name = schema_.get_vertex_label_name(std::get<0>(triplet)); + auto dst_label_name = schema_.get_vertex_label_name(std::get<1>(triplet)); + auto edge_label_name = schema_.get_edge_label_name(std::get<2>(triplet)); + append_edge_loading_progress(src_label_name, dst_label_name, + edge_label_name, LoadingStatus::kLoading); } + VLOG(1) << "Finish init edge status files"; } void BasicFragmentLoader::init_vertex_data() { @@ -157,8 +152,8 @@ IndexerType& BasicFragmentLoader::GetLFIndexer(label_t v_label) { void BasicFragmentLoader::set_csr(label_t src_label_id, label_t dst_label_id, label_t edge_label_id, DualCsrBase* dual_csr) { - size_t index = src_label_id * vertex_label_num_ * edge_label_num_ + - dst_label_id * edge_label_num_ + edge_label_id; + size_t index = + schema_.get_edge_triplet_id(src_label_id, dst_label_id, edge_label_id); dual_csr_list_[index] = dual_csr; ie_[index] = dual_csr->GetInCsr(); oe_[index] = dual_csr->GetOutCsr(); @@ -167,16 +162,16 @@ void BasicFragmentLoader::set_csr(label_t src_label_id, label_t dst_label_id, DualCsrBase* BasicFragmentLoader::get_csr(label_t src_label_id, label_t dst_label_id, label_t edge_label_id) { - size_t index = src_label_id * vertex_label_num_ * edge_label_num_ + - dst_label_id * edge_label_num_ + edge_label_id; + size_t index = + schema_.get_edge_triplet_id(src_label_id, dst_label_id, edge_label_id); return dual_csr_list_[index]; } void BasicFragmentLoader::init_edge_table(label_t src_label_id, label_t dst_label_id, label_t edge_label_id) { - size_t index = src_label_id * vertex_label_num_ * edge_label_num_ + - dst_label_id * edge_label_num_ + edge_label_id; + size_t index = + schema_.get_edge_triplet_id(src_label_id, dst_label_id, edge_label_id); auto cast_dual_csr = dynamic_cast*>(dual_csr_list_[index]); CHECK(cast_dual_csr != nullptr); diff --git a/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.h b/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.h index 338b0f713768..8b1bd1b50d67 100644 --- a/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.h +++ b/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.h @@ -107,8 +107,8 @@ class BasicFragmentLoader { template void AddNoPropEdgeBatch(label_t src_label_id, label_t dst_label_id, label_t edge_label_id) { - size_t index = src_label_id * vertex_label_num_ * edge_label_num_ + - dst_label_id * edge_label_num_ + edge_label_id; + size_t index = + schema_.get_edge_triplet_id(src_label_id, dst_label_id, edge_label_id); CHECK(ie_[index] == NULL); CHECK(oe_[index] == NULL); auto src_label_name = schema_.get_vertex_label_name(src_label_id); @@ -166,8 +166,8 @@ class BasicFragmentLoader { label_t edge_label_id, const std::vector& edges_vec, const std::vector& ie_degree, const std::vector& oe_degree, bool build_csr_in_mem) { - size_t index = src_label_id * vertex_label_num_ * edge_label_num_ + - dst_label_id * edge_label_num_ + edge_label_id; + size_t index = + schema_.get_edge_triplet_id(src_label_id, dst_label_id, edge_label_id); auto dual_csr = dual_csr_list_[index]; CHECK(dual_csr != NULL); auto casted_dual_csr = get_casted_dual_csr(dual_csr); diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc index 444c1472066f..af32b0dcc0b8 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc @@ -29,17 +29,9 @@ MutablePropertyFragment::~MutablePropertyFragment() { degree_list[i] = lf_indexers_[i].size(); vertex_data_[i].resize(degree_list[i]); } - for (size_t src_label = 0; src_label != vertex_label_num_; ++src_label) { - for (size_t dst_label = 0; dst_label != vertex_label_num_; ++dst_label) { - for (size_t e_label = 0; e_label != edge_label_num_; ++e_label) { - size_t index = src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + e_label; - if (dual_csr_list_[index] != NULL) { - dual_csr_list_[index]->Resize(degree_list[src_label], - degree_list[dst_label]); - delete dual_csr_list_[index]; - } - } + for (auto ptr : dual_csr_list_) { + if (ptr != NULL) { + delete ptr; } } } @@ -204,95 +196,73 @@ void MutablePropertyFragment::Open(const std::string& work_dir, vertex_capacities[i] = vertex_capacity; } - ie_.resize(vertex_label_num_ * vertex_label_num_ * edge_label_num_, NULL); - oe_.resize(vertex_label_num_ * vertex_label_num_ * edge_label_num_, NULL); - - dual_csr_list_.resize(vertex_label_num_ * vertex_label_num_ * edge_label_num_, - NULL); - - for (size_t src_label_i = 0; src_label_i != vertex_label_num_; - ++src_label_i) { - std::string src_label = - schema_.get_vertex_label_name(static_cast(src_label_i)); - for (size_t dst_label_i = 0; dst_label_i != vertex_label_num_; - ++dst_label_i) { - std::string dst_label = - schema_.get_vertex_label_name(static_cast(dst_label_i)); - for (size_t e_label_i = 0; e_label_i != edge_label_num_; ++e_label_i) { - std::string edge_label = - schema_.get_edge_label_name(static_cast(e_label_i)); - if (!schema_.exist(src_label, dst_label, edge_label)) { - continue; - } - size_t index = src_label_i * vertex_label_num_ * edge_label_num_ + - dst_label_i * edge_label_num_ + e_label_i; - auto& properties = - schema_.get_edge_properties(src_label, dst_label, edge_label); - EdgeStrategy oe_strategy = schema_.get_outgoing_edge_strategy( - src_label, dst_label, edge_label); - EdgeStrategy ie_strategy = schema_.get_incoming_edge_strategy( - src_label, dst_label, edge_label); - bool oe_mutable = - schema_.outgoing_edge_mutable(src_label, dst_label, edge_label); - bool ie_mutable = - schema_.incoming_edge_mutable(src_label, dst_label, edge_label); - - auto& prop_names = - schema_.get_edge_property_names(src_label, dst_label, edge_label); - - dual_csr_list_[index] = create_csr(oe_strategy, ie_strategy, properties, - oe_mutable, ie_mutable, prop_names); - ie_[index] = dual_csr_list_[index]->GetInCsr(); - oe_[index] = dual_csr_list_[index]->GetOutCsr(); - if (memory_level == 0) { - dual_csr_list_[index]->Open( - oe_prefix(src_label, dst_label, edge_label), - ie_prefix(src_label, dst_label, edge_label), - edata_prefix(src_label, dst_label, edge_label), snapshot_dir, - tmp_dir_path); - } else if (memory_level >= 2) { - dual_csr_list_[index]->OpenWithHugepages( - oe_prefix(src_label, dst_label, edge_label), - ie_prefix(src_label, dst_label, edge_label), - edata_prefix(src_label, dst_label, edge_label), snapshot_dir, - vertex_capacities[src_label_i], vertex_capacities[dst_label_i]); - } else { - dual_csr_list_[index]->OpenInMemory( - oe_prefix(src_label, dst_label, edge_label), - ie_prefix(src_label, dst_label, edge_label), - edata_prefix(src_label, dst_label, edge_label), snapshot_dir, - vertex_capacities[src_label_i], vertex_capacities[dst_label_i]); - } - dual_csr_list_[index]->Resize(vertex_capacities[src_label_i], - vertex_capacities[dst_label_i]); - } + ie_.resize(schema_.get_edge_triplet_num(), NULL); + oe_.resize(schema_.get_edge_triplet_num(), NULL); + + dual_csr_list_.resize(schema_.get_edge_triplet_num(), NULL); + + for (size_t index = 0; index < schema_.get_edge_triplet_num(); ++index) { + auto triplet = schema_.get_edge_triplet(index); + auto src_label_i = std::get<0>(triplet); + auto dst_label_i = std::get<1>(triplet); + auto src_label = schema_.get_vertex_label_name(std::get<0>(triplet)); + auto dst_label = schema_.get_vertex_label_name(std::get<1>(triplet)); + auto edge_label = schema_.get_edge_label_name(std::get<2>(triplet)); + + auto& properties = + schema_.get_edge_properties(src_label, dst_label, edge_label); + EdgeStrategy oe_strategy = + schema_.get_outgoing_edge_strategy(src_label, dst_label, edge_label); + EdgeStrategy ie_strategy = + schema_.get_incoming_edge_strategy(src_label, dst_label, edge_label); + bool oe_mutable = + schema_.outgoing_edge_mutable(src_label, dst_label, edge_label); + bool ie_mutable = + schema_.incoming_edge_mutable(src_label, dst_label, edge_label); + + auto& prop_names = + schema_.get_edge_property_names(src_label, dst_label, edge_label); + + dual_csr_list_[index] = create_csr(oe_strategy, ie_strategy, properties, + oe_mutable, ie_mutable, prop_names); + ie_[index] = dual_csr_list_[index]->GetInCsr(); + oe_[index] = dual_csr_list_[index]->GetOutCsr(); + if (memory_level == 0) { + dual_csr_list_[index]->Open( + oe_prefix(src_label, dst_label, edge_label), + ie_prefix(src_label, dst_label, edge_label), + edata_prefix(src_label, dst_label, edge_label), snapshot_dir, + tmp_dir_path); + } else if (memory_level >= 2) { + dual_csr_list_[index]->OpenWithHugepages( + oe_prefix(src_label, dst_label, edge_label), + ie_prefix(src_label, dst_label, edge_label), + edata_prefix(src_label, dst_label, edge_label), snapshot_dir, + vertex_capacities[src_label_i], vertex_capacities[dst_label_i]); + } else { + dual_csr_list_[index]->OpenInMemory( + oe_prefix(src_label, dst_label, edge_label), + ie_prefix(src_label, dst_label, edge_label), + edata_prefix(src_label, dst_label, edge_label), snapshot_dir, + vertex_capacities[src_label_i], vertex_capacities[dst_label_i]); } + dual_csr_list_[index]->Resize(vertex_capacities[src_label_i], + vertex_capacities[dst_label_i]); } } void MutablePropertyFragment::Compact(uint32_t version) { - for (size_t src_label_i = 0; src_label_i != vertex_label_num_; - ++src_label_i) { - std::string src_label = - schema_.get_vertex_label_name(static_cast(src_label_i)); - for (size_t dst_label_i = 0; dst_label_i != vertex_label_num_; - ++dst_label_i) { - std::string dst_label = - schema_.get_vertex_label_name(static_cast(dst_label_i)); - for (size_t e_label_i = 0; e_label_i != edge_label_num_; ++e_label_i) { - std::string edge_label = - schema_.get_edge_label_name(static_cast(e_label_i)); - if (!schema_.exist(src_label, dst_label, edge_label)) { - continue; - } - size_t index = src_label_i * vertex_label_num_ * edge_label_num_ + - dst_label_i * edge_label_num_ + e_label_i; - if (dual_csr_list_[index] != NULL) { - if (schema_.get_sort_on_compaction(src_label, dst_label, - edge_label)) { - dual_csr_list_[index]->SortByEdgeData(version); - } - } + for (size_t index = 0; index < schema_.get_edge_triplet_num(); ++index) { + auto src_label_name = schema_.get_vertex_label_name( + std::get<0>(schema_.get_edge_triplet(index))); + auto dst_label_name = schema_.get_vertex_label_name( + std::get<1>(schema_.get_edge_triplet(index))); + auto edge_label_name = schema_.get_edge_label_name( + std::get<2>(schema_.get_edge_triplet(index))); + if (dual_csr_list_[index] != NULL) { + if (schema_.get_sort_on_compaction(src_label_name, dst_label_name, + edge_label_name)) { + dual_csr_list_[index]->SortByEdgeData(version); } } } @@ -322,36 +292,17 @@ void MutablePropertyFragment::Dump(const std::string& work_dir, snapshot_dir_path); } - for (size_t src_label_i = 0; src_label_i != vertex_label_num_; - ++src_label_i) { - std::string src_label = - schema_.get_vertex_label_name(static_cast(src_label_i)); - for (size_t dst_label_i = 0; dst_label_i != vertex_label_num_; - ++dst_label_i) { - std::string dst_label = - schema_.get_vertex_label_name(static_cast(dst_label_i)); - for (size_t e_label_i = 0; e_label_i != edge_label_num_; ++e_label_i) { - std::string edge_label = - schema_.get_edge_label_name(static_cast(e_label_i)); - if (!schema_.exist(src_label, dst_label, edge_label)) { - continue; - } - size_t index = src_label_i * vertex_label_num_ * edge_label_num_ + - dst_label_i * edge_label_num_ + e_label_i; - if (dual_csr_list_[index] != NULL) { - dual_csr_list_[index]->Resize(vertex_num[src_label_i], - vertex_num[dst_label_i]); - if (schema_.get_sort_on_compaction(src_label, dst_label, - edge_label)) { - dual_csr_list_[index]->SortByEdgeData(version + 1); - } - dual_csr_list_[index]->Dump( - oe_prefix(src_label, dst_label, edge_label), - ie_prefix(src_label, dst_label, edge_label), - edata_prefix(src_label, dst_label, edge_label), - snapshot_dir_path); - } - } + for (size_t index = 0; index < schema_.get_edge_triplet_num(); ++index) { + if (dual_csr_list_[index] != NULL) { + auto triplet = schema_.get_edge_triplet(index); + auto src_label_name = schema_.get_vertex_label_name(std::get<0>(triplet)); + auto dst_label_name = schema_.get_vertex_label_name(std::get<1>(triplet)); + auto edge_label_name = schema_.get_edge_label_name(std::get<2>(triplet)); + dual_csr_list_[index]->Dump( + oe_prefix(src_label_name, dst_label_name, edge_label_name), + ie_prefix(src_label_name, dst_label_name, edge_label_name), + edata_prefix(src_label_name, dst_label_name, edge_label_name), + snapshot_dir_path); } } set_snapshot_version(work_dir, version); @@ -375,8 +326,7 @@ void MutablePropertyFragment::IngestEdge(label_t src_label, vid_t src_lid, label_t edge_label, timestamp_t ts, grape::OutArchive& arc, Allocator& alloc) { - size_t index = src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + edge_label; + size_t index = schema_.get_edge_triplet_id(src_label, dst_label, edge_label); dual_csr_list_[index]->IngestEdge(src_lid, dst_lid, arc, ts, alloc); } @@ -384,8 +334,7 @@ void MutablePropertyFragment::UpdateEdge(label_t src_label, vid_t src_lid, label_t dst_label, vid_t dst_lid, label_t edge_label, timestamp_t ts, const Any& arc, Allocator& alloc) { - size_t index = src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + edge_label; + size_t index = schema_.get_edge_triplet_id(src_label, dst_label, edge_label); dual_csr_list_[index]->UpdateEdge(src_lid, dst_lid, arc, ts, alloc); } const Schema& MutablePropertyFragment::schema() const { return schema_; } @@ -398,8 +347,7 @@ vid_t MutablePropertyFragment::vertex_num(label_t vertex_label) const { size_t MutablePropertyFragment::edge_num(label_t src_label, label_t edge_label, label_t dst_label) const { - size_t index = src_label * vertex_label_num_ * edge_label_num_ + - dst_label * edge_label_num_ + edge_label; + size_t index = schema_.get_edge_triplet_id(src_label, dst_label, edge_label); if (dual_csr_list_[index] != NULL) { return dual_csr_list_[index]->EdgeNum(); } else { @@ -483,24 +431,15 @@ void MutablePropertyFragment::generateStatistics( size_t edge_label_num = schema_.edge_label_num(); std::vector count_threads; std::vector edge_count_list(dual_csr_list_.size(), 0); - for (size_t src_label = 0; src_label < vertex_label_num; ++src_label) { - const auto& src_label_name = schema_.get_vertex_label_name(src_label); - for (size_t dst_label = 0; dst_label < vertex_label_num; ++dst_label) { - const auto& dst_label_name = schema_.get_vertex_label_name(dst_label); - for (size_t edge_label = 0; edge_label < edge_label_num; ++edge_label) { - const auto& edge_label_name = schema_.get_edge_label_name(edge_label); - if (schema_.exist(src_label_name, dst_label_name, edge_label_name)) { - size_t index = src_label * vertex_label_num * edge_label_num + - dst_label * edge_label_num + edge_label; - if (dual_csr_list_[index] != NULL) { - count_threads.emplace_back([&edge_count_list, index, this] { - edge_count_list[index] = dual_csr_list_[index]->EdgeNum(); - }); - } - } - } + + for (size_t index = 0; index < dual_csr_list_.size(); ++index) { + if (dual_csr_list_[index] != NULL) { + count_threads.emplace_back([&edge_count_list, index, this] { + edge_count_list[index] = dual_csr_list_[index]->EdgeNum(); + }); } } + for (auto& t : count_threads) { t.join(); } @@ -515,23 +454,23 @@ void MutablePropertyFragment::generateStatistics( ss += "\"vertex_type_pair_statistics\": [\n"; bool first = true; std::string props_content{}; - for (size_t src_label = 0; src_label < vertex_label_num; ++src_label) { - const auto& src_label_name = schema_.get_vertex_label_name(src_label); - for (size_t dst_label = 0; dst_label < vertex_label_num; ++dst_label) { - const auto& dst_label_name = schema_.get_vertex_label_name(dst_label); - size_t index = src_label * vertex_label_num * edge_label_num + - dst_label * edge_label_num + edge_label; - if (schema_.exist(src_label_name, dst_label_name, edge_label_name)) { - if (!first) { - ss += ",\n"; - } - first = false; - ss += "{\n\"source_vertex\" : \"" + src_label_name + "\", \n"; - ss += "\"destination_vertex\" : \"" + dst_label_name + "\", \n"; - ss += "\"count\" : " + std::to_string(edge_count_list[index]) + "\n"; - edge_count += edge_count_list[index]; - ss += "}"; + for (size_t index = 0; index < dual_csr_list_.size(); ++index) { + if (dual_csr_list_[index] != NULL) { + auto triplet = schema_.get_edge_triplet(index); + auto src_label = std::get<0>(triplet); + auto dst_label = std::get<1>(triplet); + + if (!first) { + ss += ",\n"; } + first = false; + ss += "{\n\"source_vertex\" : \"" + + schema_.get_vertex_label_name(src_label) + "\", \n"; + ss += "\"destination_vertex\" : \"" + + schema_.get_vertex_label_name(dst_label) + "\", \n"; + ss += "\"count\" : " + std::to_string(edge_count_list[index]) + "\n"; + edge_count += edge_count_list[index]; + ss += "}"; } } diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.h b/flex/storages/rt_mutable_graph/mutable_property_fragment.h index 8a66406d7324..0c8e552d9ce4 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.h +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.h @@ -104,29 +104,29 @@ class MutablePropertyFragment { inline CsrBase* get_oe_csr(label_t label, label_t neighbor_label, label_t edge_label) { - size_t index = label * vertex_label_num_ * edge_label_num_ + - neighbor_label * edge_label_num_ + edge_label; + size_t index = + schema_.get_edge_triplet_id(label, neighbor_label, edge_label); return oe_[index]; } inline const CsrBase* get_oe_csr(label_t label, label_t neighbor_label, label_t edge_label) const { - size_t index = label * vertex_label_num_ * edge_label_num_ + - neighbor_label * edge_label_num_ + edge_label; + size_t index = + schema_.get_edge_triplet_id(label, neighbor_label, edge_label); return oe_[index]; } inline CsrBase* get_ie_csr(label_t label, label_t neighbor_label, label_t edge_label) { - size_t index = neighbor_label * vertex_label_num_ * edge_label_num_ + - label * edge_label_num_ + edge_label; + size_t index = + schema_.get_edge_triplet_id(neighbor_label, label, edge_label); return ie_[index]; } inline const CsrBase* get_ie_csr(label_t label, label_t neighbor_label, label_t edge_label) const { - size_t index = neighbor_label * vertex_label_num_ * edge_label_num_ + - label * edge_label_num_ + edge_label; + size_t index = + schema_.get_edge_triplet_id(neighbor_label, label, edge_label); return ie_[index]; } diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index 4783b4f6b0d7..47890788d109 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -45,6 +45,8 @@ void Schema::Clear() { vprop_names_.clear(); v_primary_keys_.clear(); vprop_storage_.clear(); + e_triplet_labels_.clear(); + e_triplet_to_index_.clear(); eproperties_.clear(); eprop_names_.clear(); ie_strategy_.clear(); @@ -92,19 +94,27 @@ void Schema::add_edge_label(const std::string& src_label, label_t dst_label_id = vertex_label_to_index(dst_label); label_t edge_label_id = edge_label_to_index(edge_label); - uint32_t label_id = - generate_edge_label(src_label_id, dst_label_id, edge_label_id); - eproperties_[label_id] = properties; + auto triplet_id = + insert_edge_triplet(src_label_id, dst_label_id, edge_label_id); + eproperties_.emplace_back(properties); + assert(triplet_id == eproperties_.size() - 1); if (properties.size() > 1) { has_multi_props_edge_ = true; } - oe_strategy_[label_id] = oe; - ie_strategy_[label_id] = ie; - oe_mutability_[label_id] = oe_mutable; - ie_mutability_[label_id] = ie_mutable; - eprop_names_[label_id] = prop_names; - sort_on_compactions_[label_id] = sort_on_compaction; - e_descriptions_[label_id] = description; + oe_strategy_.emplace_back(oe); + assert(triplet_id == oe_strategy_.size() - 1); + ie_strategy_.emplace_back(ie); + assert(triplet_id == ie_strategy_.size() - 1); + oe_mutability_.emplace_back(oe_mutable); + assert(triplet_id == oe_mutability_.size() - 1); + ie_mutability_.emplace_back(ie_mutable); + assert(triplet_id == ie_mutability_.size() - 1); + eprop_names_.emplace_back(prop_names); + assert(triplet_id == eprop_names_.size() - 1); + sort_on_compactions_.emplace_back(sort_on_compaction); + assert(triplet_id == sort_on_compactions_.size() - 1); + e_descriptions_.emplace_back(description); + assert(triplet_id == e_descriptions_.size() - 1); } label_t Schema::vertex_label_num() const { @@ -203,14 +213,13 @@ bool Schema::exist(const std::string& src_label, const std::string& dst_label, label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(edge_label); - uint32_t index = generate_edge_label(src, dst, edge); - return eproperties_.find(index) != eproperties_.end(); + return exist(src, dst, edge); } bool Schema::exist(label_t src_label, label_t dst_label, label_t edge_label) const { - uint32_t index = generate_edge_label(src_label, dst_label, edge_label); - return eproperties_.find(index) != eproperties_.end(); + return e_triplet_to_index_.find(std::make_tuple( + src_label, dst_label, edge_label)) != e_triplet_to_index_.end(); } const std::vector& Schema::get_edge_properties( @@ -219,8 +228,8 @@ const std::vector& Schema::get_edge_properties( label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); - return eproperties_.at(index); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + return eproperties_[triplet_id]; } const std::vector& Schema::get_edge_properties( @@ -233,8 +242,8 @@ const std::vector& Schema::get_edge_properties( "vertex label " + std::to_string(dst_label) + " not found"); THROW_EXCEPTION_IF(label >= elabel_indexer_.size(), "edge label " + std::to_string(label) + " not found"); - uint32_t index = generate_edge_label(src_label, dst_label, label); - return eproperties_.at(index); + auto triplet_id = get_edge_triplet_id(src_label, dst_label, label); + return eproperties_[triplet_id]; } std::string Schema::get_edge_description(const std::string& src_label, @@ -256,18 +265,19 @@ std::string Schema::get_edge_description(label_t src_label, label_t dst_label, "vertex label " + std::to_string(dst_label) + " not found"); THROW_EXCEPTION_IF(label >= elabel_indexer_.size(), "edge label " + std::to_string(label) + " not found"); - uint32_t index = generate_edge_label(src_label, dst_label, label); - THROW_EXCEPTION_IF(index >= e_descriptions_.size(), - "Fail to get edge description: " + std::to_string(index) + - ", out of range of e_descriptions_ " + - std::to_string(e_descriptions_.size())); - return e_descriptions_.at(index); + auto triplet_id = get_edge_triplet_id(src_label, dst_label, label); + THROW_EXCEPTION_IF( + triplet_id >= e_descriptions_.size(), + "Fail to get edge description: " + std::to_string(triplet_id) + + ", out of range of e_descriptions_ " + + std::to_string(e_descriptions_.size())); + return e_descriptions_[triplet_id]; } PropertyType Schema::get_edge_property(label_t src, label_t dst, label_t edge) const { - uint32_t index = generate_edge_label(src, dst, edge); - auto& vec = eproperties_.at(index); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + auto& vec = eproperties_.at(triplet_id); return vec.empty() ? PropertyType::kEmpty : vec[0]; } const std::vector& Schema::get_edge_property_names( @@ -290,8 +300,8 @@ const std::vector& Schema::get_edge_property_names( "vertex label " + std::to_string(dst_label) + " not found"); THROW_EXCEPTION_IF(label >= elabel_indexer_.size(), "edge label " + std::to_string(label) + " not found"); - uint32_t index = generate_edge_label(src_label, dst_label, label); - return eprop_names_.at(index); + auto triplet_id = get_edge_triplet_id(src_label, dst_label, label); + return eprop_names_[triplet_id]; } bool Schema::valid_edge_property(const std::string& src_label, @@ -300,8 +310,8 @@ bool Schema::valid_edge_property(const std::string& src_label, label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); - return eproperties_.find(index) != eproperties_.end(); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + return eprop_names_[triplet_id].size() > 0; } EdgeStrategy Schema::get_outgoing_edge_strategy( @@ -310,8 +320,8 @@ EdgeStrategy Schema::get_outgoing_edge_strategy( label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); - return oe_strategy_.at(index); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + return oe_strategy_[triplet_id]; } EdgeStrategy Schema::get_incoming_edge_strategy( @@ -320,8 +330,8 @@ EdgeStrategy Schema::get_incoming_edge_strategy( label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); - return ie_strategy_.at(index); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + return ie_strategy_[triplet_id]; } bool Schema::outgoing_edge_mutable(const std::string& src_label, @@ -330,8 +340,8 @@ bool Schema::outgoing_edge_mutable(const std::string& src_label, label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); - return oe_mutability_.at(index); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + return oe_mutability_[triplet_id]; } bool Schema::incoming_edge_mutable(const std::string& src_label, @@ -340,8 +350,8 @@ bool Schema::incoming_edge_mutable(const std::string& src_label, label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); - return ie_mutability_.at(index); + auto triplet_id = get_edge_triplet_id(src, dst, edge); + return ie_mutability_[triplet_id]; } bool Schema::get_sort_on_compaction(const std::string& src_label, @@ -350,13 +360,13 @@ bool Schema::get_sort_on_compaction(const std::string& src_label, label_t src = get_vertex_label_id(src_label); label_t dst = get_vertex_label_id(dst_label); label_t edge = get_edge_label_id(label); - uint32_t index = generate_edge_label(src, dst, edge); + auto triplet_id = get_edge_triplet_id(src, dst, edge); THROW_EXCEPTION_IF( - sort_on_compactions_.find(index) == sort_on_compactions_.end(), - "Fail to get sort on compaction: " + std::to_string(index) + + triplet_id >= sort_on_compactions_.size(), + "Fail to get sort on compaction: " + std::to_string(triplet_id) + ", out of range of sort_on_compactions_ " + std::to_string(sort_on_compactions_.size())); - return sort_on_compactions_.at(index); + return sort_on_compactions_[triplet_id]; } label_t Schema::get_edge_label_id(const std::string& label) const { @@ -366,6 +376,25 @@ label_t Schema::get_edge_label_id(const std::string& label) const { return ret; } +label_t Schema::get_edge_triplet_id(label_t src_label, label_t dst_label, + label_t edge_label) const { + auto index = std::make_tuple(src_label, dst_label, edge_label); + THROW_EXCEPTION_IF( + e_triplet_to_index_.find(index) == e_triplet_to_index_.end(), + "Edge triplet not found: " + std::to_string(src_label) + ", " + + std::to_string(dst_label) + ", " + std::to_string(edge_label)); + return e_triplet_to_index_.at(index); +} + +std::tuple Schema::get_edge_triplet( + label_t index) const { + THROW_EXCEPTION_IF(index >= e_triplet_labels_.size(), + "Fail to get edge triplet: " + std::to_string(index) + + ", out of range of e_triplet_labels_ " + + std::to_string(e_triplet_labels_.size())); + return e_triplet_labels_[index]; +} + bool Schema::contains_edge_label(const std::string& label) const { label_t ret; return elabel_indexer_.get_index(label, ret); @@ -411,9 +440,15 @@ void Schema::Serialize(std::unique_ptr& writer) const { elabel_indexer_.Serialize(writer); grape::InArchive arc; arc << v_primary_keys_ << vproperties_ << vprop_names_ << vprop_storage_ - << eproperties_ << eprop_names_ << ie_strategy_ << oe_strategy_ - << ie_mutability_ << oe_mutability_ << sort_on_compactions_ << max_vnum_ - << v_descriptions_ << e_descriptions_ << description_ << version_; + << e_triplet_labels_ << eproperties_ << eprop_names_ << ie_strategy_ + << oe_strategy_ << ie_mutability_ << oe_mutability_ + << sort_on_compactions_ << max_vnum_ << v_descriptions_ << e_descriptions_ + << description_ << version_; + + std::vector test{true, false, true}; + grape::InArchive test_arc; + test_arc << test; + CHECK(writer->WriteArchive(arc)); } @@ -424,12 +459,13 @@ void Schema::Deserialize(std::unique_ptr& reader) { grape::OutArchive arc; CHECK(reader->ReadArchive(arc)); arc >> v_primary_keys_ >> vproperties_ >> vprop_names_ >> vprop_storage_ >> - eproperties_ >> eprop_names_ >> ie_strategy_ >> oe_strategy_ >> - ie_mutability_ >> oe_mutability_ >> sort_on_compactions_ >> max_vnum_ >> - v_descriptions_ >> e_descriptions_ >> description_ >> version_; + e_triplet_labels_ >> eproperties_ >> eprop_names_ >> ie_strategy_ >> + oe_strategy_ >> ie_mutability_ >> oe_mutability_ >> + sort_on_compactions_ >> max_vnum_ >> v_descriptions_ >> e_descriptions_ >> + description_ >> version_; has_multi_props_edge_ = false; for (auto& eprops : eproperties_) { - if (eprops.second.size() > 1) { + if (eprops.size() > 1) { has_multi_props_edge_ = true; break; } @@ -442,6 +478,10 @@ void Schema::Deserialize(std::unique_ptr& reader) { std::make_pair(vproperties_[i][j], j); } } + e_triplet_to_index_.clear(); + for (uint32_t i = 0; i < e_triplet_labels_.size(); i++) { + e_triplet_to_index_[e_triplet_labels_[i]] = i; + } } label_t Schema::vertex_label_to_index(const std::string& label) { @@ -465,14 +505,18 @@ label_t Schema::edge_label_to_index(const std::string& label) { return ret; } -uint32_t Schema::generate_edge_label(label_t src, label_t dst, - label_t edge) const { - uint32_t ret = 0; - ret |= src; - ret <<= 8; - ret |= dst; - ret <<= 8; - ret |= edge; +uint32_t Schema::insert_edge_triplet(label_t src, label_t dst, label_t edge) { + auto index = std::make_tuple(src, dst, edge); + THROW_EXCEPTION_IF( + e_triplet_to_index_.find(index) != e_triplet_to_index_.end(), + "Edge triplet already exists: " + std::to_string(src) + ", " + + std::to_string(dst) + ", " + std::to_string(edge)); + THROW_EXCEPTION_IF( + e_triplet_labels_.size() >= std::numeric_limits::max(), + "Too many edge triplets"); + uint32_t ret = e_triplet_labels_.size(); + e_triplet_labels_.emplace_back(index); + e_triplet_to_index_[index] = ret; return ret; } @@ -502,49 +546,33 @@ bool Schema::Equals(const Schema& other) const { return false; } } - for (label_t src_label = 0; src_label < vertex_label_num(); ++src_label) { - for (label_t dst_label = 0; dst_label < vertex_label_num(); ++dst_label) { - for (label_t edge_label = 0; edge_label < edge_label_num(); - ++edge_label) { - std::string src_label_name = get_vertex_label_name(src_label); - std::string dst_label_name = get_vertex_label_name(dst_label); - std::string edge_label_name = get_edge_label_name(edge_label); - auto lhs_exists = - exist(src_label_name, dst_label_name, edge_label_name); - auto rhs_exists = - other.exist(src_label_name, dst_label_name, edge_label_name); - if (lhs_exists != rhs_exists) { - return false; - } - if (lhs_exists) { - { - const auto& lhs = get_edge_properties( - src_label_name, dst_label_name, edge_label_name); - const auto& rhs = other.get_edge_properties( - src_label_name, dst_label_name, edge_label_name); - if (lhs != rhs) { - return false; - } - } - { - auto lhs = get_incoming_edge_strategy( - src_label_name, dst_label_name, edge_label_name); - auto rhs = other.get_incoming_edge_strategy( - src_label_name, dst_label_name, edge_label_name); - if (lhs != rhs) { - return false; - } - } - { - auto lhs = get_outgoing_edge_strategy( - src_label_name, dst_label_name, edge_label_name); - auto rhs = other.get_outgoing_edge_strategy( - src_label_name, dst_label_name, edge_label_name); - if (lhs != rhs) { - return false; - } - } - } + for (size_t index = 0; index < get_edge_triplet_num(); ++index) { + auto triplet = get_edge_triplet(index); + auto src_label = std::get<0>(triplet); + auto dst_label = std::get<1>(triplet); + auto edge_label = std::get<2>(triplet); + { + const auto& lhs = get_edge_properties(src_label, dst_label, edge_label); + const auto& rhs = + other.get_edge_properties(src_label, dst_label, edge_label); + if (lhs != rhs) { + return false; + } + } + { + auto lhs = get_incoming_edge_strategy(src_label, dst_label, edge_label); + auto rhs = + other.get_incoming_edge_strategy(src_label, dst_label, edge_label); + if (lhs != rhs) { + return false; + } + } + { + auto lhs = get_outgoing_edge_strategy(src_label, dst_label, edge_label); + auto rhs = + other.get_outgoing_edge_strategy(src_label, dst_label, edge_label); + if (lhs != rhs) { + return false; } } } @@ -740,7 +768,7 @@ static Status parse_vertex_schema(YAML::Node node, Schema& schema) { "Vertex label " + label_name + " already exists"); } - size_t max_num = ((size_t) 1) << 32; + size_t max_num = Schema::DEFAULT_MAX_VNUM; if (node["x_csr_params"]) { auto csr_node = node["x_csr_params"]; get_scalar(csr_node, "max_vertex_num", max_num); @@ -1425,14 +1453,14 @@ bool Schema::edge_has_property(const std::string& src_label, auto e_label_id = get_edge_label_id(edge_label); auto src_label_id = get_vertex_label_id(src_label); auto dst_label_id = get_vertex_label_id(dst_label); - auto label_id = generate_edge_label(src_label_id, dst_label_id, e_label_id); - if (eprop_names_.find(label_id) == eprop_names_.end()) { + auto triplet_id = get_edge_triplet_id(src_label_id, dst_label_id, e_label_id); + if (triplet_id >= eprop_names_.size()) { LOG(FATAL) << "edge label " << edge_label << ": (" << src_label << ", " - << dst_label << ") not found, e_label_id " - << std::to_string(label_id) + << dst_label << ") not found, e_triple_id " + << std::to_string(triplet_id) << ", total size: " << eprop_names_.size(); } - auto& e_prop_names = eprop_names_.at(label_id); + auto& e_prop_names = eprop_names_.at(triplet_id); return std::find(e_prop_names.begin(), e_prop_names.end(), prop) != e_prop_names.end(); } @@ -1461,8 +1489,8 @@ bool Schema::has_edge_label(const std::string& src_label, bool Schema::has_edge_label(label_t src_label, label_t dst_label, label_t edge_label) const { - uint32_t e_label_id = generate_edge_label(src_label, dst_label, edge_label); - return eprop_names_.find(e_label_id) != eprop_names_.end(); + return e_triplet_to_index_.find(std::make_tuple( + src_label, dst_label, edge_label)) != e_triplet_to_index_.end(); } Result Schema::LoadFromYaml(const std::string& schema_config) { diff --git a/flex/storages/rt_mutable_graph/schema.h b/flex/storages/rt_mutable_graph/schema.h index 6331b3bdcd2c..e216623401fe 100644 --- a/flex/storages/rt_mutable_graph/schema.h +++ b/flex/storages/rt_mutable_graph/schema.h @@ -16,6 +16,8 @@ #ifndef GRAPHSCOPE_FRAGMENT_SCHEMA_H_ #define GRAPHSCOPE_FRAGMENT_SCHEMA_H_ +#include + #include "flex/engines/hqps_db/core/utils/hqps_utils.h" #include "flex/storages/rt_mutable_graph/types.h" #include "flex/utils/id_indexer.h" @@ -71,6 +73,8 @@ class Schema { // An array containing all compatible versions of schema. static const std::vector COMPATIBLE_VERSIONS; static constexpr const char* DEFAULT_SCHEMA_VERSION = "v0.0"; + // By default use a relatively large number as the default max_vnum. + static constexpr const size_t DEFAULT_MAX_VNUM = 1 << 16; static bool IsBuiltinPlugin(const std::string& plugin_name); @@ -88,8 +92,7 @@ class Schema { const std::vector>& primary_key, const std::vector& strategies = {}, - size_t max_vnum = static_cast(1) << 32, - const std::string& description = ""); + size_t max_vnum = DEFAULT_MAX_VNUM, const std::string& description = ""); void add_edge_label(const std::string& src_label, const std::string& dst_label, @@ -204,15 +207,15 @@ class Schema { inline EdgeStrategy get_outgoing_edge_strategy(label_t src_label, label_t dst_label, label_t label) const { - uint32_t index = generate_edge_label(src_label, dst_label, label); - return oe_strategy_.at(index); + auto triplet_id = get_edge_triplet_id(src_label, dst_label, label); + return oe_strategy_[triplet_id]; } inline EdgeStrategy get_incoming_edge_strategy(label_t src_label, label_t dst_label, label_t label) const { - uint32_t index = generate_edge_label(src_label, dst_label, label); - return ie_strategy_.at(index); + auto triplet_id = get_edge_triplet_id(src_label, dst_label, label); + return ie_strategy_[triplet_id]; } bool outgoing_edge_mutable(const std::string& src_label, @@ -231,6 +234,15 @@ class Schema { label_t get_edge_label_id(const std::string& label) const; + label_t get_edge_triplet_id(label_t src_label, label_t dst_label, + label_t edge_label) const; + + inline label_t get_edge_triplet_num() const { + return e_triplet_labels_.size(); + } + + std::tuple get_edge_triplet(label_t index) const; + std::string get_vertex_label_name(label_t index) const; std::string get_edge_label_name(label_t index) const; @@ -281,7 +293,7 @@ class Schema { label_t edge_label_to_index(const std::string& label); - uint32_t generate_edge_label(label_t src, label_t dst, label_t edge) const; + uint32_t insert_edge_triplet(label_t src, label_t dst, label_t edge); IdIndexer vlabel_indexer_; IdIndexer elabel_indexer_; @@ -292,14 +304,18 @@ class Schema { v_primary_keys_; // the third element is the index of the property in the // vertex property list std::vector> vprop_storage_; - std::map> eproperties_; - std::map> eprop_names_; - std::map e_descriptions_; - std::map oe_strategy_; - std::map ie_strategy_; - std::map oe_mutability_; - std::map ie_mutability_; - std::map sort_on_compactions_; + std::vector> e_triplet_labels_; + std::unordered_map, uint32_t, + boost::hash>> + e_triplet_to_index_; + std::vector> eproperties_; + std::vector> eprop_names_; + std::vector e_descriptions_; + std::vector oe_strategy_; + std::vector ie_strategy_; + std::vector oe_mutability_; + std::vector ie_mutability_; + std::vector sort_on_compactions_; std::vector>> vprop_name_to_type_and_index_; std::vector max_vnum_; diff --git a/flex/storages/rt_mutable_graph/types.h b/flex/storages/rt_mutable_graph/types.h index e962ecbd39c9..c6e17f7cd8b3 100644 --- a/flex/storages/rt_mutable_graph/types.h +++ b/flex/storages/rt_mutable_graph/types.h @@ -29,7 +29,11 @@ enum class EdgeStrategy { using timestamp_t = uint32_t; using vid_t = uint32_t; +#ifdef FLEX_LABEL_TYPE +using label_t = FLEX_LABEL_TYPE; +#else using label_t = uint8_t; +#endif } // namespace gs diff --git a/flex/utils/id_indexer.h b/flex/utils/id_indexer.h index 7f1e0806dac7..777fe46a925d 100644 --- a/flex/utils/id_indexer.h +++ b/flex/utils/id_indexer.h @@ -785,7 +785,7 @@ class IdIndexer : public IdIndexerBase { arc.Clear(); if (indices_.size() > 0) { - CHECK(writer->Write(const_cast(indices_.data()), + CHECK(writer->Write(const_cast(indices_.data()), indices_.size() * sizeof(INDEX_T))); } if (distances_.size() > 0) { diff --git a/flex/utils/property/types.cc b/flex/utils/property/types.cc index b76f17294eab..277ffd072a45 100644 --- a/flex/utils/property/types.cc +++ b/flex/utils/property/types.cc @@ -532,6 +532,28 @@ grape::OutArchive& operator>>(grape::OutArchive& out_archive, LabelKey& value) { return out_archive; } +grape::InArchive& operator<<(grape::InArchive& in_archive, + const std::vector& value) { + in_archive << value.size(); + for (auto v : value) { + in_archive << (v ? (char) 1 : (char) 0); + } + return in_archive; +} + +grape::OutArchive& operator>>(grape::OutArchive& out_archive, + std::vector& value) { + size_t size; + out_archive >> size; + value.resize(size); + char v; + for (size_t i = 0; i < size; ++i) { + out_archive >> v; + value[i] = (v == 1); + } + return out_archive; +} + GlobalId::label_id_t GlobalId::get_label_id(gid_t gid) { return static_cast(gid >> label_id_offset); } diff --git a/flex/utils/property/types.h b/flex/utils/property/types.h index d30c949bb00c..3b17970b87ba 100644 --- a/flex/utils/property/types.h +++ b/flex/utils/property/types.h @@ -1327,6 +1327,11 @@ grape::InArchive& operator<<(grape::InArchive& in_archive, const GlobalId& value); grape::OutArchive& operator>>(grape::OutArchive& out_archive, GlobalId& value); +grape::InArchive& operator<<(grape::InArchive& in_archive, + const std::vector& value); +grape::OutArchive& operator>>(grape::OutArchive& out_archive, + std::vector& value); + } // namespace gs namespace boost { From 6d2e0c8ec9b21ca20d7e35c6014fb1c6453907b3 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Fri, 28 Feb 2025 16:53:18 +0800 Subject: [PATCH 02/11] fixing ci Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 2 +- flex/engines/graph_db/database/update_transaction.cc | 2 +- flex/storages/rt_mutable_graph/schema.cc | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index dcd0526ee0c7..38fd6f215577 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -55,7 +55,7 @@ jobs: git submodule update --init cd ${GITHUB_WORKSPACE}/flex mkdir build && cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG && sudo make -j$(nproc) + cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG -DLABEL_TYPE="uint16_t" && sudo make -j$(nproc) # package the build artifacts cd .. && tar -zcf build.tar.gz build diff --git a/flex/engines/graph_db/database/update_transaction.cc b/flex/engines/graph_db/database/update_transaction.cc index fc9d832942f1..4278f673c062 100644 --- a/flex/engines/graph_db/database/update_transaction.cc +++ b/flex/engines/graph_db/database/update_transaction.cc @@ -686,7 +686,7 @@ size_t UpdateTransaction::get_in_csr_index(label_t src_label, label_t dst_label, size_t UpdateTransaction::get_out_csr_index(label_t src_label, label_t dst_label, label_t edge_label) const { - return graph_.schema().get_edge_triplet_id(dst_label, src_label, edge_label) + + return graph_.schema().get_edge_triplet_id(src_label, dst_label, edge_label) + graph_.schema().get_edge_triplet_num(); } diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index 47890788d109..ae5d0a57a487 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -96,6 +96,8 @@ void Schema::add_edge_label(const std::string& src_label, auto triplet_id = insert_edge_triplet(src_label_id, dst_label_id, edge_label_id); + VLOG(10) << "insert edge triplet: " << src_label_id << ", " << dst_label_id + << ", " << edge_label_id << " -> " << triplet_id; eproperties_.emplace_back(properties); assert(triplet_id == eproperties_.size() - 1); if (properties.size() > 1) { From ce3cfd8f997fa297f0f261f35bd580c427fb8468 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 3 Mar 2025 15:42:05 +0800 Subject: [PATCH 03/11] fix ci Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 2 +- flex/engines/graph_db/grin/CMakeLists.txt | 9 ++++++ .../graph_db/grin/src/property/property.cc | 7 +++-- .../grin/src/property/propertylist.cc | 8 +++-- .../graph_db/grin/src/property/topology.cc | 8 ++--- .../graph_db/grin/src/property/type.cc | 31 +++++++++++++------ .../grin/src/topology/adjacentlist.cc | 14 ++++++--- flex/storages/rt_mutable_graph/schema.cc | 4 --- 8 files changed, 53 insertions(+), 30 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 38fd6f215577..ab0f7bd50536 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -502,7 +502,7 @@ jobs: git submodule update --init cd flex/engines/graph_db/grin mkdir build && cd build - cmake .. && sudo make -j$(nproc) + cmake .. -DLABEL_TYPE="uint16_t" && sudo make -j$(nproc) export FLEX_DATA_DIR=../../../../interactive/examples/modern_graph/ ${GITHUB_WORKSPACE}/flex/build/bin/bulk_loader -g ../../../../interactive/examples/modern_graph/graph.yaml -l ../../../../interactive/examples/modern_graph/bulk_load.yaml -d ./data/ rm -r ./data/wal diff --git a/flex/engines/graph_db/grin/CMakeLists.txt b/flex/engines/graph_db/grin/CMakeLists.txt index aea6bc6ee35a..b039906cc44f 100644 --- a/flex/engines/graph_db/grin/CMakeLists.txt +++ b/flex/engines/graph_db/grin/CMakeLists.txt @@ -6,6 +6,15 @@ set(GRIN_READER_VERSION ${GRIN_READER_MAJOR_VERSION}.${GRIN_READER_MINOR_VERSION project(grin_reader LANGUAGES C CXX VERSION ${GRIN_READER_VERSION}) +# This option should be aligned with flex/CMakeLists.txt +SET(LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") + +# check label_type in {uint8, uint16} +if(NOT LABEL_TYPE STREQUAL "uint8_t" AND NOT LABEL_TYPE STREQUAL "uint16_t") + message(FATAL_ERROR "LABEL_TYPE must be uint8 or uint16, but got ${LABEL_TYPE}") +endif() +add_compile_definitions(FLEX_LABEL_TYPE=${LABEL_TYPE}) + # Set flags set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/flex/engines/graph_db/grin/src/property/property.cc b/flex/engines/graph_db/grin/src/property/property.cc index 1a00e94aa134..6057002f97b7 100644 --- a/flex/engines/graph_db/grin/src/property/property.cc +++ b/flex/engines/graph_db/grin/src/property/property.cc @@ -394,11 +394,12 @@ void grin_destroy_edge_property(GRIN_GRAPH g, GRIN_EDGE_PROPERTY ep) {} GRIN_DATATYPE grin_get_edge_property_datatype(GRIN_GRAPH g, GRIN_EDGE_PROPERTY ep) { auto _g = static_cast(g); - auto src_label_i = (ep >> 16) & 0xff; + auto edge_triplet_tuple = _g->g.schema().get_edge_triplet(ep); + auto src_label_i = std::get<0>(edge_triplet_tuple); + auto dst_label_i = std::get<1>(edge_triplet_tuple); + auto edge_label_i = std::get<2>(edge_triplet_tuple); const auto& src_label = _g->g.schema().get_vertex_label_name(src_label_i); - auto dst_label_i = (ep >> 8) & 0xff; const auto& dst_label = _g->g.schema().get_vertex_label_name(dst_label_i); - auto edge_label_i = ep & 0xff; const auto& edge_label = _g->g.schema().get_edge_label_name(edge_label_i); const auto& type = _g->g.schema().get_edge_properties(src_label, dst_label, edge_label); diff --git a/flex/engines/graph_db/grin/src/property/propertylist.cc b/flex/engines/graph_db/grin/src/property/propertylist.cc index 1d2b6485a1f9..ddb083570c45 100644 --- a/flex/engines/graph_db/grin/src/property/propertylist.cc +++ b/flex/engines/graph_db/grin/src/property/propertylist.cc @@ -96,11 +96,13 @@ GRIN_EDGE_PROPERTY_LIST grin_get_edge_property_list_by_type(GRIN_GRAPH g, GRIN_EDGE_PROPERTY_LIST_T* p = new GRIN_EDGE_PROPERTY_LIST_T(); auto _g = static_cast(g); - auto src_label_i = et >> 16; + auto edge_triplet_tuple = _g->g.schema().get_edge_triplet(et); + auto src_label_i = std::get<0>(edge_triplet_tuple); + auto dst_label_i = std::get<1>(edge_triplet_tuple); + auto edge_label_i = std::get<2>(edge_triplet_tuple); + auto src_label = _g->g.schema().get_vertex_label_name(src_label_i); - auto dst_label_i = (et >> 8) & (0xff); auto dst_label = _g->g.schema().get_vertex_label_name(dst_label_i); - auto edge_label_i = et & 0xff; auto edge_label = _g->g.schema().get_edge_label_name(edge_label_i); auto sz = _g->g.schema() .get_edge_properties(src_label, dst_label, edge_label) diff --git a/flex/engines/graph_db/grin/src/property/topology.cc b/flex/engines/graph_db/grin/src/property/topology.cc index 5a58e07ecb03..09738b87df6b 100644 --- a/flex/engines/graph_db/grin/src/property/topology.cc +++ b/flex/engines/graph_db/grin/src/property/topology.cc @@ -23,11 +23,11 @@ size_t grin_get_vertex_num_by_type(GRIN_GRAPH g, GRIN_VERTEX_TYPE vt) { #ifdef GRIN_WITH_EDGE_PROPERTY size_t grin_get_edge_num_by_type(GRIN_GRAPH g, GRIN_EDGE_TYPE et) { - auto src_label = et >> 16; - auto dst_label = (et >> 8) & (0xff); - auto edge_label = et & (0xff); auto _g = static_cast(g); - + auto edge_triplet_tuple = _g->g.schema().get_edge_triplet(et); + auto src_label = std::get<0>(edge_triplet_tuple); + auto dst_label = std::get<1>(edge_triplet_tuple); + auto edge_label = std::get<2>(edge_triplet_tuple); auto oe = _g->g.get_oe_csr(src_label, dst_label, edge_label); auto vertex_num = _g->g.vertex_num(src_label); size_t edge_num = 0; diff --git a/flex/engines/graph_db/grin/src/property/type.cc b/flex/engines/graph_db/grin/src/property/type.cc index ea40c524b980..6e13831834de 100644 --- a/flex/engines/graph_db/grin/src/property/type.cc +++ b/flex/engines/graph_db/grin/src/property/type.cc @@ -108,10 +108,11 @@ bool grin_equal_edge_type(GRIN_GRAPH g, GRIN_EDGE_TYPE et1, } GRIN_EDGE_TYPE grin_get_edge_type(GRIN_GRAPH g, GRIN_EDGE e) { + auto _g = static_cast(g); auto _e = static_cast(e); auto src_label = _e->src >> 32; auto dst_label = _e->dst >> 32; - return _e->label + (src_label << 16) + (dst_label << 8); + return _g->g.schema().get_edge_triplet_id(src_label, dst_label, _e->label); } void grin_destroy_edge_type(GRIN_GRAPH g, GRIN_EDGE_TYPE et) { @@ -134,7 +135,8 @@ GRIN_EDGE_TYPE_LIST grin_get_edge_type_list(GRIN_GRAPH g) { const auto& edge_label = _g->g.schema().get_edge_label_name(edge_label_i); if (_g->g.schema().exist(src_label, dst_label, edge_label)) { - auto label = (src_label_i << 16) + (dst_label_i << 8) + edge_label_i; + auto label = _g->g.schema().get_edge_triplet_id( + src_label_i, dst_label_i, edge_label_i); etl->push_back(label); } } @@ -177,9 +179,10 @@ GRIN_EDGE_TYPE grin_get_edge_type_from_list(GRIN_GRAPH g, const char* grin_get_edge_type_name(GRIN_GRAPH g, GRIN_EDGE_TYPE et) { auto _g = static_cast(g); const auto& schema = _g->g.schema(); - auto edge_label_i = et & 0xff; - auto src_label_i = et >> 16; - auto dst_label_i = (et >> 8) & 0xff; + auto edge_triplet_tuple = schema.get_edge_triplet(et); + auto src_label_i = std::get<0>(edge_triplet_tuple); + auto dst_label_i = std::get<1>(edge_triplet_tuple); + auto edge_label_i = std::get<2>(edge_triplet_tuple); const auto& edge_label = schema.get_edge_label_name(edge_label_i); const auto& src_label = schema.get_vertex_label_name(src_label_i); const auto& dst_label = schema.get_vertex_label_name(dst_label_i); @@ -217,7 +220,7 @@ GRIN_EDGE_TYPE grin_get_edge_type_by_name(GRIN_GRAPH g, const char* name) { auto src_label = schema.get_vertex_label_id(vec[0]); auto dst_label = schema.get_vertex_label_id(vec[1]); auto edge_label = schema.get_edge_label_id(vec[2]); - return (src_label << 16) + (dst_label << 8) + edge_label; + return schema.get_edge_triplet_id(src_label, dst_label, edge_label); } #endif @@ -236,7 +239,10 @@ GRIN_EDGE_TYPE grin_get_edge_type_by_id(GRIN_GRAPH g, GRIN_EDGE_TYPE_ID etid) { GRIN_VERTEX_TYPE_LIST grin_get_src_types_by_edge_type(GRIN_GRAPH g, GRIN_EDGE_TYPE et) { auto vtl = new GRIN_VERTEX_TYPE_LIST_T(); - vtl->emplace_back(et >> 16); + auto _g = static_cast(g); + const auto& schema = _g->g.schema(); + auto triplet = schema.get_edge_triplet(et); + vtl->emplace_back(std::get<0>(triplet)); return vtl; } @@ -244,7 +250,10 @@ GRIN_VERTEX_TYPE_LIST grin_get_src_types_by_edge_type(GRIN_GRAPH g, GRIN_VERTEX_TYPE_LIST grin_get_dst_types_by_edge_type(GRIN_GRAPH g, GRIN_EDGE_TYPE et) { auto vtl = new GRIN_VERTEX_TYPE_LIST_T(); - vtl->emplace_back((et >> 8) & (0xff)); + auto _g = static_cast(g); + const auto& schema = _g->g.schema(); + auto triplet = schema.get_edge_triplet(et); + vtl->emplace_back(std::get<1>(triplet)); return vtl; } @@ -260,13 +269,15 @@ GRIN_EDGE_TYPE_LIST grin_get_edge_types_by_vertex_type_pair( schema.get_vertex_label_name(static_cast(vt1)); std::string dst_label = schema.get_vertex_label_name(static_cast(vt2)); - auto label = (vt1 << 16) + (vt2 << 8); + for (size_t edge_label_i = 0; edge_label_i != edge_label_num; ++edge_label_i) { std::string edge_label = schema.get_vertex_label_name(static_cast(edge_label_i)); if (schema.exist(src_label, dst_label, edge_label)) { - vtl->push_back(label + edge_label_i); + vtl->push_back(schema.get_edge_triplet_id( + static_cast(vt1), static_cast(vt2), + static_cast(edge_label_i))); } } return vtl; diff --git a/flex/engines/graph_db/grin/src/topology/adjacentlist.cc b/flex/engines/graph_db/grin/src/topology/adjacentlist.cc index ac6928a0d97e..d0661be1c19f 100644 --- a/flex/engines/graph_db/grin/src/topology/adjacentlist.cc +++ b/flex/engines/graph_db/grin/src/topology/adjacentlist.cc @@ -39,9 +39,11 @@ GRIN_ADJACENT_LIST_ITERATOR grin_get_adjacent_list_begin( auto& v = adj_list.v; iter.adj_list = adj_list; auto label = adj_list.edge_label; - auto src_label = label >> 16; - auto dst_label = (label >> 8) & 0xff; - auto edge_label = label & 0xff; + auto edge_triplet_tuple = + static_cast(g)->g.schema().get_edge_triplet(label); + auto src_label = std::get<0>(edge_triplet_tuple); + auto dst_label = std::get<1>(edge_triplet_tuple); + auto edge_label = std::get<2>(edge_triplet_tuple); auto v_label = v >> 32; auto vid = v & (0xffffffff); if (adj_list.dir == GRIN_DIRECTION::OUT) { @@ -90,11 +92,13 @@ GRIN_VERTEX grin_get_neighbor_from_adjacent_list_iter( auto edge_iter = static_cast(iter.edge_iter); auto vid = edge_iter->get_neighbor(); auto label = iter.adj_list.edge_label; + auto edge_triplet_tuple = + static_cast(g)->g.schema().get_edge_triplet(label); if (iter.adj_list.dir == GRIN_DIRECTION::OUT) { - label = (label >> 8) & 0xff; + label = std::get<1>(edge_triplet_tuple); } else { - label = label >> 16; + label = std::get<0>(edge_triplet_tuple); } return ((label * 1ull) << 32) + vid; } diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index ae5d0a57a487..f5f25e88f55d 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -447,10 +447,6 @@ void Schema::Serialize(std::unique_ptr& writer) const { << sort_on_compactions_ << max_vnum_ << v_descriptions_ << e_descriptions_ << description_ << version_; - std::vector test{true, false, true}; - grape::InArchive test_arc; - test_arc << test; - CHECK(writer->WriteArchive(arc)); } From 492603f77b49423ff00654665ff52882678b07e4 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 3 Mar 2025 15:48:20 +0800 Subject: [PATCH 04/11] format Committed-by: xiaolei.zl from Dev container --- flex/engines/hqps_db/core/utils/hqps_utils.h | 206 +++++++------------ 1 file changed, 71 insertions(+), 135 deletions(-) diff --git a/flex/engines/hqps_db/core/utils/hqps_utils.h b/flex/engines/hqps_db/core/utils/hqps_utils.h index 781693e74e49..ed567138a8a0 100644 --- a/flex/engines/hqps_db/core/utils/hqps_utils.h +++ b/flex/engines/hqps_db/core/utils/hqps_utils.h @@ -145,16 +145,14 @@ template struct group_key_on_property : public std::true_type {}; template -struct group_key_on_property< - AliasTagProp> +struct group_key_on_property> : public std::false_type {}; template struct group_key_on_property> : public std::true_type {}; template -struct group_key_on_property> - : public std::false_type {}; +struct group_key_on_property> : public std::false_type {}; // check edge_dir and vopt consistency inline bool check_edge_dir_consist_vopt(const Direction& dir, VOpt vopt) { @@ -169,24 +167,18 @@ inline bool check_edge_dir_consist_vopt(const Direction& dir, VOpt vopt) { return false; } -template < - std::size_t nth, std::size_t... Head, std::size_t... Tail, - typename... Types, - typename std::enable_if<(nth + 1 != sizeof...(Types))>::type* = nullptr> -constexpr auto remove_nth_element_impl(std::index_sequence, - std::index_sequence, +template ::type* = nullptr> +constexpr auto remove_nth_element_impl(std::index_sequence, std::index_sequence, const std::tuple& tup) { return std::tuple{std::get(tup)..., // We +1 to refer one element after the one removed std::get(tup)...}; } -template < - std::size_t nth, std::size_t... Head, std::size_t... Tail, - typename... Types, - typename std::enable_if<(nth + 1 == sizeof...(Types))>::type* = nullptr> -constexpr auto remove_nth_element_impl(std::index_sequence, - std::index_sequence, +template ::type* = nullptr> +constexpr auto remove_nth_element_impl(std::index_sequence, std::index_sequence, const std::tuple& tup) { return std::tuple{std::get(tup)...}; } @@ -194,9 +186,8 @@ constexpr auto remove_nth_element_impl(std::index_sequence, template constexpr auto remove_nth_element(const std::tuple& tup) { static_assert(nth < sizeof...(Types)); - return remove_nth_element_impl( - std::make_index_sequence(), - std::make_index_sequence(), tup); + return remove_nth_element_impl(std::make_index_sequence(), + std::make_index_sequence(), tup); } template @@ -223,8 +214,7 @@ template struct remove_ith_type> { typedef decltype(std::tuple_cat( std::declval>(), - std::declval>::type>())) - type; + std::declval>::type>())) type; }; // I != J @@ -232,15 +222,13 @@ template struct remove_ith_jth_type {}; template -struct remove_ith_jth_type, - typename std::enable_if<(I < J)>::type> { +struct remove_ith_jth_type, typename std::enable_if<(I < J)>::type> { using first_type = typename remove_ith_type>::type; using type = typename remove_ith_type::type; }; template -struct remove_ith_jth_type, - typename std::enable_if<(I > J)>::type> { +struct remove_ith_jth_type, typename std::enable_if<(I > J)>::type> { using type = typename remove_ith_jth_type>::type; }; @@ -249,20 +237,17 @@ struct Edge; template < size_t Is, typename... PROP_T, - typename std::enable_if::type* = nullptr> void - props_to_string_array( - std::tuple& props, - std::array>>& - res) { + typename std::enable_if::type* = nullptr> void props_to_string_array( + std::tuple& props, + std::array>>& res) { res[Is] = std::get(props).property_name; props_to_string_array(props, res); } template ::type* = nullptr> -void props_to_string_array( - std::tuple& props, - std::array>>& res) { +void props_to_string_array(std::tuple& props, + std::array>>& res) { res[Is] = std::get(props).property_name; } template @@ -277,8 +262,8 @@ struct tuple_element; // recursive case template -struct tuple_element> - : gs::tuple_element> {}; +struct tuple_element> : gs::tuple_element> { +}; // base case template @@ -296,8 +281,7 @@ struct tuple_element<-1, std::tuple> { template auto unwrap_future_tuple(std::tuple&& tuple) { - return unwrap_future_tuple(std::move(tuple), - std::make_index_sequence()); + return unwrap_future_tuple(std::move(tuple), std::make_index_sequence()); } template auto unwrap_future_tuple(std::tuple&& tuple, std::index_sequence) { @@ -338,9 +322,7 @@ struct FirstElement { // Create a tuple of const references to the elements of a tuple. template auto make_tuple_of_const_refs(const std::tuple& t) { - return std::apply( - [](const Args&... args) { return std::make_tuple(std::cref(args)...); }, - t); + return std::apply([](const Args&... args) { return std::make_tuple(std::cref(args)...); }, t); } template @@ -358,9 +340,8 @@ struct first_n_impl; template struct first_n_impl, Out...> { - typedef - typename first_n_impl, Out..., First>::type - type; // move first input to output. + typedef typename first_n_impl, Out..., First>::type + type; // move first input to output. }; // need First, Other... here to resolve ambiguity on n = 0 @@ -396,42 +377,34 @@ constexpr auto tuple_slice_impl(T&& t, std::index_sequence) { template constexpr auto tuple_slice(T&& t) { static_assert(r >= l, "invalid slice"); - static_assert(std::tuple_size>::value >= r, - "slice index out of bounds"); - return tuple_slice_impl(std::forward(t), - std::make_index_sequence{}); + static_assert(std::tuple_size>::value >= r, "slice index out of bounds"); + return tuple_slice_impl(std::forward(t), std::make_index_sequence{}); } // [l, tuple_size - 1] template constexpr auto tuple_slice(T&& t) { - static_assert(std::tuple_size>::value > l, - "slice index out of bounds"); + static_assert(std::tuple_size>::value > l, "slice index out of bounds"); return tuple_slice_impl( - std::forward(t), - std::make_index_sequence>::value - l>{}); + std::forward(t), std::make_index_sequence>::value - l>{}); } -template = 0)>::type* = nullptr> +template = 0)>::type* = nullptr> inline auto get_from_tuple(std::tuple& tuple) { return std::get(tuple); } -template ::type* = nullptr> +template ::type* = nullptr> inline auto get_from_tuple(std::tuple& tuple) { static constexpr size_t num = sizeof...(T); return std::get(tuple); } -template = 0)>::type* = nullptr> +template = 0)>::type* = nullptr> inline const auto& get_from_tuple(const std::tuple& tuple) { return std::get(tuple); } -template ::type* = nullptr> +template ::type* = nullptr> inline const auto& get_from_tuple(const std::tuple& tuple) { static constexpr size_t num = sizeof...(T); return std::get(tuple); @@ -440,30 +413,26 @@ inline const auto& get_from_tuple(const std::tuple& tuple) { // vertex/edge property associate with type template ::type> -auto transform_array_impl(std::array&& array, FUNC_T&& func, - std::index_sequence) { +auto transform_array_impl(std::array&& array, FUNC_T&& func, std::index_sequence) { return std::array{std::move(func(std::move(array[Is])))...}; } template auto transform_array(std::array&& array, FUNC_T&& func) { - return transform_array_impl(std::move(array), std::move(func), - std::make_index_sequence()); + return transform_array_impl(std::move(array), std::move(func), std::make_index_sequence()); } template ::type> auto transform_tuple_impl(const std::tuple&& tuple, FUNC_T&& func, std::index_sequence) { - return std::make_tuple( - std::move(func(Is, std::move(std::get(tuple))))...); + return std::make_tuple(std::move(func(Is, std::move(std::get(tuple))))...); } template auto transform_tuple(const std::tuple&& tuple, FUNC_T&& func) { static constexpr size_t N = sizeof...(T); - return transform_tuple_impl(std::move(tuple), std::move(func), - std::make_index_sequence()); + return transform_tuple_impl(std::move(tuple), std::move(func), std::make_index_sequence()); } template @@ -474,32 +443,28 @@ bool apply_on_tuple_impl(const FUNC& func, const std::tuple& tuple, template bool apply_on_tuple(const FUNC& func, const std::tuple& tuple) { - return apply_on_tuple_impl(func, tuple, - std::make_index_sequence()); + return apply_on_tuple_impl(func, tuple, std::make_index_sequence()); } template ::type> -auto apply_array_impl(const std::array& array, FUNC_T&& func, - std::index_sequence) { +auto apply_array_impl(const std::array& array, FUNC_T&& func, std::index_sequence) { return std::array{std::move(func(array[Is]))...}; } template auto apply_array(const std::array& array, FUNC_T&& func) { - return apply_array_impl(array, std::move(func), - std::make_index_sequence()); + return apply_array_impl(array, std::move(func), std::make_index_sequence()); } template -void apply_tuple_impl(const std::tuple& tuple, const FUNC_T& func, - std::index_sequence, OTHER_ARGS&... other_args) { +void apply_tuple_impl(const std::tuple& tuple, const FUNC_T& func, std::index_sequence, + OTHER_ARGS&... other_args) { ((func(std::get(tuple), std::forward(other_args)...)), ...); } template -auto apply_tuple(const std::tuple& tuple, const FUNC_T& func, - OTHER_ARGS&... other_args) { +auto apply_tuple(const std::tuple& tuple, const FUNC_T& func, OTHER_ARGS&... other_args) { static constexpr size_t N = sizeof...(T); return apply_tuple_impl(tuple, func, std::make_index_sequence(), std::forward(other_args)...); @@ -508,8 +473,8 @@ auto apply_tuple(const std::tuple& tuple, const FUNC_T& func, template constexpr auto make_array(Args&&... args) { if constexpr (std::is_same::value) { - return std::array...>, - sizeof...(Args)>{{std::forward(args)...}}; + return std::array...>, sizeof...(Args)>{ + {std::forward(args)...}}; } else { return std::array{{std::forward(args)...}}; } @@ -557,13 +522,11 @@ struct NumberLarger { }; template -using make_index_range = - decltype(add(std::make_index_sequence())); +using make_index_range = decltype(add(std::make_index_sequence())); template struct TupleCatT { - using tuple_cat_t = - decltype(std::tuple_cat(std::declval(), std::declval())); + using tuple_cat_t = decltype(std::tuple_cat(std::declval(), std::declval())); }; template @@ -577,8 +540,7 @@ struct TupleCatT> { }; template -auto make_getter_tuple(label_t label, std::tuple&& tuple, - std::index_sequence) { +auto make_getter_tuple(label_t label, std::tuple&& tuple, std::index_sequence) { return std::make_tuple(std::get(tuple).CreateGetter(label)...); } @@ -620,9 +582,8 @@ struct ColumnAccessorImpl {}; // Recursive template -struct ColumnAccessorImpl - : public SingleColumn, - public ColumnAccessorImpl {}; +struct ColumnAccessorImpl : public SingleColumn, + public ColumnAccessorImpl {}; // multiple single columns. @@ -680,8 +641,8 @@ std::vector array_to_vec(const std::array& array) { } template -static typename PRIORITY_QUEUE_T::container_type priority_queue_to_vec( - PRIORITY_QUEUE_T& pq, bool reversed = false) { +static typename PRIORITY_QUEUE_T::container_type priority_queue_to_vec(PRIORITY_QUEUE_T& pq, + bool reversed = false) { auto pq_size = pq.size(); typename PRIORITY_QUEUE_T::container_type res; res.reserve(pq_size); @@ -720,8 +681,7 @@ struct to_string_impl> { // map{key:value, ...} ss << "map{"; for (auto& [k, v] : vec) { - ss << to_string_impl::to_string(k) << ":" - << to_string_impl::to_string(v) << ","; + ss << to_string_impl::to_string(k) << ":" << to_string_impl::to_string(v) << ","; } ss << "}"; return ss.str(); @@ -741,8 +701,7 @@ struct to_string_impl> { template struct to_string_impl, M>> { - static inline std::string to_string( - const std::array, M>& empty) { + static inline std::string to_string(const std::array, M>& empty) { std::stringstream ss; ss << "["; for (auto i : empty) { @@ -770,9 +729,7 @@ struct to_string_impl { template <> struct to_string_impl { - static inline std::string to_string(const Dist& empty) { - return std::to_string(empty.dist); - } + static inline std::string to_string(const Dist& empty) { return std::to_string(empty.dist); } }; template <> @@ -784,16 +741,12 @@ struct to_string_impl { template <> struct to_string_impl { - static inline std::string to_string(const std::string_view& empty) { - return std::string(empty); - } + static inline std::string to_string(const std::string_view& empty) { return std::string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const grape::EmptyType& empty) { - return ""; - } + static inline std::string to_string(const grape::EmptyType& empty) { return ""; } }; template <> @@ -812,51 +765,37 @@ struct to_string_impl { template <> struct to_string_impl { - static inline std::string to_string(const int64_t& empty) { - return std::to_string(empty); - } + static inline std::string to_string(const int64_t& empty) { return std::to_string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const bool& empty) { - return std::to_string(empty); - } + static inline std::string to_string(const bool& empty) { return std::to_string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const unsigned long& empty) { - return std::to_string(empty); - } + static inline std::string to_string(const unsigned long& empty) { return std::to_string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const int32_t& empty) { - return std::to_string(empty); - } + static inline std::string to_string(const int32_t& empty) { return std::to_string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const uint32_t& empty) { - return std::to_string(empty); - } + static inline std::string to_string(const uint32_t& empty) { return std::to_string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const double& empty) { - return std::to_string(empty); - } + static inline std::string to_string(const double& empty) { return std::to_string(empty); } }; template <> struct to_string_impl { - static inline std::string to_string(const std::string& empty) { - return empty; - } + static inline std::string to_string(const std::string& empty) { return empty; } }; template <> @@ -942,8 +881,8 @@ struct to_string_impl> { std::apply( [&result](const auto&... v) { ((result += - (to_string_impl>>::to_string(v)) + + (to_string_impl>>::to_string( + v)) + ","), ...); }, @@ -972,11 +911,9 @@ template struct Edge { VID_T src, dst; const std::tuple& edata; - Edge(VID_T s, VID_T d, const std::tuple& data) - : src(s), dst(d), edata(data) {} + Edge(VID_T s, VID_T d, const std::tuple& data) : src(s), dst(d), edata(data) {} std::string to_string() const { - return std::to_string(src) + "->" + std::to_string(dst) + "(" + - gs::to_string(edata) + ")"; + return std::to_string(src) + "->" + std::to_string(dst) + "(" + gs::to_string(edata) + ")"; } }; @@ -994,8 +931,7 @@ template using DefaultEdge = Edge; template -inline bool operator==(const DefaultEdge& lhs, - const DefaultEdge& rhs) { +inline bool operator==(const DefaultEdge& lhs, const DefaultEdge& rhs) { return lhs.src == rhs.src && lhs.dst == rhs.dst; } @@ -1029,9 +965,9 @@ struct function_traits }; template -static std::tuple get_graph_label_pair( - Direction& direction, label_id_t query_src_label, - label_id_t query_dst_label) { +static std::tuple get_graph_label_pair(Direction& direction, + label_id_t query_src_label, + label_id_t query_dst_label) { label_id_t src_label, dst_label; if (direction == Direction::In) { src_label = query_dst_label; From 8f0760c347d8e574298ffa3d1941c004ad0d2bbd Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 3 Mar 2025 18:00:53 +0800 Subject: [PATCH 05/11] fixing ci Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 4 +-- flex/CMakeLists.txt | 10 +++--- flex/bin/load_plan_and_gen.sh | 35 +++++++++++++------ .../single_edge_insert_transaction.cc | 3 +- flex/engines/graph_db/grin/CMakeLists.txt | 8 ++--- flex/engines/hqps_db/core/utils/hqps_utils.h | 4 +-- flex/engines/http_server/codegen_proxy.cc | 6 +++- flex/resources/hqps/CMakeLists.txt.template | 7 ++++ .../mutable_property_fragment.cc | 11 ++++-- 9 files changed, 59 insertions(+), 29 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index ab0f7bd50536..aed8ed767761 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -55,7 +55,7 @@ jobs: git submodule update --init cd ${GITHUB_WORKSPACE}/flex mkdir build && cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG -DLABEL_TYPE="uint16_t" && sudo make -j$(nproc) + cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG -DFLEX_LABEL_TYPE="uint16_t" && sudo make -j$(nproc) # package the build artifacts cd .. && tar -zcf build.tar.gz build @@ -502,7 +502,7 @@ jobs: git submodule update --init cd flex/engines/graph_db/grin mkdir build && cd build - cmake .. -DLABEL_TYPE="uint16_t" && sudo make -j$(nproc) + cmake .. -DFLEX_LABEL_TYPE="uint16_t" && sudo make -j$(nproc) export FLEX_DATA_DIR=../../../../interactive/examples/modern_graph/ ${GITHUB_WORKSPACE}/flex/build/bin/bulk_loader -g ../../../../interactive/examples/modern_graph/graph.yaml -l ../../../../interactive/examples/modern_graph/bulk_load.yaml -d ./data/ rm -r ./data/wal diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index 2623c93c9256..14a6b77cb87e 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -17,8 +17,8 @@ option(USE_PTHASH "Whether to use pthash" OFF) option(OPTIMIZE_FOR_HOST "Whether to optimize on host" ON) # Whether to build optimized code on host option(USE_STATIC_ARROW "Whether to use static arrow" OFF) # Whether to link arrow statically, default is OFF option(BUILD_WITH_OTEL "Whether to build with opentelemetry-cpp" OFF) # Whether to build with opentelemetry-cpp, default is OFF -# option(LABEL_TYPE "The label type of the graph, valid values: uint16, uint8" "uint8") # The type of the label in the graph -SET(LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") +# option(FLEX_LABEL_TYPE "The label type of the graph, valid values: uint16, uint8" "uint8") # The type of the label in the graph +SET(FLEX_LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") #print options message(STATUS "Build test: ${BUILD_TEST}") @@ -61,10 +61,10 @@ if(USE_PTHASH) endif() # check label_type in {uint8, uint16} -if(NOT LABEL_TYPE STREQUAL "uint8_t" AND NOT LABEL_TYPE STREQUAL "uint16_t") - message(FATAL_ERROR "LABEL_TYPE must be uint8 or uint16, but got ${LABEL_TYPE}") +if(NOT FLEX_LABEL_TYPE STREQUAL "uint8_t" AND NOT FLEX_LABEL_TYPE STREQUAL "uint16_t") + message(FATAL_ERROR "FLEX_LABEL_TYPE must be uint8 or uint16, but got ${FLEX_LABEL_TYPE}") endif() -add_compile_definitions(FLEX_LABEL_TYPE=${LABEL_TYPE}) +add_compile_definitions(FLEX_LABEL_TYPE=${FLEX_LABEL_TYPE}) set(DEFAULT_BUILD_TYPE "Release") if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) diff --git a/flex/bin/load_plan_and_gen.sh b/flex/bin/load_plan_and_gen.sh index 5670c83ca079..11cdebc49727 100755 --- a/flex/bin/load_plan_and_gen.sh +++ b/flex/bin/load_plan_and_gen.sh @@ -200,9 +200,9 @@ EOM compile_hqps_so() { #check input params size eq 2 or 3 - if [ $# -gt 8 ] || [ $# -lt 4 ]; then + if [ $# -gt 9 ] || [ $# -lt 4 ]; then echo "Usage: $0 " - echo " [statistic_path] [output_dir] [stored_procedure_name] [stored_procedure_description]" + echo " [label_type] [output_dir] [stored_procedure_name] [stored_procedure_description] [statistic_path]" exit 1 fi input_path=$1 @@ -210,25 +210,31 @@ compile_hqps_so() { ir_compiler_properties=$3 graph_schema_path=$4 if [ $# -ge 5 ]; then - output_dir=$5 + label_type=$5 else - output_dir=${work_dir} + label_type="" fi if [ $# -ge 6 ]; then - procedure_name=$6 + output_dir=$6 else - procedure_name="" + output_dir=${work_dir} fi if [ $# -ge 7 ]; then - procedure_description=$7 + procedure_name=$7 else - procedure_description="" + procedure_name="" fi if [ $# -ge 8 ]; then - statistic_path=$8 + procedure_description=$8 + else + procedure_description="" + fi + + if [ $# -ge 9 ]; then + statistic_path=$9 else statistic_path="" fi @@ -237,6 +243,7 @@ compile_hqps_so() { info "Work dir = ${work_dir}" info "ir compiler properties = ${ir_compiler_properties}" info "graph schema path = ${graph_schema_path}" + info "Label type = ${label_type}" info "Output dir = ${output_dir}" info "Procedure name = ${procedure_name}" info "Procedure description = ${procedure_description}" @@ -327,6 +334,9 @@ compile_hqps_so() { # run cmake and make in output path. pushd ${cur_dir} cmd="cmake . -DQUERY_NAME=${procedure_name} -DFLEX_INCLUDE_PREFIX=${FLEX_INCLUDE_PREFIX}" + if [ ! -z ${label_type} ]; then + cmd="${cmd} -DFLEX_LABEL_TYPE=${label_type}" + fi # if CMAKE_CXX_COMPILER is set, use it. if [ ! -z ${CMAKE_CXX_COMPILER} ]; then cmd="${cmd} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" @@ -573,6 +583,10 @@ run() { STATISTIC_PATH="${i#*=}" shift # past argument=value ;; + --label_type=*) + LABEL_TYPE="${i#*=}" + shift # past argument=value + ;; -* | --*) err "Unknown option $i" exit 1 @@ -591,6 +605,7 @@ run() { echo "Procedure name ="${PROCEDURE_NAME} echo "Procedure description ="${PROCEDURE_DESCRIPTION} echo "Statistic path ="${STATISTIC_PATH} + echo "Label type ="${LABEL_TYPE} find_resources @@ -625,7 +640,7 @@ run() { PROCEDURE_NAME="${PROCEDURE_NAME%.cc}" PROCEDURE_NAME="${PROCEDURE_NAME%.pb}" fi - compile_hqps_so ${INPUT} ${WORK_DIR} ${IR_CONF} ${GRAPH_SCHEMA_PATH} ${OUTPUT_DIR} ${PROCEDURE_NAME} "${PROCEDURE_DESCRIPTION}" ${STATISTIC_PATH} + compile_hqps_so ${INPUT} ${WORK_DIR} ${IR_CONF} ${GRAPH_SCHEMA_PATH} ${LABEL_TYPE} ${OUTPUT_DIR} ${PROCEDURE_NAME} "${PROCEDURE_DESCRIPTION}" ${STATISTIC_PATH} # else if engine_type equals pegasus elif [ ${ENGINE_TYPE} == "pegasus" ]; then diff --git a/flex/engines/graph_db/database/single_edge_insert_transaction.cc b/flex/engines/graph_db/database/single_edge_insert_transaction.cc index f9c00dca0772..49a66e226f6a 100644 --- a/flex/engines/graph_db/database/single_edge_insert_transaction.cc +++ b/flex/engines/graph_db/database/single_edge_insert_transaction.cc @@ -127,7 +127,8 @@ bool SingleEdgeInsertTransaction::Commit() { { arc.SetSlice(arc_.GetBuffer() + sizeof(WalHeader), arc_.GetSize() - sizeof(WalHeader)); - label_t op_type, label; + uint8_t op_type; + label_t label; Any temp; arc >> op_type; deserialize_oid(graph_, arc, temp); diff --git a/flex/engines/graph_db/grin/CMakeLists.txt b/flex/engines/graph_db/grin/CMakeLists.txt index b039906cc44f..ddc5cbd25089 100644 --- a/flex/engines/graph_db/grin/CMakeLists.txt +++ b/flex/engines/graph_db/grin/CMakeLists.txt @@ -7,13 +7,13 @@ set(GRIN_READER_VERSION ${GRIN_READER_MAJOR_VERSION}.${GRIN_READER_MINOR_VERSION project(grin_reader LANGUAGES C CXX VERSION ${GRIN_READER_VERSION}) # This option should be aligned with flex/CMakeLists.txt -SET(LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") +SET(FLEX_LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") # check label_type in {uint8, uint16} -if(NOT LABEL_TYPE STREQUAL "uint8_t" AND NOT LABEL_TYPE STREQUAL "uint16_t") - message(FATAL_ERROR "LABEL_TYPE must be uint8 or uint16, but got ${LABEL_TYPE}") +if(NOT FLEX_LABEL_TYPE STREQUAL "uint8_t" AND NOT FLEX_LABEL_TYPE STREQUAL "uint16_t") + message(FATAL_ERROR "FLEX_LABEL_TYPE must be uint8 or uint16, but got ${FLEX_LABEL_TYPE}") endif() -add_compile_definitions(FLEX_LABEL_TYPE=${LABEL_TYPE}) +add_compile_definitions(FLEX_LABEL_TYPE=${FLEX_LABEL_TYPE}) # Set flags set(CMAKE_CXX_STANDARD 17) diff --git a/flex/engines/hqps_db/core/utils/hqps_utils.h b/flex/engines/hqps_db/core/utils/hqps_utils.h index ed567138a8a0..d2fcc02f36f9 100644 --- a/flex/engines/hqps_db/core/utils/hqps_utils.h +++ b/flex/engines/hqps_db/core/utils/hqps_utils.h @@ -758,9 +758,7 @@ struct to_string_impl { template <> struct to_string_impl { - static inline std::string to_string(const uint16_t& empty) { - return std::to_string((int32_t) empty); - } + static inline std::string to_string(const uint16_t& value) { return std::to_string(value); } }; template <> diff --git a/flex/engines/http_server/codegen_proxy.cc b/flex/engines/http_server/codegen_proxy.cc index 3bd7a1932602..ce7d9967836d 100644 --- a/flex/engines/http_server/codegen_proxy.cc +++ b/flex/engines/http_server/codegen_proxy.cc @@ -16,6 +16,9 @@ #include "flex/engines/http_server/graph_db_service.h" #include "flex/engines/http_server/workdir_manipulator.h" +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + namespace server { CodegenProxy& CodegenProxy::get() { static CodegenProxy instance; @@ -208,7 +211,8 @@ seastar::future> CodegenProxy::CallCodegenCmd( std::string cmd = codegen_bin + " -e=hqps " + " -i=" + plan_path + " -o=" + output_dir + " --procedure_name=" + query_name + " -w=" + work_dir + " --ir_conf=" + engine_config + - " --graph_schema_path=" + graph_schema_path; + " --graph_schema_path=" + graph_schema_path + + " --label_type=" + TOSTRING(FLEX_LABEL_TYPE); std::string desc_file = work_dir + "/" + query_name + ".desc"; if (!procedure_desc.empty()) { std::ofstream ofs(desc_file); diff --git a/flex/resources/hqps/CMakeLists.txt.template b/flex/resources/hqps/CMakeLists.txt.template index 2d093668726e..553ac0ec9f12 100644 --- a/flex/resources/hqps/CMakeLists.txt.template +++ b/flex/resources/hqps/CMakeLists.txt.template @@ -9,6 +9,13 @@ project(HighQpsCodeGen VERSION 1.0 LANGUAGES CXX) +SET(FLEX_LABEL_TYPE "uint8_t" CACHE STRING "The label type of the graph, valid values: uint16_t, uint8_t") +# check label_type in {uint8, uint16} +if(NOT FLEX_LABEL_TYPE STREQUAL "uint8_t" AND NOT FLEX_LABEL_TYPE STREQUAL "uint16_t") + message(FATAL_ERROR "FLEX_LABEL_TYPE must be uint8 or uint16, but got ${FLEX_LABEL_TYPE}") +endif() +add_compile_definitions(FLEX_LABEL_TYPE=${FLEX_LABEL_TYPE}) + # the query name, if (QUERY_NAME) message("Query name ${QUERY_NAME}") diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc index af32b0dcc0b8..ceb416ca7af1 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc @@ -29,9 +29,14 @@ MutablePropertyFragment::~MutablePropertyFragment() { degree_list[i] = lf_indexers_[i].size(); vertex_data_[i].resize(degree_list[i]); } - for (auto ptr : dual_csr_list_) { - if (ptr != NULL) { - delete ptr; + for (size_t index = 0; index < dual_csr_list_.size(); ++index) { + auto triplet = schema_.get_edge_triplet(index); + auto src_label = std::get<0>(triplet); + auto dst_label = std::get<1>(triplet); + if (dual_csr_list_[index] != NULL) { + dual_csr_list_[index]->Resize(degree_list[src_label], + degree_list[dst_label]); + delete dual_csr_list_[index]; } } } From ab7384632c9005a2c7ff363d3d9523b5c827f775 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 10 Mar 2025 15:23:14 +0800 Subject: [PATCH 06/11] fix match query test Committed-by: xiaolei.zl from Dev container --- .../loader/basic_fragment_loader.cc | 1 - flex/tests/hqps/match_query.h | 22 +++++-------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc index 731a3b985c43..c5b2b3231e33 100644 --- a/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/basic_fragment_loader.cc @@ -110,7 +110,6 @@ void BasicFragmentLoader::init_loading_status_file() { append_edge_loading_progress(src_label_name, dst_label_name, edge_label_name, LoadingStatus::kLoading); } - VLOG(1) << "Finish init edge status files"; } void BasicFragmentLoader::init_vertex_data() { diff --git a/flex/tests/hqps/match_query.h b/flex/tests/hqps/match_query.h index 20db0ce5f537..fccfcd7939e5 100644 --- a/flex/tests/hqps/match_query.h +++ b/flex/tests/hqps/match_query.h @@ -1059,22 +1059,12 @@ class MatchQuery16 : public ReadAppBase { graph, std::array{0, 2}, std::move(expr0)); auto edge_expand_opt0 = gs::make_edge_expand_multie_opt< - label_id_t, std::tuple, std::tuple, - std::tuple, std::tuple, - std::tuple, std::tuple>( + label_id_t, std::tuple, std::tuple>( gs::Direction::Out, - std::array, 6>{ - std::array{0, 1, 0}, - std::array{2, 1, 5}, - std::array{0, 1, 2}, - std::array{0, 1, 3}, - std::array{2, 0, 4}, - std::array{0, 1, 1}}, + std::array, 2>{ + std::array{0, 0, 11}, + std::array{6, 6, 13}}, std::tuple{PropTupleArrayT>{}, - PropTupleArrayT>{"rating"}, - PropTupleArrayT>{}, - PropTupleArrayT>{}, - PropTupleArrayT>{}, PropTupleArrayT>{}}); auto ctx1 = Engine::template EdgeExpandE( @@ -1090,10 +1080,8 @@ class MatchQuery16 : public ReadAppBase { std::tuple{gs::make_mapper_with_variable( gs::PropertySelector("")), gs::make_mapper_with_variable( - gs::PropertySelector("")), - gs::make_mapper_with_variable( gs::PropertySelector(""))}); - return Engine::Sink(graph, ctx3, std::array{0, 1, 2}); + return Engine::Sink(graph, ctx3, std::array{0, 1}); } // Wrapper query function for query class From b8913a2bc27c9d41226bc20ac73e64efbca143ff Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 10 Mar 2025 17:31:27 +0800 Subject: [PATCH 07/11] debug Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index aed8ed767761..33858cee0268 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -41,10 +41,6 @@ jobs: make -j$(nproc) make install - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v3 - - name: Build Interactive env: GIE_HOME: ${{ github.workspace }}/interactive_engine/ @@ -225,6 +221,10 @@ jobs: cd ${GITHUB_WORKSPACE}/flex/tests/hqps bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml + - name: Setup tmate session + if: false + uses: mxschmitt/action-tmate@v3 + - name: Sample Query test env: GS_TEST_DIR: ${{ github.workspace }}/gstest From a2bbd40a64a0ef4c60f79b4fb73a468abaf79c01 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 10 Mar 2025 19:09:25 +0800 Subject: [PATCH 08/11] debug Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 33858cee0268..58558a3647aa 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -222,7 +222,7 @@ jobs: bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml - name: Setup tmate session - if: false + if: true uses: mxschmitt/action-tmate@v3 - name: Sample Query test From b010b2e967b86ed8c864994960e0dccf69b5d897 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 11 Mar 2025 11:50:53 +0800 Subject: [PATCH 09/11] fixing Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 8 ++++---- flex/engines/hqps_db/structures/path.h | 26 ++++++++++++++------------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 58558a3647aa..aed8ed767761 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -41,6 +41,10 @@ jobs: make -j$(nproc) make install + - name: Setup tmate session + if: false + uses: mxschmitt/action-tmate@v3 + - name: Build Interactive env: GIE_HOME: ${{ github.workspace }}/interactive_engine/ @@ -221,10 +225,6 @@ jobs: cd ${GITHUB_WORKSPACE}/flex/tests/hqps bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml - - name: Setup tmate session - if: true - uses: mxschmitt/action-tmate@v3 - - name: Sample Query test env: GS_TEST_DIR: ${{ github.workspace }}/gstest diff --git a/flex/engines/hqps_db/structures/path.h b/flex/engines/hqps_db/structures/path.h index 38c6c9b06ed7..9c165a5df0da 100644 --- a/flex/engines/hqps_db/structures/path.h +++ b/flex/engines/hqps_db/structures/path.h @@ -220,11 +220,11 @@ class PathSet { std::vector GetLabels(VOpt v_opt) const { std::vector label_set(sizeof(LabelT) * 8, false); if (v_opt == VOpt::End) { - for (auto i = 0; i < paths_.size(); ++i) { + for (size_t i = 0; i < paths_.size(); ++i) { label_set[paths_[i].GetLabels().back()] = true; } } else if (v_opt == VOpt::Start) { - for (auto i = 0; i < paths_.size(); ++i) { + for (size_t i = 0; i < paths_.size(); ++i) { label_set[paths_[i].GetLabels().front()] = true; } } else { @@ -247,21 +247,23 @@ class PathSet { offsets.emplace_back(0); // intersect with req_labels - std::vector label_set(sizeof(LabelT) * 8, false); + // get num of max label num + std::set label_set; std::vector labels; - std::vector label_to_index(sizeof(LabelT) * 8, -1); + std::unordered_map label_to_index; if (req_labels.size() > 0) { for (auto label : req_labels) { - label_set[label] = true; + label_set.insert(label); + } + size_t valid_labels = 0; + for (auto label : label_set) { + labels.emplace_back(label); + label_to_index[label] = valid_labels++; } } else { - std::fill(label_set.begin(), label_set.end(), true); // empty means all - } - size_t valid_labels = 0; - for (size_t i = 0; i < label_set.size(); ++i) { - if (label_set[i]) { - labels.emplace_back(i); - label_to_index[i] = valid_labels++; + labels = GetLabels(vopt); + for (size_t i = 0; i < labels.size(); ++i) { + label_to_index[labels[i]] = i; } } From 4b8d4f6d936c06805b95108cf29238dc3c780e1d Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 11 Mar 2025 14:45:07 +0800 Subject: [PATCH 10/11] fix Committed-by: xiaolei.zl from Dev container --- flex/bin/load_plan_and_gen.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flex/bin/load_plan_and_gen.sh b/flex/bin/load_plan_and_gen.sh index 11cdebc49727..530fa03f2178 100755 --- a/flex/bin/load_plan_and_gen.sh +++ b/flex/bin/load_plan_and_gen.sh @@ -596,6 +596,10 @@ run() { esac done + if [ -z "${LABEL_TYPE}" ]; then + LABEL_TYPE="uint8_t" + fi + echo "Engine type ="${ENGINE_TYPE} echo "Input ="${INPUT} echo "Work dir ="${WORK_DIR} From 9722fe42196bc1108709ae3d1cf7d75bbeda0a19 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 11 Mar 2025 16:02:24 +0800 Subject: [PATCH 11/11] fix Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index aed8ed767761..1641ecaa2c0a 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -278,17 +278,20 @@ jobs: --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \ --procedure_name=plus_one \ --graph_schema_path=../interactive/examples/modern_graph/graph.yaml \ - --procedure_desc="This is test procedure, and the input is a number, and the output is the number plus one." + --procedure_desc="This is test procedure, and the input is a number, and the output is the number plus one." \ + --label_type="uint16_t" ./load_plan_and_gen.sh -e=hqps -i=../interactive/sdk/java/src/test/resources/sample_app.cc -w=/tmp/codegen \ --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \ --procedure_name=sample_app \ - --graph_schema_path=../interactive/examples/modern_graph/graph.yaml + --graph_schema_path=../interactive/examples/modern_graph/graph.yaml \ + --label_type="uint16_t" ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \ --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \ --procedure_name=count_vertex_num \ - --graph_schema_path=../interactive/examples/modern_graph/graph.yaml + --graph_schema_path=../interactive/examples/modern_graph/graph.yaml \ + --label_type="uint16_t" # Among the above procedures, the correct input format for each is: # count_vertex_num: () -> (num: int64), CypherProcedure.