From d95c97e6773b1ef234639b592eefcb239c640ad9 Mon Sep 17 00:00:00 2001 From: Mykhailo Kuchma Date: Mon, 7 Apr 2025 11:35:58 +0200 Subject: [PATCH] Decrease allocations in QuadTreeIndex Added a `fields` parameter to GetIndexData method. Change GetIndexData method to skip string fields which are not needed. Relates-To: DATASDK-64 Signed-off-by: Mykhailo Kuchma --- .../src/BlobDataReader.h | 34 +++- .../src/BlobDataWriter.h | 4 +- .../src/ReleaseDependencyResolver.cpp | 4 +- .../src/VersionedLayerClientImpl.cpp | 4 +- .../repositories/PrefetchTilesRepository.cpp | 15 +- .../src/repositories/QuadTreeIndex.cpp | 148 +++++++++--------- .../src/repositories/QuadTreeIndex.h | 41 +++-- 7 files changed, 147 insertions(+), 103 deletions(-) diff --git a/olp-cpp-sdk-dataservice-read/src/BlobDataReader.h b/olp-cpp-sdk-dataservice-read/src/BlobDataReader.h index 36621d478..50dfe3b93 100644 --- a/olp-cpp-sdk-dataservice-read/src/BlobDataReader.h +++ b/olp-cpp-sdk-dataservice-read/src/BlobDataReader.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 HERE Europe B.V. + * Copyright (C) 2020-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,8 +41,17 @@ class BlobDataReader { return true; } - size_t GetOffset() { return read_offset_; } - void SetOffset(size_t offset) { read_offset_ = offset; } + template + bool Skip() { + if (read_offset_ + sizeof(T) > data_.size()) { + return false; + } + read_offset_ += sizeof(T); + return true; + } + + size_t GetOffset() const { return read_offset_; } + void SetOffset(const size_t offset) { read_offset_ = offset; } private: size_t read_offset_ = 0; @@ -50,7 +59,7 @@ class BlobDataReader { }; template <> -bool BlobDataReader::Read(std::string& value) { +inline bool BlobDataReader::Read(std::string& value) { size_t end = std::numeric_limits::max(); for (size_t i = read_offset_; i < data_.size(); ++i) { if (data_[i] == 0) { @@ -69,6 +78,23 @@ bool BlobDataReader::Read(std::string& value) { return true; } +template <> +inline bool BlobDataReader::Skip() { + size_t end = std::numeric_limits::max(); + for (size_t i = read_offset_; i < data_.size(); ++i) { + if (data_[i] == 0) { + end = i; + break; + } + } + if (end == std::numeric_limits::max()) { + return false; + } + + read_offset_ = end + 1; + return true; +} + } // namespace read } // namespace dataservice } // namespace olp diff --git a/olp-cpp-sdk-dataservice-read/src/BlobDataWriter.h b/olp-cpp-sdk-dataservice-read/src/BlobDataWriter.h index 34408ef4b..3f6716657 100644 --- a/olp-cpp-sdk-dataservice-read/src/BlobDataWriter.h +++ b/olp-cpp-sdk-dataservice-read/src/BlobDataWriter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 HERE Europe B.V. + * Copyright (C) 2020-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,7 @@ class BlobDataWriter { }; template <> -bool BlobDataWriter::Write(const std::string& value) { +inline bool BlobDataWriter::Write(const std::string& value) { if (write_offset_ + value.size() + 1 > data_.size()) { return false; } diff --git a/olp-cpp-sdk-dataservice-read/src/ReleaseDependencyResolver.cpp b/olp-cpp-sdk-dataservice-read/src/ReleaseDependencyResolver.cpp index 7596eeaea..924ba0c0c 100644 --- a/olp-cpp-sdk-dataservice-read/src/ReleaseDependencyResolver.cpp +++ b/olp-cpp-sdk-dataservice-read/src/ReleaseDependencyResolver.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 HERE Europe B.V. + * Copyright (C) 2020-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -111,7 +111,7 @@ ReleaseDependencyResolver::CheckProtectedTilesInQuad( // check if quad tree has other protected keys, if not, add quad key to // release from protected list, othervise add all protected keys left // for this quad to map - auto index_data = cached_tree.GetIndexData(); + auto index_data = cached_tree.GetIndexData(QuadTreeIndex::DataHandle); TilesDataKeysType protected_keys; for (const auto& ind : index_data) { const auto tile_data_key = cache::KeyGenerator::CreateDataHandleKey( diff --git a/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp b/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp index 6b01dc3b2..7ec167c61 100644 --- a/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp +++ b/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 HERE Europe B.V. + * Copyright (C) 2019-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -743,7 +743,7 @@ client::ApiNoResponse VersionedLayerClientImpl::DeleteFromCache( return result; } - auto index_data = cached_tree.GetIndexData(); + auto index_data = cached_tree.GetIndexData(QuadTreeIndex::DataHandle); for (const auto& ind : index_data) { if (ind.tile_key != tile && data_cache_repository.IsCached(layer_id_, ind.data_handle)) { diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp index d92384566..1eb0ccc95 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 HERE Europe B.V. + * Copyright (C) 2019-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ constexpr std::int32_t kMaxQuadTreeIndexDepth = 4; SubQuadsResult FlattenTree(const QuadTreeIndex& tree) { SubQuadsResult result; - auto index_data = tree.GetIndexData(); + auto index_data = tree.GetIndexData(QuadTreeIndex::DataHandle); for (auto& data : index_data) { const auto it = result.lower_bound(data.tile_key); if (it == result.end() || result.key_comp()(data.tile_key, it->first)) { @@ -455,11 +455,12 @@ PrefetchTilesRepository::DownloadVersionedQuadTree( default_additional_fields, billing_tag_, context); if (quad_tree.GetStatus() != olp::http::HttpStatusCode::OK) { - OLP_SDK_LOG_WARNING_F(kLogTag, - "GetSubQuads failed(%s, %" PRId64 ", %" PRId32 - "), status_code='%d'", - tile_key.c_str(), version, depth, quad_tree.GetStatus()); - return {client::ApiError(quad_tree.GetStatus(), quad_tree.GetResponseAsString()), + OLP_SDK_LOG_WARNING_F( + kLogTag, + "GetSubQuads failed(%s, %" PRId64 ", %" PRId32 "), status_code='%d'", + tile_key.c_str(), version, depth, quad_tree.GetStatus()); + return {client::ApiError(quad_tree.GetStatus(), + quad_tree.GetResponseAsString()), quad_tree.GetNetworkStatistics()}; } diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.cpp index b72748757..7de0ee8ea 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 HERE Europe B.V. + * Copyright (C) 2020-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,10 +46,9 @@ constexpr auto kCompressedDataSizeKey = "compressedDataSize"; constexpr auto kCrcKey = "crc"; constexpr auto kLogTag = "QuadTreeIndex"; -olp::dataservice::read::QuadTreeIndex::IndexData ParseCommonIndexData( - rapidjson::Value& value) { - olp::dataservice::read::QuadTreeIndex::IndexData data; - auto obj = value.GetObject(); +QuadTreeIndex::IndexData ParseCommonIndexData(rapidjson::Value& value) { + QuadTreeIndex::IndexData data; + const auto obj = value.GetObject(); if (obj.HasMember(kAdditionalMetadataKey) && obj[kAdditionalMetadataKey].IsString()) { data.additional_metadata = obj[kAdditionalMetadataKey].GetString(); @@ -100,10 +99,10 @@ QuadTreeIndex::QuadTreeIndex(const cache::KeyValueCache::ValueTypePtr& data) { size_ = data->size(); } -QuadTreeIndex::QuadTreeIndex(const olp::geo::TileKey& root, int depth, +QuadTreeIndex::QuadTreeIndex(const geo::TileKey& root, int depth, std::stringstream& json_stream) { - static thread_local rapidjson::CrtAllocator crt_allocator; - static thread_local rapidjson::MemoryPoolAllocator<> pool_allocator; + thread_local rapidjson::CrtAllocator crt_allocator; + thread_local rapidjson::MemoryPoolAllocator<> pool_allocator; rapidjson::Document doc(&pool_allocator, 4096, &crt_allocator); rapidjson::IStreamWrapper stream(json_stream); @@ -138,7 +137,7 @@ QuadTreeIndex::QuadTreeIndex(const olp::geo::TileKey& root, int depth, IndexData data = ParseCommonIndexData(value); data.data_handle = obj[kDataHandleKey].GetString(); data.tile_key = - olp::geo::TileKey::FromHereTile(obj[kPartitionKey].GetString()); + geo::TileKey::FromHereTile(obj[kPartitionKey].GetString()); parents.push_back(std::move(data)); } } @@ -163,27 +162,37 @@ QuadTreeIndex::QuadTreeIndex(const olp::geo::TileKey& root, int depth, CreateBlob(root, depth, std::move(parents), std::move(subs)); } -bool QuadTreeIndex::ReadIndexData(QuadTreeIndex::IndexData& data, - uint32_t offset, uint32_t limit) const { +bool QuadTreeIndex::ReadIndexData(IndexData& data, const uint32_t offset, + const uint32_t limit, int fields) const { BlobDataReader reader(*raw_data_); reader.SetOffset(offset); - bool success = reader.Read(data.version); + auto field_required = [fields](const Field field) { + return (fields & field) > 0; + }; + + bool success = true; + success &= reader.Read(data.version); success &= reader.Read(data.data_size); success &= reader.Read(data.compressed_data_size); - success &= reader.Read(data.data_handle); - success &= reader.Read(data.checksum); - success &= reader.Read(data.additional_metadata); + success &= field_required(DataHandle) ? reader.Read(data.data_handle) + : reader.Skip(); + success &= field_required(Checksum) ? reader.Read(data.checksum) + : reader.Skip(); + success &= field_required(AdditionalMetadata) + ? reader.Read(data.additional_metadata) + : reader.Skip(); // The CRC field was added after the initial QuadTreeIndex implementation, and - // to maintain the backwards compatibility we must check that we do not read + // to maintain the backwards compatibility, we must check that we do not read // the crc from the next index block. if (reader.GetOffset() < limit) { - success &= reader.Read(data.crc); + success &= field_required(Crc) ? reader.Read(data.crc) + : reader.Skip(); } return success; } -void QuadTreeIndex::CreateBlob(olp::geo::TileKey root, int depth, +void QuadTreeIndex::CreateBlob(geo::TileKey root, int depth, std::vector parents, std::vector subs) { // quads must be sorted by their sub quad key, not by Quad::operator< @@ -197,7 +206,7 @@ void QuadTreeIndex::CreateBlob(olp::geo::TileKey root, int depth, return lhs.tile_key.ToQuadKey64() < rhs.tile_key.ToQuadKey64(); }); - // count data size(for now it is header version and data handle) + // count data size (for now it is a header version and data handle) size_t additional_data_size = 0; for (const IndexData& data : subs) { additional_data_size += @@ -237,12 +246,12 @@ void QuadTreeIndex::CreateBlob(olp::geo::TileKey root, int depth, serializer.SetOffset(const_cast(DataBegin()) - raw_data_->data()); for (const IndexData& data : subs) { - *entry_ptr++ = { - std::uint16_t(olp::geo::QuadKey64Helper{data.tile_key.ToQuadKey64()} - .GetSubkey(static_cast(data.tile_key.Level() - - root_quad_level)) - .key), - static_cast(serializer.GetOffset())}; + *entry_ptr++ = {static_cast( + geo::QuadKey64Helper{data.tile_key.ToQuadKey64()} + .GetSubkey(static_cast(data.tile_key.Level() - + root_quad_level)) + .key), + static_cast(serializer.GetOffset())}; if (!WriteIndexData(data, serializer)) { OLP_SDK_LOG_ERROR(kLogTag, "Could not write IndexData"); raw_data_ = nullptr; @@ -267,74 +276,73 @@ void QuadTreeIndex::CreateBlob(olp::geo::TileKey root, int depth, } boost::optional QuadTreeIndex::Find( - const olp::geo::TileKey& tile_key, bool aggregated) const { + const geo::TileKey& tile_key, bool aggregated_search) const { if (IsNull()) { return boost::none; } - const olp::geo::TileKey& root_tile_key = - olp::geo::TileKey::FromQuadKey64(data_->root_tilekey); + const geo::TileKey& root_tile_key = + geo::TileKey::FromQuadKey64(data_->root_tilekey); IndexData data; if (tile_key.Level() >= root_tile_key.Level()) { - auto sub = std::uint16_t(tile_key.GetSubkey64( + auto sub = static_cast(tile_key.GetSubkey64( static_cast(tile_key.Level() - root_tile_key.Level()))); const SubEntry* end = SubEntryEnd(); const SubEntry* entry = std::lower_bound(SubEntryBegin(), end, SubEntry{sub, 0}); if (entry == end || entry->sub_quadkey != sub) { - return aggregated ? FindNearestParent(tile_key) : boost::none; + return aggregated_search ? FindNearestParent(tile_key) : boost::none; } const auto offset = entry->tag_offset; - // The limit is the offset for the next entry, or the beginning of the - // parents entries. (in case there are no parents use the size of the index + // The limit is the offset for the next entry or the beginning of the + // parent's entries. (in case there are no parents use the size of the index // block.) const auto limit = [&]() { - auto next = entry + 1; + const auto next = entry + 1; if (next == end) { if (data_->parent_count == 0) { return static_cast(raw_data_->size()); - } else { - return ParentEntryBegin()->tag_offset; } + return ParentEntryBegin()->tag_offset; } return next->tag_offset; }(); - if (!ReadIndexData(data, offset, limit)) { + if (!ReadIndexData(data, offset, limit, All)) { return boost::none; } data.tile_key = tile_key; return data; - } else { - std::uint64_t key = tile_key.ToQuadKey64(); - - const ParentEntry* end = ParentEntryEnd(); - const ParentEntry* entry = - std::lower_bound(ParentEntryBegin(), end, ParentEntry{key, 0}); - if (entry == end || entry->key != key) { - return aggregated ? FindNearestParent(tile_key) : boost::none; - } - const auto offset = entry->tag_offset; + } - // The limit is the offset for the next entry, or the end of the index data. - const auto limit = [&]() { - auto next = entry + 1; - return (next == end) ? raw_data_->size() : next->tag_offset; - }(); + const std::uint64_t key = tile_key.ToQuadKey64(); - if (!ReadIndexData(data, offset, limit)) { - return boost::none; - } - data.tile_key = tile_key; - return data; + const ParentEntry* end = ParentEntryEnd(); + const ParentEntry* entry = + std::lower_bound(ParentEntryBegin(), end, ParentEntry{key, 0}); + if (entry == end || entry->key != key) { + return aggregated_search ? FindNearestParent(tile_key) : boost::none; } + const auto offset = entry->tag_offset; + + // The limit is the offset for the next entry, or the end of the index data. + const auto limit = [&]() { + const auto next = entry + 1; + return (next == end) ? raw_data_->size() : next->tag_offset; + }(); + + if (!ReadIndexData(data, offset, limit, All)) { + return boost::none; + } + data.tile_key = tile_key; + return data; } boost::optional QuadTreeIndex::FindNearestParent( geo::TileKey tile_key) const { - const olp::geo::TileKey& root_tile_key = - olp::geo::TileKey::FromQuadKey64(data_->root_tilekey); + const geo::TileKey& root_tile_key = + geo::TileKey::FromQuadKey64(data_->root_tilekey); if (tile_key.Level() >= root_tile_key.Level()) { auto parents_begin = ParentEntryBegin(); @@ -347,7 +355,7 @@ boost::optional QuadTreeIndex::FindNearestParent( if (tile_key.IsChildOf(key)) { IndexData data; data.tile_key = key; - if (!ReadIndexData(data, it->tag_offset, limit)) { + if (!ReadIndexData(data, it->tag_offset, limit, All)) { return boost::none; } return data; @@ -363,7 +371,7 @@ boost::optional QuadTreeIndex::FindNearestParent( if (tile_key.IsChildOf(key)) { IndexData data; data.tile_key = key; - if (!ReadIndexData(data, it->tag_offset, limit)) { + if (!ReadIndexData(data, it->tag_offset, limit, All)) { return boost::none; } return data; @@ -373,8 +381,9 @@ boost::optional QuadTreeIndex::FindNearestParent( return boost::none; } -std::vector QuadTreeIndex::GetIndexData() const { - std::vector result; +std::vector QuadTreeIndex::GetIndexData( + int fields) const { + std::vector result; if (IsNull()) { return result; } @@ -383,21 +392,20 @@ std::vector QuadTreeIndex::GetIndexData() const { auto limit = static_cast(raw_data_->size()); for (auto it = ParentEntryEnd(); it-- != ParentEntryBegin();) { - QuadTreeIndex::IndexData data; + IndexData data; data.tile_key = geo::TileKey::FromQuadKey64(it->key); - if (ReadIndexData(data, it->tag_offset, limit)) { + if (ReadIndexData(data, it->tag_offset, limit, fields)) { result.emplace_back(std::move(data)); } limit = it->tag_offset; } for (auto it = SubEntryEnd(); it-- != SubEntryBegin();) { - QuadTreeIndex::IndexData data; - const olp::geo::TileKey& root_tile_key = - olp::geo::TileKey::FromQuadKey64(data_->root_tilekey); - auto subtile = root_tile_key.AddedSubkey64(std::uint64_t(it->sub_quadkey)); - data.tile_key = subtile; - if (ReadIndexData(data, it->tag_offset, limit)) { + IndexData data; + const geo::TileKey& root_tile_key = + geo::TileKey::FromQuadKey64(data_->root_tilekey); + data.tile_key = root_tile_key.AddedSubkey64(it->sub_quadkey); + if (ReadIndexData(data, it->tag_offset, limit, fields)) { result.emplace_back(std::move(data)); } limit = it->tag_offset; diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.h b/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.h index c7c48c9c3..8714c461f 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.h +++ b/olp-cpp-sdk-dataservice-read/src/repositories/QuadTreeIndex.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 HERE Europe B.V. + * Copyright (C) 2020-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ class BlobDataWriter; class QuadTreeIndex { public: struct IndexData { - olp::geo::TileKey tile_key; + geo::TileKey tile_key; std::string data_handle; std::string additional_metadata; std::string crc; @@ -51,9 +51,17 @@ class QuadTreeIndex { } }; + enum Field { + DataHandle = 1 << 1, + AdditionalMetadata = 1 << 2, + Crc = 1 << 3, + Checksum = 1 << 4, + All = DataHandle | AdditionalMetadata | Crc | Checksum + }; + QuadTreeIndex() = default; explicit QuadTreeIndex(const cache::KeyValueCache::ValueTypePtr& data); - QuadTreeIndex(const olp::geo::TileKey& root, int depth, + QuadTreeIndex(const geo::TileKey& root, int depth, std::stringstream& json_stream); QuadTreeIndex(const QuadTreeIndex& other) = delete; @@ -62,20 +70,21 @@ class QuadTreeIndex { QuadTreeIndex& operator=(const QuadTreeIndex& other) = delete; ~QuadTreeIndex() = default; - inline bool IsNull() const { return data_ == nullptr; } + bool IsNull() const { return data_ == nullptr; } - boost::optional Find(const olp::geo::TileKey& tile_key, - bool aggregated) const; + boost::optional Find(const geo::TileKey& tile_key, + bool aggregated_search) const; - inline cache::KeyValueCache::ValueTypePtr GetRawData() const { - return raw_data_; - } + cache::KeyValueCache::ValueTypePtr GetRawData() const { return raw_data_; } - std::vector GetIndexData() const; + // Get the entire index data, including all parents and sub quads. With a + // `fields` parameter, the caller can specify which fields should be included + // in the result. + std::vector GetIndexData(int fields = All) const; - olp::geo::TileKey GetRootTile() const { - return data_ ? olp::geo::TileKey::FromQuadKey64(data_->root_tilekey) - : olp::geo::TileKey(); + geo::TileKey GetRootTile() const { + return data_ ? geo::TileKey::FromQuadKey64(data_->root_tilekey) + : geo::TileKey(); } private: @@ -106,8 +115,7 @@ class QuadTreeIndex { void CreateBlob(geo::TileKey root, int depth, std::vector parents, std::vector subs); - boost::optional FindNearestParent( - geo::TileKey tile_key) const; + boost::optional FindNearestParent(geo::TileKey tile_key) const; const SubEntry* SubEntryBegin() const { return data_->entries; } const SubEntry* SubEntryEnd() const { @@ -128,7 +136,8 @@ class QuadTreeIndex { return reinterpret_cast(data_) + size_; } - bool ReadIndexData(IndexData& data, uint32_t offset, uint32_t limit) const; + bool ReadIndexData(IndexData& data, uint32_t offset, uint32_t limit, + int fields) const; DataHeader* data_ = nullptr; cache::KeyValueCache::ValueTypePtr raw_data_ = nullptr;