From 15f0f0fe62dd994e06722e5299da15118e2a7f07 Mon Sep 17 00:00:00 2001 From: Mykhailo Kuchma Date: Tue, 11 Feb 2025 15:41:54 +0100 Subject: [PATCH] Add HarCaptureAdapter class Implement HarCaptureAdapter to capture network requests and generate HAR files. It is capable to capture network requests along with headers, status codes, transfer sizes and timings. Relates-To: OCMAM-418 Signed-off-by: Mykhailo Kuchma --- olp-cpp-sdk-core/CMakeLists.txt | 2 + .../core/http/adapters/HarCaptureAdapter.h | 86 ++++ .../src/http/adapters/HarCaptureAdapter.cpp | 379 ++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100644 olp-cpp-sdk-core/include/olp/core/http/adapters/HarCaptureAdapter.h create mode 100644 olp-cpp-sdk-core/src/http/adapters/HarCaptureAdapter.cpp diff --git a/olp-cpp-sdk-core/CMakeLists.txt b/olp-cpp-sdk-core/CMakeLists.txt index 8264461dc..0097fea4b 100644 --- a/olp-cpp-sdk-core/CMakeLists.txt +++ b/olp-cpp-sdk-core/CMakeLists.txt @@ -104,6 +104,7 @@ set(OLP_SDK_GENERATED_HEADERS ) set(OLP_SDK_HTTP_HEADERS + ./include/olp/core/http/adapters/HarCaptureAdapter.h ./include/olp/core/http/CertificateSettings.h ./include/olp/core/http/HttpStatusCode.h ./include/olp/core/http/Network.h @@ -288,6 +289,7 @@ set(OLP_SDK_CLIENT_SOURCES ) set(OLP_SDK_HTTP_SOURCES + ./src/http/adapters/HarCaptureAdapter.cpp ./src/http/DefaultNetwork.cpp ./src/http/DefaultNetwork.h ./src/http/Network.cpp diff --git a/olp-cpp-sdk-core/include/olp/core/http/adapters/HarCaptureAdapter.h b/olp-cpp-sdk-core/include/olp/core/http/adapters/HarCaptureAdapter.h new file mode 100644 index 000000000..b27804737 --- /dev/null +++ b/olp-cpp-sdk-core/include/olp/core/http/adapters/HarCaptureAdapter.h @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2025 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +#pragma once + +#include +#include + +#include +#include + +namespace olp { +namespace http { + +/** + * @class HarCaptureAdapter + * @brief A network adapter that captures HTTP requests and responses, + * generating a HAR (HTTP Archive) file. + * + * HarCaptureAdapter implements the olp::http::Network interface, intercepting + * network traffic for debugging, logging, and analysis. It records request + * metadata, headers, and response details, allowing developers to + * inspect network interactions in HAR format. + * + * @note Request timings are only available when Curl is used. + * @note The HAR file is produced when the instance is destroyed. + * + * Features: + * - Captures HTTP requests and responses. + * - Logs request/response details, including headers, status codes and timings. + * - Generates a HAR file for easy debugging and sharing. + * + * Example Usage: + * @code + * auto network = std::make_shared(network, "/tmp/out.har"); + * @endcode + */ +class CORE_API HarCaptureAdapter final : public Network { + public: + /** + * @brief Constructs a HarCaptureAdapter instance. + * + * @param network The underlying network implementation to forward requests + * to. + * @param har_out_path The file path where the HAR (HTTP Archive) file will be + * saved. + */ + HarCaptureAdapter(std::shared_ptr network, std::string har_out_path); + + ~HarCaptureAdapter() override; + + /** + * @copydoc Network::Send + */ + SendOutcome Send(NetworkRequest request, Payload payload, Callback callback, + HeaderCallback header_callback, + DataCallback data_callback) override; + + /** + * @copydoc Network::Cancel + */ + void Cancel(RequestId id) override; + + private: + class HarCaptureAdapterImpl; + std::shared_ptr impl_; +}; + +} // namespace http +} // namespace olp diff --git a/olp-cpp-sdk-core/src/http/adapters/HarCaptureAdapter.cpp b/olp-cpp-sdk-core/src/http/adapters/HarCaptureAdapter.cpp new file mode 100644 index 000000000..fe1939e9c --- /dev/null +++ b/olp-cpp-sdk-core/src/http/adapters/HarCaptureAdapter.cpp @@ -0,0 +1,379 @@ +/* + * Copyright (C) 2025 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "olp/core/logging/Log.h" + +namespace olp { +namespace http { +namespace { + +const char* VerbToString(const NetworkRequest::HttpVerb verb) { + switch (verb) { + case NetworkRequest::HttpVerb::GET: + return "GET"; + case NetworkRequest::HttpVerb::POST: + return "POST"; + case NetworkRequest::HttpVerb::HEAD: + return "HEAD"; + case NetworkRequest::HttpVerb::PUT: + return "PUT"; + case NetworkRequest::HttpVerb::DEL: + return "DEL"; + case NetworkRequest::HttpVerb::PATCH: + return "PATCH"; + case NetworkRequest::HttpVerb::OPTIONS: + return "OPTIONS"; + default: + return "UNKNOWN"; + } +} + +std::string FormatTime(const std::chrono::system_clock::time_point timestamp) { + std::stringstream ss; + const std::time_t time = std::chrono::system_clock::to_time_t(timestamp); + const auto ms = std::chrono::duration_cast( + timestamp.time_since_epoch()) % + 1000; + ss << std::put_time(std::localtime(&time), "%FT%T") << '.' + << std::setfill('0') << std::setw(3) << ms.count() << "Z"; + return ss.str(); +} + +class JsonFileSerializer { + public: + explicit JsonFileSerializer(std::ofstream& file) + : out_stream_(file), writer_(out_stream_) {} + + void Object(const std::function& body) { + writer_.StartObject(); + body(); + writer_.EndObject(); + } + + void Object(const char* key, const std::function& body) { + writer_.Key(key); + writer_.StartObject(); + body(); + writer_.EndObject(); + } + + void Array(const char* key, const std::function& body) { + writer_.Key(key); + writer_.StartArray(); + body(); + writer_.EndArray(); + } + + void String(const char* key, const std::string& value) { + writer_.Key(key); + writer_.String(value.c_str(), value.size()); + } + + void Int(const char* key, const int value) { + writer_.Key(key); + writer_.Int(value); + } + + void Double(const char* key, const double value) { + writer_.Key(key); + writer_.Double(value); + } + + void EmptyArray(const char* key) { + writer_.Key(key); + writer_.StartArray(); + writer_.EndArray(); + } + + private: + rapidjson::OStreamWrapper out_stream_; + rapidjson::PrettyWriter writer_{}; +}; + +} // namespace + +class HarCaptureAdapter::HarCaptureAdapterImpl final : public Network { + public: + HarCaptureAdapterImpl(std::shared_ptr network, + std::string har_out_path) + : network_{std::move(network)}, har_out_path_{std::move(har_out_path)} {} + + ~HarCaptureAdapterImpl() override { SaveSessionToFile(); } + + SendOutcome Send(NetworkRequest request, Payload payload, Callback callback, + HeaderCallback header_callback, + DataCallback data_callback) override { + const auto session_request_id = RecordRequest(request); + + const auto response_headers = std::make_shared(); + + auto header_callback_proxy = [=](std::string key, std::string value) { + header_callback(key, value); + response_headers->emplace_back(std::move(key), std::move(value)); + }; + + auto callback_proxy = [=](NetworkResponse response) { + RecordResponse(session_request_id, response, *response_headers); + callback(std::move(response)); + }; + + return network_->Send( + std::move(request), std::move(payload), std::move(callback_proxy), + std::move(header_callback_proxy), std::move(data_callback)); + } + + void Cancel(const RequestId id) override { network_->Cancel(id); } + + private: + struct RequestEntry { + size_t url{}; + std::chrono::system_clock::time_point start_time; + std::chrono::system_clock::time_point end_time; + uint8_t method{}; + uint8_t status_code{}; + uint16_t request_headers_offset{}; + uint16_t request_headers_count{}; + uint16_t response_headers_offset{}; + uint16_t response_headers_count{}; + uint32_t transfer_size{}; + }; + + RequestId RecordRequest(const NetworkRequest& request) { + std::lock_guard lock(mutex_); + constexpr std::hash hasher{}; + + const auto size = requests_.size(); + + RequestEntry request_entry; + + const auto& url = request.GetUrl(); + request_entry.url = hasher(url); + cache_[request_entry.url] = request.GetUrl(); + + request_entry.method = static_cast(request.GetVerb()); + request_entry.start_time = std::chrono::system_clock::now(); + + const auto& request_headers = request.GetHeaders(); + request_entry.request_headers_offset = headers_.size(); + request_entry.request_headers_count = request_headers.size(); + + for (const auto& header : request_headers) { + cache_[hasher(header.first)] = header.first; + cache_[hasher(header.second)] = header.second; + headers_.emplace_back(hasher(header.first), hasher(header.second)); + } + + requests_.emplace_back(request_entry); + + return size; + } + + void RecordResponse(const RequestId request_id, + const NetworkResponse& response, + const Headers& response_headers) { + std::lock_guard lock(mutex_); + constexpr std::hash hasher{}; + + RequestEntry& request_entry = requests_[request_id]; + request_entry.status_code = response.GetStatus(); + request_entry.end_time = std::chrono::system_clock::now(); + request_entry.transfer_size = + response.GetBytesUploaded() + response.GetBytesDownloaded(); + + request_entry.response_headers_offset = headers_.size(); + request_entry.response_headers_count = response_headers.size(); + + for (const auto& header : response_headers) { + cache_[hasher(header.first)] = header.first; + cache_[hasher(header.second)] = header.second; + headers_.emplace_back(hasher(header.first), hasher(header.second)); + } + + const auto& diagnostics = response.GetDiagnostics(); + if (diagnostics) { + if (diagnostics_.size() <= request_id) { + diagnostics_.resize(request_id + 1); + } + + diagnostics_[request_id] = *diagnostics; + } + } + + void SaveSessionToFile() const { + std::ofstream file(har_out_path_); + if (!file.is_open()) { + OLP_SDK_LOG_ERROR("HarCaptureAdapter::SaveSession", + "Failed to save session."); + return; + } + + JsonFileSerializer serializer{file}; + + serializer.Object([&] { + serializer.Object("log", [&] { + serializer.String("version", "1.2"); + + serializer.Object("creator", [&] { + serializer.String("name", "DataSDK"); + serializer.String("version", OLP_SDK_VERSION_STRING); + }); + + serializer.Array("entries", [&] { + for (auto request_index = 0u; request_index < requests_.size(); + ++request_index) { + const auto& request = requests_[request_index]; + const auto diagnostics = diagnostics_.size() > request_index + ? diagnostics_[request_index] + : Diagnostics{}; + + // return duration in milliseconds as float + auto duration = [&](const Diagnostics::Timings timing, + const double default_value = -1.0) { + return diagnostics.available_timings[timing] + ? diagnostics.timings[timing].count() / 1000.0 + : default_value; + }; + + const double total_time = duration( + Diagnostics::Total, + static_cast( + std::chrono::duration_cast( + request.end_time - request.start_time) + .count()) / + 1000.0); + + auto output_headers = [&](const uint16_t headers_offset, + const uint16_t headers_count) { + serializer.Array("headers", [&] { + for (auto i = 0u; i < headers_count; ++i) { + serializer.Object([&] { + auto header = headers_[headers_offset + i]; + serializer.String("name", cache_.at(header.first)); + serializer.String("value", cache_.at(header.second)); + }); + } + }); + }; + + serializer.Object([&] { + serializer.String("startedDateTime", + FormatTime(request.start_time)); + serializer.Double("time", total_time); + + serializer.Object("request", [&] { + serializer.String( + "method", + VerbToString( + static_cast(request.method))); + serializer.String("url", cache_.at(request.url)); + serializer.String("httpVersion", "UNSPECIFIED"); + serializer.EmptyArray("cookies"); + output_headers(request.request_headers_offset, + request.request_headers_count); + serializer.EmptyArray("queryString"); + serializer.Int("headersSize", -1); + serializer.Int("bodySize", -1); + }); + + // response + serializer.Object("response", [&] { + serializer.Int("status", request.status_code); + serializer.String("statusText", ""); + serializer.String("httpVersion", "UNSPECIFIED"); + serializer.EmptyArray("cookies"); + output_headers(request.response_headers_offset, + request.response_headers_count); + serializer.Object("content", [&] { + serializer.Int("size", 0); + serializer.String("mimeType", ""); + }); + serializer.String("redirectURL", ""); + serializer.Int("headersSize", -1); + serializer.Int("bodySize", -1); + serializer.Int("_transferSize", + static_cast(request.transfer_size)); + }); + + // timings + serializer.Object("timings", [&] { + using Timings = Diagnostics::Timings; + serializer.Double("blocked", duration(Timings::Queue)); + serializer.Double("dns", duration(Timings::NameLookup)); + serializer.Double("connect", duration(Timings::Connect)); + serializer.Double("ssl", duration(Timings::SSL_Handshake)); + serializer.Double("send", duration(Timings::Send, 0.0)); + serializer.Double("wait", duration(Timings::Wait, 0.0)); + serializer.Double("receive", + duration(Timings::Receive, total_time)); + }); + }); + } + }); + }); + }); + + file.close(); + + OLP_SDK_LOG_INFO("HarCaptureAdapter::SaveSession", + "Session is saved to: " << har_out_path_); + } + + std::mutex mutex_; + std::unordered_map cache_{}; + std::deque > headers_{}; + std::deque requests_{}; + std::deque diagnostics_{}; + + std::shared_ptr network_; + std::string har_out_path_; +}; + +HarCaptureAdapter::HarCaptureAdapter(std::shared_ptr network, + std::string har_out_path) + : impl_(std::make_shared(std::move(network), + std::move(har_out_path))) {} + +HarCaptureAdapter::~HarCaptureAdapter() = default; + +SendOutcome HarCaptureAdapter::Send(NetworkRequest request, Payload payload, + Callback callback, + HeaderCallback header_callback, + DataCallback data_callback) { + return impl_->Send(std::move(request), std::move(payload), + std::move(callback), std::move(header_callback), + std::move(data_callback)); +} + +void HarCaptureAdapter::Cancel(const RequestId id) { impl_->Cancel(id); } + +} // namespace http +} // namespace olp