Use fdb5::Key and fdb5::ListElement instead of custom types

Author: pgeier

src/fdb5/tools/compare/common/Types.cc

@@ -18,15 +18,6 @@ namespace compare {
 
 //---------------------------------------------------------------------------------------------------------------------
 
-std::ostream& operator<<(std::ostream& os, const KeyValueMap& km) {
-    os << "{";
-    for (const auto& [k, v] : km) {
-        os << k << "=" << v << ", ";
-    }
-    os << "}";
-    return os;
-}
-
 std::ostream& operator<<(std::ostream& os, const KeyDiffMap& km) {
     os << "{";
     for (const auto& [k, v] : km) {
@@ -46,10 +37,6 @@ std::ostream& operator<<(std::ostream& os, const KeySet& km) {
     return os;
 }
 
-std::ostream& operator<<(std::ostream& os, const DataLocation& loc) {
-    return os << "(" << loc.path << ", " << loc.offset << ", " << loc.length << ")";
-}
-
 std::ostream& operator<<(std::ostream& os, const DataIndex& idx) {
     for (const auto& [km, loc] : idx) {
         os << "Key: " << km << " -> Value: " << loc << "\n";
@@ -59,26 +46,6 @@ std::ostream& operator<<(std::ostream& os, const DataIndex& idx) {
 
 //---------------------------------------------------------------------------------------------------------------------
 
-
-void parseKeyValues(KeyValueMap& container, const std::string& keyValueStr) {
-    std::istringstream stream(keyValueStr);
-    std::string entry;
-
-    while (std::getline(stream, entry, ',')) {
-        std::istringstream pairStream(entry);
-        std::string key, value;
-        if (std::getline(pairStream, key, '=') && std::getline(pairStream, value)) {
-            container[key] = value;  // Update or insert key-value pair
-        }
-    }
-}
-
-KeyValueMap parseKeyValues(const std::string& keyValueStr) {
-    KeyValueMap container;
-    parseKeyValues(container, keyValueStr);
-    return container;
-}
-
 void parseKeySet(KeySet& container, const std::string& keyStr) {
     std::istringstream stream(keyStr);
     std::string entry;
@@ -94,11 +61,11 @@ KeySet parseKeySet(const std::string& keyStr) {
     return container;
 }
 
-compare::KeyDiffMap requestDiff(const KeyValueMap& l, const KeyValueMap& r) {
+compare::KeyDiffMap requestDiff(const fdb5::Key& l, const fdb5::Key& r) {
     compare::KeyDiffMap res;
     for (const auto& [lk, lv] : l) {
-        auto search = r.find(lk);
-        if (search != r.end()) {
+        auto [search, isValid] = r.find(lk);
+        if (isValid) {
             if (search->second != lv) {
                 res.insert({lk, {lv, search->second}});
             }
@@ -109,8 +76,8 @@ compare::KeyDiffMap requestDiff(const KeyValueMap& l, const KeyValueMap& r) {
     }
 
     for (const auto& [rk, rv] : r) {
-        auto search = l.find(rk);
-        if (search == l.end()) {
+        auto [search, isValid] = l.find(rk);
+        if (!isValid) {
             res.insert({rk, {{}, rv}});
         }
     }
@@ -120,21 +87,15 @@ compare::KeyDiffMap requestDiff(const KeyValueMap& l, const KeyValueMap& r) {
 //---------------------------------------------------------------------------------------------------------------------
 
 
-KeyValueMap applyKeyDiff(KeyValueMap k, const KeyDiffMap& diff, bool swapDiff) {
+fdb5::Key applyKeyDiff(fdb5::Key k, const KeyDiffMap& diff, bool swapDiff) {
     for (const auto& [field, val_pair] : diff) {
         const auto& val = swapDiff ? val_pair.first : val_pair.second;
         if (val) {
-            auto it = k.find(field);
-            if (it != k.end()) {
-                it->second = *val;  // replace value
-            }
-            else {
-                k.insert({field, *val});
-            }
+            k.set(field, *val);
         }
         else {
             // Delete
-            k.erase(field);
+            k.unset(field);
         }
     }
     return k;  // return modified copy
@@ -265,31 +226,26 @@ void Result::update(const Result& other) {
 //---------------------------------------------------------------------------------------------------------------------
 
 
-bool isSubset(const KeyValueMap& a, const KeyValueMap& b) {
+bool isSubset(const fdb5::Key& a, const fdb5::Key& b) {
     for (const auto& kv : a) {
-        auto it = b.find(kv.first);
-        if (it == b.end() || it->second != kv.second)
+        auto [it, isValid] = b.find(kv.first);
+        if (!isValid || it->second != kv.second)
             return false;
     }
     return true;
 }
 
 
-DataIndex assembleCompareMap(fdb5::FDB& fdb, const fdb5::FDBToolRequest& req, const KeyValueMap& ignore) {
+DataIndex assembleCompareMap(fdb5::FDB& fdb, const fdb5::FDBToolRequest& req, const fdb5::Key& ignore) {
     DataIndex out;
 
     auto list = fdb.list(req);
     fdb5::ListElement elem;
 
     while (list.next(elem)) {
-        KeyValueMap km;
-        for (const auto& bit : elem.keys()) {
-            auto d = bit.keyDict();
-            km.insert(d.begin(), d.end());
-        }
+        fdb5::Key km = elem.combinedKey();
         if (ignore.empty() || !isSubset(ignore, km)) {
-            out.emplace(km, DataLocation{elem.location().uri().path(), static_cast<long long>(elem.location().offset()),
-                                         static_cast<long long>(elem.location().length())});
+            out.emplace(km, elem);
         }
     }
     eckit::Log::info() << "[LOG] FDB request: " << req << " resulted in " << out.size() << " entries.\n";

src/fdb5/tools/compare/common/Types.h

@@ -16,6 +16,7 @@
 #pragma once
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
+#include "fdb5/api/helpers/ListElement.h"
 
 #include <iostream>
 #include <map>
@@ -28,16 +29,6 @@ namespace compare {
 
 //---------------------------------------------------------------------------------------------------------------------
 
-/// Describes the location of a message
-struct DataLocation {
-    std::string path;
-    long long offset = 0;
-    long long length = 0;
-};
-
-/// String-string map which is used to store FDB keys (e.g. {class=rd, expver=1234, ...})
-using KeyValueMap = std::map<std::string, std::string>;
-
 /// A pair describing differences between two key-value pairs in a requests, e.g. expver=1234 or expver=2345
 /// Each value is optional - to indicate if a key is missing in one the comparing pairs.
 using ValueDiff = std::pair<std::optional<std::string>, std::optional<std::string>>;
@@ -50,67 +41,46 @@ using KeySet = std::unordered_set<std::string>;
 
 }  // namespace compare
 
-template <>
-struct std::hash<compare::KeyValueMap> {
-    size_t operator()(const compare::KeyValueMap& m) const noexcept {
-        size_t h = 0;
-        std::hash<std::string> hs;
-        for (const auto& kv : m) {
-            h ^= (hs(kv.first) * 1315423911u) ^ hs(kv.second);
-        }
-        return h;
-    }
-};
-
-template <>
-struct std::equal_to<compare::KeyValueMap> {
-    bool operator()(const compare::KeyValueMap& a, const compare::KeyValueMap& b) const noexcept { return a == b; }
-};
-
 namespace compare {
 
 /// Creates a map of full specificed FDB keys (as a KeyValueMap),
 /// mapping to a data location which describes where the message can be loaded from.
-using DataIndex = std::unordered_map<KeyValueMap, DataLocation>;
+using DataIndex = std::unordered_map<fdb5::Key, fdb5::ListElement>;
 
 /// Runs a request on a fdb to create a list of filtered entries to compared on.
 ///
 /// @param fdb FDB on which to operate on
 /// @param req Request which performs a selection on the fdb
 /// @param ignore Map of key-value pairs to ignore
 /// @return filtered DataIndex
-DataIndex assembleCompareMap(fdb5::FDB& fdb, const fdb5::FDBToolRequest& req, const KeyValueMap& ignore);
+DataIndex assembleCompareMap(fdb5::FDB& fdb, const fdb5::FDBToolRequest& req, const fdb5::Key& ignore);
 
 //---------------------------------------------------------------------------------------------------------------------
 
-std::ostream& operator<<(std::ostream& os, const KeyValueMap& km);
-
 std::ostream& operator<<(std::ostream& os, const KeyDiffMap& km);
 
 std::ostream& operator<<(std::ostream& os, const KeySet& km);
 
-std::ostream& operator<<(std::ostream& os, const DataLocation& loc);
-
 std::ostream& operator<<(std::ostream& os, const DataIndex& idx);
 
 
 //---------------------------------------------------------------------------------------------------------------------
 
-void parseKeyValues(KeyValueMap& container, const std::string& keyValueStr);
-KeyValueMap parseKeyValues(const std::string& keyValueStr);
+// void parseKeyValues(KeyValueMap& container, const std::string& keyValueStr);
+// KeyValueMap parseKeyValues(const std::string& keyValueStr);
 
 void parseKeySet(KeySet& container, const std::string& keyValueStr);
 KeySet parseKeySet(const std::string& keyValueStr);
 
-compare::KeyDiffMap requestDiff(const KeyValueMap& l, const KeyValueMap& r);
+compare::KeyDiffMap requestDiff(const fdb5::Key& l, const fdb5::Key& r);
 
 
 /// @param km Map of keys with values
 /// @param diff Map of keys with differences
 /// @param swapPair Optional - If false the second pair of the difference is put in the resulting request.
 ///                 If true the first one is used.
 /// @return New KeyValueMap with differences applied
-KeyValueMap applyKeyDiff(KeyValueMap km, const KeyDiffMap& diff, bool swapPair = false);
+fdb5::Key applyKeyDiff(fdb5::Key km, const KeyDiffMap& diff, bool swapPair = false);
 
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -141,14 +111,14 @@ struct Options {
     Method method = Method::KeyByKey;
 
     // For MARS comparison:
-    KeyValueMap ignoreMarsKeys;
+    fdb5::Key ignoreMarsKeys;
 
     // Expected differences between two requets (e.g. comparing different expver)
     KeyDiffMap marsReqDiff;
 
     // Optional explicit requests to compare different subtrees, e.g. expver=1111 with expver=2222
-    std::optional<KeyValueMap> referenceRequest;
-    std::optional<KeyValueMap> testRequest;
+    std::optional<fdb5::Key> referenceRequest;
+    std::optional<fdb5::Key> testRequest;
 
     // List of specific grib keys which are used for comparison only
     KeySet gribKeysSelect;

src/fdb5/tools/compare/fdb-compare.cc

@@ -139,7 +139,7 @@ void FDBCompare::init(const CmdArgs& args) {
 
     std::string tmp = args.getString("mars-keys-ignore", "");
     if (!tmp.empty()) {
-        parseKeyValues(opts_.ignoreMarsKeys, tmp);
+        opts_.ignoreMarsKeys = fdb5::Key::parse(tmp);
     }
 
     if (args.has("reference-request")) {
@@ -218,10 +218,10 @@ void FDBCompare::execute(const CmdArgs& args) {
     })();
 
     if (refReqString_) {
-        opts_.referenceRequest = parseKeyValues(*refReqString_);
+        opts_.referenceRequest = fdb5::Key::parse(*refReqString_);
     }
     if (testReqString_) {
-        opts_.testRequest = parseKeyValues(*testReqString_);
+        opts_.testRequest = fdb5::Key::parse(*testReqString_);
     }
 
     const auto pickReq = [&](const std::optional<std::string>& s) -> fdb5::FDBToolRequest {

src/fdb5/tools/compare/grib/Compare.cc

@@ -16,6 +16,7 @@
 #include "Utils.h"
 
 #include "eckit/exception/Exceptions.h"
+#include "fdb5/api/helpers/ListElement.h"
 #include "metkit/codes/api/CodesAPI.h"
 
 
@@ -27,13 +28,13 @@ namespace compare::grib {
 using namespace eckit;
 
 
-CompareResult gribCompareBitExact(const DataLocation& gribLocRef, const DataLocation& gribLocTest, const Options& o,
+CompareResult gribCompareBitExact(const fdb5::ListElement& gribLocRef, const fdb5::ListElement& gribLocTest, const Options& o,
                                   const uint8_t* bufferRef, const uint8_t* bufferTest) {
     if (o.verbose) {
         eckit::Log::info() << "[LOG] Memory comparison (Bytestream)" << std::endl;
     }
     // Bitexact comparison directly on the Bytestream begin
-    auto res = bitComparison(bufferRef, bufferTest, (gribLocTest.length));
+    auto res = bitComparison(bufferRef, bufferTest, gribLocTest.length());
     if (res == CompareResult::OtherMismatch) {
         std::ostringstream oss;
         oss << "[GRIB COMPARISON MISMATCH: BITEXACT: HEADER ] {Location,offset,length} = " << gribLocRef;
@@ -43,7 +44,7 @@ CompareResult gribCompareBitExact(const DataLocation& gribLocRef, const DataLoca
 }
 
 
-CompareResult gribCompareHash(const DataLocation& gribLocRef, const DataLocation& gribLocTest, const Options& o,
+CompareResult gribCompareHash(const fdb5::ListElement& gribLocRef, const fdb5::ListElement& gribLocTest, const Options& o,
                               const metkit::codes::CodesHandle& hRef, const metkit::codes::CodesHandle& hTest) {
     if (!compareHeader(hRef, hTest)) {
         std::ostringstream oss;
@@ -58,7 +59,7 @@ CompareResult gribCompareHash(const DataLocation& gribLocRef, const DataLocation
 }
 
 
-CompareResult gribCompareKeyByKey(const DataLocation& gribLocRef, const DataLocation& gribLocTest, const Options& o,
+CompareResult gribCompareKeyByKey(const fdb5::ListElement& gribLocRef, const fdb5::ListElement& gribLocTest, const Options& o,
                                   const metkit::codes::CodesHandle& hRef, const metkit::codes::CodesHandle& hTest) {
     auto res = compareKeyByKey(hRef, hTest, o.gribKeysIgnore, o.gribKeysSelect);
     if (res == CompareResult::OtherMismatch) {
@@ -72,15 +73,15 @@ CompareResult gribCompareKeyByKey(const DataLocation& gribLocRef, const DataLoca
 }
 
 
-Result gribCompareSingleMessage(const DataLocation& gribLocRef, const DataLocation& gribLocTest, const Options& o) {
+Result gribCompareSingleMessage(const fdb5::ListElement& gribLocRef, const fdb5::ListElement& gribLocTest, const Options& o) {
     Result res;
     res.match = true;
 
     std::unique_ptr<uint8_t[]> bufferRef  = extractGribMessage(gribLocRef);
     std::unique_ptr<uint8_t[]> bufferTest = extractGribMessage(gribLocTest);
 
-    auto hRef  = metkit::codes::codesHandleFromMessage({bufferRef.get(), static_cast<size_t>(gribLocRef.length)});
-    auto hTest = metkit::codes::codesHandleFromMessage({bufferTest.get(), static_cast<size_t>(gribLocTest.length)});
+    auto hRef  = metkit::codes::codesHandleFromMessage({bufferRef.get(), static_cast<size_t>(gribLocRef.length())});
+    auto hTest = metkit::codes::codesHandleFromMessage({bufferTest.get(), static_cast<size_t>(gribLocTest.length())});
 
     // Non-numeric comparisons
     CompareResult methodRes = ([&]() {