[WIP] Compare jsons as dicts, add json type to tests

Author: nicolasaldecoa

data_diff/hashdiff_tables.py

@@ -10,7 +10,7 @@
 from sqeleton.abcs import ColType_UUID, NumericType, PrecisionType, StringType, Boolean
 
 from .info_tree import InfoTree
-from .utils import safezip
+from .utils import safezip, diffs_are_equiv_jsons
 from .thread_utils import ThreadedYielder
 from .table_segment import TableSegment
 
@@ -24,7 +24,10 @@
 logger = logging.getLogger("hashdiff_tables")
 
 
-def diff_sets(a: set, b: set) -> Iterator:
+def diff_sets(a: list, b: list, has_json_cols: bool = None) -> Iterator:
+    # check unless the only item is the key. TODO: pass a boolean to know whether the schema has json columns or not
+    has_json_cols = len(a[0]) > 1
+
     sa = set(a)
     sb = set(b)
 
@@ -39,6 +42,8 @@ def diff_sets(a: set, b: set) -> Iterator:
             d[row[0]].append(("+", row))
 
     for _k, v in sorted(d.items(), key=lambda i: i[0]):
+        if has_json_cols and diffs_are_equiv_jsons(v):
+            continue  # don't count this as a diff, maybe do and send a warning, maybe parametrized ??
         yield from v
 
 

data_diff/utils.py

@@ -5,6 +5,7 @@
 import operator
 import threading
 from datetime import datetime
+import json
 
 
 def safezip(*args):
@@ -72,3 +73,22 @@ def get_timestamp(_match):
         return datetime.now().isoformat("_", "seconds").replace(":", "_")
 
     return re.sub("%t", get_timestamp, name)
+
+
+def _jsons_equal(a, b):
+    try:
+        return json.loads(a) == json.loads(b)
+    except (ValueError, TypeError, json.decoder.JSONDecodeError):  # not valid jsons
+        return False
+
+
+def diffs_are_equiv_jsons(v):
+    if (len(v) != 2) or ({v[0][0], v[1][0]} != {'+', '-'}):  # ignore rows that are missing in one of the tables
+        return False
+    # check all extra columns.  TODO: would be more efficient if we pass the indices of json cols to only compare those
+    match = True
+    for col_a, col_b in safezip(v[0][1][1:], v[1][1][1:]):
+        match = (col_a == col_b) or _jsons_equal(col_a, col_b)
+        if not match:
+            break
+    return match

tests/test_database_types.py

@@ -74,6 +74,10 @@ def init_conns():
         "boolean": [
             "boolean",
         ],
+        "json": [
+            "json",
+            "jsonb"
+        ]
     },
     db.MySQL: {
         # https://dev.mysql.com/doc/refman/8.0/en/integer-types.html
@@ -199,6 +203,9 @@ def init_conns():
         "boolean": [
             "boolean",
         ],
+        "json": [
+            "super",
+        ]
     },
     db.Oracle: {
         "int": [
@@ -469,12 +476,28 @@ def __iter__(self):
         return (uuid.uuid1(i) for i in range(self.max))
 
 
+class JsonFaker:
+    MANUAL_FAKES = [
+        '{"keyText": "text", "keyInt": 3, "keyFloat": 5.4445, "keyBoolean": true}',
+    ]
+
+    def __init__(self, max):
+        self.max = max
+
+    def __iter__(self):
+        return iter(self.MANUAL_FAKES[: self.max])
+
+    def __len__(self):
+        return min(self.max, len(self.MANUAL_FAKES))
+
+
 TYPE_SAMPLES = {
     "int": IntFaker(N_SAMPLES),
     "datetime": DateTimeFaker(N_SAMPLES),
     "float": FloatFaker(N_SAMPLES),
     "uuid": UUID_Faker(N_SAMPLES),
     "boolean": BooleanFaker(N_SAMPLES),
+    "json": JsonFaker(N_SAMPLES)
 }