Added Database.normalize_uuid(); more tests (WIP)

Author: erezsh

data_diff/databases/base.py

@@ -197,8 +197,11 @@ def _refine_coltypes(self, table_path: DbPath, col_dict: Dict[str, ColType]):
         "Refine the types in the column dict, by querying the database for a sample of their values"
 
         text_columns = [k for k, v in col_dict.items() if isinstance(v, Text)]
+        if not text_columns:
+            return
 
-        samples_by_row = self.query(Select(text_columns, TableName(table_path), limit=16), list)
+        fields = [self.normalize_uuid(c, ColType_UUID()) for c in text_columns]
+        samples_by_row = self.query(Select(fields, TableName(table_path), limit=16), list)
         samples_by_col = list(zip(*samples_by_row))
         for col_name, samples in safezip(text_columns, samples_by_col):
             uuid_samples = list(filter(is_uuid, samples))
@@ -270,6 +273,9 @@ def offset_limit(self, offset: Optional[int] = None, limit: Optional[int] = None
 
         return f"LIMIT {limit}"
 
+    def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str:
+        return f"TRIM({value})"
+
 
 CHECKSUM_HEXDIGITS = 15  # Must be 15 or lower
 MD5_HEXDIGITS = 32

data_diff/databases/database_types.py

@@ -169,6 +169,15 @@ def normalize_number(self, value: str, coltype: FractionalType) -> str:
         """
         ...
 
+    @abstractmethod
+    def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str:
+        """Creates an SQL expression, that converts 'value' to a normalized uuid.
+
+        i.e. just makes sure there is no trailing whitespace.
+        """
+        ...
+
+
     def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
         """Creates an SQL expression, that converts 'value' to a normalized representation.
 
@@ -189,6 +198,8 @@ def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
             return self.normalize_timestamp(value, coltype)
         elif isinstance(coltype, FractionalType):
             return self.normalize_number(value, coltype)
+        elif isinstance(coltype, ColType_UUID):
+            return self.normalize_uuid(value, coltype)
         return self.to_string(value)
 
     def _normalize_table_path(self, path: DbPath) -> DbPath:

data_diff/databases/postgresql.py

@@ -26,6 +26,8 @@ class PostgreSQL(ThreadedDatabase):
         "numeric": Decimal,
         "bigint": Integer,
         # Text
+        "character": Text,
+        "character varying": Text,
         "varchar": Text,
         "text": Text,
     }

tests/test_database_types.py

@@ -3,6 +3,7 @@
 import time
 import re
 import math
+import uuid
 from datetime import datetime, timedelta
 from decimal import Decimal
 from parameterized import parameterized
@@ -157,11 +158,23 @@ def __next__(self) -> float:
         else:
             raise StopIteration
 
+class UUID_Faker:
+    def __init__(self, max):
+        self.max = max
+
+    def __len__(self):
+        return self.max
+
+    def __iter__(self):
+        return (uuid.uuid1(i) for i in range(self.max))
+
+
 
 TYPE_SAMPLES = {
     "int": IntFaker(N_SAMPLES),
     "datetime_no_timezone": DateTimeFaker(N_SAMPLES),
     "float": FloatFaker(N_SAMPLES),
+    "uuid": UUID_Faker(N_SAMPLES),
 }
 
 DATABASE_TYPES = {
@@ -185,6 +198,11 @@ def __next__(self) -> float:
             "double precision",
             "numeric(6,3)",
         ],
+        "uuid": [
+            "text",
+            "varchar(100)",
+            "char(100)",
+        ],
     },
     db.MySQL: {
         # https://dev.mysql.com/doc/refman/8.0/en/integer-types.html
@@ -210,6 +228,10 @@ def __next__(self) -> float:
             "numeric",
             "numeric(65, 10)",
         ],
+        "uuid": [
+            "varchar(100)",
+            "char(100)",
+        ],
     },
     db.BigQuery: {
         "int": ["int"],
@@ -222,6 +244,9 @@ def __next__(self) -> float:
             "float64",
             "bignumeric",
         ],
+        "uuid": [
+            "$uuid"
+        ],
     },
     db.Snowflake: {
         # https://docs.snowflake.com/en/sql-reference/data-types-numeric.html#int-integer-bigint-smallint-tinyint-byteint
@@ -246,6 +271,10 @@ def __next__(self) -> float:
             "float",
             "numeric",
         ],
+        "uuid": [
+            "$uuid"
+        ],
+
     },
     db.Redshift: {
         "int": [
@@ -260,6 +289,9 @@ def __next__(self) -> float:
             "float8",
             "numeric",
         ],
+        "uuid": [
+            "$uuid"
+        ],
     },
     db.Oracle: {
         "int": [
@@ -276,6 +308,9 @@ def __next__(self) -> float:
             "real",
             "double precision",
         ],
+        "uuid": [
+            "$uuid"
+        ],
     },
     db.Presto: {
         "int": [
@@ -295,6 +330,10 @@ def __next__(self) -> float:
             "decimal(10,2)",
             "decimal(30,6)",
         ],
+        "uuid": [
+            "$uuid"
+        ],
+
     },
 }
 

tests/test_diff_tables.py

@@ -263,7 +263,6 @@ def setUp(self):
         queries.append(f"INSERT INTO a VALUES ('{self.new_uuid}', 'This one is different')")
 
         # TODO test unexpected values?
-        # queries.append(f"INSERT INTO a VALUES ('unexpected', '<-- this bad value should not break us')")
 
         for query in queries:
             self.connection.query(query, None)
@@ -276,6 +275,10 @@ def test_string_keys(self):
         diff = list(differ.diff_tables(self.a, self.b))
         self.assertEqual(diff, [("-", (str(self.new_uuid), "This one is different"))])
 
+        self.connection.query(f"INSERT INTO a VALUES ('unexpected', '<-- this bad value should not break us')", None)
+
+        self.assertRaises(ValueError, differ.diff_tables, self.a, self.b)
+
 
 class TestTableSegment(TestWithConnection):
     def setUp(self) -> None: