Merge pull request #111 from datafold/fix_int_column

Split Integer from Decimal to reduce casts in SQL. Added FractionalType.

Author: erezsh

data_diff/databases/bigquery.py

@@ -87,9 +87,7 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
             f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')"
         )
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
-        if isinstance(coltype, Integer):
-            return self.to_string(value)
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return f"format('%.{coltype.precision}f', {value})"
 
     def parse_table_name(self, name: str) -> DbPath:

data_diff/databases/database_types.py

@@ -40,17 +40,19 @@ class NumericType(ColType):
     # 'precision' signifies how many fractional digits (after the dot) we want to compare
     precision: int
 
+class FractionalType(NumericType):
+    pass
 
-class Float(NumericType):
+class Float(FractionalType):
     pass
 
 
-class Decimal(NumericType):
+class Decimal(FractionalType):
     pass
 
 
 @dataclass
-class Integer(Decimal):
+class Integer(NumericType):
     def __post_init__(self):
         assert self.precision == 0
 
@@ -114,7 +116,7 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
         ...
 
     @abstractmethod
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         """Creates an SQL expression, that converts 'value' to a normalized number.
 
         The returned expression must accept any SQL int/numeric/float, and return a string.
@@ -139,18 +141,20 @@ def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
 
         The returned expression must accept any SQL value, and return a string.
 
-        The default implementation dispatches to a method according to ``coltype``:
+        The default implementation dispatches to a method according to `coltype`:
 
             TemporalType -> normalize_timestamp()
-            NumericType  -> normalize_number()
-            -else-       -> to_string()
+            FractionalType  -> normalize_number()
+            *else*       -> to_string()
+
+            (`Integer` falls in the *else* category)
 
         """
         if isinstance(coltype, TemporalType):
             return self.normalize_timestamp(value, coltype)
-        elif isinstance(coltype, NumericType):
+        elif isinstance(coltype, FractionalType):
             return self.normalize_number(value, coltype)
-        return self.to_string(f"{value}")
+        return self.to_string(value)
 
     def _normalize_table_path(self, path: DbPath) -> DbPath:
         ...

data_diff/databases/mysql.py

@@ -60,5 +60,5 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
         s = self.to_string(f"cast({value} as datetime(6))")
         return f"RPAD(RPAD({s}, {TIMESTAMP_PRECISION_POS+coltype.precision}, '.'), {TIMESTAMP_PRECISION_POS+6}, '0')"
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return self.to_string(f"cast({value} as decimal(38, {coltype.precision}))")

data_diff/databases/oracle.py

@@ -57,7 +57,7 @@ def select_table_schema(self, path: DbPath) -> str:
     def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
         return f"to_char(cast({value} as timestamp({coltype.precision})), 'YYYY-MM-DD HH24:MI:SS.FF6')"
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         # FM999.9990
         format_str = "FM" + "9" * (38 - coltype.precision)
         if coltype.precision:

data_diff/databases/postgresql.py

@@ -67,5 +67,5 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
             f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')"
         )
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return self.to_string(f"{value}::decimal(38, {coltype.precision})")

data_diff/databases/presto.py

@@ -64,7 +64,7 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
 
         return f"RPAD(RPAD({s}, {TIMESTAMP_PRECISION_POS+coltype.precision}, '.'), {TIMESTAMP_PRECISION_POS+6}, '0')"
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return self.to_string(f"cast({value} as decimal(38,{coltype.precision}))")
 
     def select_table_schema(self, path: DbPath) -> str:

data_diff/databases/redshift.py

@@ -32,7 +32,7 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
             f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')"
         )
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return self.to_string(f"{value}::decimal(38,{coltype.precision})")
 
     def select_table_schema(self, path: DbPath) -> str:

data_diff/databases/snowflake.py

@@ -86,5 +86,5 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
 
         return f"to_char({timestamp}, 'YYYY-MM-DD HH24:MI:SS.FF6')"
 
-    def normalize_number(self, value: str, coltype: NumericType) -> str:
+    def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return self.to_string(f"cast({value} as decimal(38, {coltype.precision}))")

data_diff/diff_tables.py

@@ -96,7 +96,7 @@ def _update_column(self):
 
     def _quote_column(self, c):
         if self._schema:
-            c = self._schema.get_key(c)
+            c = self._schema.get_key(c)     # Get the actual name. Might be case-insensitive.
         return self.database.quote(c)
 
     def with_schema(self) -> "TableSegment":