support PG json vs RS super matching

Author: nicolasaldecoa

sqeleton/abcs/database_types.py

@@ -134,6 +134,18 @@ class Text(StringType):
     supported = False
 
 
+class JSONType(ColType):
+    pass
+
+
+class RedShiftSuper(JSONType):
+    pass
+
+
+class PostgresqlJSON(JSONType):
+    pass
+
+
 @dataclass
 class Integer(NumericType, IKey):
     precision: int = 0

sqeleton/databases/base.py

@@ -34,6 +34,7 @@
     DbTime,
     DbPath,
     Boolean,
+    JSONType
 )
 from ..abcs.mixins import Compilable
 from ..abcs.mixins import AbstractMixin_Schema, AbstractMixin_RandomSample, AbstractMixin_NormalizeValue
@@ -246,6 +247,9 @@ def parse_type(
         elif issubclass(cls, (Text, Native_UUID)):
             return cls()
 
+        elif issubclass(cls, JSONType):
+            return cls()
+
         raise TypeError(f"Parsing {type_repr} returned an unknown type '{cls}'.")
 
     def _convert_db_precision_to_digits(self, p: int) -> int:

sqeleton/databases/postgresql.py

@@ -1,15 +1,18 @@
 from ..abcs.database_types import (
+    ColType,
     Timestamp,
     TimestampTZ,
     Float,
     Decimal,
     Integer,
     TemporalType,
+    ColType_UUID,
     Native_UUID,
     Text,
     FractionalType,
     Boolean,
     Date,
+    PostgresqlJSON
 )
 from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue
 from .base import BaseDialect, ThreadedDatabase, import_helper, ConnectError, Mixin_Schema
@@ -48,6 +51,37 @@ def normalize_number(self, value: str, coltype: FractionalType) -> str:
     def normalize_boolean(self, value: str, _coltype: Boolean) -> str:
         return self.to_string(f"{value}::int")
 
+    def normalize_json(self, value: str, _coltype: PostgresqlJSON) -> str:
+        return f"replace({value}::text, '\": \"', '\":\"')"  # minified json
+
+    def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
+        """Creates an SQL expression, that converts 'value' to a normalized representation.
+
+        The returned expression must accept any SQL value, and return a string.
+
+        The default implementation dispatches to a method according to `coltype`:
+
+        ::
+
+            TemporalType    -> normalize_timestamp()
+            FractionalType  -> normalize_number()
+            *else*          -> to_string()
+
+            (`Integer` falls in the *else* category)
+
+        """
+        if isinstance(coltype, TemporalType):
+            return self.normalize_timestamp(value, coltype)
+        elif isinstance(coltype, FractionalType):
+            return self.normalize_number(value, coltype)
+        elif isinstance(coltype, ColType_UUID):
+            return self.normalize_uuid(value, coltype)
+        elif isinstance(coltype, Boolean):
+            return self.normalize_boolean(value, coltype)
+        elif isinstance(coltype, PostgresqlJSON):
+            return self.normalize_json(value, coltype)
+        return self.to_string(value)
+
 
 class PostgresqlDialect(BaseDialect, Mixin_Schema):
     name = "PostgreSQL"
@@ -74,6 +108,8 @@ class PostgresqlDialect(BaseDialect, Mixin_Schema):
         "character varying": Text,
         "varchar": Text,
         "text": Text,
+        # JSON
+        "json": PostgresqlJSON,
         # UUID
         "uuid": Native_UUID,
         # Boolean

sqeleton/databases/redshift.py

@@ -1,5 +1,14 @@
 from typing import List, Dict
-from ..abcs.database_types import Float, TemporalType, FractionalType, DbPath
+from ..abcs.database_types import (
+    Float,
+    TemporalType,
+    FractionalType,
+    DbPath,
+    RedShiftSuper,
+    ColType,
+    ColType_UUID,
+    Boolean
+)
 from ..abcs.mixins import AbstractMixin_MD5
 from .postgresql import (
     PostgreSQL,
@@ -40,13 +49,46 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
     def normalize_number(self, value: str, coltype: FractionalType) -> str:
         return self.to_string(f"{value}::decimal(38,{coltype.precision})")
 
+    def normalize_super(self, value: str, _coltype: RedShiftSuper) -> str:
+        return f'nvl2({value}, json_serialize({value}), NULL)'  # only ::varchar causes redshift to return null
+
+    def normalize_value_by_type(self, value: str, coltype: ColType) -> str:
+        """Creates an SQL expression, that converts 'value' to a normalized representation.
+
+        The returned expression must accept any SQL value, and return a string.
+
+        The default implementation dispatches to a method according to `coltype`:
+
+        ::
+
+            TemporalType    -> normalize_timestamp()
+            FractionalType  -> normalize_number()
+            *else*          -> to_string()
+
+            (`Integer` falls in the *else* category)
+
+        """
+        if isinstance(coltype, TemporalType):
+            return self.normalize_timestamp(value, coltype)
+        elif isinstance(coltype, FractionalType):
+            return self.normalize_number(value, coltype)
+        elif isinstance(coltype, ColType_UUID):
+            return self.normalize_uuid(value, coltype)
+        elif isinstance(coltype, Boolean):
+            return self.normalize_boolean(value, coltype)
+        elif isinstance(coltype, RedShiftSuper):
+            return self.normalize_super(value, coltype)
+        return self.to_string(value)
+
 
 class Dialect(PostgresqlDialect):
     name = "Redshift"
     TYPE_CLASSES = {
         **PostgresqlDialect.TYPE_CLASSES,
         "double": Float,
         "real": Float,
+        # JSON
+        "super": RedShiftSuper
     }
     SUPPORTS_INDEXES = False