Fix(clickhouse): remove fractional seconds when time column is datetime/timestamp type (#3261)

Author: treysp

sqlmesh/core/model/definition.py

@@ -625,7 +625,15 @@ def convert_to_time_column(
 
             time_column_type = columns_to_types[self.time_column.column.name]
 
-            return to_time_column(time, time_column_type, self.time_column.format)
+            return to_time_column(
+                time,
+                time_column_type,
+                self.time_column.format,
+                include_microseconds=not (
+                    self.dialect in TIME_COLUMN_NO_MICROSECONDS_TYPES
+                    and time_column_type.is_type(*TIME_COLUMN_NO_MICROSECONDS_TYPES[self.dialect])
+                ),
+            )
         return exp.convert(time)
 
     def set_mapping_schema(self, schema: t.Dict) -> None:
@@ -2440,3 +2448,8 @@ def _meta_renderer(
 def get_model_name(path: Path) -> str:
     path_parts = list(path.parts[path.parts.index("models") + 1 : -1]) + [path.stem]
     return ".".join(path_parts[-3:])
+
+
+TIME_COLUMN_NO_MICROSECONDS_TYPES = {
+    "clickhouse": (exp.DataType.Type.DATETIME, exp.DataType.Type.TIMESTAMP)
+}

sqlmesh/utils/date.py

@@ -256,14 +256,20 @@ def to_ds(obj: TimeLike) -> str:
     return to_ts(obj)[0:10]
 
 
-def to_ts(obj: TimeLike) -> str:
+def to_ts(obj: TimeLike, include_microseconds: bool = True) -> str:
     """Converts a TimeLike object into YYYY-MM-DD HH:MM:SS formatted string."""
-    return to_datetime(obj).replace(tzinfo=None).isoformat(sep=" ")
+    obj_dt = to_datetime(obj)
+    if not include_microseconds:
+        obj_dt = obj_dt.replace(microsecond=0)
+    return obj_dt.replace(tzinfo=None).isoformat(sep=" ")
 
 
-def to_tstz(obj: TimeLike) -> str:
+def to_tstz(obj: TimeLike, include_microseconds: bool = True) -> str:
     """Converts a TimeLike object into YYYY-MM-DD HH:MM:SS+00:00 formatted string."""
-    return to_datetime(obj).isoformat(sep=" ")
+    obj_dt = to_datetime(obj)
+    if not include_microseconds:
+        obj_dt = obj_dt.replace(microsecond=0)
+    return obj_dt.isoformat(sep=" ")
 
 
 def is_date(obj: TimeLike) -> bool:
@@ -289,7 +295,7 @@ def make_inclusive(start: TimeLike, end: TimeLike) -> Interval:
     In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if
     start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps
     are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is
-    treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.
+    treated as an exclusive range and converted to inclusive by subtracting 1 microsecond.
 
     Args:
         start: Start timelike object.
@@ -347,16 +353,22 @@ def to_time_column(
     time_column: t.Union[TimeLike, exp.Null],
     time_column_type: exp.DataType,
     time_column_format: t.Optional[str] = None,
+    include_microseconds: bool = True,
 ) -> exp.Expression:
     """Convert a TimeLike object to the same time format and type as the model's time column."""
     if isinstance(time_column, exp.Null):
         return exp.cast(time_column, to=time_column_type)
     if time_column_type.is_type(exp.DataType.Type.DATE):
         return exp.cast(exp.Literal.string(to_ds(time_column)), to="date")
     if time_column_type.is_type(*TEMPORAL_TZ_TYPES):
-        return exp.cast(exp.Literal.string(to_tstz(time_column)), to=time_column_type)
+        return exp.cast(
+            exp.Literal.string(to_tstz(time_column, include_microseconds)),
+            to=time_column_type,
+        )
     if time_column_type.is_type(*exp.DataType.TEMPORAL_TYPES):
-        return exp.cast(exp.Literal.string(to_ts(time_column)), to=time_column_type)
+        return exp.cast(
+            exp.Literal.string(to_ts(time_column, include_microseconds)), to=time_column_type
+        )
 
     if time_column_format:
         time_column = to_datetime(time_column).strftime(time_column_format)

tests/core/engine_adapter/test_clickhouse.py

@@ -10,6 +10,8 @@
 from sqlmesh.core.schema_diff import SchemaDiffer
 from datetime import datetime
 from pytest_mock.plugin import MockerFixture
+from sqlmesh.core import dialect as d
+from sqlmesh.utils.date import to_datetime
 
 pytestmark = [pytest.mark.clickhouse, pytest.mark.engine]
 
@@ -808,3 +810,49 @@ def test_scd_type_2_by_column(
     """,
         dialect=adapter.dialect,
     ).sql(adapter.dialect)
+
+
+def test_to_time_column():
+    # datetime/timestamp data type should remove fractional seconds
+    expressions = d.parse(
+        """
+        MODEL (
+            name db.table,
+            kind INCREMENTAL_BY_TIME_RANGE(
+                time_column (ds)
+            ),
+            dialect clickhouse
+        );
+
+        SELECT ds::datetime
+    """
+    )
+    model = load_sql_based_model(expressions)
+    assert model.convert_to_time_column("2022-01-01 00:00:00.000001").this == d.parse_one(
+        "'2022-01-01 00:00:00'"
+    )
+    assert model.convert_to_time_column(
+        to_datetime("2022-01-01 00:00:00.000001")
+    ).this == d.parse_one("'2022-01-01 00:00:00'")
+
+    # DateTime64 data type should retain fractional seconds
+    expressions = d.parse(
+        """
+        MODEL (
+            name db.table,
+            kind INCREMENTAL_BY_TIME_RANGE(
+                time_column (ds)
+            ),
+            dialect clickhouse
+        );
+
+        SELECT ds::DateTime64
+    """
+    )
+    model = load_sql_based_model(expressions)
+    assert model.convert_to_time_column("2022-01-01 00:00:00.000001").this == d.parse_one(
+        "'2022-01-01 00:00:00.000001'"
+    )
+    assert model.convert_to_time_column(
+        to_datetime("2022-01-01 00:00:00.000001")
+    ).this == d.parse_one("'2022-01-01 00:00:00.000001'")