Fix: convert NaN/NaT (np.nan) values to None (#2774)

Author: georgesittas

sqlmesh/core/model/definition.py

@@ -10,6 +10,7 @@
 from pathlib import Path
 
 import pandas as pd
+import numpy as np
 from astor import to_source
 from pydantic import Field
 from sqlglot import diff, exp
@@ -1172,6 +1173,7 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
         datetime_columns = []
         bool_columns = []
         string_columns = []
+
         for name, tpe in (self.columns_to_types_ or {}).items():
             if tpe.this in (exp.DataType.Type.DATE, exp.DataType.Type.DATE32):
                 date_columns.append(name)
@@ -1196,7 +1198,7 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
                 cond=lambda x: x.notna(),  # type: ignore
                 other=df[string_columns].astype(str),  # type: ignore
             )
-            yield df
+            yield df.replace({np.nan: None})
 
     @property
     def columns_to_types(self) -> t.Optional[t.Dict[str, exp.DataType]]:

tests/core/test_model.py

@@ -851,8 +851,8 @@ def test_seed_model_custom_types(tmp_path):
 
     with open(model_csv_path, "w", encoding="utf-8") as fd:
         fd.write(
-            """key,ds_date,ds_timestamp,b_a,b_b,i,i_str
-123,2022-01-01,2022-01-01,false,0,321,321
+            """key,ds_date,ds_timestamp,b_a,b_b,i,i_str,empty_date
+123,2022-01-01,2022-01-01,false,0,321,321,
 """
         )
 
@@ -867,6 +867,7 @@ def test_seed_model_custom_types(tmp_path):
             "b_b": "boolean",
             "i": "int",
             "i_str": "text",
+            "empty_date": "date",
         },
     )
 
@@ -893,6 +894,9 @@ def test_seed_model_custom_types(tmp_path):
     assert df["i_str"].dtype == "object"
     assert df["i_str"].iloc[0] == "321"
 
+    assert df["empty_date"].dtype == "object"
+    assert df["empty_date"].iloc[0] is None
+
 
 def test_seed_with_special_characters_in_column(tmp_path, assert_exp_eq):
     config = Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))