Skip to content

Commit 1c4d0a8

Browse files
authored
Chore!: add migration script to warn about dbt data_type-related diffs (#5245)
1 parent bba2e3a commit 1c4d0a8

File tree

1 file changed

+48
-0
lines changed

1 file changed

+48
-0
lines changed
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""
2+
Warns dbt users about potential diffs due to corrected data_type handling.
3+
4+
SQLMesh previously treated dbt's schema.yml data_type field as columns_to_types, which
5+
doesn't match dbt's behavior. dbt only uses data_type for contracts/validation, not DDL.
6+
This fix may cause diffs if tables were created with incorrect types.
7+
8+
More context: https://github.com/TobikoData/sqlmesh/pull/5231
9+
"""
10+
11+
import json
12+
13+
from sqlglot import exp
14+
15+
from sqlmesh.core.console import get_console
16+
17+
SQLMESH_DBT_PACKAGE = "sqlmesh.dbt"
18+
19+
20+
def migrate(state_sync, **kwargs): # type: ignore
21+
engine_adapter = state_sync.engine_adapter
22+
schema = state_sync.schema
23+
snapshots_table = "_snapshots"
24+
if schema:
25+
snapshots_table = f"{schema}.{snapshots_table}"
26+
27+
warning = (
28+
"SQLMesh previously misinterpreted dbt's schema.yml 'data_type' field as actual "
29+
"column types, but dbt only uses these for contracts/validation, not in actual "
30+
"DDL statements. This has been fixed to match dbt's actual behavior. Your existing "
31+
"tables may have been created with incorrect column types. After this migration, run "
32+
"'sqlmesh diff prod' to check for column type differences, and if any are found, "
33+
"apply a plan to correct the table schemas. For more details, see: "
34+
"https://github.com/TobikoData/sqlmesh/pull/5231."
35+
)
36+
37+
for (snapshot,) in engine_adapter.fetchall(
38+
exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
39+
):
40+
parsed_snapshot = json.loads(snapshot)
41+
node = parsed_snapshot["node"]
42+
43+
jinja_macros = node.get("jinja_macros") or {}
44+
create_builtins_module = jinja_macros.get("create_builtins_module") or ""
45+
46+
if create_builtins_module == SQLMESH_DBT_PACKAGE and node.get("columns"):
47+
get_console().log_warning(warning)
48+
return

0 commit comments

Comments
 (0)