Fix: select * from external models for dbt, skip private variables when deep copying JinjaMacroRegistry, and properly parse jinja pre and post expressions (#943)

Author: crericha

sqlmesh/core/model/common.py

@@ -3,7 +3,7 @@
 import typing as t
 
 from pydantic import validator
-from sqlglot import exp, maybe_parse
+from sqlglot import exp
 from sqlglot.helper import seq_get, split_num_words
 
 from sqlmesh.core.dialect import parse
@@ -33,7 +33,11 @@ def parse_expression(
         return v
 
     if isinstance(v, list):
-        return [e for e in (maybe_parse(i) for i in v) if e]
+        return [
+            e
+            for expressions in (parse(i) if not isinstance(i, exp.Expression) else [i] for i in v)
+            for e in expressions
+        ]
 
     if isinstance(v, str):
         return seq_get(parse(v), 0)

sqlmesh/core/model/definition.py

@@ -623,8 +623,8 @@ def render_post_statements(
     def render_definition(self, include_python: bool = True) -> t.List[exp.Expression]:
         result = super().render_definition(include_python=include_python)
         result.extend(self.pre_statements)
-        result.append(self.query)
         result.extend(self.post_statements)
+        result.append(self.query)
         return result
 
     @property
@@ -1480,4 +1480,5 @@ def _single_value_or_tuple(values: t.Sequence) -> exp.Identifier | exp.Tuple:
     ),
     "tags": _single_value_or_tuple,
     "grain": _single_value_or_tuple,
+    "hash_raw_query": exp.convert,
 }

sqlmesh/core/renderer.py

@@ -275,6 +275,8 @@ def _optimize_query(self, query: exp.Expression) -> exp.Expression:
         # We don't want to normalize names in the schema because that's handled by the optimizer
         schema = ensure_schema(self.schema, dialect=self._dialect, normalize=False)
         query = t.cast(exp.Subqueryable, query.copy())
+        for node, *_ in query.walk():
+            node.type = None
 
         try:
             qualify(

sqlmesh/utils/jinja.py

@@ -309,6 +309,9 @@ def merge(self, other: JinjaMacroRegistry) -> JinjaMacroRegistry:
             top_level_packages=[*self.top_level_packages, *other.top_level_packages],
         )
 
+    def __deepcopy__(self, memo: t.Dict[int, t.Any]) -> JinjaMacroRegistry:
+        return JinjaMacroRegistry.parse_obj(self.dict())
+
     def _parse_macro(self, name: str, package: t.Optional[str]) -> Template:
         cache_key = (package, name)
         if cache_key not in self._parser_cache:

tests/core/test_model.py

@@ -21,6 +21,7 @@
     load_model,
     model,
 )
+from sqlmesh.core.model.common import parse_expression
 from sqlmesh.utils.date import to_date, to_datetime, to_timestamp
 from sqlmesh.utils.errors import ConfigError
 from sqlmesh.utils.metaprogramming import Executable
@@ -1185,3 +1186,8 @@ def test_is_breaking_change():
         )
         is None
     )
+
+
+def test_parse_expression_list_with_jinja():
+    input = ["{{ log('log message') }}", "GRANT SELECT ON TABLE foo TO DEV"]
+    assert input == [val.sql() for val in parse_expression(input)]

tests/fixtures/dbt/sushi_test/models/waiters.sql

@@ -5,7 +5,6 @@
 }}
 
 SELECT DISTINCT
-  waiter_id::INT AS waiter_id,
-  ds::TEXT AS ds
+   *
 FROM {{ source('streaming', 'orders') }}
 {{ incremental_by_time('ds', 'ds') }}

tests/fixtures/dbt/sushi_test/schema.yaml

@@ -0,0 +1,21 @@
+- name: raw.items
+  columns:
+    id: BIGINT
+    name: VARCHAR
+    price: DOUBLE
+    ds: DATE
+- name: raw.order_items
+  columns:
+    id: BIGINT
+    order_id: BIGINT
+    item_id: BIGINT
+    quantity: BIGINT
+    ds: DATE
+- name: raw.orders
+  columns:
+    id: BIGINT
+    customer_id: BIGINT
+    waiter_id: BIGINT
+    start_ts: TIMESTAMP
+    end_ts: TIMESTAMP
+    ds: DATE