Fix: Make dbt adapter macros available in the local scope (#3219)

Author: izeigerman

sqlmesh/dbt/manifest.py

@@ -167,6 +167,19 @@ def _load_macros(self) -> None:
                 path=Path(macro.original_file_path),
             )
 
+        # This is a workaround for dbt adapter macros (eg. "spark__dateadd") whcih are expected to be
+        # available in the global scope regardless of the package they came from.
+        adapter_macro_names = {
+            name[name.find("__") + 2 :]
+            for name in self._macros_per_package.get("dbt", {})
+            if "__" in name
+        }
+        for macros in self._macros_per_package.values():
+            for name, macro_config in macros.items():
+                pos = name.find("__")
+                if pos > 0 and name[pos + 2 :] in adapter_macro_names:
+                    macro_config.info.is_top_level = True
+
     def _load_tests(self) -> None:
         for node in self._manifest.nodes.values():
             if node.resource_type != "test":

sqlmesh/utils/jinja.py

@@ -52,6 +52,7 @@ class MacroInfo(PydanticModel):
 
     definition: str
     depends_on: t.List[MacroReference]
+    is_top_level: bool = False
 
 
 class MacroReturnVal(Exception):
@@ -298,10 +299,11 @@ def build_environment(self, **kwargs: t.Any) -> Environment:
 
         package_macros: t.Dict[str, t.Any] = defaultdict(AttributeDict)
         for package_name, macros in self.packages.items():
-            for macro_name in macros:
-                package_macros[package_name][macro_name] = self._MacroWrapper(
-                    macro_name, package_name, self, context
-                )
+            for macro_name, macro in macros.items():
+                macro_wrapper = self._MacroWrapper(macro_name, package_name, self, context)
+                package_macros[package_name][macro_name] = macro_wrapper
+                if macro.is_top_level and macro_name not in root_macros:
+                    root_macros[macro_name] = macro_wrapper
 
         if self.root_package_name is not None:
             package_macros[self.root_package_name].update(root_macros)

tests/dbt/test_manifest.py

@@ -206,3 +206,30 @@ def test_source_meta_external_location():
     )
     assert relation.identifier == "items"
     assert relation.render() == "read_parquet('path/to/external/items.parquet')"
+
+
+@pytest.mark.xdist_group("dbt_manifest")
+def test_top_level_dbt_adapter_macros():
+    project_path = Path("tests/fixtures/dbt/sushi_test")
+    profile = Profile.load(DbtContext(project_path))
+
+    helper = ManifestHelper(
+        project_path,
+        project_path,
+        "sushi",
+        profile.target,
+        variable_overrides={"start": "2020-01-01"},
+    )
+
+    # Adapter macros must be marked as top-level
+    dbt_macros = helper.macros("dbt")
+    dbt_duckdb_macros = helper.macros("dbt_duckdb")
+    assert dbt_macros["default__dateadd"].info.is_top_level
+    assert dbt_macros["default__datediff"].info.is_top_level
+    assert dbt_duckdb_macros["duckdb__datediff"].info.is_top_level
+    assert dbt_duckdb_macros["duckdb__dateadd"].info.is_top_level
+
+    # Project dispatch macros should not be marked as top-level
+    customers_macros = helper.macros("customers")
+    assert not customers_macros["default__current_engine"].info.is_top_level
+    assert not customers_macros["duckdb__current_engine"].info.is_top_level

tests/utils/test_jinja.py

@@ -280,3 +280,21 @@ def test_find_call_names():
         ("package", "package_macro"),
         ("'stringval'", "function"),
     ]
+
+
+def test_dbt_adapter_macro_scope():
+    package_a = """
+{% macro spark__macro_a() %}
+macro_a
+{% endmacro %}"""
+
+    extractor = MacroExtractor()
+    registry = JinjaMacroRegistry()
+
+    macros = extractor.extract(package_a)
+    macros["spark__macro_a"].is_top_level = True
+
+    registry.add_macros(macros, package="package_a")
+
+    rendered = registry.build_environment().from_string("{{ spark__macro_a() }}").render()
+    assert rendered.strip() == "macro_a"