From 7aa8898919bdd9e7bb22763a6700482174761deb Mon Sep 17 00:00:00 2001 From: Chris Rericha Date: Wed, 10 Sep 2025 16:39:01 -0400 Subject: [PATCH 1/3] Fix: Update dbt loader to support loading multiple models from same file --- sqlmesh/dbt/loader.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/sqlmesh/dbt/loader.py b/sqlmesh/dbt/loader.py index 695aff3c45..f7d97e74c8 100644 --- a/sqlmesh/dbt/loader.py +++ b/sqlmesh/dbt/loader.py @@ -5,6 +5,7 @@ import typing as t import sqlmesh.core.dialect as d from pathlib import Path +from collections import defaultdict from sqlmesh.core.config import ( Config, ConnectionConfig, @@ -137,16 +138,22 @@ def _to_sqlmesh(config: BMC, context: DbtContext) -> Model: package_context.set_and_render_variables(package.variables, package.name) package_models: t.Dict[str, BaseModelConfig] = {**package.models, **package.seeds} + package_models_by_path: t.Dict[Path, t.List[BaseModelConfig]] = defaultdict(list) for model in package_models.values(): if isinstance(model, ModelConfig) and not model.sql.strip(): logger.info(f"Skipping empty model '{model.name}' at path '{model.path}'.") continue + package_models_by_path[model.path].append(model) - sqlmesh_model = cache.get_or_load_models( - model.path, loader=lambda: [_to_sqlmesh(model, package_context)] - )[0] - - models[sqlmesh_model.fqn] = sqlmesh_model + for path, path_models in package_models_by_path.items(): + sqlmesh_models = cache.get_or_load_models( + path, + loader=lambda: [ + _to_sqlmesh(model, package_context) for model in path_models + ], + ) + for sqlmesh_model in sqlmesh_models: + models[sqlmesh_model.fqn] = sqlmesh_model models.update(self._load_external_models(audits, cache)) From 4473bc0e37c7a6e8a0c80957fc015614bc5950f4 Mon Sep 17 00:00:00 2001 From: Chris Rericha Date: Thu, 11 Sep 2025 13:59:28 -0400 Subject: [PATCH 2/3] Add test case for multiple dbt snapshots in one file --- .../snapshots/items_check_snapshot.sql | 15 --------------- .../{items_snapshot.sql => items_snapshots.sql} | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 tests/fixtures/dbt/sushi_test/snapshots/items_check_snapshot.sql rename tests/fixtures/dbt/sushi_test/snapshots/{items_snapshot.sql => items_snapshots.sql} (53%) diff --git a/tests/fixtures/dbt/sushi_test/snapshots/items_check_snapshot.sql b/tests/fixtures/dbt/sushi_test/snapshots/items_check_snapshot.sql deleted file mode 100644 index fdda412e7f..0000000000 --- a/tests/fixtures/dbt/sushi_test/snapshots/items_check_snapshot.sql +++ /dev/null @@ -1,15 +0,0 @@ -{% snapshot items_check_snapshot %} - -{{ - config( - target_schema='snapshots', - unique_key='id', - strategy='check', - check_cols=['ds'], - invalidate_hard_deletes=True, - ) -}} - -select * from {{ source('streaming', 'items') }} - -{% endsnapshot %} diff --git a/tests/fixtures/dbt/sushi_test/snapshots/items_snapshot.sql b/tests/fixtures/dbt/sushi_test/snapshots/items_snapshots.sql similarity index 53% rename from tests/fixtures/dbt/sushi_test/snapshots/items_snapshot.sql rename to tests/fixtures/dbt/sushi_test/snapshots/items_snapshots.sql index c5c922d217..77d79d03ba 100644 --- a/tests/fixtures/dbt/sushi_test/snapshots/items_snapshot.sql +++ b/tests/fixtures/dbt/sushi_test/snapshots/items_snapshots.sql @@ -14,3 +14,19 @@ select * from {{ source('streaming', 'items') }} {% endsnapshot %} + +{% snapshot items_check_snapshot %} + +{{ + config( + target_schema='snapshots', + unique_key='id', + strategy='check', + check_cols=['ds'], + invalidate_hard_deletes=True, + ) +}} + +select * from {{ source('streaming', 'items') }} + +{% endsnapshot %} From 6c52f8c9ea8070e67dad49be2391489aa99fbbaa Mon Sep 17 00:00:00 2001 From: Chris Rericha Date: Fri, 12 Sep 2025 09:42:10 -0400 Subject: [PATCH 3/3] Add test for snapshot loading --- tests/dbt/test_model.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/dbt/test_model.py b/tests/dbt/test_model.py index dc2ebc492b..caa409807f 100644 --- a/tests/dbt/test_model.py +++ b/tests/dbt/test_model.py @@ -733,6 +733,18 @@ def test_load_microbatch_with_ref_no_filter( ) +@pytest.mark.slow +def test_load_multiple_snapshots_defined_in_same_file(sushi_test_dbt_context: Context) -> None: + context = sushi_test_dbt_context + assert context.get_model("snapshots.items_snapshot") + assert context.get_model("snapshots.items_check_snapshot") + + # Make sure cache works too + context.load() + assert context.get_model("snapshots.items_snapshot") + assert context.get_model("snapshots.items_check_snapshot") + + @pytest.mark.slow def test_dbt_jinja_macro_undefined_variable_error(create_empty_project): project_dir, model_dir = create_empty_project()