diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index c4b7bcbd53..100e17bbd6 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -300,8 +300,8 @@ workflows: name: cloud_engine_<< matrix.engine >> context: - sqlmesh_cloud_database_integration - requires: - - engine_tests_docker + # requires: + # - engine_tests_docker matrix: parameters: engine: @@ -313,10 +313,10 @@ workflows: - athena - fabric - gcp-postgres - filters: - branches: - only: - - main + # filters: + # branches: + # only: + # - main - ui_style - ui_test - vscode_test diff --git a/sqlmesh/dbt/model.py b/sqlmesh/dbt/model.py index 41cea9b9ae..3925fe364a 100644 --- a/sqlmesh/dbt/model.py +++ b/sqlmesh/dbt/model.py @@ -745,11 +745,10 @@ def to_sqlmesh( ) else: model_kwargs["start"] = begin - # If user explicitly disables concurrent batches then we want to set depends on past to true which we - # will do by including the model in the depends_on + # If user explicitly disables concurrent batches then force sequential execution by setting batch_concurrency to 1 if self.concurrent_batches is not None and self.concurrent_batches is False: - depends_on = model_kwargs.get("depends_on", set()) - depends_on.add(self.canonical_name(context)) + if hasattr(kind, "batch_concurrency"): + kind.batch_concurrency = 1 model_kwargs["start"] = model_kwargs.get( "start", context.sqlmesh_config.model_defaults.start @@ -768,6 +767,7 @@ def to_sqlmesh( allow_partials=allow_partials, virtual_environment_mode=virtual_environment_mode, dbt_node_info=self.node_info, + optimize_query=False, **optional_kwargs, **model_kwargs, ) diff --git a/tests/core/integration/test_dbt.py b/tests/core/integration/test_dbt.py index 6f23acb97e..9c01c89a45 100644 --- a/tests/core/integration/test_dbt.py +++ b/tests/core/integration/test_dbt.py @@ -2,7 +2,6 @@ import typing as t import pytest -from sqlmesh.core.model.common import ParsableSql import time_machine from sqlmesh.core.context import Context @@ -11,7 +10,6 @@ ) from sqlmesh.core.snapshot import ( DeployabilityIndex, - SnapshotChangeCategory, ) if t.TYPE_CHECKING: @@ -20,27 +18,6 @@ pytestmark = pytest.mark.slow -@time_machine.travel("2023-01-08 15:00:00 UTC") -def test_dbt_select_star_is_directly_modified(sushi_test_dbt_context: Context): - context = sushi_test_dbt_context - - model = context.get_model("sushi.simple_model_a") - context.upsert_model( - model, - query_=ParsableSql(sql="SELECT 1 AS a, 2 AS b"), - ) - - snapshot_a_id = context.get_snapshot("sushi.simple_model_a").snapshot_id # type: ignore - snapshot_b_id = context.get_snapshot("sushi.simple_model_b").snapshot_id # type: ignore - - plan = context.plan_builder("dev", skip_tests=True).build() - assert plan.directly_modified == {snapshot_a_id, snapshot_b_id} - assert {i.snapshot_id for i in plan.missing_intervals} == {snapshot_a_id, snapshot_b_id} - - assert plan.snapshots[snapshot_a_id].change_category == SnapshotChangeCategory.NON_BREAKING - assert plan.snapshots[snapshot_b_id].change_category == SnapshotChangeCategory.NON_BREAKING - - @time_machine.travel("2023-01-08 15:00:00 UTC") def test_dbt_is_incremental_table_is_missing(sushi_test_dbt_context: Context): context = sushi_test_dbt_context diff --git a/tests/core/integration/test_multi_repo.py b/tests/core/integration/test_multi_repo.py index 6477b08741..32934a903b 100644 --- a/tests/core/integration/test_multi_repo.py +++ b/tests/core/integration/test_multi_repo.py @@ -413,7 +413,7 @@ def test_multi_hybrid(mocker): sqlmesh_rendered = ( 'SELECT "e"."col_a" AS "col_a", "e"."col_b" AS "col_b" FROM "memory"."dbt_repo"."e" AS "e"' ) - dbt_rendered = 'SELECT DISTINCT ROUND(CAST(("b"."col_a" / NULLIF(100, 0)) AS DECIMAL(16, 2)), 2) AS "rounded_col_a" FROM "memory"."sqlmesh_repo"."b" AS "b"' + dbt_rendered = 'SELECT DISTINCT ROUND(CAST(("col_a" / NULLIF(100, 0)) AS DECIMAL(16, 2)), 2) AS "rounded_col_a" FROM "memory"."sqlmesh_repo"."b" AS "b"' assert sqlmesh_model_a.render_query().sql() == sqlmesh_rendered assert dbt_model_c.render_query().sql() == dbt_rendered diff --git a/tests/dbt/test_model.py b/tests/dbt/test_model.py index 6d100e6aa5..c6e0c5b9f8 100644 --- a/tests/dbt/test_model.py +++ b/tests/dbt/test_model.py @@ -373,7 +373,8 @@ def test_load_microbatch_all_defined_diff_values( column=exp.to_column("blah", quoted=True), format="%Y-%m-%d" ) assert model.kind.batch_size == 1 - assert model.depends_on_self is True + assert model.kind.batch_concurrency == 1 + assert model.depends_on_self is False @pytest.mark.slow @@ -630,7 +631,7 @@ def test_load_microbatch_with_ref( ) assert ( context.render(microbatch_two_snapshot_fqn, start="2025-01-01", end="2025-01-10").sql() - == 'SELECT "_q_0"."cola" AS "cola", "_q_0"."ds" AS "ds" FROM (SELECT "microbatch"."cola" AS "cola", "microbatch"."ds" AS "ds" FROM "local"."main"."microbatch" AS "microbatch" WHERE "microbatch"."ds" < \'2025-01-11 00:00:00+00:00\' AND "microbatch"."ds" >= \'2025-01-01 00:00:00+00:00\') AS "_q_0"' + == 'SELECT "cola" AS "cola", "ds" AS "ds" FROM (SELECT * FROM "local"."main"."microbatch" AS "microbatch" WHERE "ds" >= \'2025-01-01 00:00:00+00:00\' AND "ds" < \'2025-01-11 00:00:00+00:00\') AS "_q_0"' ) @@ -696,7 +697,7 @@ def test_load_microbatch_with_ref_no_filter( ) assert ( context.render(microbatch_two_snapshot_fqn, start="2025-01-01", end="2025-01-10").sql() - == 'SELECT "microbatch"."cola" AS "cola", "microbatch"."ds" AS "ds" FROM "local"."main"."microbatch" AS "microbatch"' + == 'SELECT "cola" AS "cola", "ds" AS "ds" FROM "local"."main"."microbatch" AS "microbatch"' ) @@ -1104,7 +1105,7 @@ def test_conditional_ref_in_unexecuted_branch(copy_to_temp_path: t.Callable): assert rendered is not None assert ( rendered.sql() - == 'WITH "source" AS (SELECT "simple_model_a"."a" AS "a" FROM "memory"."sushi"."simple_model_a" AS "simple_model_a") SELECT "source"."a" AS "a" FROM "source" AS "source"' + == 'WITH "source" AS (SELECT * FROM "memory"."sushi"."simple_model_a" AS "simple_model_a") SELECT * FROM "source" AS "source"' ) # And run plan with this conditional model for good measure