Skip to content

Commit db6183a

Browse files
committed
fix: dbt microbatch parameter conversion
1 parent a3e7bda commit db6183a

File tree

2 files changed

+12
-11
lines changed

2 files changed

+12
-11
lines changed

sqlmesh/dbt/model.py

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -337,15 +337,8 @@ def model_kind(self, context: DbtContext) -> ModelKind:
337337
raise ConfigError(
338338
f"{self.canonical_name(context)}: 'event_time' is required for microbatch incremental strategy."
339339
)
340-
concurrent_batches = self._get_field_value("concurrent_batches")
341-
if concurrent_batches is True:
342-
if incremental_by_kind_kwargs.get("batch_size"):
343-
get_console().log_warning(
344-
f"'concurrent_batches' is set to True and 'batch_size' are defined in '{self.canonical_name(context)}'. The batch size will be set to the value of `batch_size`."
345-
)
346-
incremental_by_kind_kwargs["batch_size"] = incremental_by_kind_kwargs.get(
347-
"batch_size", 1
348-
)
340+
# dbt microbatch always processes batches in a size of 1
341+
incremental_by_kind_kwargs["batch_size"] = 1
349342
else:
350343
if not self.time_column:
351344
raise ConfigError(
@@ -674,6 +667,11 @@ def to_sqlmesh(
674667
)
675668
else:
676669
model_kwargs["start"] = begin
670+
# If user explicitly disables concurrent batches then we want to set depends on past to true which we
671+
# will do by including the model in the depends_on
672+
if self.concurrent_batches is not None and self.concurrent_batches is False:
673+
depends_on = model_kwargs.get("depends_on", set())
674+
depends_on.add(self.canonical_name(context))
677675

678676
model_kwargs["start"] = model_kwargs.get(
679677
"start", context.sqlmesh_config.model_defaults.start

tests/dbt/test_model.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ def test_load_microbatch_all_defined(
219219
column=exp.to_column("ds", quoted=True), format="%Y-%m-%d"
220220
)
221221
assert model.kind.batch_size == 1
222+
assert model.depends_on_self is False
222223

223224

224225
@pytest.mark.slow
@@ -259,7 +260,8 @@ def test_load_microbatch_all_defined_diff_values(
259260
assert model.kind.time_column == TimeColumn(
260261
column=exp.to_column("blah", quoted=True), format="%Y-%m-%d"
261262
)
262-
assert model.kind.batch_size is None
263+
assert model.kind.batch_size == 1
264+
assert model.depends_on_self is True
263265

264266

265267
@pytest.mark.slow
@@ -297,4 +299,5 @@ def test_load_microbatch_required_only(
297299
assert model.kind.time_column == TimeColumn(
298300
column=exp.to_column("ds", quoted=True), format="%Y-%m-%d"
299301
)
300-
assert model.kind.batch_size is None
302+
assert model.kind.batch_size == 1
303+
assert model.depends_on_self is False

0 commit comments

Comments
 (0)