Fix: Unexpected backfill when the parent model's cron is before its child's (#2179)

Author: izeigerman

sqlmesh/core/snapshot/definition.py

@@ -771,7 +771,6 @@ def missing_intervals(
         if self.is_symbolic or (self.is_seed and intervals):
             return []
 
-        execution_time = execution_time or now()
         allow_partials = not end_bounded and self.is_model and self.model.allow_partials
         start_ts, end_ts = (
             to_timestamp(ts)
@@ -785,6 +784,10 @@ def missing_intervals(
 
         interval_unit = self.node.interval_unit
 
+        execution_time = execution_time or now()
+        if end_bounded:
+            execution_time = min(to_timestamp(execution_time), end_ts)
+
         if not allow_partials:
             upper_bound_ts = to_timestamp(
                 self.node.cron_floor(execution_time) if not ignore_cron else execution_time
@@ -794,9 +797,6 @@ def missing_intervals(
             upper_bound_ts = to_timestamp(execution_time)
             end_ts = min(end_ts, upper_bound_ts)
 
-        if end_bounded:
-            upper_bound_ts = end_ts
-
         lookback = self.model.lookback if self.is_model else 0
 
         return compute_missing_intervals(

tests/core/test_integration.py

@@ -379,6 +379,48 @@ def test_hourly_model_with_lookback_no_backfill_in_dev(init_and_plan_context: t.
         ]
 
 
+@freeze_time("2023-01-08 00:00:00")
+def test_parent_cron_before_child(init_and_plan_context: t.Callable):
+    context, plan = init_and_plan_context("examples/sushi")
+
+    model = context.get_model("sushi.waiter_revenue_by_day")
+    model = SqlModel.parse_obj(
+        {
+            **model.dict(),
+            "cron": "50 23 * * *",
+        }
+    )
+    context.upsert_model(model)
+
+    plan = context.plan("prod", no_prompts=True, skip_tests=True)
+    context.apply(plan)
+
+    top_waiters_model = context.get_model("sushi.top_waiters")
+    top_waiters_model = add_projection_to_model(t.cast(SqlModel, top_waiters_model), literal=True)
+    context.upsert_model(top_waiters_model)
+
+    snapshot = context.get_snapshot(model, raise_if_missing=True)
+    top_waiters_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
+
+    with freeze_time("2023-01-08 23:55:00"):  # Past parent's cron, but before child's
+        plan = context.plan("dev", no_prompts=True, skip_tests=True)
+        # Make sure the waiter_revenue_by_day model is not backfilled.
+        assert plan.missing_intervals == [
+            SnapshotIntervals(
+                snapshot_id=top_waiters_snapshot.snapshot_id,
+                intervals=[
+                    (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")),
+                    (to_timestamp("2023-01-02"), to_timestamp("2023-01-03")),
+                    (to_timestamp("2023-01-03"), to_timestamp("2023-01-04")),
+                    (to_timestamp("2023-01-04"), to_timestamp("2023-01-05")),
+                    (to_timestamp("2023-01-05"), to_timestamp("2023-01-06")),
+                    (to_timestamp("2023-01-06"), to_timestamp("2023-01-07")),
+                    (to_timestamp("2023-01-07"), to_timestamp("2023-01-08")),
+                ],
+            ),
+        ]
+
+
 @freeze_time("2023-01-08 15:00:00")
 def test_forward_only_parent_created_in_dev_child_created_in_prod(
     init_and_plan_context: t.Callable,

tests/core/test_snapshot.py

@@ -339,10 +339,13 @@ def test_missing_intervals_end_bounded_with_ignore_cron(make_snapshot):
         (to_timestamp("2023-01-02"), to_timestamp(end)),
     ]
     assert (
-        snapshot.missing_intervals(start, end, execution_time=execution_ts, end_bounded=True) == []
+        snapshot.missing_intervals(
+            start, to_datetime(end), execution_time=execution_ts, end_bounded=True
+        )
+        == []
     )
     assert snapshot.missing_intervals(
-        start, end, execution_time=execution_ts, ignore_cron=True, end_bounded=True
+        start, to_datetime(end), execution_time=execution_ts, ignore_cron=True, end_bounded=True
     ) == [
         (to_timestamp("2023-01-02"), to_timestamp(end)),
     ]