Feat: make automatic python dependency inference opt-out (#4575)

Author: georgesittas

docs/reference/configuration.md

@@ -36,6 +36,7 @@ Configuration options for SQLMesh environment creation and promotion.
 | `physical_schema_mapping`     | A mapping from regular expressions to names of schemas in which physical tables for the corresponding models [will be placed](../guides/configuration.md#physical-table-schemas). (Default physical schema name: `sqlmesh__[model schema]`)                                                        | dict[string, string] | N        |
 | `environment_suffix_target`   | Whether SQLMesh views should append their environment name to the `schema` or `table` - [additional details](../guides/configuration.md#view-schema-override). (Default: `schema`)                                                                                                                 | string               | N        |
 | `gateway_managed_virtual_layer`   | Whether SQLMesh views of the virtual layer will be created by the default gateway or model specified gateways - [additional details](../guides/multi_engine.md#gateway-managed-virtual-layer). (Default: False)                                                                                                                 | boolean               | N        |
+| `infer_python_dependencies`   | Whether SQLMesh will statically analyze Python code to automatically infer Python package requirements. (Default: True)                                                                                                                 | boolean               | N        |
 | `environment_catalog_mapping` | A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.                                                                                                                                                             | dict[string, string] | N        |
 | `log_limit`                   | The default number of logs to keep (Default: `20`)                                                                                                                                                                                                                                                 | int                  | N        |
 

sqlmesh/core/config/root.py

@@ -107,6 +107,7 @@ class Config(BaseConfig):
         physical_schema_mapping: A mapping from regular expressions to names of schemas in which physical tables for corresponding models will be placed.
         environment_suffix_target: Indicates whether to append the environment name to the schema or table name.
         gateway_managed_virtual_layer: Whether the models' views in the virtual layer are created by the model-specific gateway rather than the default gateway.
+        infer_python_dependencies: Whether to statically analyze Python code to automatically infer Python package requirements.
         environment_catalog_mapping: A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.
         default_target_environment: The name of the environment that will be the default target for the `sqlmesh plan` and `sqlmesh run` commands.
         log_limit: The default number of logs to keep.
@@ -146,6 +147,7 @@ class Config(BaseConfig):
         default=EnvironmentSuffixTarget.default
     )
     gateway_managed_virtual_layer: bool = False
+    infer_python_dependencies: bool = True
     environment_catalog_mapping: RegexKeyDict = {}
     default_target_environment: str = c.PROD
     log_limit: int = c.DEFAULT_LOG_LIMIT

sqlmesh/core/context.py

@@ -388,7 +388,6 @@ def __init__(
         self.pinned_environments = Environment.sanitize_names(self.config.pinned_environments)
         self.auto_categorize_changes = self.config.plan.auto_categorize_changes
         self.selected_gateway = gateway or self.config.default_gateway_name
-        self.gateway_managed_virtual_layer = self.config.gateway_managed_virtual_layer
 
         gw_model_defaults = self.config.gateways[self.selected_gateway].model_defaults
         if gw_model_defaults:
@@ -2617,7 +2616,8 @@ def _context_diff(
             ensure_finalized_snapshots=ensure_finalized_snapshots,
             diff_rendered=diff_rendered,
             environment_statements=self._environment_statements,
-            gateway_managed_virtual_layer=self.gateway_managed_virtual_layer,
+            gateway_managed_virtual_layer=self.config.gateway_managed_virtual_layer,
+            infer_python_dependencies=self.config.infer_python_dependencies,
         )
 
     def _destroy(self) -> None:

sqlmesh/core/context_diff.py

@@ -102,6 +102,7 @@ def create(
         diff_rendered: bool = False,
         environment_statements: t.Optional[t.List[EnvironmentStatements]] = [],
         gateway_managed_virtual_layer: bool = False,
+        infer_python_dependencies: bool = True,
     ) -> ContextDiff:
         """Create a ContextDiff object.
 
@@ -116,6 +117,12 @@ def create(
                 the environment is not finalized.
             provided_requirements: Python dependencies sourced from the lock file.
             excluded_requirements: Python dependencies to exclude.
+            diff_rendered: Whether to compute the diff of the rendered version of the compared expressions.
+            environment_statements: A list of `before_all` or `after_all` statements associated with the environment.
+            gateway_managed_virtual_layer: Whether the models' views in the virtual layer are created by the
+                model-specific gateway rather than the default gateway.
+            infer_python_dependencies: Whether to statically analyze Python code to automatically infer Python
+                package requirements.
 
         Returns:
             The ContextDiff object.
@@ -208,6 +215,7 @@ def create(
             provided_requirements or {},
             excluded_requirements or set(),
             snapshots.values(),
+            infer_python_dependencies=infer_python_dependencies,
         )
 
         previous_environment_statements = state_reader.get_environment_statements(environment)
@@ -475,29 +483,41 @@ def _build_requirements(
     provided_requirements: t.Dict[str, str],
     excluded_requirements: t.Set[str],
     snapshots: t.Collection[Snapshot],
+    infer_python_dependencies: bool = True,
 ) -> t.Dict[str, str]:
     requirements = {
         k: v for k, v in provided_requirements.items() if k not in excluded_requirements
     }
+
+    if not infer_python_dependencies:
+        return requirements
+
     distributions = metadata.packages_distributions()
 
     for snapshot in snapshots:
-        if snapshot.is_model:
-            for executable in snapshot.model.python_env.values():
-                if executable.kind == "import":
-                    try:
-                        start = "from " if executable.payload.startswith("from ") else "import "
-                        lib = executable.payload.split(start)[1].split()[0].split(".")[0]
-                        if lib in distributions:
-                            for dist in distributions[lib]:
-                                if (
-                                    dist not in requirements
-                                    and dist not in IGNORED_PACKAGES
-                                    and dist not in excluded_requirements
-                                ):
-                                    requirements[dist] = metadata.version(dist)
-                    except metadata.PackageNotFoundError:
-                        from sqlmesh.core.console import get_console
-
-                        get_console().log_warning(f"Failed to find package for {lib}.")
+        if not snapshot.is_model:
+            continue
+
+        for executable in snapshot.model.python_env.values():
+            if executable.kind != "import":
+                continue
+
+            try:
+                start = "from " if executable.payload.startswith("from ") else "import "
+                lib = executable.payload.split(start)[1].split()[0].split(".")[0]
+                if lib not in distributions:
+                    continue
+
+                for dist in distributions[lib]:
+                    if (
+                        dist not in requirements
+                        and dist not in IGNORED_PACKAGES
+                        and dist not in excluded_requirements
+                    ):
+                        requirements[dist] = metadata.version(dist)
+            except metadata.PackageNotFoundError:
+                from sqlmesh.core.console import get_console
+
+                get_console().log_warning(f"Failed to find package for {lib}.")
+
     return requirements

tests/core/test_context.py

@@ -1240,6 +1240,19 @@ def test_requirements(copy_to_temp_path: t.Callable):
     assert set(diff.requirements) == {"numpy", "pandas"}
 
 
+def test_deactivate_automatic_requirement_inference(copy_to_temp_path: t.Callable):
+    context_path = copy_to_temp_path("examples/sushi")[0]
+    config = next(iter(load_configs("config", Config, paths=context_path).values()))
+
+    config.infer_python_dependencies = False
+    context = Context(paths=context_path, config=config)
+    environment = context.plan(
+        "dev", no_prompts=True, skip_tests=True, skip_backfill=True, auto_apply=True
+    ).environment
+
+    assert environment.requirements == {"pandas": "2.2.2"}
+
+
 @pytest.mark.slow
 def test_rendered_diff():
     ctx = Context(config=Config())

tests/core/test_integration.py

@@ -5050,7 +5050,7 @@ def test_multi_virtual_layer(copy_to_temp_path):
     assert len(prod_environment.snapshots_) == 3
 
     # Changing the flag should show a diff
-    context.gateway_managed_virtual_layer = False
+    context.config.gateway_managed_virtual_layer = False
     plan = context.plan_builder().build()
     assert not plan.requires_backfill
     assert (