From 636152c685ff868d3d5bbaa2154d51def3341b4a Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:32:43 +0300 Subject: [PATCH 1/4] Fix(databricks): Materialized view drop failure due to incorrect type --- sqlmesh/core/engine_adapter/databricks.py | 4 +- tests/core/engine_adapter/test_databricks.py | 72 +++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/sqlmesh/core/engine_adapter/databricks.py b/sqlmesh/core/engine_adapter/databricks.py index da70163db4..2571cb7214 100644 --- a/sqlmesh/core/engine_adapter/databricks.py +++ b/sqlmesh/core/engine_adapter/databricks.py @@ -266,7 +266,9 @@ def _get_data_objects( exp.column("table_catalog").as_("catalog"), exp.case(exp.column("table_type")) .when(exp.Literal.string("VIEW"), exp.Literal.string("view")) - .when(exp.Literal.string("MATERIALIZED_VIEW"), exp.Literal.string("view")) + .when( + exp.Literal.string("MATERIALIZED_VIEW"), exp.Literal.string("materialized_view") + ) .else_(exp.Literal.string("table")) .as_("type"), ) diff --git a/tests/core/engine_adapter/test_databricks.py b/tests/core/engine_adapter/test_databricks.py index cd4c8c4074..fcd7aec0fa 100644 --- a/tests/core/engine_adapter/test_databricks.py +++ b/tests/core/engine_adapter/test_databricks.py @@ -8,7 +8,7 @@ from sqlmesh.core import dialect as d from sqlmesh.core.engine_adapter import DatabricksEngineAdapter -from sqlmesh.core.engine_adapter.shared import DataObject +from sqlmesh.core.engine_adapter.shared import DataObject, DataObjectType from sqlmesh.core.node import IntervalUnit from tests.core.engine_adapter import to_sql_calls @@ -219,3 +219,73 @@ def test_create_table_clustered_by(mocker: MockFixture, make_mocked_engine_adapt assert sql_calls == [ "CREATE TABLE IF NOT EXISTS `test_table` (`cola` INT, `colb` STRING) CLUSTER BY (`cola`)", ] + + +def test_get_data_objects_distinguishes_view_types(mocker): + adapter = DatabricksEngineAdapter(lambda: None, default_catalog="test_catalog") + + # (Databricks requires DBSQL Serverless or Pro warehouse to test materialized views which we do not have setup) + # so this mocks the fetchdf call to simulate the response we would expect from the correct SQL query + mock_df = pd.DataFrame( + [ + { + "name": "regular_view", + "schema": "test_schema", + "catalog": "test_catalog", + "type": "view", + }, + { + "name": "mat_view", + "schema": "test_schema", + "catalog": "test_catalog", + "type": "materialized_view", + }, + { + "name": "regular_table", + "schema": "test_schema", + "catalog": "test_catalog", + "type": "table", + }, + ] + ) + + mocker.patch.object(adapter, "fetchdf", return_value=mock_df) + + data_objects = adapter._get_data_objects( + schema_name=exp.Table(db="test_schema", catalog="test_catalog") + ) + + adapter.fetchdf.assert_called_once() + call_args = adapter.fetchdf.call_args + sql_query_exp = call_args[0][0] + + # _get_data_objects query should distinguish between VIEW and MATERIALIZED_VIEW types + sql_query = sql_query_exp.sql(dialect="databricks") + assert ( + "CASE table_type WHEN 'VIEW' THEN 'view' WHEN 'MATERIALIZED_VIEW' THEN 'materialized_view' ELSE 'table' END AS type" + in sql_query + ) + + objects_by_name = {obj.name: obj for obj in data_objects} + assert objects_by_name["regular_view"].type == DataObjectType.VIEW + assert objects_by_name["mat_view"].type == DataObjectType.MATERIALIZED_VIEW + assert objects_by_name["regular_table"].type == DataObjectType.TABLE + + +def test_drop_data_object_materialized_view_calls_correct_drop(mocker: MockFixture): + adapter = DatabricksEngineAdapter(lambda: None, default_catalog="test_catalog") + + mv_data_object = DataObject( + catalog="test_catalog", + schema="test_schema", + name="test_mv", + type=DataObjectType.MATERIALIZED_VIEW, + ) + + drop_view_mock = mocker.patch.object(adapter, "drop_view") + adapter.drop_data_object(mv_data_object) + + # Ensure drop_view is called with materialized=True + drop_view_mock.assert_called_once_with( + mv_data_object.to_table(), ignore_if_not_exists=True, materialized=True + ) From 8b7b541f0824e8eebf77487ac94ca743abb8fea0 Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:58:33 +0300 Subject: [PATCH 2/4] attempt to run databrick test --- .circleci/continue_config.yml | 2 +- tests/core/engine_adapter/integration/test_integration.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index e21f3d869b..8a831d7bc5 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -303,7 +303,7 @@ workflows: parameters: engine: - snowflake - - databricks + # - databricks - redshift - bigquery - clickhouse-cloud diff --git a/tests/core/engine_adapter/integration/test_integration.py b/tests/core/engine_adapter/integration/test_integration.py index ccea105bcf..f6e64260a1 100644 --- a/tests/core/engine_adapter/integration/test_integration.py +++ b/tests/core/engine_adapter/integration/test_integration.py @@ -403,10 +403,10 @@ def test_materialized_view(ctx_query_and_df: TestContext): ctx = ctx_query_and_df if not ctx.engine_adapter.SUPPORTS_MATERIALIZED_VIEWS: pytest.skip(f"Engine adapter {ctx.engine_adapter} doesn't support materialized views") - if ctx.engine_adapter.dialect == "databricks": - pytest.skip( - "Databricks requires DBSQL Serverless or Pro warehouse to test materialized views which we do not have setup" - ) + # if ctx.engine_adapter.dialect == "databricks": + # pytest.skip( + # "Databricks requires DBSQL Serverless or Pro warehouse to test materialized views which we do not have setup" + # ) if ctx.engine_adapter.dialect == "snowflake": pytest.skip("Snowflake requires enterprise edition which we do not have setup") input_data = pd.DataFrame( From c9bc8110234e48a8e4ea743f2aff5978d442aa1d Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:16:23 +0300 Subject: [PATCH 3/4] filter out correctly --- .circleci/continue_config.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 8a831d7bc5..9c7ad4b79e 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -302,18 +302,18 @@ workflows: matrix: parameters: engine: - - snowflake - # - databricks - - redshift - - bigquery - - clickhouse-cloud - - athena - - fabric - - gcp-postgres - filters: - branches: - only: - - main + # - snowflake + - databricks + # - redshift + # - bigquery + # - clickhouse-cloud + # - athena + # - fabric + # - gcp-postgres + # filters: + # branches: + # only: + # - main - ui_style - ui_test - vscode_test From d6cc220e3711f8f5809737de4e554908f9215df2 Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:44:49 +0300 Subject: [PATCH 4/4] reinstate tests --- .circleci/continue_config.yml | 22 +++++++++---------- .../integration/test_integration.py | 8 +++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 9c7ad4b79e..e21f3d869b 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -302,18 +302,18 @@ workflows: matrix: parameters: engine: - # - snowflake + - snowflake - databricks - # - redshift - # - bigquery - # - clickhouse-cloud - # - athena - # - fabric - # - gcp-postgres - # filters: - # branches: - # only: - # - main + - redshift + - bigquery + - clickhouse-cloud + - athena + - fabric + - gcp-postgres + filters: + branches: + only: + - main - ui_style - ui_test - vscode_test diff --git a/tests/core/engine_adapter/integration/test_integration.py b/tests/core/engine_adapter/integration/test_integration.py index f6e64260a1..ccea105bcf 100644 --- a/tests/core/engine_adapter/integration/test_integration.py +++ b/tests/core/engine_adapter/integration/test_integration.py @@ -403,10 +403,10 @@ def test_materialized_view(ctx_query_and_df: TestContext): ctx = ctx_query_and_df if not ctx.engine_adapter.SUPPORTS_MATERIALIZED_VIEWS: pytest.skip(f"Engine adapter {ctx.engine_adapter} doesn't support materialized views") - # if ctx.engine_adapter.dialect == "databricks": - # pytest.skip( - # "Databricks requires DBSQL Serverless or Pro warehouse to test materialized views which we do not have setup" - # ) + if ctx.engine_adapter.dialect == "databricks": + pytest.skip( + "Databricks requires DBSQL Serverless or Pro warehouse to test materialized views which we do not have setup" + ) if ctx.engine_adapter.dialect == "snowflake": pytest.skip("Snowflake requires enterprise edition which we do not have setup") input_data = pd.DataFrame(