From d0c88082bdc4cfb2454f3f58412fd2b792a91141 Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:37:08 +0200 Subject: [PATCH 1/4] Fix(databricks): Add column type only along materialized view comments --- sqlmesh/core/engine_adapter/base.py | 9 +++++- sqlmesh/core/engine_adapter/databricks.py | 7 +++-- sqlmesh/core/engine_adapter/trino.py | 1 + tests/core/engine_adapter/test_databricks.py | 30 ++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index f04ad4fd71..3a331a43fe 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -811,6 +811,7 @@ def _build_schema_exp( column_descriptions: t.Optional[t.Dict[str, str]] = None, expressions: t.Optional[t.List[exp.PrimaryKey]] = None, is_view: bool = False, + is_materialized_view: bool = False, ) -> exp.Schema: """ Build a schema expression for a table, columns, column comments, and additional schema properties. @@ -823,6 +824,7 @@ def _build_schema_exp( target_columns_to_types=target_columns_to_types, column_descriptions=column_descriptions, is_view=is_view, + is_materialized_view=is_materialized_view, ) + expressions, ) @@ -832,6 +834,7 @@ def _build_column_defs( target_columns_to_types: t.Dict[str, exp.DataType], column_descriptions: t.Optional[t.Dict[str, str]] = None, is_view: bool = False, + is_materialized_view: bool = False, ) -> t.List[exp.ColumnDef]: engine_supports_schema_comments = ( self.COMMENT_CREATION_VIEW.supports_schema_def @@ -1260,7 +1263,11 @@ def create_view( schema: t.Union[exp.Table, exp.Schema] = exp.to_table(view_name) if target_columns_to_types: schema = self._build_schema_exp( - exp.to_table(view_name), target_columns_to_types, column_descriptions, is_view=True + exp.to_table(view_name), + target_columns_to_types, + column_descriptions, + is_view=True, + is_materialized_view=materialized, ) properties = create_kwargs.pop("properties", None) diff --git a/sqlmesh/core/engine_adapter/databricks.py b/sqlmesh/core/engine_adapter/databricks.py index 5c12c5f6cb..a444ada1d2 100644 --- a/sqlmesh/core/engine_adapter/databricks.py +++ b/sqlmesh/core/engine_adapter/databricks.py @@ -400,11 +400,14 @@ def _build_column_defs( target_columns_to_types: t.Dict[str, exp.DataType], column_descriptions: t.Optional[t.Dict[str, str]] = None, is_view: bool = False, + is_materialized_view: bool = False, ) -> t.List[exp.ColumnDef]: # Databricks requires column types to be specified when adding column comments # in CREATE MATERIALIZED VIEW statements. Override is_view to False to force # column types to be included when comments are present. - if is_view and column_descriptions: + if is_materialized_view and column_descriptions: is_view = False - return super()._build_column_defs(target_columns_to_types, column_descriptions, is_view) + return super()._build_column_defs( + target_columns_to_types, column_descriptions, is_view, is_materialized_view + ) diff --git a/sqlmesh/core/engine_adapter/trino.py b/sqlmesh/core/engine_adapter/trino.py index 272c45c193..1199b23b0c 100644 --- a/sqlmesh/core/engine_adapter/trino.py +++ b/sqlmesh/core/engine_adapter/trino.py @@ -284,6 +284,7 @@ def _build_schema_exp( column_descriptions: t.Optional[t.Dict[str, str]] = None, expressions: t.Optional[t.List[exp.PrimaryKey]] = None, is_view: bool = False, + is_materialized_view: bool = False, ) -> exp.Schema: if "delta_lake" in self.get_catalog_type_from_table(table): target_columns_to_types = self._to_delta_ts(target_columns_to_types) diff --git a/tests/core/engine_adapter/test_databricks.py b/tests/core/engine_adapter/test_databricks.py index ca36b9856f..de91fd3b70 100644 --- a/tests/core/engine_adapter/test_databricks.py +++ b/tests/core/engine_adapter/test_databricks.py @@ -406,6 +406,36 @@ def test_materialized_view_with_column_comments( ] +def test_regular_view_with_column_comments( + mocker: MockFixture, make_mocked_engine_adapter: t.Callable +): + mocker.patch( + "sqlmesh.core.engine_adapter.databricks.DatabricksEngineAdapter.set_current_catalog" + ) + adapter = make_mocked_engine_adapter(DatabricksEngineAdapter, default_catalog="test_catalog") + mocker.patch.object(adapter, "get_current_catalog", return_value="test_catalog") + + adapter.create_view( + "test_view", + parse_one("SELECT a, b FROM source_table"), + target_columns_to_types={ + "a": exp.DataType.build("INT"), + "b": exp.DataType.build("STRING"), + }, + materialized=False, + column_descriptions={ + "a": "column a description", + "b": "column b description", + }, + ) + + sql_calls = to_sql_calls(adapter) + # Regular views should NOT include column types even when column comments are present + assert sql_calls == [ + "CREATE OR REPLACE VIEW `test_view` (`a` COMMENT 'column a description', `b` COMMENT 'column b description') AS SELECT `a`, `b` FROM `source_table`", + ] + + def test_create_table_clustered_by(mocker: MockFixture, make_mocked_engine_adapter: t.Callable): mocker.patch( "sqlmesh.core.engine_adapter.databricks.DatabricksEngineAdapter.set_current_catalog" From 9b2d8e6b4fb7605b86a4b4e2650445f89fc152b5 Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:40:58 +0200 Subject: [PATCH 2/4] test cloud --- .circleci/continue_config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5a240b85e4..ea518c7c63 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -309,18 +309,18 @@ workflows: matrix: parameters: engine: - - snowflake + # - snowflake - databricks - - redshift - - bigquery - - clickhouse-cloud - - athena - - fabric - - gcp-postgres - filters: - branches: - only: - - main + # - redshift + # - bigquery + # - clickhouse-cloud + # - athena + # - fabric + # - gcp-postgres + # filters: + # branches: + # only: + # - main - ui_style - ui_test - vscode_test From 6ab4cc7be7393142d765bd7023fe9bad22daab23 Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Wed, 3 Dec 2025 12:08:24 +0200 Subject: [PATCH 3/4] Reinstate circleci filters --- .circleci/continue_config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index ea518c7c63..5a240b85e4 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -309,18 +309,18 @@ workflows: matrix: parameters: engine: - # - snowflake + - snowflake - databricks - # - redshift - # - bigquery - # - clickhouse-cloud - # - athena - # - fabric - # - gcp-postgres - # filters: - # branches: - # only: - # - main + - redshift + - bigquery + - clickhouse-cloud + - athena + - fabric + - gcp-postgres + filters: + branches: + only: + - main - ui_style - ui_test - vscode_test From 491c5231a11dc7843e2002f458300afbaf8c22b8 Mon Sep 17 00:00:00 2001 From: Themis Valtinos <73662635+themisvaltinos@users.noreply.github.com> Date: Wed, 3 Dec 2025 12:58:20 +0200 Subject: [PATCH 4/4] rename flag --- sqlmesh/core/engine_adapter/base.py | 8 ++++---- sqlmesh/core/engine_adapter/databricks.py | 6 +++--- sqlmesh/core/engine_adapter/trino.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 3a331a43fe..a7a8e2f707 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -811,7 +811,7 @@ def _build_schema_exp( column_descriptions: t.Optional[t.Dict[str, str]] = None, expressions: t.Optional[t.List[exp.PrimaryKey]] = None, is_view: bool = False, - is_materialized_view: bool = False, + materialized: bool = False, ) -> exp.Schema: """ Build a schema expression for a table, columns, column comments, and additional schema properties. @@ -824,7 +824,7 @@ def _build_schema_exp( target_columns_to_types=target_columns_to_types, column_descriptions=column_descriptions, is_view=is_view, - is_materialized_view=is_materialized_view, + materialized=materialized, ) + expressions, ) @@ -834,7 +834,7 @@ def _build_column_defs( target_columns_to_types: t.Dict[str, exp.DataType], column_descriptions: t.Optional[t.Dict[str, str]] = None, is_view: bool = False, - is_materialized_view: bool = False, + materialized: bool = False, ) -> t.List[exp.ColumnDef]: engine_supports_schema_comments = ( self.COMMENT_CREATION_VIEW.supports_schema_def @@ -1267,7 +1267,7 @@ def create_view( target_columns_to_types, column_descriptions, is_view=True, - is_materialized_view=materialized, + materialized=materialized, ) properties = create_kwargs.pop("properties", None) diff --git a/sqlmesh/core/engine_adapter/databricks.py b/sqlmesh/core/engine_adapter/databricks.py index a444ada1d2..97190492f2 100644 --- a/sqlmesh/core/engine_adapter/databricks.py +++ b/sqlmesh/core/engine_adapter/databricks.py @@ -400,14 +400,14 @@ def _build_column_defs( target_columns_to_types: t.Dict[str, exp.DataType], column_descriptions: t.Optional[t.Dict[str, str]] = None, is_view: bool = False, - is_materialized_view: bool = False, + materialized: bool = False, ) -> t.List[exp.ColumnDef]: # Databricks requires column types to be specified when adding column comments # in CREATE MATERIALIZED VIEW statements. Override is_view to False to force # column types to be included when comments are present. - if is_materialized_view and column_descriptions: + if is_view and materialized and column_descriptions: is_view = False return super()._build_column_defs( - target_columns_to_types, column_descriptions, is_view, is_materialized_view + target_columns_to_types, column_descriptions, is_view, materialized ) diff --git a/sqlmesh/core/engine_adapter/trino.py b/sqlmesh/core/engine_adapter/trino.py index 1199b23b0c..74df3667ff 100644 --- a/sqlmesh/core/engine_adapter/trino.py +++ b/sqlmesh/core/engine_adapter/trino.py @@ -284,7 +284,7 @@ def _build_schema_exp( column_descriptions: t.Optional[t.Dict[str, str]] = None, expressions: t.Optional[t.List[exp.PrimaryKey]] = None, is_view: bool = False, - is_materialized_view: bool = False, + materialized: bool = False, ) -> exp.Schema: if "delta_lake" in self.get_catalog_type_from_table(table): target_columns_to_types = self._to_delta_ts(target_columns_to_types)