-
Notifications
You must be signed in to change notification settings - Fork 194
feature: Metric views #1285
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
feature: Metric views #1285
Changes from all commits
Commits
Show all changes
20 commits
Select commit
Hold shift + click to select a range
4663ba8
Support databricks_tags for MV/STs
ericj-db a84ceb7
Update changelog
ericj-db 76a0588
Fix unit tests
ericj-db c3c33d5
Wire in TagsProcessor to MV/ST macros
ericj-db ffd41de
move upper() to DatabricksRelationType.render()
ericj-db e6ac9f3
fix unit tests
ericj-db 70fab84
Add CREATE OR REPLACE branch for materialized views
ericj-db 9809385
Set can_be_replaced to true for materialized views
ericj-db aabc754
minor fixes for relation
ericj-db 32f9d38
fix: ST set tags were being applied when empty (#1169)
benc-db 0c7ecb4
refactor: Migrating to SDK for every rest call (#1185)
benc-db 8172085
metric view
benc-db 79c51a8
Merge 1.12.latest into metric_views branch
benc-db b174f3d
Remove debug test file for metric views
benc-db 8453a68
changelog
benc-db c60e4f2
adding option for alter behavior
benc-db 38d86e2
Address PR review comments for metric views
benc-db 2543419
Fix ruff formatting (remove extra blank line)
benc-db 2f42a24
Merge branch '1.12.latest' into metric_views
benc-db 65d3ee5
fix issue
benc-db File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
101 changes: 101 additions & 0 deletions
101
dbt/adapters/databricks/relation_configs/metric_view.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| from typing import ClassVar, Optional | ||
|
|
||
| from dbt.adapters.contracts.relation import RelationConfig | ||
| from dbt.adapters.relation_configs.config_base import RelationResults | ||
| from dbt_common.exceptions import DbtRuntimeError | ||
|
|
||
| from dbt.adapters.databricks.relation_configs.base import ( | ||
| DatabricksComponentConfig, | ||
| DatabricksComponentProcessor, | ||
| DatabricksRelationConfigBase, | ||
| ) | ||
| from dbt.adapters.databricks.relation_configs.tags import TagsProcessor | ||
| from dbt.adapters.databricks.relation_configs.tblproperties import TblPropertiesProcessor | ||
|
|
||
|
|
||
| class MetricViewQueryConfig(DatabricksComponentConfig): | ||
| """Component encapsulating the YAML definition of a metric view.""" | ||
|
|
||
| query: str | ||
|
|
||
| def get_diff(self, other: "MetricViewQueryConfig") -> Optional["MetricViewQueryConfig"]: | ||
| # Normalize whitespace for comparison | ||
| self_normalized = " ".join(self.query.split()) | ||
| other_normalized = " ".join(other.query.split()) | ||
| if self_normalized != other_normalized: | ||
| return self | ||
| return None | ||
|
|
||
|
|
||
| class MetricViewQueryProcessor(DatabricksComponentProcessor[MetricViewQueryConfig]): | ||
| """Processor for metric view YAML definitions. | ||
|
|
||
| Metric views store their YAML definitions in information_schema.views, but wrapped | ||
| in $$ delimiters. This processor extracts and compares the YAML content. | ||
| """ | ||
|
|
||
| name: ClassVar[str] = "query" | ||
|
|
||
| @classmethod | ||
| def from_relation_results(cls, result: RelationResults) -> MetricViewQueryConfig: | ||
| from dbt.adapters.databricks.logging import logger | ||
|
|
||
| # Get the view text from DESCRIBE EXTENDED output | ||
| describe_extended = result.get("describe_extended") | ||
| if not describe_extended: | ||
| raise DbtRuntimeError( | ||
| f"Cannot find metric view description. Result keys: {list(result.keys())}" | ||
| ) | ||
|
|
||
| # Find the "View Text" row in DESCRIBE EXTENDED output | ||
| view_definition = None | ||
| for row in describe_extended: | ||
| if row[0] == "View Text": | ||
| view_definition = row[1] | ||
| break | ||
|
|
||
| logger.debug( | ||
| f"MetricViewQueryProcessor: view_definition = " | ||
| f"{view_definition[:200] if view_definition else 'None'}" | ||
| ) | ||
|
|
||
| if not view_definition: | ||
| raise DbtRuntimeError("Metric view has no 'View Text' in DESCRIBE EXTENDED output") | ||
|
|
||
| view_definition = view_definition.strip() | ||
|
|
||
| # Extract YAML content from $$ delimiters if present | ||
| # Format: $$ yaml_content $$ | ||
| # Check start/end explicitly to avoid issues with embedded $$ in YAML content | ||
| trimmed = view_definition.strip() | ||
| if trimmed.startswith("$$") and trimmed.endswith("$$"): | ||
| # Strip the leading and trailing $$ markers | ||
| view_definition = trimmed[2:-2].strip() | ||
|
|
||
| return MetricViewQueryConfig(query=view_definition) | ||
|
|
||
| @classmethod | ||
| def from_relation_config(cls, relation_config: RelationConfig) -> MetricViewQueryConfig: | ||
| query = relation_config.compiled_code | ||
|
|
||
| if query: | ||
| return MetricViewQueryConfig(query=query.strip()) | ||
| else: | ||
| raise DbtRuntimeError( | ||
| f"Cannot compile metric view {relation_config.identifier} with no YAML definition" | ||
| ) | ||
|
|
||
|
|
||
| class MetricViewConfig(DatabricksRelationConfigBase): | ||
| """Config for metric views. | ||
|
|
||
| Metric views use YAML definitions stored in information_schema.views wrapped in $$ delimiters. | ||
| Changes to the YAML definition can be applied via ALTER VIEW AS. | ||
| Tags and tblproperties can also be altered incrementally. | ||
| """ | ||
|
|
||
| config_components = [ | ||
| TagsProcessor, | ||
| TblPropertiesProcessor, | ||
| MetricViewQueryProcessor, | ||
| ] | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
dbt/include/databricks/macros/materializations/metric_view.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| {% materialization metric_view, adapter='databricks' -%} | ||
| {%- set existing_relation = load_relation_with_metadata(this) -%} | ||
| {%- set target_relation = this.incorporate(type='metric_view') -%} | ||
| {% set grant_config = config.get('grants') %} | ||
| {% set tags = config.get('databricks_tags') %} | ||
| {% set sql = adapter.clean_sql(sql) %} | ||
|
|
||
| {{ run_pre_hooks() }} | ||
|
|
||
| {% if existing_relation %} | ||
| {#- Only use alter path if existing relation is actually a metric_view -#} | ||
| {% if existing_relation.is_metric_view and relation_should_be_altered(existing_relation) %} | ||
| {% set configuration_changes = get_configuration_changes(existing_relation) %} | ||
| {% if configuration_changes and configuration_changes.changes %} | ||
| {% if configuration_changes.requires_full_refresh %} | ||
| {{ replace_with_metric_view(existing_relation, target_relation) }} | ||
| {% else %} | ||
| {{ alter_metric_view(target_relation, configuration_changes.changes) }} | ||
| {% endif %} | ||
| {% else %} | ||
| {# No changes detected - run a no-op statement for dbt tracking #} | ||
| {% call statement('main') %} | ||
| select 1 | ||
| {% endcall %} | ||
| {% endif %} | ||
| {% else %} | ||
| {{ replace_with_metric_view(existing_relation, target_relation) }} | ||
| {% endif %} | ||
| {% else %} | ||
| {% call statement('main') -%} | ||
| {{ get_create_metric_view_as_sql(target_relation, sql) }} | ||
| {%- endcall %} | ||
| {{ apply_tags(target_relation, tags) }} | ||
| {% endif %} | ||
|
|
||
| {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke(existing_relation, full_refresh_mode=True)) %} | ||
|
|
||
| {{ run_post_hooks() }} | ||
|
|
||
| {{ return({'relations': [target_relation]}) }} | ||
| {%- endmaterialization %} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
51 changes: 51 additions & 0 deletions
51
dbt/include/databricks/macros/relations/metric_view/alter.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| {% macro alter_metric_view(target_relation, changes) %} | ||
| {{ log("Updating metric view via ALTER") }} | ||
| {{ adapter.dispatch('alter_metric_view', 'dbt')(target_relation, changes) }} | ||
| {% endmacro %} | ||
|
|
||
| {% macro databricks__alter_metric_view(target_relation, changes) %} | ||
| {% set tags = changes.get("tags") %} | ||
| {% set tblproperties = changes.get("tblproperties") %} | ||
| {% set query = changes.get("query") %} | ||
|
|
||
| {# Handle YAML definition changes via ALTER VIEW AS #} | ||
| {% if query %} | ||
| {% call statement('main') %} | ||
| {{ get_alter_metric_view_as_sql(target_relation, query.query) }} | ||
| {% endcall %} | ||
| {% else %} | ||
| {# Ensure statement('main') is called for dbt to track the run #} | ||
| {% call statement('main') %} | ||
| select 1 | ||
| {% endcall %} | ||
| {% endif %} | ||
|
|
||
| {% if tags %} | ||
| {{ apply_tags(target_relation, tags.set_tags) }} | ||
| {% endif %} | ||
| {% if tblproperties %} | ||
| {{ apply_tblproperties(target_relation, tblproperties.tblproperties) }} | ||
| {% endif %} | ||
| {% endmacro %} | ||
|
|
||
| {% macro get_alter_metric_view_as_sql(relation, yaml_content) -%} | ||
| {{ adapter.dispatch('get_alter_metric_view_as_sql', 'dbt')(relation, yaml_content) }} | ||
| {%- endmacro %} | ||
|
|
||
| {% macro databricks__get_alter_metric_view_as_sql(relation, yaml_content) %} | ||
| alter view {{ relation.render() }} as $$ | ||
| {{ yaml_content }} | ||
| $$ | ||
| {% endmacro %} | ||
|
|
||
| {% macro replace_with_metric_view(existing_relation, target_relation) %} | ||
| {% set sql = adapter.clean_sql(sql) %} | ||
| {% set tags = config.get('databricks_tags') %} | ||
| {% set tblproperties = config.get('tblproperties') %} | ||
| {{ execute_multiple_statements(get_replace_sql(existing_relation, target_relation, sql)) }} | ||
| {%- do apply_tags(target_relation, tags) -%} | ||
|
|
||
| {% if tblproperties %} | ||
| {{ apply_tblproperties(target_relation, tblproperties) }} | ||
| {% endif %} | ||
| {% endmacro %} |
12 changes: 12 additions & 0 deletions
12
dbt/include/databricks/macros/relations/metric_view/create.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| {% macro get_create_metric_view_as_sql(relation, sql) -%} | ||
| {{ adapter.dispatch('get_create_metric_view_as_sql', 'dbt')(relation, sql) }} | ||
| {%- endmacro %} | ||
|
|
||
| {% macro databricks__get_create_metric_view_as_sql(relation, sql) %} | ||
| create or replace view {{ relation.render() }} | ||
| with metrics | ||
| language yaml | ||
| as $$ | ||
| {{ sql }} | ||
| $$ | ||
| {% endmacro %} |
7 changes: 7 additions & 0 deletions
7
dbt/include/databricks/macros/relations/metric_view/replace.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| {% macro get_replace_metric_view_sql(target_relation, sql) %} | ||
| {{ adapter.dispatch('get_replace_metric_view_sql', 'dbt')(target_relation, sql) }} | ||
| {% endmacro %} | ||
|
|
||
| {% macro databricks__get_replace_metric_view_sql(target_relation, sql) %} | ||
| {{ get_create_metric_view_as_sql(target_relation, sql) }} | ||
| {% endmacro %} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.