Merge branch 'update-dbt-artifact-versions' of https://github.com/datafold/data-diff into update-dbt-artifact-versions

Author: Sung Won Chung

.github/workflows/formatter.yml

@@ -21,3 +21,6 @@ jobs:
         uses: reviewdog/action-suggester@v1
         with:
           tool_name: blackfmt
+      - name: Fail if there are formatting suggestions
+        if: steps.black_formatter.outputs.is_formatted == 'true'
+        run: exit 1

data_diff/__main__.py

@@ -267,6 +267,20 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
     metavar="PATH",
     help="Specify manifest to utilize for 'prod' comparison paths instead of using configuration.",
 )
+@click.option(
+    "-pd",
+    "--prod-database",
+    "prod_database",
+    default=None,
+    help="Override the dbt production database configuration within dbt_project.yml",
+)
+@click.option(
+    "-ps",
+    "--prod-schema",
+    "prod_schema",
+    default=None,
+    help="Override the dbt production schema configuration within dbt_project.yml",
+)
 def main(conf, run, **kw):
     log_handlers = _get_log_handlers(kw["dbt"])
     if kw["table2"] is None and kw["database2"]:
@@ -323,6 +337,8 @@ def main(conf, run, **kw):
                 where_flag=kw["where"],
                 stats_flag=kw["stats"],
                 columns_flag=kw["columns"],
+                production_database_flag=kw["prod_database"],
+                production_schema_flag=kw["prod_schema"],
             )
         else:
             return _data_diff(
@@ -366,6 +382,8 @@ def _data_diff(
     cloud,
     dbt_profiles_dir,
     dbt_project_dir,
+    prod_database,
+    prod_schema,
     select,
     state,
     threads1=None,

data_diff/dbt.py

@@ -76,6 +76,8 @@ def dbt_diff(
     where_flag: Optional[str] = None,
     stats_flag: bool = False,
     columns_flag: Optional[Tuple[str]] = None,
+    production_database_flag: Optional[str] = None,
+    production_schema_flag: Optional[str] = None,
 ) -> None:
     print_version_info()
     diff_threads = []
@@ -115,7 +117,16 @@ def dbt_diff(
             if log_status_handler:
                 log_status_handler.set_prefix(f"Diffing {model.alias} \n")
 
-            diff_vars = _get_diff_vars(dbt_parser, config, model, where_flag, stats_flag, columns_flag)
+            diff_vars = _get_diff_vars(
+                dbt_parser,
+                config,
+                model,
+                where_flag,
+                stats_flag,
+                columns_flag,
+                production_database_flag,
+                production_schema_flag,
+            )
 
             # we won't always have a prod path when using state
             # when the model DNE in prod manifest, skip the model diff
@@ -169,6 +180,8 @@ def _get_diff_vars(
     where_flag: Optional[str] = None,
     stats_flag: bool = False,
     columns_flag: Optional[Tuple[str]] = None,
+    production_database_flag: Optional[str] = None,
+    production_schema_flag: Optional[str] = None,
 ) -> TDiffVars:
     cli_columns = list(columns_flag) if columns_flag else []
     dev_database = model.database
@@ -182,6 +195,10 @@ def _get_diff_vars(
     else:
         prod_database, prod_schema = _get_prod_path_from_config(config, model, dev_database, dev_schema)
 
+    # cli flags take precedence over any project level config
+    prod_database = production_database_flag or prod_database
+    prod_schema = production_schema_flag or prod_schema
+
     if dbt_parser.requires_upper:
         dev_qualified_list = [x.upper() for x in [dev_database, dev_schema, dev_alias] if x]
         prod_qualified_list = [x.upper() for x in [prod_database, prod_schema, prod_alias] if x]

pyproject.toml

@@ -6,7 +6,6 @@ authors = ["Datafold <data-diff@datafold.com>"]
 license = "MIT"
 readme = "README.md"
 repository = "https://github.com/datafold/data-diff"
-documentation = ""
 classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Information Technology",

tests/test_dbt.py

@@ -964,11 +964,23 @@ def test_get_diff_vars_cli_columns(self, mock_prod_path_from_manifest, mock_prod
         mock_dbt_parser.prod_manifest_obj = None
         mock_prod_path_from_config.return_value = ("prod_db", "prod_schema")
         cli_columns = ("col1", "col2")
-
-        diff_vars = _get_diff_vars(mock_dbt_parser, config, mock_model, where_flag=None, columns_flag=cli_columns)
+        production_database_flag_override = "prod_db_override"
+        production_schema_flag_override = "prod_schema_override"
+
+        diff_vars = _get_diff_vars(
+            mock_dbt_parser,
+            config,
+            mock_model,
+            where_flag=None,
+            columns_flag=cli_columns,
+            production_database_flag=production_database_flag_override,
+            production_schema_flag=production_schema_flag_override,
+        )
 
         mock_dbt_parser.get_pk_from_model.assert_called_once()
         mock_prod_path_from_config.assert_called_once_with(config, mock_model, mock_model.database, mock_model.schema_)
         mock_prod_path_from_manifest.assert_not_called()
         self.assertEqual(diff_vars.include_columns, list(cli_columns))
         self.assertEqual(diff_vars.exclude_columns, [])
+        self.assertEqual(diff_vars.prod_path[0], production_database_flag_override)
+        self.assertEqual(diff_vars.prod_path[1], production_schema_flag_override)

tests/test_dbt_config_validators.py

@@ -15,7 +15,7 @@
 class TestRunResultsJsonConfig(unittest.TestCase):
     def test_run_results(self):
         # https://docs.getdbt.com/reference/artifacts/run-results-json
-        versions = ["v4","v5"]
+        versions = ["v4", "v5"]
 
         for version in versions:
             with self.subTest(version=version):