From e9171d898ce684b2229ea31c456b133b7766071a Mon Sep 17 00:00:00 2001 From: AGibson <4319494+annajgibson@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:07:16 +0100 Subject: [PATCH 1/2] Set partition mode to 'overwrite' --- scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py index b1bc2ac0e..14d651840 100644 --- a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py +++ b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py @@ -125,7 +125,7 @@ def main(): dataset=True, database=target_database, table=target_table, - mode="overwrite_partitions", + mode="overwrite", partition_cols=partition_keys, dtype=dtype_dict, schema_evolution=True From ffb38e6b8bc29329c79b68eaf66b480d12a0ae8a Mon Sep 17 00:00:00 2001 From: AGibson <4319494+annajgibson@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:11:11 +0100 Subject: [PATCH 2/2] Remove partition reference --- .../jobs/housing/housing_nec_migration_apply_gx_dq_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py index 14d651840..a3d10ff03 100644 --- a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py +++ b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py @@ -10,7 +10,7 @@ import great_expectations as gx import pandas as pd from pyathena import connect -from scripts.helpers.housing_nec_migration_gx_dq_inputs import sql_config, table_list, partition_keys +from scripts.helpers.housing_nec_migration_gx_dq_inputs import sql_config, table_list import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite logging.basicConfig(level=logging.INFO) @@ -126,7 +126,6 @@ def main(): database=target_database, table=target_table, mode="overwrite", - partition_cols=partition_keys, dtype=dtype_dict, schema_evolution=True )