From ccd1a920eb27eb3cd7c9638154ac9cd878f28017 Mon Sep 17 00:00:00 2001 From: AGibson <4319494+annajgibson@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:26:20 +0000 Subject: [PATCH 1/3] - add dq dimension type for new test --- scripts/helpers/housing_gx_dq_inputs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helpers/housing_gx_dq_inputs.py b/scripts/helpers/housing_gx_dq_inputs.py index cb69da74c..f285a62f5 100644 --- a/scripts/helpers/housing_gx_dq_inputs.py +++ b/scripts/helpers/housing_gx_dq_inputs.py @@ -57,6 +57,7 @@ 'expect_tenure_code_column_not_to_be_null': 'COMPLETENESS', 'expect_target_id_and_value_column_values_to_be_unique_within_record': 'UNIQUENESS', 'expect_sub_type_column_values_to_be_in_set': 'CONSISTENCY', + 'expect_tenure_values_to_be_in_set_housing_dwellings_list': 'CONSISTENCY', 'expect_contact_value_column_values_to_be_unique': 'UNIQUENESS', 'expect_contact_value_column_values_to_not_be_null': 'COMPLETENESS', 'expect_asset_type_not_to_be_null': 'COMPLETENESS', From dad89ab956d33826927f9b19e3877ba1350e0e66 Mon Sep 17 00:00:00 2001 From: AGibson <4319494+annajgibson@users.noreply.github.com> Date: Wed, 12 Mar 2025 16:34:57 +0000 Subject: [PATCH 2/3] - amend dq dimension types for new test --- scripts/helpers/housing_gx_dq_inputs.py | 93 +++++++++++++++++-------- 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/scripts/helpers/housing_gx_dq_inputs.py b/scripts/helpers/housing_gx_dq_inputs.py index f285a62f5..07dc77236 100644 --- a/scripts/helpers/housing_gx_dq_inputs.py +++ b/scripts/helpers/housing_gx_dq_inputs.py @@ -25,42 +25,77 @@ partition_keys = ['import_year', 'import_month', 'import_day', 'import_date'] dq_dimensions_map = { - 'expect_first_name_column_value_lengths': 'VALIDITY', - 'expect_first_name_column_value_length': 'VALIDITY', - 'expect_uprn_column_values_to_match_regex': 'VALIDITY', - 'expect_person_type_values_to_be_in_set': 'CONSISTENCY', - 'expect_preferred_title_values_to_be_in_set': 'CONSISTENCY', - 'expect_uprn_not_to_be_null': 'COMPLETENESS', + 'expect_arr_patch_not_to_be_null': 'COMPLETENESS', + 'expect_asset_id_not_to_be_null': 'COMPLETENESS', + 'expect_asset_type_not_to_be_null': 'COMPLETENESS', + 'expect_asset_type_values_to_be_in_set': 'CONSISTENCY', + 'expect_block_ref_no_column_values_to_match_regex': 'ACCURACY', + 'expect_co_t_not_to_be_null': 'COMPLETENESS', + 'expect_co_t_to_be_between': 'VALIDITY', + 'expect_column_value_lengths_to_be_between': 'VALIDITY', + 'expect_column_values_to_be_in_set': 'CONSISTENCY', + 'expect_column_values_to_be_unique': 'UNIQUENESS', + 'expect_column_values_to_match_regex': 'ACCURACY', + 'expect_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_contact_type_column_values_to_be_in_set': 'CONSISTENCY', + 'expect_contact_type_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_contact_value_column_values_to_be_unique': 'UNIQUENESS', + 'expect_contact_value_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_date_of_birth_column_values_to_not_be_null': 'COMPLETENESS', 'expect_date_of_birth_to_be_between': 'VALIDITY', + 'expect_description_values_to_be_in_set': 'CONSISTENCY', + 'expect_end_of_tenure_date_column_to_be_null': 'COMPLETENESS', + 'expect_eo_t_to_be_between': 'VALIDITY', + 'expect_estate_ref_no_column_values_to_match_regex': 'ACCURACY', + 'expect_first_name_column_value_length': 'ACCURACY', + 'expect_firstname_column_value_length': 'ACCURACY', + 'expect_is_organisation_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_is_organisation_values_to_be_in_set': 'CONSISTENCY', + 'expect_llpg_and_prop_ref_column_values_to_be_unique_within_record': 'UNIQUENESS', + 'expect_llpg_column_value_lengths_between': 'VALIDITY', + 'expect_llpg_column_values_to_be_unique': 'UNIQUENESS', + 'expect_llpg_column_values_to_match_regex': 'ACCURACY', + 'expect_llpg_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_member_full_name_column_value_lengths_between': 'VALIDITY', + 'expect_member_is_responsible_values_to_be_in_set': 'CONSISTENCY', + 'expect_payment_ref_no_column_values_to_match_regex': 'ACCURACY', + 'expect_payment_reference_column_not_to_be_null': 'COMPLETENESS', + 'expect_person_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS', 'expect_person_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS', + 'expect_person_id_column_values_to_be_unique': 'UNIQUENESS', + 'expect_person_id_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_person_type_values_to_be_in_set': 'CONSISTENCY', + 'expect_preferred_title_values_to_be_in_set': 'CONSISTENCY', + 'expect_prop_no_and_payment_ref_column_values_to_be_unique_within_record': 'UNIQUENESS', + 'expect_prop_no_column_values_to_be_unique': 'UNIQUENESS', + 'expect_prop_no_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_prop_no_no_column_values_to_match_regex': 'ACCURACY', + 'expect_prop_ref_column_values_to_be_unique': 'UNIQUENESS', + 'expect_prop_ref_not_to_be_null': 'COMPLETENESS', 'expect_property_ref_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_member_full_name_column_value_lengths_between': 'VALIDITY', - 'expect_description_values_to_be_in_set': 'CONSISTENCY', - 'expect_tenancy_id_column_not_to_be_null': 'COMPLETENESS', + 'expect_rent_group_ref_not_to_be_null': 'COMPLETENESS', + 'expect_saff_rent_acc_not_to_be_null': 'COMPLETENESS', + 'expect_select_column_values_to_be_unique_within_record': 'UNIQUENESS', 'expect_start_of_tenure_date_column_not_to_be_null': 'COMPLETENESS', - 'expect_contact_type_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_surname_column_value_length': 'VALIDITY', - 'expect_surname_column_value_lengths': 'VALIDITY', - 'expect_person_id_column_values_to_be_unique': 'UNIQUENESS', - 'expect_person_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS', - 'expect_date_of_birth_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_asset_type_values_to_be_in_set': 'CONSISTENCY', - 'expect_tenancy_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS', - 'expect_payment_reference_column_not_to_be_null': 'COMPLETENESS', - 'expect_target_id_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_contact_type_column_values_to_be_in_set': 'CONSISTENCY', + 'expect_sub_type_column_values_to_be_in_set': 'CONSISTENCY', 'expect_sub_type_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_asset_id_not_to_be_null': 'COMPLETENESS', - 'expect_uprn_column_value_lengths_between': 'VALIDITY', - 'expect_person_id_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_surname_column_value_length': 'ACCURACY', + 'expect_tag_ref_column_not_to_be_null': 'COMPLETENESS', + 'expect_tag_ref_column_values_to_be_unique': 'UNIQUENESS', + 'expect_tag_ref_not_to_be_null': 'COMPLETENESS', + 'expect_target_id_and_value_column_values_to_be_unique_within_record': 'UNIQUENESS', + 'expect_target_id_column_values_to_not_be_null': 'COMPLETENESS', + 'expect_target_type_column_values_to_be_in_set': 'CONSISTENCY', 'expect_tenancy_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS', + 'expect_tenancy_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS', + 'expect_tenancy_id_column_not_to_be_null': 'COMPLETENESS', 'expect_tenure_code_column_not_to_be_null': 'COMPLETENESS', - 'expect_target_id_and_value_column_values_to_be_unique_within_record': 'UNIQUENESS', - 'expect_sub_type_column_values_to_be_in_set': 'CONSISTENCY', + 'expect_tenure_code_values_to_be_in_set': 'CONSISTENCY', + 'expect_tenure_not_to_be_null': 'COMPLETENESS', + 'expect_tenure_type_column_values_to_be_in_set': 'CONSISTENCY', 'expect_tenure_values_to_be_in_set_housing_dwellings_list': 'CONSISTENCY', - 'expect_contact_value_column_values_to_be_unique': 'UNIQUENESS', - 'expect_contact_value_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_asset_type_not_to_be_null': 'COMPLETENESS', + 'expect_uprn_column_value_lengths_between': 'VALIDITY', + 'expect_uprn_column_values_to_match_regex': 'ACCURACY', 'expect_uprn_column_values_to_not_be_null': 'COMPLETENESS', - 'expect_target_type_column_values_to_be_in_set': 'CONSISTENCY', + 'expect_uprn_not_to_be_null': 'COMPLETENESS' } From 7aa9abb01688fd45c686a25dd9b9679234fda6a4 Mon Sep 17 00:00:00 2001 From: AGibson <4319494+annajgibson@users.noreply.github.com> Date: Thu, 13 Mar 2025 10:10:47 +0000 Subject: [PATCH 3/3] - amend dq dimension types for expect_tenure_values_to_be_in_set --- scripts/helpers/housing_gx_dq_inputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helpers/housing_gx_dq_inputs.py b/scripts/helpers/housing_gx_dq_inputs.py index 07dc77236..44df0a6e8 100644 --- a/scripts/helpers/housing_gx_dq_inputs.py +++ b/scripts/helpers/housing_gx_dq_inputs.py @@ -93,7 +93,7 @@ 'expect_tenure_code_values_to_be_in_set': 'CONSISTENCY', 'expect_tenure_not_to_be_null': 'COMPLETENESS', 'expect_tenure_type_column_values_to_be_in_set': 'CONSISTENCY', - 'expect_tenure_values_to_be_in_set_housing_dwellings_list': 'CONSISTENCY', + 'expect_tenure_values_to_be_in_set': 'CONSISTENCY', 'expect_uprn_column_value_lengths_between': 'VALIDITY', 'expect_uprn_column_values_to_match_regex': 'ACCURACY', 'expect_uprn_column_values_to_not_be_null': 'COMPLETENESS',