diff --git a/scripts/helpers/housing_nec_migration_gx_dq_inputs.py b/scripts/helpers/housing_nec_migration_gx_dq_inputs.py index a941e4780..4d00835b9 100644 --- a/scripts/helpers/housing_nec_migration_gx_dq_inputs.py +++ b/scripts/helpers/housing_nec_migration_gx_dq_inputs.py @@ -2,6 +2,7 @@ "properties": {"id_field": "lpro_propref"}, "tenancies": {"id_field": "ltcy_alt_ref"}, "people": {"id_field": "lpar_per_alt_ref"}, + "debit_breakdowns": {"id_field": "ldbr_pay_ref"}, "contacts": {"id_field": "lcde_legacy_ref"}, "arrears_actions": {"id_field": "laca_pay_ref"}, "revenue_accounts": {"id_field": "lrac_pay_ref"}, @@ -12,7 +13,8 @@ data_load_list = [ "properties", "tenancies", - "people" + "people", + "debit_breakdowns" # "contacts", # "arrears_actions", # "revenue_accounts", @@ -22,38 +24,14 @@ table_list = { "properties": [ - # "properties_1a", - # "properties_1b", - # "properties_1c", - # "properties_1d", - # "properties_1e", - # "properties_2a", - # "properties_3a", - # "properties_4a", - # "properties_4b", - # "properties_4c", - # "properties_7a", "full_dq_full_dq_hem_pro_all_dq", ], "tenancies": [ - # "tenancies_1a", - # "tenancies_1c", - # "tenancies_2a", - # "tenancies_other", "full_dq_full_dq_hem_tcy_all_dq", ], - "people": [ - # "people_1a", - # "people_1b", - # "people_1c", - # "people_2a", - "full_dq_full_dq_hem_per_all_dq" - ], + "people": ["full_dq_full_dq_hem_per_all_dq"], + "debit_breakdowns": ["full_dq_full_dq_hra_dbr_all_dq"], "contacts": [ - # "contacts_1a", - # "contacts_1b", - # "contacts_1c", - # "contacts_2a", "contacts_all", ], "arrears_actions": [ @@ -69,10 +47,6 @@ "revenue_accounts_other", ], "transactions": [ - # "transactions_1a", - # "transactions_1c", - # "transactions_2a", - # "transactions_other", "transactions_all", ], "addresses": ["addresses_1a", "addresses_1b", "addresses_2a"], diff --git a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py index 5cc144690..581c98b41 100644 --- a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py +++ b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py @@ -18,6 +18,7 @@ import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite import scripts.jobs.housing.housing_nec_migration_tenancies_data_load_gx_suite import scripts.jobs.housing.housing_nec_migration_people_data_load_gx_suite +import scripts.jobs.housing.housing_nec_migration_debit_breakdowns_data_load_gx_suite # import scripts.jobs.housing.housing_nec_migration_contacts_data_load_gx_suite # import scripts.jobs.housing.housing_nec_migration_arrears_actions_data_load_gx_suite diff --git a/scripts/jobs/housing/housing_nec_migration_debit_breakdowns_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_debit_breakdowns_data_load_gx_suite.py new file mode 100644 index 000000000..c40a3e9e8 --- /dev/null +++ b/scripts/jobs/housing/housing_nec_migration_debit_breakdowns_data_load_gx_suite.py @@ -0,0 +1,73 @@ +# flake8: noqa: F821 + +import sys + +from awsglue.utils import getResolvedOptions +import great_expectations as gx +import great_expectations.expectations as gxe + + +class DebitBreakdownsExpectPayRefColumnValuesToNotBeNull( + gxe.ExpectColumnValuesToNotBeNull +): + column: str = "ldbr_pay_ref" + description: str = "Expect ldbr_pay_ref values to not be Null" + + +class DebitBreakdownsExpectElementCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): + column: str = "ldbr_ele_code" + value_set: list = [ + "DBR", + "DCA", + "DCC", + "DCE", + "DCO", + "DCP", + "DGA", + "DGM", + "DHA", + "DHE", + "DHM", + "DLL", + "DR2", + "DSC", + "DTA", + "DTC", + "DTL", + "DVA", + "DWR", + "DWS", + ] + description: str = "Expect element code (ldbr_ele_code) to be one of the set" + + +class DebitBreakdownsExpectDBRColumnsToMatchOrderedList( + gxe.ExpectTableColumnsToMatchOrderedList +): + column_list = [ + "ldbr_pay_ref", + "ldbr_pro_refno", + "ldbr_ele_code", + "ldbr_start_date", + "ldbr_end_date", + "ldbr_att_code", + "ldbr_ele_value", + "tranche", + ] + description: str = "Expect columns to match ordered list exactly; tranche at end" + + +arg_key = ["s3_target_location"] +args = getResolvedOptions(sys.argv, arg_key) +locals().update(args) + +# add to GX context +context = gx.get_context(mode="file", project_root_dir=s3_target_location) + +suite = gx.ExpectationSuite(name="debit_breakdowns_data_load_suite") + +suite.add_expectation(DebitBreakdownsExpectPayRefColumnValuesToNotBeNull()) +suite.add_expectation(DebitBreakdownsExpectElementCodeToBeInSet()) +suite.add_expectation(DebitBreakdownsExpectDBRColumnsToMatchOrderedList()) + +suite = context.suites.add(suite) diff --git a/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py index d04f26a6b..63ea4fd1f 100644 --- a/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py +++ b/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py @@ -36,11 +36,9 @@ class PropertiesExpectOwnTypeToBeInSet(gxe.ExpectColumnValuesToBeInSet): value_set: list = [ "ASSOC", "COUN", - "LEASH", - "LEASL", - "LEASHOUT", - "LEASLOUT", + "LEASEHOLD", "PRIVATE", + "ENFRAN" ] description: str = ( "Expect ownership type code (lpro_hou_hrv_hot_code) to be one of the set" diff --git a/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py index ac81fdd83..bff26f14e 100644 --- a/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py +++ b/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py @@ -33,7 +33,7 @@ class TenanciesExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): "LHS", "LTA", "MPA", - "NONSEC", + "NONSECHR", "PVG", "RTM", "SEC",