Add dq tests

scripts/helpers/housing_nec_migration_gx_dq_inputs.py

@@ -2,6 +2,7 @@
     "properties": {"id_field": "lpro_propref"},
     "tenancies": {"id_field": "ltcy_alt_ref"},
     "people": {"id_field": "lpar_per_alt_ref"},
+    "debit_breakdowns": {"id_field": "ldbr_pay_ref"},
     "contacts": {"id_field": "lcde_legacy_ref"},
     "arrears_actions": {"id_field": "laca_pay_ref"},
     "revenue_accounts": {"id_field": "lrac_pay_ref"},
@@ -12,7 +13,8 @@
 data_load_list = [
     "properties",
     "tenancies",
-    "people"
+    "people",
+    "debit_breakdowns"
     # "contacts",
     # "arrears_actions",
     # "revenue_accounts",
@@ -22,38 +24,14 @@
 
 table_list = {
     "properties": [
-        # "properties_1a",
-        # "properties_1b",
-        # "properties_1c",
-        # "properties_1d",
-        # "properties_1e",
-        # "properties_2a",
-        # "properties_3a",
-        # "properties_4a",
-        # "properties_4b",
-        # "properties_4c",
-        # "properties_7a",
         "full_dq_full_dq_hem_pro_all_dq",
     ],
     "tenancies": [
-        # "tenancies_1a",
-        # "tenancies_1c",
-        # "tenancies_2a",
-        # "tenancies_other",
         "full_dq_full_dq_hem_tcy_all_dq",
     ],
-    "people": [
-        # "people_1a",
-        #        "people_1b",
-        #        "people_1c",
-        #        "people_2a",
-        "full_dq_full_dq_hem_per_all_dq"
-    ],
+    "people": ["full_dq_full_dq_hem_per_all_dq"],
+    "debit_breakdowns": ["full_dq_full_dq_hra_dbr_all_dq"],
     "contacts": [
-        # "contacts_1a",
-        # "contacts_1b",
-        # "contacts_1c",
-        # "contacts_2a",
         "contacts_all",
     ],
     "arrears_actions": [
@@ -69,10 +47,6 @@
         "revenue_accounts_other",
     ],
     "transactions": [
-        # "transactions_1a",
-        # "transactions_1c",
-        # "transactions_2a",
-        # "transactions_other",
         "transactions_all",
     ],
     "addresses": ["addresses_1a", "addresses_1b", "addresses_2a"],

scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py

@@ -18,6 +18,7 @@
 import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite
 import scripts.jobs.housing.housing_nec_migration_tenancies_data_load_gx_suite
 import scripts.jobs.housing.housing_nec_migration_people_data_load_gx_suite
+import scripts.jobs.housing.housing_nec_migration_debit_breakdowns_data_load_gx_suite
 
 # import scripts.jobs.housing.housing_nec_migration_contacts_data_load_gx_suite
 # import scripts.jobs.housing.housing_nec_migration_arrears_actions_data_load_gx_suite

scripts/jobs/housing/housing_nec_migration_debit_breakdowns_data_load_gx_suite.py

@@ -0,0 +1,73 @@
+# flake8: noqa: F821
+
+import sys
+
+from awsglue.utils import getResolvedOptions
+import great_expectations as gx
+import great_expectations.expectations as gxe
+
+
+class DebitBreakdownsExpectPayRefColumnValuesToNotBeNull(
+    gxe.ExpectColumnValuesToNotBeNull
+):
+    column: str = "ldbr_pay_ref"
+    description: str = "Expect ldbr_pay_ref values to not be Null"
+
+
+class DebitBreakdownsExpectElementCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
+    column: str = "ldbr_ele_code"
+    value_set: list = [
+        "DBR",
+        "DCA",
+        "DCC",
+        "DCE",
+        "DCO",
+        "DCP",
+        "DGA",
+        "DGM",
+        "DHA",
+        "DHE",
+        "DHM",
+        "DLL",
+        "DR2",
+        "DSC",
+        "DTA",
+        "DTC",
+        "DTL",
+        "DVA",
+        "DWR",
+        "DWS",
+    ]
+    description: str = "Expect element code (ldbr_ele_code) to be one of the set"
+
+
+class DebitBreakdownsExpectDBRColumnsToMatchOrderedList(
+    gxe.ExpectTableColumnsToMatchOrderedList
+):
+    column_list = [
+        "ldbr_pay_ref",
+        "ldbr_pro_refno",
+        "ldbr_ele_code",
+        "ldbr_start_date",
+        "ldbr_end_date",
+        "ldbr_att_code",
+        "ldbr_ele_value",
+        "tranche",
+    ]
+    description: str = "Expect columns to match ordered list exactly; tranche at end"
+
+
+arg_key = ["s3_target_location"]
+args = getResolvedOptions(sys.argv, arg_key)
+locals().update(args)
+
+# add to GX context
+context = gx.get_context(mode="file", project_root_dir=s3_target_location)
+
+suite = gx.ExpectationSuite(name="debit_breakdowns_data_load_suite")
+
+suite.add_expectation(DebitBreakdownsExpectPayRefColumnValuesToNotBeNull())
+suite.add_expectation(DebitBreakdownsExpectElementCodeToBeInSet())
+suite.add_expectation(DebitBreakdownsExpectDBRColumnsToMatchOrderedList())
+
+suite = context.suites.add(suite)

scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py

@@ -36,11 +36,9 @@ class PropertiesExpectOwnTypeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
     value_set: list = [
         "ASSOC",
         "COUN",
-        "LEASH",
-        "LEASL",
-        "LEASHOUT",
-        "LEASLOUT",
+        "LEASEHOLD",
         "PRIVATE",
+        "ENFRAN"
     ]
     description: str = (
         "Expect ownership type code (lpro_hou_hrv_hot_code) to be one of the set"

scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py

@@ -33,7 +33,7 @@ class TenanciesExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
         "LHS",
         "LTA",
         "MPA",
-        "NONSEC",
+        "NONSECHR",
         "PVG",
         "RTM",
         "SEC",