Gx test updates

scripts/helpers/housing_nec_migration_gx_dq_inputs.py

@@ -5,7 +5,7 @@
     "contacts": {"id_field": "LCDE_LEGACY_REF"},
 }
 
-data_load_list = ["properties", "tenancies", "people", "contacts"]
+data_load_list = ["properties", "tenancies", "people", "contacts", "arrears_actions"]
 
 table_list = {
     "properties": [
@@ -29,8 +29,9 @@
         "tenancies_all",
         "tenancies_other",
     ],
-    "people": ["people_1a", "people_1b", "people_1c", "people_2a"],
-    "contacts": ["contacts_1a", "contacts_1b", "contacts_2a"]
+    "people": ["people_1a", "people_1b", "people_1c", "people_2a", "people_all"],
+    "contacts": ["contacts_1a", "contacts_1b", "contacts_2a", "contacts_all"],
+    "arrears_actions": ["arrears_actions_1a", "arrears_actions_1c", "arrears_actions_2a"],
 }
 
 partition_keys = ["import_date"]

scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py

@@ -19,6 +19,7 @@
 import scripts.jobs.housing.housing_nec_migration_tenancies_data_load_gx_suite
 import scripts.jobs.housing.housing_nec_migration_people_data_load_gx_suite
 import scripts.jobs.housing.housing_nec_migration_contacts_data_load_gx_suite
+import scripts.jobs.housing.housing_nec_migration_arrears_actions_data_load_gx_suite
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

scripts/jobs/housing/housing_nec_migration_arrears_actions_data_load_gx_suite.py

@@ -0,0 +1,143 @@
+# flake8: noqa: F821
+
+import sys
+
+from awsglue.utils import getResolvedOptions
+import great_expectations as gx
+import great_expectations.expectations as gxe
+
+
+class ExpectPayRefColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull):
+    column: str = "LACA_PAY_REF"
+    description: str = (
+        "Expect LACA_PAY_REF (pay ref) values to not be Null in contacts load"
+    )
+
+
+class ExpectValueColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull):
+    column: str = "LACA_CREATED_DATE"
+    description: str = "Expect LCDE_CONTACT_VALUE (contact value) to not be Null"
+
+
+class ExpectArrearCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
+    column: str = "LACA_ARA_CODE"
+    value_set: list = [
+        "BAGF",
+        "BAG1",
+        "RTRN",
+        "FRA2",
+        "ADVR",
+        "RDCN",
+        "DWPR",
+        "DWPN",
+        "EDCL",
+        "NOTE",
+        "RHCN",
+        "CORT",
+        "IUPO",
+        "IRA1",
+        "NTQ",
+        "CON3",
+        "RREQ",
+        "RRFN",
+        "STAT",
+        "SHB",
+        "SNW",
+        "ARRN",
+        "POP",
+        "FRA1",
+        "BCOL",
+        "DWPC",
+        "DWPT",
+        "COUT",
+        "NFA",
+        "NRA2",
+        "FRET",
+        "SCH",
+        "SNP",
+        "VISN",
+        "WOA",
+        "WOC",
+        "WOH",
+        "WON",
+        "CDAT",
+        "CNOK",
+        "ADVC",
+        "EVIC",
+        "FINC",
+        "HBN",
+        "SRA1",
+        "TRA1",
+        "NRA1",
+        "IRA1",
+        "MRA1",
+        "TELO",
+        "RPAN",
+        "RRHB",
+        "RRF",
+        "SAR",
+        "SBA",
+        "SCM",
+        "SSA",
+        "VISI",
+        "WOF",
+        "RCHN",
+        "RDDN",
+        "CDL",
+        "FINI",
+        "GRA1",
+        "AGRL",
+        "SRA2",
+        "TRA2",
+        "NRA2",
+        "IRA2",
+        "MRA2",
+        "CWAL",
+        "TELI",
+        "RELI",
+        "LREF",
+        "NOSP",
+        "INTV",
+        "SUP",
+        "UCC"
+    ]
+    description: str = "Expect arrear code to be one of the set"
+
+
+class ExpectArrearsActionsColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList):
+    column_list = [
+        "LACA_BALANCE",
+        "LACA_PAY_REF",
+        "LACA_TYPE",
+        "LACA_CREATED_BY",
+        "LACA_CREATED_DATE",
+        "LACA_ARREARS_DISPUTE_IND",
+        "LACA_ARA_CODE",
+        "LACA_STATUS",
+        "LACA_HRV_ADL_CODE",
+        "LACA_EAC_EPO_CODE",
+        "LACA_EFFECTIVE_DATE",
+        "LACA_EXPIRY_DATE",
+        "LACA_NEXT_ACTION_DATE",
+        "LACA_AUTH_DATE",
+        "LACA_AUTH_USERNAME",
+        "LACA_PRINT_DATE",
+        "LACA_DEL_"
+    ]
+    description: str = "Expect columns to match ordered list exactly"
+
+
+arg_key = ["s3_target_location"]
+args = getResolvedOptions(sys.argv, arg_key)
+locals().update(args)
+
+# add to GX context
+context = gx.get_context(mode="file", project_root_dir=s3_target_location)
+
+suite = gx.ExpectationSuite(name="arrears_actions_data_load_suite")
+
+suite.add_expectation(ExpectArrearsActionsColumnsToMatchOrderedList())
+suite.add_expectation(ExpectArrearCodeToBeInSet())
+suite.add_expectation(ExpectPayRefColumnValuesToNotBeNull())
+suite.add_expectation(ExpectValueColumnValuesToNotBeNull())
+suite = context.suites.add(suite)

scripts/jobs/housing/housing_nec_migration_contacts_data_load_gx_suite.py

@@ -9,7 +9,9 @@
 
 class ExpectPersonRefColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull):
     column: str = "LCDE_LEGACY_REF"
-    description: str = "Expect LCDE_LEGACY_REF (person ref) values to not be Null in contacts load"
+    description: str = (
+        "Expect LCDE_LEGACY_REF (person ref) values to not be Null in contacts load"
+    )
 
 
 class ExpectValueColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull):
@@ -25,21 +27,20 @@ class ExpectContactTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
 
 class ExpectContactsColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList):
     column_list = [
-        "LCDE_START_DATE",
-        "LCDE_PRIMARY_REF",
-        "LCDE_SECONDARY_REF",
-        "LCDE_PRECEDENCE",
-        "LCDE_LEGACY_TYPE",
         "LCDE_LEGACY_REF",
-        "LCDE_FRV_COMM_PREF_CODE",
-        "LCDE_FRV_CME_CODE",
-        "LCDE_END_DATE",
+        "LCDE_LEGACY_TYPE",
+        "LCDE_START_DATE",
         "LCDE_CREATED_DATE",
         "LCDE_CREATED_BY",
         "LCDE_CONTACT_VALUE",
+        "LCDE_FRV_CME_CODE",
         "LCDE_CONTACT_NAME",
-        "LCDE_COMMENTS",
+        "LCDE_END_DATE",
+        "LCDE_PRECEDENCE",
+        "LCDE_FRV_COMM_PREF_CODE",
         "LCDE_ALLOW_TEXTS",
+        "LCDE_SECONDARY_REF",
+        "LCDE_COMMENTS",
     ]
     description: str = "Expect columns to match ordered list exactly"
 

scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py

@@ -96,6 +96,7 @@ class ExpectTenancyColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrdere
         "LTCY_RTB_WITHDRAWN_DATE",
         "LTCY_RTB_APP_EXPECTED_END_DATE",
         "LTCY_HRV_TST_CODE",
+        "LTCY_HRV_TTR_CODE",
         "LTCY_HRV_TNR_CODE",
         "LTCY_HRV_RHR_CODE",
         "LTCY_HRV_RWR_CODE",