diff --git a/scripts/helpers/housing_nec_migration_gx_dq_inputs.py b/scripts/helpers/housing_nec_migration_gx_dq_inputs.py index ad3d9f300..a941e4780 100644 --- a/scripts/helpers/housing_nec_migration_gx_dq_inputs.py +++ b/scripts/helpers/housing_nec_migration_gx_dq_inputs.py @@ -1,18 +1,18 @@ sql_config = { - "properties": {"id_field": "LPRO_PROPREF"}, - "tenancies": {"id_field": "LTCY_ALT_REF"}, - "people": {"id_field": "LPAR_PER_ALT_REF"}, - "contacts": {"id_field": "LCDE_LEGACY_REF"}, - "arrears_actions": {"id_field": "LACA_PAY_REF"}, - "revenue_accounts": {"id_field": "LRAC_PAY_REF"}, - "transactions": {"id_field": "LTRN_ALT_REF"}, - "addresses": {"id_field": "LAUS_LEGACY_REF"}, + "properties": {"id_field": "lpro_propref"}, + "tenancies": {"id_field": "ltcy_alt_ref"}, + "people": {"id_field": "lpar_per_alt_ref"}, + "contacts": {"id_field": "lcde_legacy_ref"}, + "arrears_actions": {"id_field": "laca_pay_ref"}, + "revenue_accounts": {"id_field": "lrac_pay_ref"}, + "transactions": {"id_field": "ltrn_alt_ref"}, + "addresses": {"id_field": "laus_legacy_ref"}, } data_load_list = [ - "properties" - # "tenancies", - # "people", + "properties", + "tenancies", + "people" # "contacts", # "arrears_actions", # "revenue_accounts", @@ -33,21 +33,21 @@ # "properties_4b", # "properties_4c", # "properties_7a", - "full_full_hem_pro_all_dq", + "full_dq_full_dq_hem_pro_all_dq", ], "tenancies": [ # "tenancies_1a", # "tenancies_1c", # "tenancies_2a", # "tenancies_other", - "tenancies_all", + "full_dq_full_dq_hem_tcy_all_dq", ], "people": [ # "people_1a", # "people_1b", # "people_1c", # "people_2a", - "people_all" + "full_dq_full_dq_hem_per_all_dq" ], "contacts": [ # "contacts_1a", diff --git a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py index 04d43054a..5cc144690 100644 --- a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py +++ b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py @@ -16,8 +16,9 @@ table_list, ) import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite -# import scripts.jobs.housing.housing_nec_migration_tenancies_data_load_gx_suite -# import scripts.jobs.housing.housing_nec_migration_people_data_load_gx_suite +import scripts.jobs.housing.housing_nec_migration_tenancies_data_load_gx_suite +import scripts.jobs.housing.housing_nec_migration_people_data_load_gx_suite + # import scripts.jobs.housing.housing_nec_migration_contacts_data_load_gx_suite # import scripts.jobs.housing.housing_nec_migration_arrears_actions_data_load_gx_suite # import scripts.jobs.housing.housing_nec_migration_revenue_accounts_data_load_gx_suite @@ -162,7 +163,7 @@ def main(): ) ] - table_results_df["unexpected_id_list"] = pd.Series(dtype="object") + table_results_df["unexpected_id_list"] = pd.Series(dtype="string") for i, row in query_df.iterrows(): try: diff --git a/scripts/jobs/housing/housing_nec_migration_people_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_people_data_load_gx_suite.py index 0bedec7fd..d4ee229c6 100644 --- a/scripts/jobs/housing/housing_nec_migration_people_data_load_gx_suite.py +++ b/scripts/jobs/housing/housing_nec_migration_people_data_load_gx_suite.py @@ -8,17 +8,17 @@ class PeopleExpectPersonRefColumnValuesToBeUnique(gxe.ExpectColumnValuesToBeUnique): - column: str = "LPAR_PER_ALT_REF" - description: str = "Expect LPAR_PER_ALT_REF (person ref) values to be unique" + column: str = "lpar_per_alt_ref" + description: str = "Expect lpar_per_alt_ref (person ref) values to be unique" class PeopleExpectPersonRefColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull): - column: str = "LPAR_PER_ALT_REF" - description: str = "Expect LPAR_PER_ALT_REF (person ref) values to not be Null" + column: str = "lpar_per_alt_ref" + description: str = "Expect lpar_per_alt_ref (person ref) values to not be Null" class PeopleExpectTitleToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPAR_PER_TITLE" + column: str = "lpar_per_title" value_set: list = [ "DAME", "DR", @@ -39,37 +39,74 @@ class PeopleExpectTitleToBeInSet(gxe.ExpectColumnValuesToBeInSet): description: str = "Expect title to be one of the set" -class PeopleExpectPeopleColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList): +class PeopleExpectPeopleColumnsToMatchOrderedList( + gxe.ExpectTableColumnsToMatchOrderedList +): column_list = [ - "LPAR_HOP_START_DATE", - "LPAR_PER_SURNAME", - "LPAR_TCY_ALT_REF", - "LPAR_PER_FORENAME", - "LPAR_HOP_HPSR_CODE", - "LPAR_PER_TITLE", - "LPAR_PER_INITIALS", - "LPAR_PER_DATE_OF_BIRTH", - "LPAR_PER_HOU_DISABLED_IND", - "LPAR_PER_HOU_OAP_IND", - "LPAR_PER_FRV_FGE_CODE", - "LPAR_HOP_HRV_REL_CODE", - "LPAR_PER_HOU_EMPLOYER", - "LPAR_PER_HOU_HRV_HMS_CODE", - "LPAR_PHONE", - "LPAR_HOP_END_DATE", - "LPAR_HOP_HPER_CODE", - "LPAR_TCY_IND", - "LPAR_TIN_MAIN_TENANT_IND", - "LPAR_TIN_START_DATE", - "LPAR_TIN_END_DATE", - "PAR_TIN_HRV_TIR_CODE", - "LPAR_TIN_STAT_SUCCESSOR_IND", - "LPAR_PER_ALT_REF", - "LPAR_PER_FRV_FEO_CODE", - "LPAR_PER_NI_NO", - "LPAR_PER_FRV_HGO_CODE", - "LPAR_PER_FRV_FNL_CODE", - "LPAR_PER_OTHER_NAME" + "lpar_per_surname", + "lpar_per_forename", + "lpar_hop_start_date", + "lpar_tcy_alt_ref", + "lpar_hop_hpsr_code", + "lpar_per_title", + "lpar_per_initials", + "lpar_per_date_of_birth", + "lpar_per_hou_disabled_ind", + "lpar_per_hou_oap_ind", + "lpar_per_frv_fge_code", + "lpar_hop_hrv_rel_code", + "lpar_per_hou_employer", + "lpar_per_hou_hrv_hms_code", + "lpar_phone", + "lpar_hop_end_date", + "lpar_hop_hper_code", + "lpar_tcy_ind", + "lpar_tin_main_tenant_ind", + "lpar_tin_start_date", + "lpar_tin_end_date", + "lpar_tin_hrv_tir_code", + "lpar_tin_stat_successor_ind", + "lpar_per_alt_ref", + "lpar_per_frv_feo_code", + "lpar_per_ni_no", + "lpar_per_frv_hgo_code", + "lpar_per_frv_fnl_code", + "lpar_per_other_name", + "lpar_per_hou_surname_prefix", + "lpar_hou_legacy_ref", + "lpar_ipp_shortname", + "lpar_ipp_placement_ind", + "lpar_ipp_current_ind", + "lpar_ipp_ipt_code", + "lpar_ipp_usr_username", + "lpar_ipp_spr_printer_name", + "lpar_ipp_comments", + "lpar_ipp_vca_code", + "lpar_ipu_aun_code", + "lpar_ipp_staff_id", + "lpar_ipp_cos_code", + "lpar_ipp_hrv_fit_code", + "lpar_type", + "lpar_org_sort_code", + "lpar_org_name", + "lpar_org_short_name", + "lpar_org_frv_oty_code", + "lpar_per_hou_at_risk_ind", + "lpar_per_hou_hrv_ntly_code", + "lpar_per_hou_hrv_sexo_code", + "lpar_per_hou_hrv_rlgn_code", + "lpar_per_hou_hrv_ecst_code", + "lpar_org_current_ind", + "lpar_hop_head_hhold_ind", + "lpar_hhold_group_no", + "lpar_created_date", + "lpar_created_by", + "lpar_modified_date", + "lpar_modified_by", + "lpar_per_hou_end_date", + "lpar_per_hou_hrv_hpe_code", + "lpar_org_dup", + "tranche", ] description: str = "Expect people load columns to match ordered list exactly" diff --git a/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py index 5d8440a83..d04f26a6b 100644 --- a/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py +++ b/scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py @@ -8,29 +8,31 @@ class PropertiesExpectPropRefColumnValuesToBeUnique(gxe.ExpectColumnValuesToBeUnique): - column: str = "LPRO_PROPREF" - description: str = "Expect UPRN (LPRO_PROPREF) values to be unique" + column: str = "lpro_propref" + description: str = "Expect UPRN (lpro_propref) values to be unique" class PropertiesExpectPropRefColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull): - column: str = "LPRO_PROPREF" - description: str = "Expect LPRO_PROPREF (prop ref) values to not be Null" + column: str = "lpro_propref" + description: str = "Expect lpro_propref (prop ref) values to not be Null" class PropertiesExpectOccStatusCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPRO_SCO_CODE" + column: str = "lpro_sco_code" value_set: list = ["OCC", "VOI", "CLO"] - description: str = "Expect status codes to be one of the set" + description: str = "Expect status codes (lpro_sco_code) to be one of the set" class PropertiesExpectOrgIndicatorToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPRO_ORGANISATION_IND" + column: str = "lpro_organisation_ind" value_set: list = ["Y", "N"] - description: str = "Expect organisation indicator to be one of the set" + description: str = ( + "Expect organisation indicator (lpro_organisation_ind) to be one of the set" + ) class PropertiesExpectOwnTypeToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPRO_HOU_HRV_HOT_CODE" + column: str = "lpro_hou_hrv_hot_code" value_set: list = [ "ASSOC", "COUN", @@ -40,11 +42,13 @@ class PropertiesExpectOwnTypeToBeInSet(gxe.ExpectColumnValuesToBeInSet): "LEASLOUT", "PRIVATE", ] - description: str = "Expect ownership type code to be one of the set" + description: str = ( + "Expect ownership type code (lpro_hou_hrv_hot_code) to be one of the set" + ) class PropertiesExpectPropSourceToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPRO_HOU_HRV_HRS_CODE" + column: str = "lpro_hou_hrv_hrs_code" value_set: list = [ "LEASED", "NEWBUILD", @@ -52,80 +56,89 @@ class PropertiesExpectPropSourceToBeInSet(gxe.ExpectColumnValuesToBeInSet): "STOCKTRANS", "BUYBACK", "ACQUIRED", + "ENFRAN", ] - description: str = "Expect property source code to be one of the set" + description: str = ( + "Expect property source code (lpro_hou_hrv_hrs_code) to be one of the set" + ) class PropertiesExpectResIndicatorToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPRO_HOU_RESIDENTIAL_IND" + column: str = "lpro_hou_residential_ind" value_set: list = ["Y", "N"] - description: str = "Expect resdidential indicator to be one of the set" + description: str = ( + "Expect residential indicator (lpro_hou_residential_ind) to be one of the set" + ) class PropertiesExpectPropTypeValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LPRO_HOU_PTV_CODE" + column: str = "lpro_hou_ptv_code" value_set: list = [ - "CMC", - "CMC", - "GAR", - "FLT", - "HOU", - "MAI", + "BOI", "BUN", - "TRV", - "STD", - "ROM", + "CMC", + "CMF", "COM", - "PSP", - "PRA", + "CON", "CYC", "DUP", - "BOI", - "CMC", - "CON", + "FLT", + "GAR", + "HOU", "LFT", - "PLY" + "MAI", + "PLY", + "PRA", + "PSP", + "ROM", + "STD", + "TRV", ] - description: str = "Expect property type values to be one of the set" + description: str = ( + "Expect property type values (lpro_hou_ptv_code) to be one of the set" + ) -class PropertiesExpectPropColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList): +class PropertiesExpectPropColumnsToMatchOrderedList( + gxe.ExpectTableColumnsToMatchOrderedList +): column_list = [ - "LPRO_PROPREF", - "LPRO_HOU_FRB", - "LPRO_SCO_CODE", - "LPRO_ORGANISATION_IND", - "LPRO_HOU_HRV_HOT_CODE", - "LPRO_HOU_HRV_HRS_CODE", - "LPRO_HOU_HRV_HBU_CODE", - "LPRO_HOU_HRV_HLT_CODE", - "LPRO_PARENT_PROPREF", - "LPRO_HOU_SALE_DATE", - "LPRO_SERVICE_PROP_IND", - "LPRO_HOU_ACQUIRED_DATE", - "LPRO_HOU_DEFECTS_IND", - "LPRO_HOU_RESIDENTIAL_IND", - "LPRO_HOU_ALT_REF", - "LPRO_HOU_LEASE_START_DATE", - "LPRO_HOU_LEASE_REVIEW_DATE", - "LPRO_HOU_CONSTRUCTION_DATE", - "LPRO_HOU_PTV_CODE", - "LPRO_HOU_HRV_PST_CODE", - "LPRO_HOU_HRV_HMT_CODE", - "LPRO_HOU_MANAGEMENT_END_DATE", - "LPRO_FREE_REFNO", - "LPRO_FREE_NAME", - "LPRO_PROP_STATUS", - "LPRO_STATUS_START", - "LPRO_HOU_ALLOW_PLACEMENT_IND", - "LPRO_HOU_DEBIT_TO_DATE", - "LPRO_ON_DEBIT_START_DATE", - "LPRO_PHONE", - "LPRO_AGENT_PAR_REFNO", - "LPRO_PLD_COMMENTS", - "LPRO_REFNO", + "lpro_propref", + "lpro_hou_frb", + "lpro_sco_code", + "lpro_organisation_ind", + "lpro_hou_hrv_hot_code", + "lpro_hou_hrv_hrs_code", + "lpro_hou_hrv_hbu_code", + "lpro_hou_hrv_hlt_code", + "lpro_parent_propref", + "lpro_hou_sale_date", + "lpro_service_prop_ind", + "lpro_hou_acquired_date", + "lpro_hou_defects_ind", + "lpro_hou_residential_ind", + "lpro_hou_alt_ref", + "lpro_hou_lease_start_date", + "lpro_hou_lease_review_date", + "lpro_hou_construction_date", + "lpro_hou_ptv_code", + "lpro_hou_hrv_pst_code", + "lpro_hou_hrv_hmt_code", + "lpro_hou_management_end_date", + "lpro_free_refno", + "lpro_free_name", + "lpro_prop_status", + "lpro_status_start", + "lpro_hou_allow_placement_ind", + "lpro_hou_debit_to_date", + "lpro_on_debit_start_date", + "lpro_phone", + "lpro_agent_par_refno", + "lpro_pld_comments", + "lpro_refno", + "tranche", ] - description: str = "Expect columns to match ordered list exactly" + description: str = "Expect columns to match ordered list exactly; tranche at end" arg_key = ["s3_target_location"] @@ -146,4 +159,4 @@ class PropertiesExpectPropColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatc suite.add_expectation(PropertiesExpectPropTypeValuesToBeInSet()) suite.add_expectation(PropertiesExpectPropColumnsToMatchOrderedList()) suite.add_expectation(PropertiesExpectPropRefColumnValuesToNotBeNull()) -suite = context.suites.add(suite) +suite = context.suites.add(suite) \ No newline at end of file diff --git a/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py index 8420e0a9f..ac81fdd83 100644 --- a/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py +++ b/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py @@ -8,17 +8,17 @@ class TenanciesExpectTagRefColumnValuesToBeUnique(gxe.ExpectColumnValuesToBeUnique): - column: str = "LTCY_ALT_REF" - description: str = "Expect LTCY_ALT_REF (tenancy ref) values to be unique" + column: str = "ltcy_alt_ref" + description: str = "Expect ltcy_alt_ref (tenancy ref) values to be unique" class TenanciesExpectTenancyRefColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull): - column: str = "LTCY_ALT_REF" - description: str = "Expect LTCY_ALT_REF (tenancy ref) values to not be Null" + column: str = "ltcy_alt_ref" + description: str = "Expect ltcy_alt_ref (tenancy ref) values to not be Null" class TenanciesExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LTCY_TTY_CODE" + column: str = "ltcy_tty_code" value_set: list = [ "ASH", "ASY", @@ -58,7 +58,7 @@ class TenanciesExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): class TenanciesExpectTenureTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LTCY_HRV_TTYP_CODE" + column: str = "ltcy_hrv_ttyp_code" value_set: list = [ "SECURE", "NONSEC", @@ -73,62 +73,63 @@ class TenanciesExpectTenureTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): class TenanciesExpectTenancyStatusCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): - column: str = "LTCY_HRV_TST_CODE" + column: str = "ltcy_hrv_tst_code" value_set: list = ["DECANT", "NOTICE", "UNAUTHOCC"] - description: str = "Expect tenancy status code to be one of the set" + description: str = "Expect tenancy status code (ltcy_hrv_tst_code) to be one of the set" class TenanciesExpectTenancyColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList): column_list = [ - "LTCY_ALT_REF", - "LTCY_TTY_CODE", - "LTCY_ACT_START_DATE", - "LTCY_CORRESPOND_NAME", - "LTCY_HRV_TTYP_CODE", - "LTCY_HRV_TSO_CODE", - "LTCY_ACT_END_DATE", - "LTCY_NOTICE_GIVEN_DATE", - "LTCY_NOTICE_REC_DATE", - "LTCY_EXPECTED_END_DATE", - "LTCY_RTB_RECEIVED_DATE", - "LTCY_RTB_ADMITTED_DATE", - "LTCY_RTB_HELD_DATE", - "LTCY_RTB_WITHDRAWN_DATE", - "LTCY_RTB_APP_EXPECTED_END_DATE", - "LTCY_HRV_TST_CODE", - "LTCY_HRV_TTR_CODE", - "LTCY_HRV_TNR_CODE", - "LTCY_HRV_RHR_CODE", - "LTCY_HRV_RWR_CODE", - "LTCY_RTB_APP_REFERENCE", - "LTCY_THO_PROPREF1", - "LTCY_THO_START_DATE1", - "LTCY_THO_END_DATE1", - "LTCY_THO_TTR_CODE1", - "LTCY_THO_PROPREF2", - "LTCY_THO_START_DATE2", - "LTCY_THO_END_DATE2", - "LTCY_THO_HRV_TTR_CODE2", - "LTCY_THO_PROPREF3", - "LTCY_THO_START_DATE3", - "LTCY_THO_END_DATE3", - "LTCY_THO_HRV_TTR_CODE3", - "LTCY_THO_PROPREF4", - "LTCY_THO_START_DATE4", - "LTCY_THO_END_DATE4", - "LTCY_THO_HRV_TTR_CODE4", - "LTCY_THO_PROPREF5", - "LTCY_THO_START_DATE5", - "LTCY_THO_END_DATE5", - "LTCY_THO_HRV_TTR_CODE5", - "LTCY_THO_PROPREF6", - "LTCY_THO_START_DATE6", - "LTCY_THO_END_DATE6", - "LTCY_THO_HRV_TTR_CODE6", - "LTCY_PHONE", - "LTCY_REVIEW_DATE", + 'ltcy_act_end_date', + 'ltcy_act_start_date', + 'ltcy_alt_ref', + 'ltcy_correspond_name', + 'ltcy_expected_end_date', + 'ltcy_hrv_rhr_code', + 'ltcy_hrv_rwr_code', + 'ltcy_hrv_tnr_code', + 'ltcy_hrv_tso_code', + 'ltcy_hrv_tst_code', + 'ltcy_hrv_ttr_code', + 'ltcy_hrv_ttyp_code', + 'ltcy_notice_given_date', + 'ltcy_notice_rec_date', + 'ltcy_phone', + 'ltcy_review_date', + 'ltcy_rtb_admitted_date', + 'ltcy_rtb_app_expected_end_date', + 'ltcy_rtb_app_reference', + 'ltcy_rtb_held_date', + 'ltcy_rtb_received_date', + 'ltcy_rtb_withdrawn_date', + 'ltcy_tho_end_date1', + 'ltcy_tho_end_date2', + 'ltcy_tho_end_date3', + 'ltcy_tho_end_date4', + 'ltcy_tho_end_date5', + 'ltcy_tho_end_date6', + 'ltcy_tho_hrv_ttr_code2', + 'ltcy_tho_hrv_ttr_code3', + 'ltcy_tho_hrv_ttr_code4', + 'ltcy_tho_hrv_ttr_code5', + 'ltcy_tho_hrv_ttr_code6', + 'ltcy_tho_propref1', + 'ltcy_tho_propref2', + 'ltcy_tho_propref3', + 'ltcy_tho_propref4', + 'ltcy_tho_propref5', + 'ltcy_tho_propref6', + 'ltcy_tho_start_date1', + 'ltcy_tho_start_date2', + 'ltcy_tho_start_date3', + 'ltcy_tho_start_date4', + 'ltcy_tho_start_date5', + 'ltcy_tho_start_date6', + 'ltcy_tho_ttr_code1', + 'ltcy_tty_code', + 'tranche' ] - description: str = "Expect tenancy load columns to match ordered list exactly" + description: str = "Expect tenancy load columns to match ordered list exactly; tranche at end" arg_key = ["s3_target_location"] @@ -147,4 +148,4 @@ class TenanciesExpectTenancyColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMa suite.add_expectation(TenanciesExpectTenancyColumnsToMatchOrderedList()) suite.add_expectation(TenanciesExpectTenancyRefColumnValuesToNotBeNull()) -context.suites.add(suite) +context.suites.add(suite) \ No newline at end of file