diff --git a/scripts/helpers/housing_nec_migration_gx_dq_inputs.py b/scripts/helpers/housing_nec_migration_gx_dq_inputs.py index 22a9ecb34..1dc63a6ab 100644 --- a/scripts/helpers/housing_nec_migration_gx_dq_inputs.py +++ b/scripts/helpers/housing_nec_migration_gx_dq_inputs.py @@ -1,6 +1,9 @@ -sql_config = {"properties": {"id_field": "LPRO_PROPREF"}} +sql_config = { + "properties": {"id_field": "LPRO_PROPREF"}, + "tenancies": {"id_field": "LTCY_ALT_REF"}, +} -data_load_list = ["properties"] +data_load_list = ["properties", "tenancies"] table_list = { "properties": [ @@ -11,8 +14,10 @@ "properties_1e", "properties_2a", "properties_4a", - "properties_4c", - ] + "properties_4b", + "properties_4c" + ], + "tenancies": ["tenancies_1a"], } partition_keys = ["import_date"] diff --git a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py index 6bca3cb83..4515387fa 100644 --- a/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py +++ b/scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py @@ -16,6 +16,8 @@ table_list, ) import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite +import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py b/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py new file mode 100644 index 000000000..deb92e62a --- /dev/null +++ b/scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py @@ -0,0 +1,91 @@ +# flake8: noqa: F821 + +import sys + +from awsglue.utils import getResolvedOptions +import great_expectations as gx +import great_expectations.expectations as gxe + + +class ExpectTagRefColumnValuesToBeUnique(gxe.ExpectColumnValuesToBeUnique): + column: str = "LTCY_ALT_REF" + description: str = "Expect LTCY_ALT_REF (tenancy ref) values to be unique" + + +class ExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): + column: str = "LTCY_TTY_CODE" + value_set: list = [ + "ASH", + "ASY", + "Demoted", + "FRS", + "HAL", + "INT", + "LEA", + "LTA", + "LHS", + "MPA", + "PVG", + "SPS", + "RTM", + "SEC", + "SSE", + "SHO", + "SLL", + "TLA", + "TBB", + "TBBFam", + "DEC", + "THGF", + "THO", + "THL", + "TPL", + "TRA", + "TACCFLAT", + "TGA", + "UNDER18", + "NONSECTA", + "NONSECHR", + "OFFICESE", + "LIVINGRT", + "FRE", + ] + description: str = "Expect tenancy type code to contain one of the set" + + +class ExpectTenureTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): + column: str = "LTCY_HRV_TTYP_CODE" + value_set: list = [ + "Secure", + "NonSec", + "NonRes", + "Leasehold", + "Temporary", + "Freehold", + "Commercial", + "LivingRent" + ] + description: str = "Expect tenure type code to be one of the set" + + +class ExpectTenancyStatusCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet): + column: str = "LTCY_HRV_TST_CODE" + value_set: list = ["Notice", "Decant", "UnautOcc"] + description: str = "Expect tenancy status code to be one of the set" + + + +arg_key = ["s3_target_location"] +args = getResolvedOptions(sys.argv, arg_key) +locals().update(args) + +# add to GX context +context = gx.get_context(mode="file", project_root_dir=s3_target_location) + +suite = gx.ExpectationSuite(name="tenancies_data_load_suite") + +suite.add_expectation(ExpectTagRefColumnValuesToBeUnique()) +suite.add_expectation(ExpectTenancyTypeCodeToBeInSet()) +suite.add_expectation(ExpectTenureTypeCodeToBeInSet()) +suite.add_expectation(ExpectTenancyStatusCodeToBeInSet()) +suite = context.suites.add(suite)