diff --git a/scripts/helpers/housing_gx_dq_inputs.py b/scripts/helpers/housing_gx_dq_inputs.py index 5f5b5cbc1..e0811b8e2 100644 --- a/scripts/helpers/housing_gx_dq_inputs.py +++ b/scripts/helpers/housing_gx_dq_inputs.py @@ -1,21 +1,93 @@ sql_config = {'person_reshape': { 'sql': """SELECT *, substr(startdate, 1, 10) as startdate_parsed, substr(enddate, 1, 10) as enddate_parsed, - substr(dateofbirth, 1, 10) as dateofbirth_parsed FROM "housing-refined-zone"."person_reshape" WHERE import_date = (SELECT max(import_date) FROM "housing-refined-zone"."person_reshape") AND enddate IS NULL AND type IN ('Secure', 'Introductory')""", +substr(dateofbirth, 1, 10) as dateofbirth_parsed FROM "housing-refined-zone"."person_reshape" WHERE import_date = (SELECT max(import_date) FROM "housing-refined-zone"."person_reshape") AND enddate IS NULL AND type IN ('Secure', 'Introductory')""", 'id_field': 'person_id'}, 'tenure_reshape': { 'sql': """SELECT * FROM "housing-refined-zone"."tenure_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."tenure_reshape") and description in ('Secure', 'Introductory', 'Mesne Profit Ac', 'Non-Secure') and (endoftenuredate is null or substr(endoftenuredate, 1, 11) = '1900-01-01')""", 'id_field': 'tenancy_id'}, 'contacts_reshape': { - 'sql': """SELECT id, targetid, substr(createdat, 1, 10) as createdat, contacttype, subtype, value, substr(lastmodified, 1, 10) as lastmodified, targettype, isactive, person_id, import_date FROM "housing-refined-zone"."contacts_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."contacts_reshape") and isactive=True""", + 'sql': """SELECT + DATE_FORMAT(from_iso8601_timestamp(createdat), '%Y-%b-%d') AS createdat, + contacttype, + subtype, + cast(value as varchar) as contact_value, + DATE_FORMAT(from_iso8601_timestamp(lastmodified), '%Y-%b-%d') AS lastmodified, + targettype, + person_id, + import_year, + import_month, + import_day, + import_date + FROM "housing-refined-zone"."contacts_reshape" + where targettype = 'person' + and import_date = (select max(import_date) from "housing-refined-zone"."contacts_reshape")""", 'id_field': 'id'}, 'housing_homeowner_record_sheet': { 'sql': """SELECT * FROM "housing-raw-zone"."housing_homeowner_record_sheet" where import_date=(select max(import_date) from "housing-raw-zone"."housing_homeowner_record_sheet")""", 'id_field': 'property_no'}, 'housing_dwellings_list': { - 'sql': """SELECT * FROM "housing-raw-zone"."housing_dwellings_list" where import_date=(select max(import_date) from "housing-raw-zone"."housing_homeowner_record_sheet")""", + 'sql': """SELECT + cast(property_dwelling_reference_number as varchar) as property_dwelling_reference_number, + cast(llpg as varchar) as llpg, + cast(estate_reference_number as varchar) as estate_reference_number, + estate_name, + block_name_as_per_naming_convention_procedure as block_name, + cast(block_reference_number as varchar) as block_reference_number, + sub_block_name, + cast(sub_block_reference as varchar) as sub_block_reference, + block_total_stories, + block_typology_based_on_dwelling_type_defined_list as block_typology, + building_heights_h1, + communal_heat_network, + dwelling_front_door_access_floor_level, + dwelling_no_of_bedrooms, + fra_block, + no_of_floors_within_dwelling, + registered_building, + year_built_block + FROM "housing-raw-zone"."housing_dwellings_list" where import_date = (select max(import_date) from "housing-raw-zone"."housing_dwellings_list")""", 'id_field': 'property_dwelling_reference_number'}, 'assets_reshape': { - 'sql': """SELECT * FROM "housing-refined-zone"."assets_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."assets_reshape") and assettype = 'Dwelling'""", + 'sql': """select + cast(assetid as varchar) as assetid, + assettype, + cast(uprn as varchar) as uprn, + owner as property_owner, + agent, + iscouncilproperty, + istmomanaged, + managingorganisation, + cast(managingorganisationid as varchar) as managingorganisationid, + totalblockfloors, + floorno, + numberoflifts, + numberoflivingrooms, + numberofkitchens, + numberofbedrooms, + numberofbedspaces, + numberofshowers, + numberoffloors, + numberofdoublebeds, + numberofsinglebeds, + numberofbathrooms, + yearconstructed, + hascommunalareas, + hasprivatekitchen, + hasprivatebathroom, + hasrampaccess, + isstepfree, + hasstairs, + heating, + numberofcots, + windowtype, + estate_name, + cast(estate_id as varchar) as estate_id, + import_year, + import_month, + import_day, + import_date + FROM "housing-refined-zone"."assets_reshape" where import_date = (select max(import_date) from "housing-refined-zone"."assets_reshape") + and assettype in ('BoilerHouse','BoosterPump','CleanersFacilities','CombinedHeatAndPowerUnit','CommunityHall','Concierge','Dwelling','LettableNonDwelling','Lift','NA','NBD','OutBuilding','TravellerSite')""", 'id_field': 'asset_id'}, 'matenancyagreement': { 'sql': """SELECT *, substr(cast(eot as varchar), 1, 10) as eot_parsed, substr(cast(cot as varchar), 1, 10) as cot_parsed FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement" where import_date=(select max(import_date) FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement")""",