Amend sql queries to only include ke columns and rows

scripts/helpers/housing_gx_dq_inputs.py

@@ -1,21 +1,93 @@
 sql_config = {'person_reshape': {
     'sql': """SELECT *, substr(startdate, 1, 10) as startdate_parsed, substr(enddate, 1, 10) as enddate_parsed,
-    substr(dateofbirth, 1, 10) as dateofbirth_parsed FROM "housing-refined-zone"."person_reshape" WHERE import_date = (SELECT max(import_date) FROM "housing-refined-zone"."person_reshape") AND enddate IS NULL AND type IN ('Secure', 'Introductory')""",
+substr(dateofbirth, 1, 10) as dateofbirth_parsed FROM "housing-refined-zone"."person_reshape" WHERE import_date = (SELECT max(import_date) FROM "housing-refined-zone"."person_reshape") AND enddate IS NULL AND type IN ('Secure', 'Introductory')""",
     'id_field': 'person_id'},
     'tenure_reshape': {
         'sql': """SELECT * FROM "housing-refined-zone"."tenure_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."tenure_reshape") and description in ('Secure', 'Introductory', 'Mesne Profit Ac', 'Non-Secure') and (endoftenuredate is null or substr(endoftenuredate, 1, 11) = '1900-01-01')""",
         'id_field': 'tenancy_id'},
     'contacts_reshape': {
-        'sql': """SELECT id, targetid, substr(createdat, 1, 10) as createdat, contacttype, subtype, value, substr(lastmodified, 1, 10) as lastmodified, targettype, isactive, person_id, import_date  FROM "housing-refined-zone"."contacts_reshape"  where import_date=(select max(import_date) from "housing-refined-zone"."contacts_reshape") and isactive=True""",
+        'sql': """SELECT
+            DATE_FORMAT(from_iso8601_timestamp(createdat), '%Y-%b-%d') AS createdat,
+            contacttype,
+            subtype,
+            cast(value as varchar) as contact_value,
+            DATE_FORMAT(from_iso8601_timestamp(lastmodified), '%Y-%b-%d') AS lastmodified,
+            targettype,
+            person_id,
+            import_year,
+            import_month,
+            import_day,
+            import_date
+            FROM "housing-refined-zone"."contacts_reshape"
+            where targettype = 'person'
+            and import_date = (select max(import_date) from "housing-refined-zone"."contacts_reshape")""",
         'id_field': 'id'},
     'housing_homeowner_record_sheet': {
         'sql': """SELECT * FROM "housing-raw-zone"."housing_homeowner_record_sheet" where import_date=(select max(import_date) from "housing-raw-zone"."housing_homeowner_record_sheet")""",
         'id_field': 'property_no'},
     'housing_dwellings_list': {
-        'sql': """SELECT * FROM "housing-raw-zone"."housing_dwellings_list" where import_date=(select max(import_date) from "housing-raw-zone"."housing_homeowner_record_sheet")""",
+        'sql': """SELECT
+            cast(property_dwelling_reference_number as varchar) as property_dwelling_reference_number,
+            cast(llpg as varchar) as llpg,
+            cast(estate_reference_number as varchar) as estate_reference_number,
+            estate_name,
+            block_name_as_per_naming_convention_procedure as block_name,
+            cast(block_reference_number as varchar) as block_reference_number,
+            sub_block_name,
+            cast(sub_block_reference as varchar) as sub_block_reference,
+            block_total_stories,
+            block_typology_based_on_dwelling_type_defined_list as block_typology,
+            building_heights_h1,
+            communal_heat_network,
+            dwelling_front_door_access_floor_level,
+            dwelling_no_of_bedrooms,
+            fra_block,
+            no_of_floors_within_dwelling,
+            registered_building,
+            year_built_block
+            FROM "housing-raw-zone"."housing_dwellings_list" where import_date = (select max(import_date) from "housing-raw-zone"."housing_dwellings_list")""",
         'id_field': 'property_dwelling_reference_number'},
     'assets_reshape': {
-        'sql': """SELECT * FROM "housing-refined-zone"."assets_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."assets_reshape") and assettype = 'Dwelling'""",
+        'sql': """select
+            cast(assetid as varchar) as assetid,
+            assettype,
+            cast(uprn as varchar) as uprn,
+            owner as property_owner,
+            agent,
+            iscouncilproperty,
+            istmomanaged,
+            managingorganisation,
+            cast(managingorganisationid as varchar) as managingorganisationid,
+            totalblockfloors,
+            floorno,
+            numberoflifts,
+            numberoflivingrooms,
+            numberofkitchens,
+            numberofbedrooms,
+            numberofbedspaces,
+            numberofshowers,
+            numberoffloors,
+            numberofdoublebeds,
+            numberofsinglebeds,
+            numberofbathrooms,
+            yearconstructed,
+            hascommunalareas,
+            hasprivatekitchen,
+            hasprivatebathroom,
+            hasrampaccess,
+            isstepfree,
+            hasstairs,
+            heating,
+            numberofcots,
+            windowtype,
+            estate_name,
+            cast(estate_id as varchar) as estate_id,
+            import_year,
+            import_month,
+            import_day,
+            import_date
+            FROM "housing-refined-zone"."assets_reshape" where import_date = (select max(import_date) from "housing-refined-zone"."assets_reshape")
+            and assettype in ('BoilerHouse','BoosterPump','CleanersFacilities','CombinedHeatAndPowerUnit','CommunityHall','Concierge','Dwelling','LettableNonDwelling','Lift','NA','NBD','OutBuilding','TravellerSite')""",
         'id_field': 'asset_id'},
     'matenancyagreement': {
         'sql': """SELECT *, substr(cast(eot as varchar), 1, 10) as eot_parsed, substr(cast(cot as varchar), 1, 10) as cot_parsed FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement" where import_date=(select max(import_date) FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement")""",