From db16bce5f891deabf3b3f7684959f30ab7dadf5b Mon Sep 17 00:00:00 2001 From: Tian Chen <38001883+Tian-2017@users.noreply.github.com> Date: Fri, 30 May 2025 12:53:03 +0100 Subject: [PATCH] tidy up - remove the parking transformation python scripts and terrafrom file --- ...rking_interim_cycle_hangar_waiting_list.py | 119 --- .../parking/parking_bailiff_allocation.py | 308 ------- .../jobs/parking/parking_bailiff_return.py | 292 ------ .../jobs/parking/parking_bailiff_totals.py | 69 -- .../parking_bailiff_warrant_figures.py | 214 ----- .../parking_cedar_backing_data_summary.py | 151 ---- .../parking_cedar_fulling_total_summary.py | 364 -------- .../jobs/parking/parking_cedar_payments.py | 160 ---- .../parking_copy_ringgo_sftp_data_to_raw.py | 56 -- ...customer_services_permit_activity_users.py | 756 ---------------- .../parking_cycle_hangar_allocation_detail.py | 106 --- .../parking_cycle_hangar_allocation_update.py | 361 -------- .../parking_cycle_hangar_allocation_wait.py | 332 ------- .../parking_cycle_hangar_interim_wait_list.py | 47 - ...ng_cycle_hangar_met_fail_monthly_format.py | 369 -------- .../parking_cycle_hangars_denormalisation.py | 167 ---- ...ng_cycle_hangars_denormalisation_update.py | 208 ----- ...hangars_denormalised_inservice_live_gds.py | 278 ------ .../parking_cycle_hangars_waiting_list.py | 205 ----- ...rking_dc_liberator_latest_permit_status.py | 105 --- ...dc_liberator_permit_llpg_street_records.py | 137 --- .../jobs/parking/parking_defect_met_fail.py | 127 --- .../parking_defect_met_fail_monthly_format.py | 281 ------ ...tate_waiting_list_live_permits_type_gds.py | 141 --- .../parking_gds_permit_change_comparison.py | 107 --- ...permit_change_comparison_specific_dates.py | 103 --- .../parking/parking_market_licence_totals.py | 210 ----- .../parking_markets_denormalisation.py | 268 ------ ..._with_address_match_llpg_nlpg_postcodes.py | 837 ------------------ ...king_motorcycle_monthly_ringgo_payments.py | 84 -- .../parking/parking_motorcycle_permits_480.py | 63 -- .../parking_pcn_daily_print_monitoring.py | 198 ----- .../parking_pcn_daily_print_monitoring_all.py | 158 ---- .../parking_permit_denormalisation_mc.py | 562 ------------ ...ing_permit_denormalised_gds_street_llpg.py | 683 -------------- ...parking_reps_and_appeals_correspondance.py | 121 --- ..._appeals_correspondance_kpi_gds_summary.py | 114 --- ...eals_correspondance_kpi_gds_summary_qtr.py | 118 --- .../parking/parking_school_street_vrms.py | 109 --- .../parking_shop_front_licence_totals.py | 189 ---- .../parking_visitor_voucher_qtrly_review.py | 323 ------- .../parking/parking_voucher_de_normalised.py | 149 ---- .../parking_vouchers_approved_summary_gds.py | 571 ------------ ...rking_vouchers_approved_summary_gds_lbh.py | 566 ------------ ...spatial-enrichment-dictionary-preprod.json | 13 - .../spatial-enrichment-dictionary.json | 13 - terraform/etl/38-aws-glue-job-parking.tf | 85 -- 47 files changed, 10997 deletions(-) delete mode 100644 scripts/jobs/parking/Parking_interim_cycle_hangar_waiting_list.py delete mode 100644 scripts/jobs/parking/parking_bailiff_allocation.py delete mode 100644 scripts/jobs/parking/parking_bailiff_return.py delete mode 100644 scripts/jobs/parking/parking_bailiff_totals.py delete mode 100644 scripts/jobs/parking/parking_bailiff_warrant_figures.py delete mode 100644 scripts/jobs/parking/parking_cedar_backing_data_summary.py delete mode 100644 scripts/jobs/parking/parking_cedar_fulling_total_summary.py delete mode 100644 scripts/jobs/parking/parking_cedar_payments.py delete mode 100644 scripts/jobs/parking/parking_copy_ringgo_sftp_data_to_raw.py delete mode 100644 scripts/jobs/parking/parking_customer_services_permit_activity_users.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangar_allocation_detail.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangar_allocation_update.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangar_allocation_wait.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangar_interim_wait_list.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangar_met_fail_monthly_format.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangars_denormalisation.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangars_denormalisation_update.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangars_denormalised_inservice_live_gds.py delete mode 100644 scripts/jobs/parking/parking_cycle_hangars_waiting_list.py delete mode 100644 scripts/jobs/parking/parking_dc_liberator_latest_permit_status.py delete mode 100644 scripts/jobs/parking/parking_dc_liberator_permit_llpg_street_records.py delete mode 100644 scripts/jobs/parking/parking_defect_met_fail.py delete mode 100644 scripts/jobs/parking/parking_defect_met_fail_monthly_format.py delete mode 100644 scripts/jobs/parking/parking_estate_waiting_list_live_permits_type_gds.py delete mode 100644 scripts/jobs/parking/parking_gds_permit_change_comparison.py delete mode 100644 scripts/jobs/parking/parking_gds_permit_change_comparison_specific_dates.py delete mode 100644 scripts/jobs/parking/parking_market_licence_totals.py delete mode 100644 scripts/jobs/parking/parking_markets_denormalisation.py delete mode 100644 scripts/jobs/parking/parking_match_pcn_permit_vrm_with_address_match_llpg_nlpg_postcodes.py delete mode 100644 scripts/jobs/parking/parking_motorcycle_monthly_ringgo_payments.py delete mode 100644 scripts/jobs/parking/parking_motorcycle_permits_480.py delete mode 100644 scripts/jobs/parking/parking_pcn_daily_print_monitoring.py delete mode 100644 scripts/jobs/parking/parking_pcn_daily_print_monitoring_all.py delete mode 100644 scripts/jobs/parking/parking_permit_denormalisation_mc.py delete mode 100644 scripts/jobs/parking/parking_permit_denormalised_gds_street_llpg.py delete mode 100644 scripts/jobs/parking/parking_reps_and_appeals_correspondance.py delete mode 100644 scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary.py delete mode 100644 scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary_qtr.py delete mode 100644 scripts/jobs/parking/parking_school_street_vrms.py delete mode 100644 scripts/jobs/parking/parking_shop_front_licence_totals.py delete mode 100644 scripts/jobs/parking/parking_visitor_voucher_qtrly_review.py delete mode 100644 scripts/jobs/parking/parking_voucher_de_normalised.py delete mode 100644 scripts/jobs/parking/parking_vouchers_approved_summary_gds.py delete mode 100644 scripts/jobs/parking/parking_vouchers_approved_summary_gds_lbh.py delete mode 100644 scripts/jobs/parking/spatial-enrichment-dictionary-preprod.json delete mode 100644 scripts/jobs/parking/spatial-enrichment-dictionary.json delete mode 100644 terraform/etl/38-aws-glue-job-parking.tf diff --git a/scripts/jobs/parking/Parking_interim_cycle_hangar_waiting_list.py b/scripts/jobs/parking/Parking_interim_cycle_hangar_waiting_list.py deleted file mode 100644 index 2ec723096..000000000 --- a/scripts/jobs/parking/Parking_interim_cycle_hangar_waiting_list.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -Note: python file name should be the same as the table name -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "Parking_interim_cycle_hangar_waiting_list" - -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ -/********************************************************************************* -Parking_interim_cycle_hangar_waiting_list - -Process the interim cycle hangar waiting list (supplied by Michael W) to add -additional fields (telephone Number, etc). - -23/12/2024 - Create SQL -06/01/2025 - add unsubscribed email -13/01/2025 - add opt-in data -21/01/2025 - change to use new telephone & email field in Michael's data -27/01/2025 - add the additional fields, Date Registered, in_hackney, gform -*********************************************************************************/ -With Interim_Wait as ( - SELECT - distinct forename - ,surname - ,email - ,party_id_to - ,party_id - ,cast(uprn as decimal) as uprn - ,address1 - ,address2 - ,post_code - ,x - ,y - ,lat - ,long - ,telephone - ,date_registered - ,in_hackney, gform - FROM "parking-raw-zone".interim_cycle_wait_list - WHERE import_date = (select max(import_date) - from "parking-raw-zone".interim_cycle_wait_list)), - -/*** Obtain the llpg data ***/ -FULL_LLPG as ( - SELECT * From "parking-refined-zone".spatially_enriched_liberator_permit_llpg - WHERE import_date = (select max(import_date) - from "parking-refined-zone".spatially_enriched_liberator_permit_llpg) - AND address1 not like '%STREET RECORD%'), - -STREET_LLPG as ( - SELECT * From "parking-refined-zone".spatially_enriched_liberator_permit_llpg - WHERE import_date = (select max(import_date) - from "parking-refined-zone".spatially_enriched_liberator_permit_llpg) - AND address1 like '%STREET RECORD%'), - -/*** Obtain the Party details, where available ***/ -Party as ( - SELECT - * - From "dataplatform-stg-liberator-raw-zone".liberator_licence_party - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), - -/*** 06/01/2024 - obtain the emails (from Tom) of those parties that are NOT interested in a Hangar***/ -unsubscribed_emails as ( - SELECT *, - ROW_NUMBER() OVER ( PARTITION BY email_address ORDER BY email_address DESC) row1 - FROM "parking-raw-zone".parking_parking_cycle_hangar_unsubscribed_emails - WHERE import_date = (select max(import_date) - from "parking-raw-zone".parking_parking_cycle_hangar_unsubscribed_emails)), - -/*** 13/01/2025 added ***/ -opt_in_emails as ( - SELECT - please_note_your_email_address_has_been_prefilled_based_on_your_account_registration_please_do_not_amend_this as email, - please_select_one_of_the_options_below, - ROW_NUMBER() OVER ( PARTITION BY please_note_your_email_address_has_been_prefilled_based_on_your_account_registration_please_do_not_amend_this - ORDER BY please_note_your_email_address_has_been_prefilled_based_on_your_account_registration_please_do_not_amend_this - DESC) row1 - FROM "parking-raw-zone".parking_parking_opt_in_form_responses - WHERE import_date = (select max(import_date) - from "parking-raw-zone".parking_parking_opt_in_form_responses) - AND please_select_one_of_the_options_below like 'No.%') - -/*** 21/01/2025 - Update the selected list to use Michael's data (Telephone & Email)***/ -SELECT - A.forename, A.surname, A.email, A.party_id_to, A.party_id, A.uprn, A.address1, A.address2, - A.post_code, A.x, A.y, A.lat, A.long, cast(telephone as varchar) as telephone_number, - C.address2 as Street, B.housing_estate, A.email as email_address, - date_registered, in_hackney, gform, - - format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), - 'yyyy-MM-dd HH:mm:ss') AS import_date_timestamp, - - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date - -FROM Interim_Wait as A -LEFT JOIN FULL_LLPG as B ON A.uprn = B.UPRN -LEFT JOIN STREET_LLPG as C ON B.USRN = C.USRN -LEFT JOIN Party as D ON A.party_id = D.business_party_id -LEFT JOIN unsubscribed_emails as E ON upper(ltrim(rtrim(A.email))) = upper(ltrim(rtrim(E.email_address))) - AND E.row1 = 1 -LEFT JOIN opt_in_emails as F ON upper(ltrim(rtrim(A.email))) = upper(ltrim(rtrim(F.email))) - AND F.row1 = 1 -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_bailiff_allocation.py b/scripts/jobs/parking/parking_bailiff_allocation.py deleted file mode 100644 index ce5065f72..000000000 --- a/scripts/jobs/parking/parking_bailiff_allocation.py +++ /dev/null @@ -1,308 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1625732038443 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_marston", - transformation_ctx="AmazonS3_node1625732038443", -) - -# Script generated for node S3 bucket -S3bucket_node1 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_equita", - transformation_ctx="S3bucket_node1", -) - -# Script generated for node Amazon S3 -AmazonS3_node1625732651466 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_newlyn", - transformation_ctx="AmazonS3_node1625732651466", -) - -# Script generated for node Amazon S3 -AmazonS3_node1625732674746 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_whyte", - transformation_ctx="AmazonS3_node1625732674746", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************************* -Bailiff Allocation - -The SQL builds the Bailiff allocation summary data - -07/07/2021 - Create SQL - -*************************************************************************************************************************/ - -/*** Find the latest Created Date, in each of the Bailiff files ***/ -WITH MAX_Date as ( -SELECT MAX(Created_Date) as MaxDate -FROM bailiffreturn_marston as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_marston) -UNION ALL -SELECT MAX(Created_Date) -FROM bailiffreturn_equita as B -where B.import_Date = (Select MAX(import_date) from bailiffreturn_equita) -UNION ALL -SELECT MAX(Created_Date) -FROM bailiffreturn_newlyn as C -where C.import_Date = (Select MAX(import_date) from bailiffreturn_newlyn) -UNION ALL -SELECT MAX(Created_Date) -FROM bailiffreturn_whyte as D -where D.import_Date = (Select MAX(import_date) from bailiffreturn_whyte)), - -/*** Collect the latest Bailiff data ***/ -BailiffReturn AS ( -SELECT 'Marston' as EA - ,Status as Status_Code - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN Payment_Date != 'NULL' AND Payment_Date > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - /** Format the remaining balence from string **/ - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN Payment_Date != 'NULL' Then CAST(Payment_Date as date) - ELSE CAST(NULL as date) - END as Payment_Date - ,CASE - WHEN Payment_Date != 'NULL' Then CONCAT(substring(Payment_Date,1,5), - CAST(extract(week from CAST(Payment_Date as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status = 'D' then 1 else 0 end as Returned - ,CASE When Status = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_marston as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_marston) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_marston) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date) - -UNION ALL -SELECT 'Equita' as EA - ,Status - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN PaymentDate != 'NULL' AND PaymentDate > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN PaymentDate != 'NULL' Then CAST(PaymentDate as date) - ELSE CAST(NULL as date) - END as PaymentDate - ,CASE - WHEN PaymentDate != 'NULL' Then CONCAT(substring(PaymentDate,1,5), - CAST(extract(week from CAST(PaymentDate as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status = 'D' then 1 else 0 end as Returned - ,CASE When Status = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_equita as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_equita) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_equita) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date) - -UNION ALL -SELECT 'Newlyn' as EA - ,Status_code - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN Payment_Date != 'NULL' AND Payment_Date > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN Payment_Date != 'NULL' Then CAST(Payment_Date as date) - ELSE CAST(NULL as date) - END as PaymentDate - ,CASE - WHEN Payment_Date != 'NULL' Then CONCAT(substring(Payment_Date,1,5), - CAST(extract(week from CAST(Payment_Date as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status_code = 'D' then 1 else 0 end as Returned - ,CASE When Status_code = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_newlyn as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_newlyn) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_newlyn) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date) -UNION ALL -SELECT 'Whyte' as EA - ,Status_code - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN Payment_Date != 'NULL' AND Payment_Date > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN Payment_Date != 'NULL' Then CAST(Payment_Date as date) - ELSE CAST(NULL as date) - END as PaymentDate - ,CASE - WHEN Payment_Date != 'NULL' Then CONCAT(substring(Payment_Date,1,5), - CAST(extract(week from CAST(Payment_Date as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status_code = 'D' then 1 else 0 end as Returned - ,CASE When Status_code = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_whyte as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_whyte) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_whyte) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date)), - -/************************************************************************************************************************* -Process the Data collected, summerise to Michael W's requirements -*************************************************************************************************************************/ -ProcessBailiffReturn as ( -select EA, MonthYear, count(*) as TotalWarrants, Sum(Returned) as ReturnedWarrant, - SUM(NotReturned) AS AcceptedWarrants, - CAST(SUM(Case When Status_Code != 'D' then PCN_Fee else 0 end) as decimal(11,2)) AS AcceptedTotalValue, - CAST(SUM(Case When Status_Code != 'D' then Amount_Remitted_to_Client else 0 end) as decimal(11,2)) AS TotalIncome, - CAST(SUM(Case When Status_Code IN ('B','C','H','PP','V','Z') Then PCN_Fee else 0 end) as decimal(11,2)) AS Excluded, - SUM(Case When Status_Code IN ('B','C','H','PP','V','Z') Then 1 else 0 end) AS ExcludedCount, - SUM(Case When Status_Code not IN ('B','C','H','PP','V','Z') Then PCN_Fee else 0 end) AS WarrantsValue, - SUM(Case When Status_Code IN ('A','AH','AT','AP') Then PCN_Fee else 0 end) AS Live, - - - CAST((SUM(Case When Status_Code != 'D' then Amount_Remitted_to_Client else 0 end) / - SUM(Case When Status_Code not IN ('B','C','H','PP','V','Z') Then PCN_Fee else 0 end) * 100) as decimal(11,2)) AS RecoveryRate - -FROM BailiffReturn -group by EA, MonthYear -Order by EA, MonthYear), - -Warrant_totals as ( - SELECT MonthYear, SUM(TotalWarrants) as Monthly_W_Total FROM ProcessBailiffReturn - Group By MonthYear) - - -SELECT - A.MonthYear, - A.EA, - A.TotalWarrants, - A.ExcludedCount, - cast(A.live as decimal(11,2)) as Live, - CAST((cast(A.TotalWarrants as double)/B.Monthly_W_Total)* 100 as decimal(11,2)) as Percentage_Allocation_of_Warrants_to_EA, - CASE - When A.ExcludedCount > 0 THEN CAST((cast(A.ExcludedCount as double)/A.TotalWarrants) * 100 as decimal(11,2)) - Else 0 - End as Percentage_Excluded_by_EA, - - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -from ProcessBailiffReturn as A -INNER JOIN Warrant_totals as B ON A.MonthYear = B.MonthYear -Order by Monthyear, EA - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "bailiffreturn_marston": AmazonS3_node1625732038443, - "bailiffreturn_equita": S3bucket_node1, - "bailiffreturn_newlyn": AmazonS3_node1625732651466, - "bailiffreturn_whyte": AmazonS3_node1625732674746, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Bailiff_Allocation/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Bailiff_Allocation", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_bailiff_return.py b/scripts/jobs/parking/parking_bailiff_return.py deleted file mode 100644 index 430dd1698..000000000 --- a/scripts/jobs/parking/parking_bailiff_return.py +++ /dev/null @@ -1,292 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1625732038443 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_marston", - transformation_ctx="AmazonS3_node1625732038443", -) - -# Script generated for node S3 bucket -S3bucket_node1 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_equita", - transformation_ctx="S3bucket_node1", -) - -# Script generated for node Amazon S3 -AmazonS3_node1625732651466 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_newlyn", - transformation_ctx="AmazonS3_node1625732651466", -) - -# Script generated for node Amazon S3 -AmazonS3_node1625732674746 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="bailiffreturn_whyte", - transformation_ctx="AmazonS3_node1625732674746", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************************* -Bailiff Return - -The SQL builds the Bailiff performance summary data - -07/07/2021 - Create SQL - -*************************************************************************************************************************/ - -/*** Find the latest Created Date, in each of the Bailiff files ***/ -WITH MAX_Date as ( -SELECT MAX(Created_Date) as MaxDate -FROM bailiffreturn_marston as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_marston) -UNION ALL -SELECT MAX(Created_Date) -FROM bailiffreturn_equita as B -where B.import_Date = (Select MAX(import_date) from bailiffreturn_equita) -UNION ALL -SELECT MAX(Created_Date) -FROM bailiffreturn_newlyn as C -where C.import_Date = (Select MAX(import_date) from bailiffreturn_newlyn) -UNION ALL -SELECT MAX(Created_Date) -FROM bailiffreturn_whyte as D -where D.import_Date = (Select MAX(import_date) from bailiffreturn_whyte)), - -/*** Collect the latest Bailiff data ***/ -BailiffReturn AS ( -SELECT 'Marston' as EA - ,Status as Status_Code - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN Payment_Date != 'NULL' AND Payment_Date > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - /** Format the remaining balence from string **/ - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN Payment_Date != 'NULL' Then CAST(Payment_Date as date) - ELSE CAST(NULL as date) - END as Payment_Date - ,CASE - WHEN Payment_Date != 'NULL' Then CONCAT(substring(Payment_Date,1,5), - CAST(extract(week from CAST(Payment_Date as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status = 'D' then 1 else 0 end as Returned - ,CASE When Status = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_marston as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_marston) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_marston) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date) - -UNION ALL -SELECT 'Equita' as EA - ,Status - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN PaymentDate != 'NULL' AND PaymentDate > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN PaymentDate != 'NULL' Then CAST(PaymentDate as date) - ELSE CAST(NULL as date) - END as PaymentDate - ,CASE - WHEN PaymentDate != 'NULL' Then CONCAT(substring(PaymentDate,1,5), - CAST(extract(week from CAST(PaymentDate as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status = 'D' then 1 else 0 end as Returned - ,CASE When Status = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_equita as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_equita) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_equita) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date) - -UNION ALL -SELECT 'Newlyn' as EA - ,Status_code - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN Payment_Date != 'NULL' AND Payment_Date > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN Payment_Date != 'NULL' Then CAST(Payment_Date as date) - ELSE CAST(NULL as date) - END as PaymentDate - ,CASE - WHEN Payment_Date != 'NULL' Then CONCAT(substring(Payment_Date,1,5), - CAST(extract(week from CAST(Payment_Date as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status_code = 'D' then 1 else 0 end as Returned - ,CASE When Status_code = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_newlyn as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_newlyn) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_newlyn) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date) -UNION ALL -SELECT 'Whyte' as EA - ,Status_code - ,Status_description - ,CAST(Warrant_Date as date) AS Warrant_Date - ,CAST(CONCAT(substring(Warrant_Date, 1, 8), '01') as date) AS MonthYear - ,CAST(PCN_Fee as double) AS PCN_Fee - - /*** check the payments, if payment greater than the min created date, ignore the payments */ - ,CASE - WHEN Payment_Date != 'NULL' AND Payment_Date > (SELECT MIN(MaxDate) FROM MAX_Date) THEN 0 - ELSE - CASE When Amount_Remitted_to_Client = 'NULL' Then 0 - ELSE CAST(Amount_Remitted_to_Client as double) - END - END AS Amount_Remitted_to_Client - ,CASE When Client_Balance_Remaining = 'NULL' Then 0 ELSE - CAST(Client_Balance_Remaining as double) END as Client_Balance_Remaining - /* Format the Dates */ - ,CASE - WHEN Payment_Date != 'NULL' Then CAST(Payment_Date as date) - ELSE CAST(NULL as date) - END as PaymentDate - ,CASE - WHEN Payment_Date != 'NULL' Then CONCAT(substring(Payment_Date,1,5), - CAST(extract(week from CAST(Payment_Date as date)) as varchar(2))) - ELSE '' - END as PaidYearWeek - /* Status Flags */ - ,CASE When Status_code = 'D' then 1 else 0 end as Returned - ,CASE When Status_code = 'D' then 0 else 1 end as NotReturned -FROM bailiffreturn_whyte as A -where A.import_Date = (Select MAX(import_date) from bailiffreturn_whyte) AND - Created_Date = (Select MAX(Created_Date) FROM bailiffreturn_whyte) AND - Warrant_Date Between '2021-03-01' AND (SELECT MIN(MaxDate) FROM MAX_Date)), - -/************************************************************************************************************************* -Process the Data collected, summerise to Michael W's requirements -*************************************************************************************************************************/ -ProcessBailiffReturn as ( -select EA, MonthYear, count(*) as TotalWarrants, Sum(Returned) as ReturnedWarrant, - SUM(NotReturned) AS AcceptedWarrants, - CAST(SUM(Case When Status_Code != 'D' then PCN_Fee else 0 end) as decimal(11,2)) AS AcceptedTotalValue, - CAST(SUM(Case When Status_Code != 'D' then Amount_Remitted_to_Client else 0 end) as decimal(11,2)) AS TotalIncome, - CAST(SUM(Case When Status_Code IN ('B','C','H','PP','V','Z') Then PCN_Fee else 0 end) as decimal(11,2)) AS Excluded, - SUM(Case When Status_Code IN ('B','C','H','PP','V','Z') Then 1 else 0 end) AS ExcludedCount, - SUM(Case When Status_Code not IN ('B','C','H','PP','V','Z') Then PCN_Fee else 0 end) AS WarrantsValue, - SUM(Case When Status_Code IN ('A','AH','AT','AP') Then PCN_Fee else 0 end) AS Live, - - - CAST((SUM(Case When Status_Code != 'D' then Amount_Remitted_to_Client else 0 end) / - SUM(Case When Status_Code not IN ('B','C','H','PP','V','Z') Then PCN_Fee else 0 end) * 100) as decimal(11,2)) AS RecoveryRate - -FROM BailiffReturn -group by EA, MonthYear -Order by EA, MonthYear) - -/*** Output the Bailiff Return ***/ -SELECT *, - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM ProcessBailiffReturn -ORDER BY Monthyear, EA - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "bailiffreturn_marston": AmazonS3_node1625732038443, - "bailiffreturn_equita": S3bucket_node1, - "bailiffreturn_newlyn": AmazonS3_node1625732651466, - "bailiffreturn_whyte": AmazonS3_node1625732674746, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Bailiff_Return/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Bailiff_Return", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_bailiff_totals.py b/scripts/jobs/parking/parking_bailiff_totals.py deleted file mode 100644 index 3ef8ec636..000000000 --- a/scripts/jobs/parking/parking_bailiff_totals.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "parking_bailiff_totals" - -# The exact same query prototyped in pre-prod(stg) orprod Athena -query_on_athena = """ -/********************************************************************************* -Parking_Bailiff_Totals - -This SQL summeries the number of Warrants issues against the total Income - -20/11/2024 - Create Query -*********************************************************************************/ -/*** Warrants ***/ -with bailiff_warrants as ( - SELECT *, cast(substr(cast(warrantissuedate as varchar), 1, 8)||'01' as date) as Bailiff_Date - FROM "dataplatform-stg-liberator-raw-zone".liberator_pcn_bailiff - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), - -/*** Obtain the Cedar payments ***/ -cedar_pay as ( - SELECT *, - CAST(substr(cast(trandate as varchar), 7, 4)||'-'|| - substr(cast(trandate as varchar), 4, 2)||'-01' as date) as cedar_month - FROM "parking-raw-zone".cedar_parking_payments - WHERE import_Date = (Select MAX(import_date) from "parking-raw-zone".cedar_parking_payments) - AND description = 'NBI'), - -/*** Summerise the payments ***/ -Cedar_Summ as ( - SELECT - cedar_month, - -(sum(cast(financialvalue as double))) as Monthly_Paid - FROM cedar_pay - group by cedar_month), - -/*** Summerise the warrants ***/ -warrant_sum as ( - SELECT - Bailiff_Date, - count(*) as Monthly_Totals - FROM bailiff_warrants - group by Bailiff_Date) - -/*** Bring the two togather ***/ -SELECT - A.*, B.*, - - format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS importdatetime, - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date -FROM warrant_sum as A -LEFT JOIN Cedar_Summ as B ON A.Bailiff_Date = B.cedar_month -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_bailiff_warrant_figures.py b/scripts/jobs/parking/parking_bailiff_warrant_figures.py deleted file mode 100644 index 5bb516211..000000000 --- a/scripts/jobs/parking/parking_bailiff_warrant_figures.py +++ /dev/null @@ -1,214 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1628173244776 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="pcnfoidetails_pcn_foi_full", - transformation_ctx="AmazonS3_node1628173244776", - push_down_predicate=create_pushdown_predicate("import_date", 1), -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/****************************************************************************************************************************** -Parking_Bailiff_Warrant_Figures - -This SQL creates the Warrant return figures on the basis of the following spec:- - -1) Firstly identify all warrants issued to enforcement agents in the month in question. -2) Exclude from this group PCNs that were cancelled -3) Then identify the payment status of the PCN - fully paid, partially paid, not paid -4) Produce a dataset for each month, that shows the number and % of the overall number of warrants (excluding cancelled) that fall into the above three categories. - - -24/11/2021 - create SQL -25/11/2021 - Amend to add cancellation reasons from Alan, to identify PCN as NOT cancelled but NOT PAID -12/01/2022 - Amend the Cancellation reason list inline with an email from Nohaad -20/01/2022 - Amend the cancellation reason list after review by Michael Benn -19/05/2023 - change calculation of Paid/Part Paid -28/07/2028 - add additional cancel reasons and rework the calculation for paid/part paid, etc -******************************************************************************************************************************/ -/*** Collect those PCNs that have a Warrant date ***/ -WITH Bailiff_PCNs_Int as ( - SELECT - PCN, warrantissuedate as warrant_issued, - substr(cast(cast(warrant_issued as date) as string), 1, 8)||'01' as MonthYear, bailiff, pcn_canx_date, - cancellationreason, - lib_initial_debt_amount, lib_payment_received, lib_write_off_amount, - /*** This is a flag that identifies those cancelled PCNs that should be treated and not cancelled? ***/ - CASE - When cancellationreason IN ('Cancelled - Court Claim', - 'Closed file - Nulla Bona - No access', - 'Council Tennant - Sublet', - 'DEBT RECOVERY', - 'Gesture of Goodwill', - 'Gone away', - 'Gone Away-S100 ', - 'I - Insufficient Goods', - 'I - Unable to find effects', - 'I - Unable to Recover Debt', - 'Incomplete address', - 'L-Gone Away ', - 'L-Not valid address', - 'L-PP-Sold ', - 'L-Vulnerable ', - 'M - Gone Away No Trace', - 'M - Goneaway', - 'M - Moved abroad', - 'N - Not Contacted', - 'New occupiers, no info on defaulters', - 'No trace', - 'Not applicable', - 'Not registered keeper', - 'Not resident in England and Wales', - 'Parents address - not resident', - 'Part paid no more expected', - 'Payment', - 'PCN written off', - 'PP - Persistent Evader', - 'PP -Part Payment Received - Persistent Evader', - 'Property Empty enquiries negative', - 'Property empty, fire damaged', - 'Refused Contact', - 'U - Unable to enforce', - 'U - Unable to Recover Debt', - 'Unenforceable', - 'Unsuccessful', - 'V - Vulnerable Group', - 'Vulnerable', - 'Vulnerable Debtor', - 'X - Warrant has expired' - 'X - Warrant expired', - 'Warrant out of date') Then 1 - When pcn_canx_date is NULL Then 2 - When cast(lib_payment_received as double) > 0 Then 3 /* Added 28/07/2023 */ - ELSE 0 - END as Write_Off_Flag - FROM pcnfoidetails_pcn_foi_full - WHERE import_Date = (Select MAX(import_date) from pcnfoidetails_pcn_foi_full) and - length(cast(warrantissuedate as string)) > 0 ), - -Bailiff_PCNs as ( - SELECT - PCN, warrant_issued, - substr(cast(cast(warrant_issued as date) as string), 1, 8)||'01' as MonthYear, - lib_initial_debt_amount, lib_payment_received, lib_write_off_amount,pcn_canx_date,cancellationreason, - - /*** Create the payment flag ***/ - /** 19-05-2023 update calculation to add write off and payment **/ - /** 28/07/2023 - change the calculation...again! **/ - CASE - When Write_Off_Flag = 1 AND - CAST(lib_payment_received as double) = 0 Then 'Not Paid' - When cast(lib_payment_received as double) = 0 AND - CAST(lib_write_off_amount as double) = 0 Then 'Not Paid' - When (cast(lib_payment_received as double) + cast(lib_write_off_amount as double)) - >= cast(lib_initial_debt_amount as double) Then 'Paid' - When (cast(lib_payment_received as double) + cast(lib_write_off_amount as double)) - < cast(lib_initial_debt_amount as double) Then 'Part Payment' - END as Payment_Status - - FROM Bailiff_PCNs_Int - WHERE Write_Off_Flag != 0), - -/*** Get the distinct Months ***/ -Basic_Month as ( - SELECT distinct MonthYear - FROM Bailiff_PCNs), - -/*** Summerise the number of PCNs ***/ -Summary_PCN as ( - SELECT - MonthYear, - Payment_Status, - count(*) as No_of_Warrents - - FROM Bailiff_PCNs - GROUP BY MonthYear,Payment_Status - ORDER BY MonthYear, Payment_Status), - -/*** Bring the data togather into a single record ***/ -Merge_Month as ( - SELECT - A.MonthYear as WarrantMonth, - coalesce(B.No_of_Warrents,0) as Paid_Total, - coalesce(C.No_of_Warrents,0) as Part_Paid_Total, - coalesce(D.No_of_Warrents,0) as Not_Paid_Total, - - (coalesce(B.No_of_Warrents,0)+ - coalesce(C.No_of_Warrents,0)+ - coalesce(D.No_of_Warrents,0)) as Total_No_PCN - - from Basic_Month as A - LEFT JOIN Summary_PCN as B ON A.MonthYear = B.MonthYear AND B.payment_status = 'Paid' - LEFT JOIN Summary_PCN as C ON A.MonthYear = C.MonthYear AND C.payment_status = 'Part Payment' - LEFT JOIN Summary_PCN as D ON A.MonthYear = D.MonthYear AND D.payment_status = 'Not Paid' - order by A.MonthYear) - -/*** Output the data and create a % figure for each of the Payment flags ***/ -SELECT - A.*, - round((cast(Paid_Total as double)/Total_No_PCN)*100,2) as Paid_Total_Percentage, - round((cast(Part_Paid_Total as double)/Total_No_PCN)*100,2) as Part_Paid_Total_Percentage, - round((cast(Not_Paid_Total as double)/Total_No_PCN)*100,2) as Not_Paid_Total_Percentage, - - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM Merge_Month as A -Order by A.WarrantMonth -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={"pcnfoidetails_pcn_foi_full": AmazonS3_node1628173244776}, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Bailiff_Warrant_Figures/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Bailiff_Warrant_Figures", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_cedar_backing_data_summary.py b/scripts/jobs/parking/parking_cedar_backing_data_summary.py deleted file mode 100644 index 8a5c798ad..000000000 --- a/scripts/jobs/parking/parking_cedar_backing_data_summary.py +++ /dev/null @@ -1,151 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1625732038443 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="cedar_backing_data", - transformation_ctx="AmazonS3_node1625732038443", -) - -# Script generated for node Amazon S3 -AmazonS3_node1631704526786 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="calendar", - transformation_ctx="AmazonS3_node1631704526786", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************** -Parking_Cedar_Backing_Data_Summary - -This query summerises the Cedar backing data - -08/11/2021 - create the query -08/05/2024 - update for different backing data date format and calendar format -****************************************************************************************************************/ -/*** Collect & Format the backing data ***/ -WITH CedarBackingData as ( - SELECT - CAST(CASE - When trandate like '%/%'Then - substr(trandate, 7, 4)||'-'||substr(trandate, 4, 2)||'-'||'01' - ELSE - substr(trandate, 1, 8)||'01' - END as date) as IssueMonthYear, - reftype, sub_reftype, cast(REPLACE(REPLACE(financialvalue, '£',''),',','') as decimal(15,2)) as financialvalue - FROM cedar_backing_data - Where import_Date = (Select MAX(import_date) from cedar_backing_data)), - -Before_ICalendar as ( - SELECT - CAST( - CASE - When date like '%/%'Then - substr(date, 7, 4)||'-'||substr(date, 4, 2)||'-'||substr(date, 1, 2) - ELSE date - END as date) as Date, - dow, - fin_year, fin_year_startdate, fin_year_enddate - FROM calendar), - -/** 08/05/2024 - Add de-dupe of calendar **/ -ICalendar as ( - Select *, - ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date DESC) R1 - FROM Before_ICalendar), - -LatestYear as ( - SELECT MAX(fin_year) as LYear from Calendar), - -/*** Summerise the data ***/ -Payment_Summary as ( - SELECT - IssueMonthYear, reftype, sub_reftype, SUM(financialvalue) as Total_Payments - FROM CedarBackingData - GROUP BY IssueMonthYear, reftype, sub_reftype - ORDER BY IssueMonthYear, reftype, sub_reftype) - -/*** OUTPUT THE DATA ***/ -SELECT A.*, - /* Identify the Financial year, this, last, previous to last */ - CASE - When cast(substr(cast(current_date as string), 6, 2) as int) = 4 Then - CASE - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-1 as varchar(4)) Then 'Current' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-2 as varchar(4)) Then 'Previous' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-3 as varchar(4)) Then 'Before_Previous' - END - ELSE - CASE - When B.fin_year = (Select LYear from LatestYear) Then 'Current' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-1 as varchar(4)) Then 'Previous' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-2 as varchar(4)) Then 'Before_Previous' - Else '' - END - END as Year_Type, - - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - - -FROM Payment_Summary as A -LEFT JOIN ICalendar as B ON A.IssueMonthYear = date AND R1 = 1 -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "cedar_backing_data": AmazonS3_node1625732038443, - "calendar": AmazonS3_node1631704526786, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Cedar_Backing_Data_Summary/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Cedar_Backing_Data_Summary", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_cedar_fulling_total_summary.py b/scripts/jobs/parking/parking_cedar_fulling_total_summary.py deleted file mode 100644 index a2a26058b..000000000 --- a/scripts/jobs/parking/parking_cedar_fulling_total_summary.py +++ /dev/null @@ -1,364 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var - - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1625732038443 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_cedar_payments", - transformation_ctx="AmazonS3_node1625732038443", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1631887639199 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="calendar", - transformation_ctx="AmazonS3_node1631887639199", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************************* -Parking_Cedar_Total_Monthly_Summary - -The SQL as been created to create monthly values & % difference between last financial year and this financial year - -17/09/2021 - Create SQL -06/10/2021 - change the calculation of year -on - year % -07/10/2021 - change the calc of % back...there seems to be a problem -18/11/2021 - add code to format the calendar date and to identify payments beore last year -02/12/2021 - change import_date to string -19/04/2022 - change calculation of the current/past/etc Fin Year -29/04/2024 add additional de-dupe for the calendar year -*************************************************************************************************************************/ -/*** Format the date because Calendar data imported with different date formats!!! ***/ -With BeforeCalendarFormat as ( - SELECT - CAST(CASE - When date like '%/%'Then substr(date, 7, 4)||'-'||substr(date, 4,2)||'-'||substr(date, 1,2) - ELSE substr(date, 1, 10) - end as date) as date, - workingday, - holiday, - dow, - fin_year, - fin_year_startdate, - fin_year_enddate - FROM calendar), - -/* 29/04/2024 - new code to de-dupe the calendar data */ -CalendarFormat as ( - Select *, - ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date DESC) R1 - FROM BeforeCalendarFormat), - -LatestYear as ( -SELECT MAX(fin_year) as LYear from CalendarFormat -Where date <= current_date), - - -Full_Fin_Data as ( -SELECT - cast(paymonthyear as date) as Pay_Year, - cast(-(totalpayments) as decimal(10,2)) as Total_Paid, - CASE - When date_format(paymonthyear, "M") = '1' Then 'Jan' - When date_format(paymonthyear, "M") = '2' Then 'Feb' - When date_format(paymonthyear, "M") = '3' Then 'Mar' - When date_format(paymonthyear, "M") = '4' Then 'Apr' - When date_format(paymonthyear, "M") = '5' Then 'May' - When date_format(paymonthyear, "M") = '6' Then 'Jun' - When date_format(paymonthyear, "M") = '7' Then 'Jul' - When date_format(paymonthyear, "M") = '8' Then 'Aug' - When date_format(paymonthyear, "M") = '9' Then 'Sep' - When date_format(paymonthyear, "M") = '10' Then 'Oct' - When date_format(paymonthyear, "M") = '11' Then 'Nov' - When date_format(paymonthyear, "M") = '12' Then 'Dec' - END as Pay_Month, - - /* Identify the Financial year, this, last, previous to last */ - CASE - When cast(substr(cast(current_date as string), 6, 2) as int) = 4 Then - CASE - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-1 as varchar(4)) Then 'Current' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-2 as varchar(4)) Then 'Previous' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-3 as varchar(4)) Then 'PreviousPrevious' - END - ELSE - CASE - When B.fin_year = (Select LYear from LatestYear) Then 'Current' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-1 as varchar(4)) Then 'Previous' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-2 as varchar(4)) Then 'PreviousPrevious' - Else 'NULL' - END - END as Year_Type - -FROM parking_cedar_payments as A -LEFT JOIN CalendarFormat as B ON A.PayMonthYear = Cast(Cast(B.date as varchar(10)) as date) and R1 = 1 -WHERE A.import_Date = (Select MAX(import_date) from parking_cedar_payments) and -paymenttype = 'Parking'), - -/***********************************************************************************************************************************/ -/*** Total the months payments ***/ -Curent_ToDate as ( -SELECT Pay_Month as Month, A.Pay_Year, - CASE - When A.Pay_Month = 'Apr' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month = 'Apr' - AND Year_Type = 'Current') - When A.Pay_Month = 'May' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May') - AND Year_Type = 'Current') - When A.Pay_Month = 'Jun' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun') - AND Year_Type = 'Current') - When A.Pay_Month = 'Jul' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul') - AND Year_Type = 'Current') - When A.Pay_Month = 'Aug' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug') - AND Year_Type = 'Current') - When A.Pay_Month = 'Sep' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep') - AND Year_Type = 'Current') - When A.Pay_Month = 'Oct' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct') - AND Year_Type = 'Current') - When A.Pay_Month = 'Nov' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov') - AND Year_Type = 'Current') - When A.Pay_Month = 'Dec' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec') - AND Year_Type = 'Current') - When A.Pay_Month = 'Jan' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan') - AND Year_Type = 'Current') - When A.Pay_Month = 'Feb' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan','Feb') - AND Year_Type = 'Current') - When A.Pay_Month = 'Mar' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan','Feb','Mar') - AND Year_Type = 'Current') - - END as TotalToDate, A.Total_Paid -FROM Full_Fin_Data as A -WHERE Year_Type = 'Current' -ORDER BY A.Pay_Year), - -/*** Previous Month(s) ***/ -Previous_ToDate as ( -SELECT Pay_Month as Month, A.Pay_Year, - CASE - When A.Pay_Month = 'Apr' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month = 'Apr' - AND Year_Type = 'Previous') - When A.Pay_Month = 'May' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Jun' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Jul' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Aug' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Sep' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Oct' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Nov' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Dec' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Jan' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Feb' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan','Feb') - AND Year_Type = 'Previous') - When A.Pay_Month = 'Mar' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan','Feb','Mar') - AND Year_Type = 'Previous') - - END as TotalToDate, A.Total_Paid -FROM Full_Fin_Data as A -WHERE Year_Type = 'Previous' -ORDER BY A.Pay_Year), - -/*** Previous- previous Month(s) ***/ -PreviousPrevious_ToDate as ( -SELECT Pay_Month as Month, A.Pay_Year, - CASE - When A.Pay_Month = 'Apr' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month = 'Apr' - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'May' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Jun' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Jul' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Aug' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Sep' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Oct' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Nov' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Dec' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Jan' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Feb' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan','Feb') - AND Year_Type = 'PreviousPrevious') - When A.Pay_Month = 'Mar' Then - (Select SUM(Z.Total_Paid) From Full_Fin_Data as Z - Where Z.Pay_Month IN ('Apr', 'May', 'Jun','Jul','Aug','Sep','Oct','Nov','Dec','Jan','Feb','Mar') - AND Year_Type = 'PreviousPrevious') - - END as TotalToDate, A.Total_Paid -FROM Full_Fin_Data as A -WHERE Year_Type = 'PreviousPrevious' -ORDER BY A.Pay_Year), - -DeDupe_Data as ( -SELECT - A.Month, - A.TotalToDate as Current_Monthly_Total, - B.TotalToDate as Previous_Monthly_Total, - C.TotalToDate as PreviousPrevious_Monthly_Total, - A.Pay_Year, - - /** try and trap duplicates **/ - ROW_NUMBER() OVER ( PARTITION BY A.Month, A.TotalToDate, B.TotalToDate,C.TotalToDate, A.Pay_Year - ORDER BY A.Month, A.TotalToDate, B.TotalToDate,C.TotalToDate, A.Pay_Year DESC) row_num -FROM Curent_ToDate as A -LEFT JOIN Previous_ToDate as B ON A.Month = B.Month -LEFT JOIN PreviousPrevious_ToDate as C ON A.Month = C.Month) - -/*** Ouput Summary data & Apply a percentage diff ***/ -SELECT - Month, - Current_Monthly_Total, - Previous_Monthly_Total, - PreviousPrevious_Monthly_Total, - - CAST((Previous_Monthly_Total/Current_Monthly_Total)*100 as decimal(10,2)) as PercentageDiff, - - Pay_Year, - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM DeDupe_Data -WHERE row_num = 1 -ORDER BY Pay_Year -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_cedar_payments": AmazonS3_node1625732038443, - "calendar": AmazonS3_node1631887639199, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Cedar_Total_Monthly_Summary/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Cedar_Total_Monthly_Summary", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_cedar_payments.py b/scripts/jobs/parking/parking_cedar_payments.py deleted file mode 100644 index 3fc69aa45..000000000 --- a/scripts/jobs/parking/parking_cedar_payments.py +++ /dev/null @@ -1,160 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1625732038443 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="cedar_parking_payments", - transformation_ctx="AmazonS3_node1625732038443", -) - -# Script generated for node Amazon S3 -AmazonS3_node1631704526786 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="calendar", - transformation_ctx="AmazonS3_node1631704526786", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************** -Parking_Cedar_Payments - -This query summerises the Cedar Parking payments, into Parking and Cycle Hanger payments - -14/09/2021 - create the query -15/09/2021 - Add code to identify the current and previous financial year -05/10/2021 - Remove the WHERE against Cedar_Parking_Payments because I am importing the - .csv data a month at a time -06/10/2021 - update for incorrect September date format import -08/11/2021 - update to collect dates BEFORE last Financial year -19/11/2021 - change code to tidy date formating -01/12/2021 - add where statement when collecting parking_payments - and set import_date to string -26/04/2024 - add further code to de-dupe the calendar data -****************************************************************************************************************/ -WITH Cedar_Payments as ( -SELECT - cc, subj,analysis, trandate, - CASE - When trandate like '%/%' Then - CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date) - When substr(trandate, 1, 4) like '%-%' Then - CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date) - ELSE cast(concat(substr(Cast(trandate as varchar(10)),1, 7), '-01') as date) - END as PayMonthYear, - description, o_description, cast(financialvalue as decimal(10,2)) as financialvalue -FROM cedar_parking_payments -WHERE import_Date = (Select MAX(import_date) from cedar_parking_payments)), - -Before_ICalendar as ( - SELECT - CAST(CASE - When date like '%/%'Then substr(date, 7, 4)||'-'||substr(date, 4,2)||'-'||substr(date, 1,2) - ELSE substr(date, 1, 10) - end as date) as date, - workingday, - holiday, - dow, - fin_year, - fin_year_startdate, - fin_year_enddate - FROM calendar), - -/* 26/04/2024 - new code to de-dupe the calendar data */ -ICalendar as ( - Select *, - ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date DESC) row_num - FROM Before_ICalendar), - -Cedar_Summary as ( -SELECT - PayMonthYear, CASE When Description != 'Cycle Hangar' Then 'Parking' Else 'Cycle Hangar' END as PaymentType, - SUM(financialvalue) as TotalPayments -FROM Cedar_Payments as A -group by PayMonthYear,CASE When Description != 'Cycle Hangar' Then 'Parking' Else 'Cycle Hangar' END), - -LatestYear as ( - SELECT MAX(fin_year) as LYear from Calendar) - -SELECT A.*, - /* Identify the Financial year, this, last, previous to last */ - CASE - When cast(substr(cast(current_date as string), 6, 2) as int) = 4 Then - CASE - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-1 as varchar(4)) Then 'Current' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-2 as varchar(4)) Then 'Previous' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-3 as varchar(4)) Then 'PreviousPrevious' - END - ELSE - CASE - When B.fin_year = (Select LYear from LatestYear) Then 'Current' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-1 as varchar(4)) Then 'Previous' - When B.fin_year = cast((Select cast(LYear as int) from LatestYear)-2 as varchar(4)) Then 'Before_Previous' - Else 'NULL' - END - END as Year_Type, - - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM Cedar_Summary as A -LEFT JOIN ICalendar as B ON A.PayMonthYear = date and row_num = 1 -order by A.PayMonthYear -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "cedar_parking_payments": AmazonS3_node1625732038443, - "calendar": AmazonS3_node1631704526786, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Cedar_Payments/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Cedar_Payments", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_copy_ringgo_sftp_data_to_raw.py b/scripts/jobs/parking/parking_copy_ringgo_sftp_data_to_raw.py deleted file mode 100644 index 268050e5d..000000000 --- a/scripts/jobs/parking/parking_copy_ringgo_sftp_data_to_raw.py +++ /dev/null @@ -1,56 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from pyspark.sql.functions import col, lit, when, concat -from pyspark.sql.types import IntegerType -from pyspark.sql import SparkSession -import boto3 - -from scripts.helpers.helpers import get_glue_env_var, add_import_time_columns, get_s3_subfolders, PARTITION_KEYS, \ - clean_column_names, get_latest_partitions - -s3_client = boto3.client('s3') -sc = SparkContext.getOrCreate() -spark = SparkSession.builder.getOrCreate() -glue_context = GlueContext(sc) - -logger = glue_context.get_logger() - - -def data_source_landing_to_raw(bucket_source, bucket_target, s3_prefix): - logger.info("bucket_target" + bucket_target) - logger.info("s3_prefix" + s3_prefix) - data_source = spark.read.option("header", "true").csv(bucket_source + "/" + s3_prefix) - latest_data = get_latest_partitions(data_source) - logger.info(f"latest_data: {latest_data}") - - logger.info(f"Retrieved data source from s3 path {bucket_source}/{s3_prefix}") - - data_frame = clean_column_names(latest_data) - logger.info("Using Columns: " + str(data_frame.columns)) - - date_partition_formatted = data_frame.withColumn("import_month", col("import_month").cast(IntegerType())) \ - .withColumn("import_day", col("import_day").cast(IntegerType())) \ - .withColumn("import_month", when(col("import_month") < 10, concat(lit("0"), col("import_month"))).otherwise( - col("import_month"))) \ - .withColumn("import_day", - when(col("import_day") < 10, concat(lit("0"), col("import_day"))).otherwise(col("import_day"))) - - date_partition_formatted.write.mode("append").partitionBy(*PARTITION_KEYS).parquet(bucket_target + "/" + s3_prefix) - - -args = getResolvedOptions(sys.argv, ['JOB_NAME']) - -job = Job(glue_context) -job.init(args['JOB_NAME'], args) - -s3_bucket_target = get_glue_env_var('s3_bucket_target', '') -s3_prefix = get_glue_env_var('s3_prefix', '') -s3_bucket_source = get_glue_env_var('s3_bucket_source', '') - -data_source_landing_to_raw("s3://" + s3_bucket_source, "s3://" + s3_bucket_target, s3_prefix) - -job.commit() diff --git a/scripts/jobs/parking/parking_customer_services_permit_activity_users.py b/scripts/jobs/parking/parking_customer_services_permit_activity_users.py deleted file mode 100644 index d79430d37..000000000 --- a/scripts/jobs/parking/parking_customer_services_permit_activity_users.py +++ /dev/null @@ -1,756 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 - liberator_permit_activity -AmazonS3liberator_permit_activity_node1715091109305 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="liberator_permit_activity", - transformation_ctx="AmazonS3liberator_permit_activity_node1715091109305", -) - -# Script generated for node SQL Query - parking_customer_services_permit_activity_users -SqlQuery0 = """ -/*Parking Customer Services - Permit Activity summarised for specific users by -- Actvity Financial year -- Actvity Month year -- Actvity Date -- Actvity Hour -- Product Type -- Product Category - -07/05/2024 - CREATED - -*/ - -With act_data as ( -Select -Case -when cast(activity_date as varchar(10)) like '2031-03-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2031-02-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2031-01-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-12-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-11-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-10-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-09-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-08-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-07-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-06-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-05-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-04-%' THEN '2027' --2030 to 2031 - -when cast(activity_date as varchar(10)) like '2030-03-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-02-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2030-01-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-12-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-11-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-10-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-09-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-08-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-07-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-06-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-05-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-04-%' THEN '2027' --2029 to 2030 - -when cast(activity_date as varchar(10)) like '2029-03-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-02-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2029-01-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-12-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-11-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-10-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-09-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-08-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-07-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-06-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-05-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-04-%' THEN '2027' --2028 to 2029 - -when cast(activity_date as varchar(10)) like '2028-03-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-02-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2028-01-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-12-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-11-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-10-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-09-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-08-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-07-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-06-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-05-%' THEN '2027' -when cast(activity_date as varchar(10)) like '2027-04-%' THEN '2027' --2027 to 2028 -when cast(activity_date as varchar(10)) like '2027-03-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2027-02-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2027-01-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-12-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-11-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-10-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-09-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-08-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-07-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-06-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-05-%' THEN '2026' -when cast(activity_date as varchar(10)) like '2026-04-%' THEN '2026' --2026 to 2027 -when cast(activity_date as varchar(10)) like '2026-03-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2026-02-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2026-01-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-12-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-11-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-10-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-09-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-08-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-07-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-06-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-05-%' THEN '2025' -when cast(activity_date as varchar(10)) like '2025-04-%' THEN '2025' --2025 to 2026 -when cast(activity_date as varchar(10)) like '2025-03-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2025-02-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2025-01-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-12-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-11-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-10-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-09-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-08-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-07-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-06-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-05-%' THEN '2024' -when cast(activity_date as varchar(10)) like '2024-04-%' THEN '2024' --2024 to 2025 -when cast(activity_date as varchar(10)) like '2024-03-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2024-02-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2024-01-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-12-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-11-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-10-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-09-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-08-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-07-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-06-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-05-%' THEN '2023' -when cast(activity_date as varchar(10)) like '2023-04-%' THEN '2023' --2023 to 2024 -when cast(activity_date as varchar(10)) like '2023-03-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2023-02-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2023-01-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-12-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-11-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-10-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-09-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-08-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-07-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-06-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-05-%' THEN '2022' -when cast(activity_date as varchar(10)) like '2022-04-%' THEN '2022' --2022 to 2023 -when cast(activity_date as varchar(10)) like '2022-03-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2022-02-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2022-01-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-12-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-11-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-10-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-09-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-08-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-07-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-06-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-05-%' THEN '2021' -when cast(activity_date as varchar(10)) like '2021-04-%' THEN '2021' --2021 to 2022 -when cast(activity_date as varchar(10)) like '2021-03-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2021-02-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2021-01-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-12-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-11-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-10-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-09-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-08-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-07-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-06-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-05-%' THEN '2020' -when cast(activity_date as varchar(10)) like '2020-04-%' THEN '2020' --2020 to 2021 -when cast(activity_date as varchar(10)) like '2020-03-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2020-02-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2020-01-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-12-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-11-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-10-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-09-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-08-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-07-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-06-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-05-%' THEN '2019' -when cast(activity_date as varchar(10)) like '2019-04-%' THEN '2019' --2019 to 2020 -when cast(activity_date as varchar(10)) like '2019-03-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2019-02-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2019-01-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-12-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-11-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-10-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-09-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-08-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-07-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-06-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-05-%' THEN '2018' -when cast(activity_date as varchar(10)) like '2018-04-%' THEN '2018' --2018 to 2019 -when cast(activity_date as varchar(10)) like '2018-03-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2018-02-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2018-01-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-12-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-11-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-10-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-09-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-08-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-07-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-06-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-05-%' THEN '2017' -when cast(activity_date as varchar(10)) like '2017-04-%' THEN '2017' --2017 to 2018 -when cast(activity_date as varchar(10)) like '2017-03-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2017-02-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2017-01-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-12-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-11-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-10-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-09-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-08-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-07-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-06-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-05-%' THEN '2016' -when cast(activity_date as varchar(10)) like '2016-04-%' THEN '2016' --2016 to 2017 -when cast(activity_date as varchar(10)) like '2016-03-%' THEN '2015' -when cast(activity_date as varchar(10)) like '2016-02-%' THEN '2015' --2015 to 2016 -when cast(activity_date as varchar(10)) like '1970-01-01' THEN '1969' -when cast(activity_date as varchar(10)) like '1970-01-26' THEN '1969' --1969 -else '1900' -end as fy -,concat(substr(Cast(activity_date as varchar(10)),1, 7), '-01') as act_MonthYear -,substr(cast(activity_date as varchar(40)), 1, 10) as act_date ,substr(cast(activity_date as varchar(40)), 12, 2) as act_hour -,case -when substr(permit_referece,1,3) = 'HYA' then 'All zone' -when substr(permit_referece,1,3) = 'HYG' then 'Companion Badge' -when substr(permit_referece,1,3) = 'HYB' then 'Business' -when substr(permit_referece,1,3) = 'HYD' then 'Doctor' -when substr(permit_referece,1,3) = 'HYP' then 'Estate resident' -when substr(permit_referece,1,3) = 'HYH' then 'Health and social care' -when substr(permit_referece,1,3) = 'HYR' then 'Residents' -when substr(permit_referece,1,3) = 'HYL' then 'Leisure Centre Permit' -when substr(permit_referece,1,3) = 'HYE' then 'All Zone Business Voucher' -when substr(permit_referece,1,3) = 'HYF' then 'Film Voucher' -when substr(permit_referece,1,3) = 'HYJ' then 'Health and Social Care Voucher' -when substr(permit_referece,1,3) = 'HYQ' then 'Estate Visitor Voucher' -when substr(permit_referece,1,3) = 'HYV' then 'Resident Visitor Voucher' -when substr(permit_referece,1,3) = 'HYN' then 'Dispensation' -when permit_referece ='HY50300' then 'Dispensation' -when substr(permit_referece,1,3) = 'HYS' then 'Suspension' -else substr(permit_referece,1,3) -end as product_type -,case -when substr(permit_referece,1,3) = 'HYE' then 'Voucher' -when substr(permit_referece,1,3) = 'HYF' then 'Voucher' -when substr(permit_referece,1,3) = 'HYQ' then 'Voucher' -when substr(permit_referece,1,3) = 'HYV' then 'Voucher' -when substr(permit_referece,1,3) = 'HYJ' then 'Voucher' -when substr(permit_referece,1,3) = 'HYN' then 'Dispensation' -when permit_referece ='HY50300' then 'Dispensation' -when substr(permit_referece,1,3) = 'HYS' then 'Suspension' else 'Permit' -end as product_category - -/*Activity tag group*/ -,case -when upper(activity) like 'ACTIVATE VOUCHER MAIL SENT FOR CONFIRMATION.' then 'ACTIVATION' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: A JOB DESCRIPTION FROM CUSTOMER%' then 'JOB DESCRIPTION' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER%' then 'BLUE BADGE - FRONT AND BACK' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE BACK FROM CUSTOMER%' then 'BLUE BADGE - BACK' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE FRONT FROM CUSTOMER%' then 'BLUE BADGE - FRONT' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: CORPORATE TAX OR VAT RETURN FROM CUSTOMER%' then 'TAX OR VAT' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: DOCUMENTATION REQUESTED BY EMAIL FROM CUSTOMER%' then 'DOCUMENTATION' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: DRIVING LICENCE FROM CUSTOMER%' then 'DRIVING LICENCE' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: LETTER FROM RELEVANT RELIGIOUS LEADER FROM CUSTOMER%' then 'RELIGIOUS LEADER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: LETTER FROM YOUR ORGANISATION, SIGNED BY YOUR LINE MANAGER IN SUPPORT OF PERMIT APPLICANT%' then 'LINE MANAGER IN SUPPORT' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: LGP CONVERSION CERTIFICATE FROM CUSTOMER%' then 'LGP CONVERSION CERTIFICATE' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: REMOVAL COMPANY DETAILS FROM CUSTOMER%' then 'REMOVAL COMPANY DETAILS' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: VEHICLE LOG BOOK FROM CUSTOMER%' then 'VEHICLE LOG BOOK' -when upper(activity) like 'APPLICATION SENT TO ASSISTANT DIRECTOR%' then 'APPLICATION SENT TO ASSISTANT DIRECTOR' -when upper(activity) like 'BOOK VOUCHER MAIL SENT FOR CONFIRMATION.' then 'BOOK VOUCHER' --Book voucher mail sent for confirmation. -when upper(activity) like 'CANCEL VOUCHER MAIL SENT FOR CONFIRMATION.' then 'CANCEL VOUCHER' -when upper(activity) like 'CHANGE ADDRESS REQUEST HAS BEEN REJECTED.' then 'REJECT ADDRESS CHANGE' -when upper(activity) like 'CHANGED PERMIT VRM%' then 'VRM CHANGE' -when upper(activity) like 'CITYPAY REFUND PROCESSED, REFUND' then 'CITYPAY REFUND' -when upper(activity) like 'CITYPAY REFUND PROCESSED, REFUND % FAILED%' then 'CITYPAY REFUND - FAILED' -when upper(activity) like 'CITYPAY REFUND PROCESSED, REFUND % PROCESSED%' then 'CITYPAY REFUND - PROCESSED' -when upper(activity) like 'CONFIRMATION OF VOUCHER BOOKING.' then 'BOOK VOUCHER' -when upper(activity) like 'CONTACT CENTRE ACTION CREATED%' then 'CREATED' -when upper(activity) like 'CONTACT CENTRE ACTION REQUEST FOR PERMIT REFERENCE%' then 'REQUEST FOR PERMIT REFERENCE' -when upper(activity) like 'DELETE VOUCHER MAIL SENT FOR CONFIRMATION%' then 'DELETE VOUCHER' -when upper(activity) like 'DISCOUNTED VOUCHER BOOKS REFUNED%' then 'DISCOUNTED VOUCHER REFUND' -when upper(activity) like 'DISPENSATION REMOVED' then 'REMOVED DISPENSATION' -when upper(activity) like 'DISPENSATION VRM REQUIRED REMINDER SENT' then 'VRM REQUIRED' -when upper(activity) like 'E VOUCHER MAIL SENT FOR ACTIVATION.' then 'ACTIVATION' -when upper(activity) like 'EMAI OF REJECTION SENT%' then 'REJECTION SENT' -when upper(activity) like 'EMAIL OF APPROVAL SENT%' then 'APPROVAL SENT' -when upper(activity) like 'EMAIL OF CANCELLATION SENT%' then 'CANCELLATION SENT' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOU HAVE PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL' then 'PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOU HAVE UNPAID INVOICES FOR PREVIOUS SUSPENSION(S)' then 'UNPAID INVOICES FOR PREVIOUS SUSPENSION(S)' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION' then 'APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION APPLICATION IS NOT A GENUINE EMERGENCY' then 'APPLICATION IS NOT A GENUINE EMERGENCY' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS (SEE BELOW FOR DETAILS)' then 'APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION WOULD CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA' then 'CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION WOULD CREATE UNACCEPTABLE PARKING STRESS IN THE AREA' then 'CREATE UNACCEPTABLE PARKING STRESS IN THE AREA' -when upper(activity) like 'EMAIL OF REJECTION SENT%' then 'REJECTION SENT' -when upper(activity) like 'EMAIL SENT TO FILMING OFFICE%' then 'EMAIL SENT TO FILMING OFFICE' --email sent to Filming office11/04/2017 -when upper(activity) like 'EVIDENCE REMINDER SENT' then 'EVIDENCE REMINDER' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 A LETTER FROM YOUR ORGANISATION, SIGNED BY YOUR LINE MANAGER IN SUPPORT OF PERMIT APPLICANT% APPLICATION FROM CUSTOMER FOR%' then 'LINE MANAGER IN SUPPORT' --Evidence requested again: 1 A Letter from your organisation, signed by your line manager in support of permit applicant’s application from Customer for HYH12690247 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (BACK OF BLUE BADGE) FROM CUSTOMER FOR %' then 'BLUE BADGE - BACK' --Evidence requested again: 1 Blue badge (back of blue badge) from Customer for HYG9063640 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER FOR%' then 'BLUE BADGE - FRONT AND BACK' --Evidence requested again: 1 Blue badge (front and back of blue badge) from Customer for HYG0999184 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (FRONT OF BLUE BADGE) FROM CUSTOMER FOR %' then 'BLUE BADGE - FRONT' --Evidence requested again: 1 Blue badge (front of blue badge) from Customer for HYG9303744 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 LPG CONVERSION CERTIFICATE FROM CUSTOMER FOR%' then 'LGP CONVERSION CERTIFICATE' --Evidence requested again: 1 LPG Conversion Certificate from Customer for HYR1380158 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PERMIT APPLICANT% JOB DESCRIPTION FROM CUSTOMER FOR%' then 'JOB DESCRIPTION' --Evidence requested again: 1 Permit applicant’s job description from Customer for HYH12690247 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF BUSINESS AT THIS ADDRESS FROM CUSTOMER FOR %' then 'PROOF BUSINESS AT THIS ADDRESS' --Evidence requested again: 1 proof business at this address from Customer for HYB7312888 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF ADDRESS FROM CUSTOMER FOR%' then 'PROOF OF BUSINESS WORKS' --Evidence requested again: null proof of address from Customer for HYR4946492 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF BUSINESS WORKS FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF BUSINESS WORKS FROM CUSTOMER' --Evidence requested again: 1 proof of business works from Customer for HYN11136834 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF VEHICLE INFORMATION FROM CUSTOMER%' then 'PROOF OF VEHICLE INFORMATION' --Evidence requested again: 1 proof of vehicle information from Customer for HYR9942251 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF WEDDING FROM CUSTOMER%' then 'PROOF OF WEDDING ' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (BACK OF BLUE BADGE) FROM CUSTOMER%' then 'BLUE BADGE - BACK' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER%' then 'BLUE BADGE - FRONT AND BACK' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (FRONT OF BLUE BADGE) FROM CUSTOMER FOR %' then 'BLUE BADGE - FRONT' --Evidence requested again: 1 Blue badge (back of blue badge) from Customer for HYG7461744 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL DOCUMENTATION REQUESTED BY EMAIL FROM CUSTOMER FOR %' then 'DOCUMENTATION' --Evidence requested again: null Documentation requested by email from Customer for HYS11582596 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL LPG CONVERSION CERTIFICATE FROM CUSTOMER FOR%' then 'LGP CONVERSION CERTIFICATE' --Evidence requested again: null LPG Conversion Certificate from Customer for HYP4205978 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL PROOF BUSINESS AT THIS ADDRESS FROM CUSTOMER%' then 'PROOF BUSINESS AT THIS ADDRESS' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL PROOF OF ADDRESS FROM CUSTOMER FOR%' then 'PROOF OF ADDRESS' --Evidence requested again: null proof of address from Customer for HYG4978685 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL PROOF OF VEHICLE INFORMATION FROM CUSTOMER FOR%' then 'PROOF OF VEHICLE INFORMATION' --Evidence requested again: null proof of vehicle information from Customer for HYB5776458 -when upper(activity) like 'EXTEND VOUCHER MAIL SENT FOR CONFIRMATION.' then 'EXTEND VOUCHER' --Extend voucher mail sent for confirmation. -when upper(activity) like 'EXTENSION REQUEST IS APPROVED BY : ' then 'APPROVAL' --Extension request is approved by : Extension request is approved by : -when upper(activity) like 'EXTENSION REQUEST IS REJECTED: ' then 'REJECTION' --Extension request is rejected: -when upper(activity) like 'FULL PRICE VOUCHER BOOKS REFUNED' then 'FULL PRICE VOUCHER REFUND' -when upper(activity) like 'FULL PRICE VOUCHER BOOKS REFUNED %. REFUND AMOUNT%' then 'FULL PRICE VOUCHER REFUND' --Full price voucher books refuned 1. Refund amount: £20.50 -when upper(activity) like 'MANUAL REFUND PROCESSED, REFUND OF % HAS BEEN PROCESSED ON %' then 'MANUAL REFUND PROCESSED' --Manual refund processed, Refund of £7.75 has been processed on 17/11/2022 00:00 -when upper(activity) like 'ON LBH INSTRUCTION PERMIT STATUS CHANGED TO ORDER RENEW' then 'LBH INSTRUCT PERMIT STATUS CHANGE' -when upper(activity) like 'ORDER QUANTITY SET TO%' then 'ORDER QUANTITY SET' -when upper(activity) like 'PERMIT ADDRESS WILL BE CHANGED%' then 'PERMIT ADDRESS WILL BE CHANGED' -when upper(activity) like 'PERMIT HAS BEEN PRINTED LOCALLY%' then 'PERMIT HAS BEEN PRINTED LOCALLY' -when upper(activity) like 'PERMIT HAS BEEN SENT FOR PRINTING%' then 'PERMIT HAS BEEN SENT FOR PRINTING' -when upper(activity) like 'PERMIT HAS BEEN SENT FOR VIRTUAL COVER%' then 'PERMIT HAS BEEN SENT FOR VIRTUAL COVER' -when upper(activity) like 'PERMIT RENEWAL REJECTED PAYMENT FAILURE%' then 'REJECTED PAYMENT FAILURE' -when upper(activity) like 'PERMIT RENEWAL REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED%' then 'REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' -when upper(activity) like 'PERMIT RENEWAL REJECTED WITH REFUND%' then 'REJECTED WITH REFUND' -when upper(activity) like 'PERMIT RENEWAL REJECTED YOUR APPLICATION NOT MEETING THIS PERMIT%S CRITERIA%' then 'REJECTED YOUR APPLICATION NOT MEETING THIS PERMITS CRITERIA' -when upper(activity) like 'PERMIT RENEWAL ACCEPTED%' then 'ACCEPTED' -when upper(activity) like 'PERMIT RENEWAL REJECTED%' then 'REJECTED' -when upper(activity) like 'PERMIT RENEWED%' then 'RENEWED' -when upper(activity) like 'PERMIT STATUS CHANGED' then 'CHANGED PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO ' then 'CHANGED PERMIT STATUS' --Permit status changed to -when upper(activity) like 'PERMIT STATUS CHANGED TO APPROVED' then 'APPROVED PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO CANCELLED' then 'CANCELLED PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO LOST' then 'LOST PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO PERMIT STATUS CHANGED TO LOST %WITH ADMIN FEE PAYMENT OF %' then 'LOST PERMIT STATUS' -- Permit status changed to Permit status changed to lost with admin fee payment of £10 received -when upper(activity) like 'PERMIT STATUS CHANGED TO PERMIT STATUS CHANGED TO DAMAGED WITH ADMIN FEE PAYMENT' then 'DAMAGED PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED.' OR upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED' then 'REJECTED PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF VEHICLE REGISTRATIONS NOT PROVIDED' then 'REJECTED PERMIT STATUS - VEHICLE REGISTRATIONS NOT PROVIDED' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF REQUESTED EVIDENCE NOT BEING PROVIDED' then 'REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' --Permit status changed to rejected because of requested evidence not being provided -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR APPLICATION NOT MEETING THIS PERMIT%S CRITERIA' then 'REJECTED YOUR APPLICATION NOT MEETING THIS PERMITS CRITERIA' --Permit status changed to rejected because of your application not meeting this permit's criteria -when upper(activity) like 'PERMIT STATUS CHANGED TO RENEWAL - EVIDENCE SUBMITTED%' then 'RENEWAL PERMIT STATUS - EVIDENCE SUBMITTED' -when upper(activity) like 'PERMIT STATUS CHANGED TO STOLEN%' then 'STOLEN PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO VRM CHANGE%' then 'VRM CHANGE PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO CANCELLED% ' then 'CANCELLED PERMIT STATUS ' -when upper(activity) like 'PERMIT STATUS CHANGED TO LOST%' then 'LOST PERMIT STATUS' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED AS REQUESTED%' then 'REJECTED PERMIT STATUS - REQUESTED' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF PAYMENT FAILURE' then 'REJECTED PERMIT STATUS - PAYMENT FAILURE' --Permit status changed to rejected because of payment failure -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR VEHICLE NOT REQUIRING A PERMIT' then 'REJECTED PERMIT STATUS - VEHICLE NOT REQUIRING A PERMIT' --Permit status changed to rejected because of your vehicle not requiring a permit -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF THE FILM OFFICE DECLINING THE VOUCHER APPLICATION' then 'REJECTED PERMIT STATUS - FILM OFFICE DECLINING THE VOUCHER APPLICATION' --Permit status changed to rejected because of the film office declining the voucher application -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOU HAVING 3 OR MORE OUTSTANDING PCNS THAT REQUIRE PAYMENT' then 'REJECTED PERMIT STATUS - OUTSTANDING PCNS' --Permit status changed to rejected because of you having 3 or more outstanding PCNs that require payment -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOU NOT LIVING IN THE BOROUGH' then 'REJECTED PERMIT STATUS - NOT LIVING IN THE BOROUGH' --Permit status changed to rejected because of you not living in the borough -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING IN A PARKING ZONE' then 'REJECTED PERMIT STATUS - ADDRESS NOT BEING IN A PARKING ZONE' --Permit status changed to rejected because of your address not being in a parking zone -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING LOCATED WITHIN A CONTROLLED PARKING ZONE IN HACKNEY' then 'REJECTED PERMIT STATUS - ADDRESS NOT BEING LOCATED WITHIN A CONTROLLED PARKING ZONE IN HACKNEY' --Permit status changed to rejected because of your address not being located within a controlled parking zone in Hackney -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING LOCATED WITHIN THE LONDON BOROUGH OF HACKNEY' then 'REJECTED PERMIT STATUS - ADDRESS NOT BEING LOCATED WITHIN THE LONDON BOROUGH OF HACKNEY' --Permit status changed to rejected because of your address not being located within the London borough of Hackney -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR PERMIT ADDRESS BEING IN A CAR FREE DEVELOPMENT' then 'REJECTED PERMIT STATUS - ADDRESS BEING IN A CAR FREE DEVELOPMENT'--Permit status changed to rejected because of your permit address being in a car free development -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS' then 'REJECTED PERMIT STATUS - VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS'--Permit status changed to rejected because of your vehicle exceeding the maximum dimensions -when upper(activity) like 'PERMIT THEFT REPORT HAS BEEN ACCEPTED FOR REFERENCE%' then 'PERMIT THEFT REPORT' -when upper(activity) like 'PERMIT%S CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN ACCEPTED%' then 'ACCEPTED VEHICLE CHANGE' -when upper(activity) like 'PERMIT%S CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN ACCEPTED FOR%' then 'ACCEPTED VEHICLE CHANGE' -when upper(activity) like 'PERMIT%S CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN REJECTED%' then 'REJECTED VEHICLE CHANGE' -when upper(activity) like 'PERSONAL BAY ADDED REFERENCE NUMBER % INSTALLATION DATE%' then 'PERSONAL BAY - ADDED' --Personal Bay added Reference Number 00156 Installation Date 9/8/2021 -when upper(activity) like 'PERSONAL BAY REMOVED REFERENCE NUMBER % REMOVAL DATE%' then 'PERSONAL BAY - REMOVED' --Personal Bay removed Reference Number 00013 Removal Date 2/8/2022 -when upper(activity) like 'REASON FOR THE REJECTION: YOU HAVE PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL. REFUND AMOUNT%' then 'PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL' --Reason for the rejection: you have previously failed to adhere to the terms and conditions of suspension requests granted by Hackney Council. Refund amount: £184.50 -when upper(activity) like 'REASON FOR THE REJECTION: YOU HAVE UNPAID INVOICES FOR PREVIOUS SUSPENSION(S). REFUND AMOUNT%' then 'UNPAID INVOICES FOR PREVIOUS SUSPENSION(S)' -when upper(activity) like 'REASON FOR THE REJECTION: YOUR APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION. REFUND AMOUNT%' then 'APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION' --Reason for the rejection: your application contains insufficient or contradictory information. Refund amount: £213.00 -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION APPLICATION IS NOT A GENUINE EMERGENCY. REFUND AMOUNT%' then 'APPLICATION IS NOT A GENUINE EMERGENCY' -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS (SEE BELOW FOR DETAILS). REFUND AMOUNT%' then 'APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS' --Reason for the rejection: your suspension application needs to be re-applied for with updated details (see below for details). Refund amount: £212.50 ---Reason for the rejection: your suspension application needs to be re-applied for with updated details (see below for details). Refund amount: £395.50 -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION WOULD CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA. REFUND AMOUNT%' then 'CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA' -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION WOULD CREATE UNACCEPTABLE PARKING STRESS IN THE AREA. REFUND AMOUNT%' then 'CREATE UNACCEPTABLE PARKING STRESS IN THE AREA' -when upper(activity) like 'REFUND DUE TO CUSTOMER' then 'REFUND DUE TO CUSTOMER' -when upper(activity) like 'REFUND DUE TO CUSTOMER PERMIT WAS AUTO-APPROVED INCORRECTLY. A FULL REFUND DONE BY MINDMILL MANUALLY.' then 'REFUND DUE TO CUSTOMER - MANUALLY - AUTO-APPROVED INCORRECTLY' -when upper(activity) like 'REFUND WAS SUPRESSED' then 'REFUND SUPRESSED' -when upper(activity) like 'REFUND OF % WAS SUPRESSED BY %' then 'REFUND SUPRESSED' --Refund of £84.92 was supressed by dmoses -when upper(activity) like 'REFUND OF % DUE TO CUSTOMER' then 'REFUND DUE TO CUSTOMER' --Refund of £20.00 due to customer -when upper(activity) like 'RENEWAL REMINDER SENT TO CUSTOMER FOR PERMIT REFERENCE%' then 'REMINDER SENT - CUSTOMER' -when upper(activity) like 'RENEWAL REMINDER IS NOT SENT FOR PERMIT REFERENCE%' then 'REMINDER IS NOT SENT' -when upper(activity) like 'RENEWAL REMINDER SENT TO CUSTOMER%' then 'REMINDER SENT - CUSTOMER' -when upper(activity) like 'RENEWAL REMINDER SENT' then 'REMINDER SENT' -when upper(activity) like 'ROLE ADDED, USER HAS AN APPROVED HASC ACCOUNT' then 'ACCOUNT ROLE ADDED - HASC' -when upper(activity) like 'SUCCESSFUL APPLICATION WITH PAYMENT%' then 'SUCCESSFUL APPLICATION - PAYMENT' -when upper(activity) like 'SUSPENSION' then 'SUSPENSION' -when upper(activity) like '%SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION REFUND IN COST% START AND END DATES ARE%' then 'AMEND SUSPENSION REFUND IN COST' -- Suspension amended and approved. Amend suspension refund in cost: £408. Start and end dates are: 2021-05-17 00:00:00.0 and 2021-05-20 00:00:00.0 -- Suspension amended and approved. Amend suspension refund in cost: £170. Start and end dates are: 2021-12-06 00:00:00.0 and 2021-12-10 00:00:00.0 -when upper(activity) like '%SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION WITH INCREASE IN COST AND PAYMENT RECEIVED% START AND END DATES ARE%' then 'AMEND SUSPENSION WITH INCREASE IN COST AND PAYMENT RECEIVED' -- Suspension amended and approved. Amend suspension with increase in cost and payment received: £68. Start and end dates are: 2021-10-12 00:00:00.0 and 2021-10-13 00:00:00.0 -- Suspension amended and approved. Amend suspension with increase in cost and payment received: £136. Start and end dates are: 2021-04-15 00:00:00.0 and 2021-04-18 00:00:00.0 -when upper(activity) like '%SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION WITH NO CHANGE IN COST. START AND END DATES ARE%' then 'AMEND SUSPENSION WITH NO CHANGE IN COST' --Suspension amended and approved. Amend suspension with no change in cost. Start and end dates are: 2023-09-11 00:00:00.0 and 2023-09-11 00:00:00.0 -when upper(activity) like 'SUSPENSION DUE TO BE CANCELLED ON %. REFUND AMOUNT%' then 'CANCELLED SUSPENSION' -- Suspension due to be cancelled on 2022-04-13 00:00:00.0. Refund amount: £114.50 -when upper(activity) like 'SUSPENSION DURATION CHANGED. CHANGE DURATION WITH INCREASE IN COST%' then 'CHANGE DURATION WITH INCREASE IN COST' -when upper(activity) like '%SUSPENSION DURATION CHANGED. CHANGE DURATION WITH REFUND IN COST:%OLD SUSPENSION START AND END DATES ARE%' then 'CHANGE DURATION WITH REFUND IN COST' -- Suspension duration changed. Change duration with refund in cost: £102.000. Old suspension start and end dates are : 2020-11-14 00:00:00.0 and 2020-11-15 00:00:00.0 -- Suspension duration changed. Change duration with refund in cost: £102.000. Old suspension start and end dates are : 2020-11-14 00:00:00.0 and 2020-11-15 00:00:00.0 -when upper(activity) like '%SUSPENSION DURATION CHANGED. OLD SUSPENSION START AND END DATES ARE%' then 'OLD SUSPENSION START AND END DATES' -- Suspension duration changed. Old suspension start and end dates are : 2017-04-03 00:00:00.0 and 2017-04-07 00:00:00.0 -- Suspension duration changed. Old suspension start and end dates are : 2017-04-03 00:00:00.0 and 2017-04-07 00:00:00.0 -when upper(activity) like 'SUSPENSION EXTENSION APPROVED' then 'SUSPENSION - EXTENSION APPROVED' --Suspension extension approved -when upper(activity) like 'SUSPENSION UPDATE TO SIGN UP' then 'SUSPENSION - SIGN UP' -when upper(activity) like 'SUSPENSION UPDATE TO SUSPENSION REJECT' then 'SUSPENSION - REJECT' -when upper(activity) like '%SUSPENSION UPDATE TO SUSPENSION APPROVED%' then 'SUSPENSION - UPDATE TO APPROVED' --Suspension update to Suspension Approved -when upper(activity) like 'TEMPORARY PERMIT PRINTED STARTING % ENDING %' then 'PRINTED TEMPORARY PERMIT' --Temporary permit printed starting 04/05/2017 ending 04/06/2017 -when upper(activity) like 'THE CRIME REFERENCE NUMBER HAS NOT BEEN CONFIRMED BY THE POLICE AND SO A FREE REPLACEMENT IS NOT POSSIBLE FOR REFERENCE%' then 'CRIME REFERENCE NUMBER HAS NOT BEEN CONFIRMED BY THE POLICE AND SO A FREE REPLACEMENT IS NOT POSSIBLE' -when upper(activity) like 'THE PROTECTED VEHICLE WAS CHANGED%' then 'PROTECTED VEHICLE WAS CHANGED' -when upper(activity) like 'THE SUSPENSION MANUALLY SIGN HAS BEEN UPDATED. THE IDENTIFIER IS%' then 'SUSPENSION MANUALLY SIGN' -when upper(activity) like 'UPDATE VOUCHER MAIL SENT FOR CONFIRMATION.' then 'UPDATE VOUCHER' -when upper(activity) like 'VEHICLE REGISTRATION NUMBER:%' then 'VEHICLE REGISTRATION NUMBER' -when upper(activity) like 'VEHICLE UPDATED ON PERMIT . OLD VEHILCE DETAILS ARE : VRM%' OR upper(activity) like ' VEHICLE UPDATED ON PERMIT . OLD VEHILCE DETAILS ARE : VRM :%' then 'VEHICLE UPDATED' -- Vehicle updated on permit . Old vehilce details are : vrm : AO05COH Emission (CO2) : 180 manufacturer : VOLKSWAGEN model : POLO TWIST AUTO. Permit charge were 112.000 --Vehicle updated on permit . Old vehilce details are : vrm : DG11VUM Emission (CO2) : 132 manufacturer : HONDA model : CIVIC TYPE-S I-VTEC S-A. Permit charge were 43.000 -when upper(activity) like 'VEHICLES ON DISPENSATION CHANGED FROM NULL%' then 'VEHICLES ON DISPENSATION CHANGED' -when upper(activity) like 'VEHICLES ON DISPENSATION CHANGED%' then 'VEHICLES ON DISPENSATION CHANGED' -when upper(activity) like 'VOUCHER HAS BEEN MARKED AS SENT TO PRINT%' then 'MARKED AS SENT TO PRINT' -when upper(activity) like 'VOUCHER COVER LETTER HAS BEEN PRINTED' then 'COVER LETTER HAS BEEN PRINTED' -when upper(activity) like 'VOUCHER SENT TO LIBERTY' then 'SENT TO LIBERTY' -when upper(activity) like 'VOUCHER SERIAL NUMBERS UPDATED%' then 'SERIAL NUMBERS UPDATED' -when upper(activity) like 'VOUCHER SESSIONS OVERRIDDEN%' then 'VOUCHER SESSIONS OVERRIDDEN' -when upper(activity) like 'VOUCHER SHIPPED LOCALLY' then 'SHIPPED LOCALLY' -when upper(activity) like 'VRM % WILL BE REMOVED WITH EFFECT FROM%' then 'VRM WILL BE REMOVED - EFFECT FROM' --VRM GJ52XEM will be removed with effect from 04/05/2017 -when upper(activity) like 'VRM % WILL BE REMOVED WITH REFUND' or upper(activity) like 'VRM % WILL BE REMOVED WITH REFUND OF % WITH EFFECT FROM%' then 'VRM WILL BE REMOVED - REFUND' --VRM T321NMH WILL BE REMOVED WITH REFUND OF £-181.26 WITH EFFECT FROM 09/02/2024 -when upper(activity) like '%VRM CHANGE ACCEPTED%' then 'VRM CHANGE - ACCEPTED' --VRM CHANGE ACCEPTED -when upper(activity) like 'VRM CHANGE APPLICATION' then 'VRM CHANGE - APPLICATION' -when upper(activity) like 'VRM CHANGE APPLICATION % SENT ON%' OR upper(activity) like 'VRM CHANGE APPLICATION SENT ON%' then 'VRM CHANGE - APPLICATION SENT' --Vrm change application £null sent on 31/05/2017 --Vrm change application sent on 01/03/2017 -when upper(activity) like 'VRM CHANGE APPLICATION WITH PAYMENT OF % SENT ON %' then 'VRM CHANGE - APPLICATION WITH PAYMENT' --Vrm change application with payment of £8.96 sent on 19/12/2023 -when upper(activity) like 'VRM CHANGE REJECTED PAYMENT FAILURE' then 'VRM CHANGE - REJECTED PAYMENT FAILURE' -when upper(activity) like 'VRM CHANGE REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' then 'VRM CHANGE - REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' -when upper(activity) like 'VRM CHANGE REJECTED YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS' then 'VRM CHANGE - REJECTED YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS' -when upper(activity) like 'VRM CHANGE REJECTED YOUR VEHICLE NOT REQUIRING A PERMIT' then 'VRM CHANGE - REJECTED YOUR VEHICLE NOT REQUIRING A PERMIT' -when upper(activity) like 'VRM CHANGED' then 'VRM CHANGED' -when upper(activity) like 'VRM CHANGED FROM%' then 'VRM CHANGED' -when upper(activity) like 'WILL BE REMOVED WITH PAYMENT' then 'REMOVED WITH PAYMENT' -when upper(activity) like '' or upper(activity) like ' ' or upper(activity) is null then - upper(case - when substr(permit_referece,1,3) = 'HYA' then 'All zone' - when substr(permit_referece,1,3) = 'HYG' then 'Companion Badge' - when substr(permit_referece,1,3) = 'HYB' then 'Business' - when substr(permit_referece,1,3) = 'HYD' then 'Doctor' - when substr(permit_referece,1,3) = 'HYP' then 'Estate resident' - when substr(permit_referece,1,3) = 'HYH' then 'Health and social care' - when substr(permit_referece,1,3) = 'HYR' then 'Residents' - when substr(permit_referece,1,3) = 'HYL' then 'Leisure Centre Permit' - when substr(permit_referece,1,3) = 'HYE' then 'All Zone Business Voucher' - when substr(permit_referece,1,3) = 'HYF' then 'Film Voucher' - when substr(permit_referece,1,3) = 'HYJ' then 'Health and Social Care Voucher' - when substr(permit_referece,1,3) = 'HYQ' then 'Estate Visitor Voucher' - when substr(permit_referece,1,3) = 'HYV' then 'Resident Visitor Voucher' - when substr(permit_referece,1,3) = 'HYN' then 'Dispensation' - when permit_referece ='HY50300' then 'Dispensation' - when substr(permit_referece,1,3) = 'HYS' then 'Suspension' - else substr(permit_referece,1,3) - end) -end as activity_tag_group - - -/*Activity Tag*/ -,case -when upper(activity) like 'ACTIVATE VOUCHER MAIL SENT FOR CONFIRMATION.' then 'ACTIVATE VOUCHER MAIL SENT FOR CONFIRMATION' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: A JOB DESCRIPTION FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: A JOB DESCRIPTION FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE BACK FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE BACK FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE FRONT FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: BLUE BADGE FRONT FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: CORPORATE TAX OR VAT RETURN FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: CORPORATE TAX OR VAT RETURN FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: DOCUMENTATION REQUESTED BY EMAIL FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: DOCUMENTATION REQUESTED BY EMAIL FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: DRIVING LICENCE FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: DRIVING LICENCE FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: LETTER FROM RELEVANT RELIGIOUS LEADER FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: LETTER FROM RELEVANT RELIGIOUS LEADER FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: LETTER FROM YOUR ORGANISATION, SIGNED BY YOUR LINE MANAGER IN SUPPORT OF PERMIT APPLICANT%' then 'ADDITIONAL EVIDENCE REQUESTED: LETTER FROM YOUR ORGANISATION, SIGNED BY YOUR LINE MANAGER IN SUPPORT OF PERMIT APPLICANT' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: LGP CONVERSION CERTIFICATE FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: LGP CONVERSION CERTIFICATE FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: REMOVAL COMPANY DETAILS FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: REMOVAL COMPANY DETAILS FROM CUSTOMER' -when upper(activity) like 'ADDITIONAL EVIDENCE REQUESTED: VEHICLE LOG BOOK FROM CUSTOMER%' then 'ADDITIONAL EVIDENCE REQUESTED: VEHICLE LOG BOOK FROM CUSTOMER' -when upper(activity) like 'APPLICATION SENT TO ASSISTANT DIRECTOR%' then 'APPLICATION SENT TO ASSISTANT DIRECTOR' -when upper(activity) like 'BOOK VOUCHER MAIL SENT FOR CONFIRMATION.' then 'BOOK VOUCHER MAIL SENT FOR CONFIRMATION' --Book voucher mail sent for confirmation. -when upper(activity) like 'CANCEL VOUCHER MAIL SENT FOR CONFIRMATION.' then 'CANCEL VOUCHER MAIL SENT FOR CONFIRMATION' -when upper(activity) like 'CHANGE ADDRESS REQUEST HAS BEEN REJECTED.' then 'CHANGE ADDRESS REQUEST HAS BEEN REJECTED' -when upper(activity) like 'CHANGED PERMIT VRM%' then 'CHANGED PERMIT VRM' -when upper(activity) like 'CITYPAY REFUND PROCESSED, REFUND' then 'CITYPAY REFUND PROCESSED, REFUND' -when upper(activity) like 'CITYPAY REFUND PROCESSED, REFUND % FAILED%' then 'CITYPAY REFUND PROCESSED, REFUND FAILED' -when upper(activity) like 'CITYPAY REFUND PROCESSED, REFUND % PROCESSED%' then 'CITYPAY REFUND PROCESSED, REFUND PROCESSED' -when upper(activity) like 'CONFIRMATION OF VOUCHER BOOKING.' then 'CONFIRMATION OF VOUCHER BOOKING' -when upper(activity) like 'CONTACT CENTRE ACTION CREATED%' then 'CONTACT CENTRE ACTION CREATED' -when upper(activity) like 'CONTACT CENTRE ACTION REQUEST FOR PERMIT REFERENCE%' then 'CONTACT CENTRE ACTION REQUEST FOR PERMIT REFERENCE' -when upper(activity) like 'DELETE VOUCHER MAIL SENT FOR CONFIRMATION%' then 'DELETE VOUCHER MAIL SENT FOR CONFIRMATION' -when upper(activity) like 'DISCOUNTED VOUCHER BOOKS REFUNED%' then 'DISCOUNTED VOUCHER BOOKS REFUNED' -when upper(activity) like 'DISPENSATION REMOVED' then 'DISPENSATION REMOVED' -when upper(activity) like 'DISPENSATION VRM REQUIRED REMINDER SENT' then 'DISPENSATION VRM REQUIRED REMINDER SENT' -when upper(activity) like 'E VOUCHER MAIL SENT FOR ACTIVATION.' then 'E VOUCHER MAIL SENT FOR ACTIVATION' -when upper(activity) like 'EMAI OF REJECTION SENT%' then 'EMAIL OF REJECTION SENT' -when upper(activity) like 'EMAIL OF APPROVAL SENT%' then 'EMAIL OF APPROVAL SENT' -when upper(activity) like 'EMAIL OF CANCELLATION SENT%' then 'EMAIL OF CANCELLATION SENT' - -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOU HAVE PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOU HAVE PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOU HAVE UNPAID INVOICES FOR PREVIOUS SUSPENSION(S)' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOU HAVE UNPAID INVOICES FOR PREVIOUS SUSPENSION(S)' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION APPLICATION IS NOT A GENUINE EMERGENCY' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION APPLICATION IS NOT A GENUINE EMERGENCY' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS (SEE BELOW FOR DETAILS)' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS (SEE BELOW FOR DETAILS)' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION WOULD CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION WOULD CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA' -when upper(activity) like 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION WOULD CREATE UNACCEPTABLE PARKING STRESS IN THE AREA' then 'EMAIL OF REJECTION SENT. REASON FOR REJECTION YOUR SUSPENSION WOULD CREATE UNACCEPTABLE PARKING STRESS IN THE AREA' - -when upper(activity) like 'EMAIL OF REJECTION SENT%' then 'EMAIL OF REJECTION SENT' -when upper(activity) like 'EMAIL SENT TO FILMING OFFICE%' then 'EMAIL SENT TO FILMING OFFICE' --email sent to Filming office11/04/2017 -when upper(activity) like 'EVIDENCE REMINDER SENT' then 'EVIDENCE REMINDER SENT' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 A LETTER FROM YOUR ORGANISATION, SIGNED BY YOUR LINE MANAGER IN SUPPORT OF PERMIT APPLICANT% APPLICATION FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 A LETTER FROM YOUR ORGANISATION, SIGNED BY YOUR LINE MANAGER IN SUPPORT OF PERMIT APPLICANT APPLICATION FROM CUSTOMER' --Evidence requested again: 1 A Letter from your organisation, signed by your line manager in support of permit applicant’s application from Customer for HYH12690247 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (BACK OF BLUE BADGE) FROM CUSTOMER FOR %' then 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (BACK OF BLUE BADGE) FROM CUSTOMER' --Evidence requested again: 1 Blue badge (back of blue badge) from Customer for HYG9063640 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER' --Evidence requested again: 1 Blue badge (front and back of blue badge) from Customer for HYG0999184 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (FRONT OF BLUE BADGE) FROM CUSTOMER FOR %' then 'EVIDENCE REQUESTED AGAIN: 1 BLUE BADGE (FRONT OF BLUE BADGE) FROM CUSTOMER' --Evidence requested again: 1 Blue badge (front of blue badge) from Customer for HYG9303744 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 LPG CONVERSION CERTIFICATE FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 LPG CONVERSION CERTIFICATE FROM CUSTOMER' --Evidence requested again: 1 LPG Conversion Certificate from Customer for HYR1380158 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PERMIT APPLICANT% JOB DESCRIPTION FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 PERMIT APPLICANT JOB DESCRIPTION FROM CUSTOMER' --Evidence requested again: 1 Permit applicant’s job description from Customer for HYH12690247 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF BUSINESS AT THIS ADDRESS FROM CUSTOMER FOR %' then 'EVIDENCE REQUESTED AGAIN: 1 PROOF BUSINESS AT THIS ADDRESS FROM CUSTOMER' --Evidence requested again: 1 proof business at this address from Customer for HYB7312888 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF ADDRESS FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF ADDRESS FROM CUSTOMER' --Evidence requested again: null proof of address from Customer for HYR4946492 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF BUSINESS WORKS FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF BUSINESS WORKS FROM CUSTOMER' --Evidence requested again: 1 proof of business works from Customer for HYN11136834 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF VEHICLE INFORMATION FROM CUSTOMER%' then 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF VEHICLE INFORMATION FROM CUSTOMER' --Evidence requested again: 1 proof of vehicle information from Customer for HYR9942251 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF WEDDING FROM CUSTOMER%' then 'EVIDENCE REQUESTED AGAIN: 1 PROOF OF WEDDING FROM CUSTOMER' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (BACK OF BLUE BADGE) FROM CUSTOMER%' then 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (BACK OF BLUE BADGE) FROM CUSTOMER' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER%' then 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (FRONT AND BACK OF BLUE BADGE) FROM CUSTOMER' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (FRONT OF BLUE BADGE) FROM CUSTOMER FOR %' then 'EVIDENCE REQUESTED AGAIN: NULL BLUE BADGE (FRONT OF BLUE BADGE) FROM CUSTOMER' --Evidence requested again: 1 Blue badge (back of blue badge) from Customer for HYG7461744 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL DOCUMENTATION REQUESTED BY EMAIL FROM CUSTOMER FOR %' then 'EVIDENCE REQUESTED AGAIN: NULL DOCUMENTATION REQUESTED BY EMAIL FROM CUSTOMER' --Evidence requested again: null Documentation requested by email from Customer for HYS11582596 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL LPG CONVERSION CERTIFICATE FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: NULL LPG CONVERSION CERTIFICATE FROM CUSTOMER' --Evidence requested again: null LPG Conversion Certificate from Customer for HYP4205978 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL PROOF BUSINESS AT THIS ADDRESS FROM CUSTOMER%' then 'EVIDENCE REQUESTED AGAIN: NULL PROOF BUSINESS AT THIS ADDRESS FROM CUSTOMER' -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL PROOF OF ADDRESS FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: NULL PROOF OF ADDRESS FROM CUSTOMER' --Evidence requested again: null proof of address from Customer for HYG4978685 -when upper(activity) like 'EVIDENCE REQUESTED AGAIN: NULL PROOF OF VEHICLE INFORMATION FROM CUSTOMER FOR%' then 'EVIDENCE REQUESTED AGAIN: NULL PROOF OF VEHICLE INFORMATION FROM CUSTOMER' --Evidence requested again: null proof of vehicle information from Customer for HYB5776458 -when upper(activity) like 'EXTEND VOUCHER MAIL SENT FOR CONFIRMATION.' then 'EXTEND VOUCHER MAIL SENT FOR CONFIRMATION' --Extend voucher mail sent for confirmation. -when upper(activity) like 'EXTENSION REQUEST IS APPROVED BY : ' then 'EXTENSION REQUEST IS APPROVED BY' --Extension request is approved by : Extension request is approved by : -when upper(activity) like 'EXTENSION REQUEST IS REJECTED: ' then 'EXTENSION REQUEST IS REJECTED:' --Extension request is rejected: -when upper(activity) like 'FULL PRICE VOUCHER BOOKS REFUNED' then 'FULL PRICE VOUCHER BOOKS REFUNED' -when upper(activity) like 'FULL PRICE VOUCHER BOOKS REFUNED %. REFUND AMOUNT%' then 'FULL PRICE VOUCHER BOOKS REFUNED' --Full price voucher books refuned 1. Refund amount: £20.50 -when upper(activity) like 'MANUAL REFUND PROCESSED, REFUND OF % HAS BEEN PROCESSED ON %' then 'MANUAL REFUND PROCESSED, REFUND' --Manual refund processed, Refund of £7.75 has been processed on 17/11/2022 00:00 -when upper(activity) like 'ON LBH INSTRUCTION PERMIT STATUS CHANGED TO ORDER RENEW' then 'ON LBH INSTRUCTION PERMIT STATUS CHANGED TO ORDER RENEW' -when upper(activity) like 'ORDER QUANTITY SET TO%' then 'ORDER QUANTITY SET TO' -when upper(activity) like 'PERMIT ADDRESS WILL BE CHANGED%' then 'PERMIT ADDRESS WILL BE CHANGED' -when upper(activity) like 'PERMIT HAS BEEN PRINTED LOCALLY%' then 'PERMIT HAS BEEN PRINTED LOCALLY' -when upper(activity) like 'PERMIT HAS BEEN SENT FOR PRINTING%' then 'PERMIT HAS BEEN SENT FOR PRINTING' -when upper(activity) like 'PERMIT HAS BEEN SENT FOR VIRTUAL COVER%' then 'PERMIT HAS BEEN SENT FOR VIRTUAL COVER' - -when upper(activity) like 'PERMIT RENEWAL REJECTED PAYMENT FAILURE%' then 'PERMIT RENEWAL REJECTED PAYMENT FAILURE' -when upper(activity) like 'PERMIT RENEWAL REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED%' then 'PERMIT RENEWAL REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' -when upper(activity) like 'PERMIT RENEWAL REJECTED WITH REFUND%' then 'PERMIT RENEWAL REJECTED WITH REFUND' -when upper(activity) like 'PERMIT RENEWAL REJECTED YOUR APPLICATION NOT MEETING THIS PERMIT%S CRITERIA%' then 'PERMIT RENEWAL REJECTED YOUR APPLICATION NOT MEETING THIS PERMITS CRITERIA' -when upper(activity) like 'PERMIT RENEWAL ACCEPTED%' then 'PERMIT RENEWAL ACCEPTED' -when upper(activity) like 'PERMIT RENEWAL REJECTED%' then 'PERMIT RENEWAL REJECTED' -when upper(activity) like 'PERMIT RENEWED%' then 'PERMIT RENEWED' -when upper(activity) like 'PERMIT STATUS CHANGED' then 'PERMIT STATUS CHANGED' -when upper(activity) like 'PERMIT STATUS CHANGED TO ' then 'PERMIT STATUS CHANGED' --Permit status changed to -when upper(activity) like 'PERMIT STATUS CHANGED TO APPROVED' then 'PERMIT STATUS CHANGED TO APPROVED' -when upper(activity) like 'PERMIT STATUS CHANGED TO CANCELLED' then 'PERMIT STATUS CHANGED TO CANCELLED' -when upper(activity) like 'PERMIT STATUS CHANGED TO LOST' then 'PERMIT STATUS CHANGED TO LOST' -when upper(activity) like 'PERMIT STATUS CHANGED TO PERMIT STATUS CHANGED TO LOST %WITH ADMIN FEE PAYMENT OF %' then 'PERMIT STATUS CHANGED TO LOST WITH ADMIN FEE PAYMENT' -- Permit status changed to Permit status changed to lost with admin fee payment of £10 received -when upper(activity) like 'PERMIT STATUS CHANGED TO PERMIT STATUS CHANGED TO DAMAGED WITH ADMIN FEE PAYMENT' then 'PERMIT STATUS CHANGED TO DAMAGED WITH ADMIN FEE PAYMENT' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED.' OR upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED' then 'PERMIT STATUS CHANGED TO REJECTED' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF VEHICLE REGISTRATIONS NOT PROVIDED' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF VEHICLE REGISTRATIONS NOT PROVIDED' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF REQUESTED EVIDENCE NOT BEING PROVIDED' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF REQUESTED EVIDENCE NOT BEING PROVIDED' --Permit status changed to rejected because of requested evidence not being provided -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR APPLICATION NOT MEETING THIS PERMIT%S CRITERIA' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR APPLICATION NOT MEETING THIS PERMITS CRITERIA' --Permit status changed to rejected because of your application not meeting this permit's criteria -when upper(activity) like 'PERMIT STATUS CHANGED TO RENEWAL - EVIDENCE SUBMITTED%' then 'PERMIT STATUS CHANGED TO RENEWAL - EVIDENCE SUBMITTED' -when upper(activity) like 'PERMIT STATUS CHANGED TO STOLEN%' then 'PERMIT STATUS CHANGED TO STOLEN' -when upper(activity) like 'PERMIT STATUS CHANGED TO VRM CHANGE%' then 'PERMIT STATUS CHANGED TO VRM CHANGE' -when upper(activity) like 'PERMIT STATUS CHANGED TO CANCELLED% ' then 'PERMIT STATUS CHANGED TO CANCELLED ' -when upper(activity) like 'PERMIT STATUS CHANGED TO LOST%' then 'PERMIT STATUS CHANGED TO LOST' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED AS REQUESTED%' then 'PERMIT STATUS CHANGED TO REJECTED AS REQUESTED' -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF PAYMENT FAILURE' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF PAYMENT FAILURE' --Permit status changed to rejected because of payment failure -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR VEHICLE NOT REQUIRING A PERMIT' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE VEHICLE DOES NOT REQUIRING A PERMIT' --Permit status changed to rejected because of your vehicle not requiring a permit -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF THE FILM OFFICE DECLINING THE VOUCHER APPLICATION' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF THE FILM OFFICE DECLINING THE VOUCHER APPLICATION' --Permit status changed to rejected because of the film office declining the voucher application -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOU HAVING 3 OR MORE OUTSTANDING PCNS THAT REQUIRE PAYMENT' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOU HAVING 3 OR MORE OUTSTANDING PCNS THAT REQUIRE PAYMENT' --Permit status changed to rejected because of you having 3 or more outstanding PCNs that require payment -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOU NOT LIVING IN THE BOROUGH' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOU NOT LIVING IN THE BOROUGH' --Permit status changed to rejected because of you not living in the borough -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING IN A PARKING ZONE' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING IN A PARKING ZONE' --Permit status changed to rejected because of your address not being in a parking zone -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING LOCATED WITHIN A CONTROLLED PARKING ZONE IN HACKNEY' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING LOCATED WITHIN A CONTROLLED PARKING ZONE IN HACKNEY' --Permit status changed to rejected because of your address not being located within a controlled parking zone in Hackney -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING LOCATED WITHIN THE LONDON BOROUGH OF HACKNEY' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR ADDRESS NOT BEING LOCATED WITHIN THE LONDON BOROUGH OF HACKNEY' --Permit status changed to rejected because of your address not being located within the London borough of Hackney -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR PERMIT ADDRESS BEING IN A CAR FREE DEVELOPMENT' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR PERMIT ADDRESS BEING IN A CAR FREE DEVELOPMENT'--Permit status changed to rejected because of your permit address being in a car free development -when upper(activity) like 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS' then 'PERMIT STATUS CHANGED TO REJECTED BECAUSE OF YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS'--Permit status changed to rejected because of your vehicle exceeding the maximum dimensions -when upper(activity) like 'PERMIT THEFT REPORT HAS BEEN ACCEPTED FOR REFERENCE%' then 'PERMIT THEFT REPORT HAS BEEN ACCEPTED FOR REFERENCE' -when upper(activity) like 'PERMIT%S CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN ACCEPTED%' then 'PERMITS CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN ACCEPTED' -when upper(activity) like 'PERMIT%S CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN ACCEPTED FOR%' then 'PERMITS CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN ACCEPTED FOR' -when upper(activity) like 'PERMIT%S CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN REJECTED%' then 'PERMITS CHANGE OF VEHICLE DETAILS REQUEST HAS BEEN REJECTED' -when upper(activity) like 'PERSONAL BAY ADDED REFERENCE NUMBER % INSTALLATION DATE%' then 'PERSONAL BAY ADDED REFERENCE NUMBER WITH INSTALLATION DATE' --Personal Bay added Reference Number 00156 Installation Date 9/8/2021 -when upper(activity) like 'PERSONAL BAY REMOVED REFERENCE NUMBER % REMOVAL DATE%' then 'PERSONAL BAY REMOVED REFERENCE NUMBER' --Personal Bay removed Reference Number 00013 Removal Date 2/8/2022 -when upper(activity) like 'REASON FOR THE REJECTION: YOU HAVE PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL. REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOU HAVE PREVIOUSLY FAILED TO ADHERE TO THE TERMS AND CONDITIONS OF SUSPENSION REQUESTS GRANTED BY HACKNEY COUNCIL. REFUND AMOUNT' --Reason for the rejection: you have previously failed to adhere to the terms and conditions of suspension requests granted by Hackney Council. Refund amount: £184.50 -when upper(activity) like 'REASON FOR THE REJECTION: YOU HAVE UNPAID INVOICES FOR PREVIOUS SUSPENSION(S). REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOU HAVE UNPAID INVOICES FOR PREVIOUS SUSPENSION(S). REFUND AMOUNT' -when upper(activity) like 'REASON FOR THE REJECTION: YOUR APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION. REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOUR APPLICATION CONTAINS INSUFFICIENT OR CONTRADICTORY INFORMATION. REFUND' --Reason for the rejection: your application contains insufficient or contradictory information. Refund amount: £213.00 -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION APPLICATION IS NOT A GENUINE EMERGENCY. REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOUR SUSPENSION APPLICATION IS NOT A GENUINE EMERGENCY. REFUND AMOUNT' -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS (SEE BELOW FOR DETAILS). REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOUR SUSPENSION APPLICATION NEEDS TO BE RE-APPLIED FOR WITH UPDATED DETAILS (SEE BELOW FOR DETAILS)' --Reason for the rejection: your suspension application needs to be re-applied for with updated details (see below for details). Refund amount: £212.50 ---Reason for the rejection: your suspension application needs to be re-applied for with updated details (see below for details). Refund amount: £395.50 -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION WOULD CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA. REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOUR SUSPENSION WOULD CLASH WITH ANOTHER SUSPENSION IN THE IMMEDIATE AREA. REFUND AMOUNT' -when upper(activity) like 'REASON FOR THE REJECTION: YOUR SUSPENSION WOULD CREATE UNACCEPTABLE PARKING STRESS IN THE AREA. REFUND AMOUNT%' then 'REASON FOR THE REJECTION: YOUR SUSPENSION WOULD CREATE UNACCEPTABLE PARKING STRESS IN THE AREA. REFUND AMOUNT' -when upper(activity) like 'REFUND DUE TO CUSTOMER' then 'REFUND DUE TO CUSTOMER' -when upper(activity) like 'REFUND DUE TO CUSTOMER PERMIT WAS AUTO-APPROVED INCORRECTLY. A FULL REFUND DONE BY MINDMILL MANUALLY.' then 'REFUND DUE TO CUSTOMER PERMIT WAS AUTO-APPROVED INCORRECTLY. A FULL REFUND DONE BY MINDMILL MANUALLY.' -when upper(activity) like 'REFUND WAS SUPRESSED' then 'REFUND WAS SUPRESSED' -when upper(activity) like 'REFUND OF % WAS SUPRESSED BY %' then 'REFUND WAS SUPRESSED' --Refund of £84.92 was supressed by dmoses -when upper(activity) like 'REFUND OF % DUE TO CUSTOMER' then 'REFUND DUE TO CUSTOMER' --Refund of £20.00 due to customer -when upper(activity) like 'RENEWAL REMINDER SENT TO CUSTOMER FOR PERMIT REFERENCE%' then 'RENEWAL REMINDER SENT TO CUSTOMER FOR PERMIT REFERENCE' -when upper(activity) like 'RENEWAL REMINDER IS NOT SENT FOR PERMIT REFERENCE%' then 'RENEWAL REMINDER IS NOT SENT FOR PERMIT REFERENCE' - -when upper(activity) like 'RENEWAL REMINDER SENT TO CUSTOMER%' then 'RENEWAL REMINDER SENT TO CUSTOMER' - -when upper(activity) like 'RENEWAL REMINDER SENT' then 'RENEWAL REMINDER SENT' -when upper(activity) like 'ROLE ADDED, USER HAS AN APPROVED HASC ACCOUNT' then 'ROLE ADDED, USER HAS AN APPROVED HASC ACCOUNT' -when upper(activity) like 'SUCCESSFUL APPLICATION WITH PAYMENT%' then 'SUCCESSFUL APPLICATION WITH PAYMENT' -when upper(activity) like 'SUSPENSION' then 'SUSPENSION' -when upper(activity) like '%SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION REFUND IN COST% START AND END DATES ARE%' then 'SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION REFUND IN COST' -- Suspension amended and approved. Amend suspension refund in cost: £408. Start and end dates are: 2021-05-17 00:00:00.0 and 2021-05-20 00:00:00.0 -- Suspension amended and approved. Amend suspension refund in cost: £170. Start and end dates are: 2021-12-06 00:00:00.0 and 2021-12-10 00:00:00.0 -when upper(activity) like '%SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION WITH INCREASE IN COST AND PAYMENT RECEIVED% START AND END DATES ARE%' then 'SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION WITH INCREASE IN COST AND PAYMENT RECEIVED' -- Suspension amended and approved. Amend suspension with increase in cost and payment received: £68. Start and end dates are: 2021-10-12 00:00:00.0 and 2021-10-13 00:00:00.0 -- Suspension amended and approved. Amend suspension with increase in cost and payment received: £136. Start and end dates are: 2021-04-15 00:00:00.0 and 2021-04-18 00:00:00.0 - -when upper(activity) like '%SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION WITH NO CHANGE IN COST. START AND END DATES ARE%' then 'SUSPENSION AMENDED AND APPROVED. AMEND SUSPENSION WITH NO CHANGE IN COST' --Suspension amended and approved. Amend suspension with no change in cost. Start and end dates are: 2023-09-11 00:00:00.0 and 2023-09-11 00:00:00.0 -when upper(activity) like 'SUSPENSION DUE TO BE CANCELLED ON %. REFUND AMOUNT%' then 'SUSPENSION DUE TO BE CANCELLED WITH REFUND' -- Suspension due to be cancelled on 2022-04-13 00:00:00.0. Refund amount: £114.50 -when upper(activity) like 'SUSPENSION DURATION CHANGED. CHANGE DURATION WITH INCREASE IN COST%' then 'SUSPENSION DURATION CHANGED. CHANGE DURATION WITH INCREASE IN COST' -when upper(activity) like '%SUSPENSION DURATION CHANGED. CHANGE DURATION WITH REFUND IN COST:%OLD SUSPENSION START AND END DATES ARE%' then 'SUSPENSION DURATION CHANGED. CHANGE DURATION WITH REFUND IN COST' -- Suspension duration changed. Change duration with refund in cost: £102.000. Old suspension start and end dates are : 2020-11-14 00:00:00.0 and 2020-11-15 00:00:00.0 -- Suspension duration changed. Change duration with refund in cost: £102.000. Old suspension start and end dates are : 2020-11-14 00:00:00.0 and 2020-11-15 00:00:00.0 -when upper(activity) like '%SUSPENSION DURATION CHANGED. OLD SUSPENSION START AND END DATES ARE%' then 'SUSPENSION DURATION CHANGED. OLD SUSPENSION START AND END DATES' -- Suspension duration changed. Old suspension start and end dates are : 2017-04-03 00:00:00.0 and 2017-04-07 00:00:00.0 -- Suspension duration changed. Old suspension start and end dates are : 2017-04-03 00:00:00.0 and 2017-04-07 00:00:00.0 -when upper(activity) like 'SUSPENSION EXTENSION APPROVED' then 'SUSPENSION EXTENSION APPROVED' --Suspension extension approved -when upper(activity) like 'SUSPENSION UPDATE TO SIGN UP' then 'SUSPENSION UPDATE TO SIGN UP' -when upper(activity) like 'SUSPENSION UPDATE TO SUSPENSION REJECT' then 'SUSPENSION UPDATE TO SUSPENSION REJECT' -when upper(activity) like '%SUSPENSION UPDATE TO SUSPENSION APPROVED%' then 'SUSPENSION UPDATE TO SUSPENSION APPROVED' --Suspension update to Suspension Approved - -when upper(activity) like 'TEMPORARY PERMIT PRINTED STARTING % ENDING %' then 'TEMPORARY PERMIT PRINTED' --Temporary permit printed starting 04/05/2017 ending 04/06/2017 -when upper(activity) like 'THE CRIME REFERENCE NUMBER HAS NOT BEEN CONFIRMED BY THE POLICE AND SO A FREE REPLACEMENT IS NOT POSSIBLE FOR REFERENCE%' then 'THE CRIME REFERENCE NUMBER HAS NOT BEEN CONFIRMED BY THE POLICE AND SO A FREE REPLACEMENT IS NOT POSSIBLE FOR REFERENCE' -when upper(activity) like 'THE PROTECTED VEHICLE WAS CHANGED%' then 'THE PROTECTED VEHICLE WAS CHANGED' -when upper(activity) like 'THE SUSPENSION MANUALLY SIGN HAS BEEN UPDATED. THE IDENTIFIER IS%' then 'THE SUSPENSION MANUALLY SIGN HAS BEEN UPDATED. THE IDENTIFIER IS' -when upper(activity) like 'UPDATE VOUCHER MAIL SENT FOR CONFIRMATION.' then 'UPDATE VOUCHER MAIL SENT FOR CONFIRMATION' -when upper(activity) like 'VEHICLE REGISTRATION NUMBER:%' then 'VEHICLE REGISTRATION NUMBER' -when upper(activity) like 'VEHICLE UPDATED ON PERMIT . OLD VEHILCE DETAILS ARE : VRM%' OR upper(activity) like ' VEHICLE UPDATED ON PERMIT . OLD VEHILCE DETAILS ARE : VRM :%' then 'VEHICLE UPDATED ON PERMIT' -- Vehicle updated on permit . Old vehilce details are : vrm : AO05COH Emission (CO2) : 180 manufacturer : VOLKSWAGEN model : POLO TWIST AUTO. Permit charge were 112.000 --Vehicle updated on permit . Old vehilce details are : vrm : DG11VUM Emission (CO2) : 132 manufacturer : HONDA model : CIVIC TYPE-S I-VTEC S-A. Permit charge were 43.000 -when upper(activity) like 'VEHICLES ON DISPENSATION CHANGED FROM NULL%' then 'VEHICLES ON DISPENSATION CHANGED FROM NULL' -when upper(activity) like 'VEHICLES ON DISPENSATION CHANGED%' then 'VEHICLES ON DISPENSATION CHANGED' - -when upper(activity) like 'VOUCHER HAS BEEN MARKED AS SENT TO PRINT%' then 'VOUCHER HAS BEEN MARKED AS SENT TO PRINT' -when upper(activity) like 'VOUCHER COVER LETTER HAS BEEN PRINTED' then 'VOUCHER COVER LETTER HAS BEEN PRINTED' - -when upper(activity) like 'VOUCHER SENT TO LIBERTY' then 'VOUCHER SENT TO LIBERTY' -when upper(activity) like 'VOUCHER SERIAL NUMBERS UPDATED%' then 'VOUCHER SERIAL NUMBERS UPDATED' -when upper(activity) like 'VOUCHER SESSIONS OVERRIDDEN%' then 'VOUCHER SESSIONS OVERRIDDEN' -when upper(activity) like 'VOUCHER SHIPPED LOCALLY' then 'VOUCHER SHIPPED LOCALLY' -when upper(activity) like 'VRM % WILL BE REMOVED WITH EFFECT FROM%' then 'VRM WILL BE REMOVED WITH EFFECT FROM' --VRM GJ52XEM will be removed with effect from 04/05/2017 -when upper(activity) like 'VRM % WILL BE REMOVED WITH REFUND' or upper(activity) like 'VRM % WILL BE REMOVED WITH REFUND OF % WITH EFFECT FROM%' then 'VRM WILL BE REMOVED WITH REFUND' --VRM T321NMH WILL BE REMOVED WITH REFUND OF £-181.26 WITH EFFECT FROM 09/02/2024 -when upper(activity) like '%VRM CHANGE ACCEPTED%' then 'VRM CHANGE ACCEPTED' --VRM CHANGE ACCEPTED -when upper(activity) like 'VRM CHANGE APPLICATION' then 'VRM CHANGE APPLICATION' -when upper(activity) like 'VRM CHANGE APPLICATION % SENT ON%' OR upper(activity) like 'VRM CHANGE APPLICATION SENT ON%' then 'VRM CHANGE APPLICATION SENT' --Vrm change application £null sent on 31/05/2017 --Vrm change application sent on 01/03/2017 -when upper(activity) like 'VRM CHANGE APPLICATION WITH PAYMENT OF % SENT ON %' then 'VRM CHANGE APPLICATION WITH PAYMENT' --Vrm change application with payment of £8.96 sent on 19/12/2023 -when upper(activity) like 'VRM CHANGE REJECTED PAYMENT FAILURE' then 'VRM CHANGE REJECTED PAYMENT FAILURE' -when upper(activity) like 'VRM CHANGE REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' then 'VRM CHANGE REJECTED REQUESTED EVIDENCE NOT BEING PROVIDED' -when upper(activity) like 'VRM CHANGE REJECTED YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS' then 'VRM CHANGE REJECTED YOUR VEHICLE EXCEEDING THE MAXIMUM DIMENSIONS' -when upper(activity) like 'VRM CHANGE REJECTED YOUR VEHICLE NOT REQUIRING A PERMIT' then 'VRM CHANGE REJECTED YOUR VEHICLE NOT REQUIRING A PERMIT' -when upper(activity) like 'VRM CHANGED' then 'VRM CHANGED' -when upper(activity) like 'VRM CHANGED FROM%' then 'VRM CHANGED FROM' -when upper(activity) like 'WILL BE REMOVED WITH PAYMENT' then 'WILL BE REMOVED WITH PAYMENT' -when upper(activity) like '' or upper(activity) like ' ' or upper(activity) is null then - upper(case - when substr(permit_referece,1,3) = 'HYA' then 'All zone' - when substr(permit_referece,1,3) = 'HYG' then 'Companion Badge' - when substr(permit_referece,1,3) = 'HYB' then 'Business' - when substr(permit_referece,1,3) = 'HYD' then 'Doctor' - when substr(permit_referece,1,3) = 'HYP' then 'Estate resident' - when substr(permit_referece,1,3) = 'HYH' then 'Health and social care' - when substr(permit_referece,1,3) = 'HYR' then 'Residents' - when substr(permit_referece,1,3) = 'HYL' then 'Leisure Centre Permit' - when substr(permit_referece,1,3) = 'HYE' then 'All Zone Business Voucher' - when substr(permit_referece,1,3) = 'HYF' then 'Film Voucher' - when substr(permit_referece,1,3) = 'HYJ' then 'Health and Social Care Voucher' - when substr(permit_referece,1,3) = 'HYQ' then 'Estate Visitor Voucher' - when substr(permit_referece,1,3) = 'HYV' then 'Resident Visitor Voucher' - when substr(permit_referece,1,3) = 'HYN' then 'Dispensation' - when permit_referece ='HY50300' then 'Dispensation' - when substr(permit_referece,1,3) = 'HYS' then 'Suspension' - else substr(permit_referece,1,3) - end) -end as activity_tag - -,* from liberator_permit_activity where import_date = (select max(import_date) from liberator_permit_activity ) and activity_by not like '%@%' --232 users ---and cast(activity_date as date) > current_date - interval '18' month --Last 400 days from todays date ---limit 75 - -) -select -act_data.fy -,act_data.act_MonthYear -,act_data.act_date -,act_data.act_hour -,act_data.product_type -,act_data.product_category -,act_data.activity_by -,act_data.activity_tag_group -,act_data.activity_tag ---,act_data.activity_id ---,act_data.permit_referece ---,act_data.activity_date ---,act_data.activity ---,act_data.record_created ---,act_data.import_timestamp -,act_data.import_year -,act_data.import_month -,act_data.import_day -,act_data.import_date - -/*summary total*/ -,count(distinct act_data.activity_id) as num_activity_id - -from act_data -where cast(act_data.act_date as date) > current_date - interval '50' month --Last 50 months from todays date -and upper(activity_by) in ('ADIRIE','BOLUFEYISAN','DMOSES','EUTOMI','LEDWARDS','OOLAGUNJU','PSHAKES','RISLAM','SHMULLA','YNGUYEN','SMCCABE') - -group by -act_data.fy -,act_data.act_MonthYear -,act_data.act_date -,act_data.act_hour -,act_data.product_type -,act_data.product_category -,act_data.activity_by -,act_data.activity_tag_group -,act_data.activity_tag - -,act_data.import_date -,act_data.import_year -,act_data.import_month -,act_data.import_day - -order by -act_data.fy desc -,act_data.act_MonthYear desc -""" -SQLQueryparking_customer_services_permit_activity_users_node1715091122327 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "liberator_permit_activity": AmazonS3liberator_permit_activity_node1715091109305 - }, - transformation_ctx="SQLQueryparking_customer_services_permit_activity_users_node1715091122327", -) - -# Script generated for node Amazon S3 - parking_customer_services_permit_activity_users -AmazonS3parking_customer_services_permit_activity_users_node1715091127108 = glueContext.getSink( - path=f"s3://dataplatform-{environment}-refined-zone/parking/liberator/parking_customer_services_permit_activity_users/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="AmazonS3parking_customer_services_permit_activity_users_node1715091127108", -) -AmazonS3parking_customer_services_permit_activity_users_node1715091127108.setCatalogInfo( - catalogDatabase=f"dataplatform-{environment}-liberator-refined-zone", - catalogTableName="parking_customer_services_permit_activity_users", -) -AmazonS3parking_customer_services_permit_activity_users_node1715091127108.setFormat( - "glueparquet", compression="snappy" -) -AmazonS3parking_customer_services_permit_activity_users_node1715091127108.writeFrame( - SQLQueryparking_customer_services_permit_activity_users_node1715091122327 -) -job.commit() diff --git a/scripts/jobs/parking/parking_cycle_hangar_allocation_detail.py b/scripts/jobs/parking/parking_cycle_hangar_allocation_detail.py deleted file mode 100644 index c8deb01b4..000000000 --- a/scripts/jobs/parking/parking_cycle_hangar_allocation_detail.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "parking_cycle_hangar_allocation_detail" - -# The exact same query prototyped in pre-prod(stg) orprod Athena -query_on_athena = """ -/******************************************************************************************************************* -parking_cycle_hangar_allocation_Details - -The SQL details the number of cycle spaces that are occupied -in each cycle hangar. It also identifies the number of Parties -that are on the waiting list. The code has been amended to use -Tom's hangar list - -29/10/2024 - Create Query -******************************************************************************************************************/ -With TomHangar as ( - SELECT - asset_no, asset_type, street_or_estate, zone, status, key_number, fob, location_description, - road_name, postcode, date_installed, easting, northing, road_or_pavement, - case - When asset_no like '%Bikehangar_1577%' Then '1577' - When asset_no like '%Bikehangar_H1439%' Then 'H1439' - When asset_no like '%Bikehangar_H1440%' Then 'Hangar_H1440' - When asset_no like '%Bikehangar_1435%' Then 'Bikehangar_H1435' - ELSE replace(asset_no, ' ','_') - END as HangarID - from "parking-raw-zone".parking_parking_ops_cycle_hangar_list - WHERE import_Date = (Select MAX(import_date) from "parking-raw-zone".parking_parking_ops_cycle_hangar_list) - AND asset_type = 'Hangar' AND lower(status) IN ('active', 'estate locked gate issue') - /** 20-08-2024 add check for blank records **/ - AND Length(trim(asset_no))> 2), - -/*** Get the Hangar details from Liberator ***/ -HangarDetails AS ( - SELECT A.ID , - HANGAR_TYPE , - HANGAR_ID , - IN_SERVICE , - MAINTENANCE_KEY , - SPACES , - HANGAR_LOCATION , - USRN , - LATITUDE , - LONGITUDE , - START_OF_LIFE , - END_OF_LIFE, - ROW_NUMBER() OVER (PARTITION BY HANGAR_TYPE, HANGAR_ID, IN_SERVICE, MAINTENANCE_KEY, SPACES, - HANGAR_LOCATION, USRN, LATITUDE,LONGITUDE, START_OF_LIFE, END_OF_LIFE - ORDER BY HANGAR_TYPE, HANGAR_ID, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, LONGITUDE, - START_OF_LIFE, END_OF_LIFE DESC) AS ROW - FROM "dataplatform-stg-liberator-raw-zone".liberator_hangar_details as A - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - ORDER BY HANGAR_ID), - -/*** Party List ***/ -Licence_Party as ( - SELECT * from "dataplatform-stg-liberator-raw-zone".liberator_licence_party - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), -/******************************************************************************* -Cycle Hangar allocation details -*******************************************************************************/ -Cycle_Hangar_allocation as ( - SELECT - *, - ROW_NUMBER() OVER ( PARTITION BY hanger_id, trim(upper(space)) - ORDER BY hanger_id, trim(upper(space)), date_of_allocation DESC, fee_due_date DESC, id DESC) row_num - FROM "dataplatform-stg-liberator-raw-zone".liberator_hangar_allocations - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - AND key_issued = 'Y' AND length(key_id) > 2 AND space != 'null') - -SELECT - hanger_id, asset_no, asset_type, street_or_estate, zone, status,location_description, road_name, - C.postcode as Hangar_Postcode, date_installed, easting, northing road_or_pavement, key_id,USRN, - space, key_issued, date_of_allocation, allocation_status, fee_due_date, - A.party_id, title, first_name, surname, - address1, address2, address3, D.postcode, uprn, telephone_number, email_address, - - format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), - 'yyyy-MM-dd HH:mm:ss') AS importdatetime, - - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date - -from Cycle_Hangar_allocation as A -LEFT JOIN TomHangar as C ON A.hanger_id = C.HangarID -LEFT JOIN Licence_Party as D ON A.party_id = D.business_party_id -LEFT JOIN HangarDetails as E ON A.hanger_id = E.HANGAR_ID -where row_num = 1 AND allocation_status not IN ('cancelled', 'key_returned','key_not_returned') -ORDER BY hanger_id, space -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_cycle_hangar_allocation_update.py b/scripts/jobs/parking/parking_cycle_hangar_allocation_update.py deleted file mode 100644 index 9ed6c931e..000000000 --- a/scripts/jobs/parking/parking_cycle_hangar_allocation_update.py +++ /dev/null @@ -1,361 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -Note: python file name should be the same as the table name -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "parking_cycle_hangar_allocation_update" - -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ -/******************************************************************************************************************* -parking_cycle_hangar_allocation_update - -The SQL details the number of cycle spaces that are occupied -in each cycle hangar. It also identifies the number of Parties -that are on the waiting list. The code has been amended to use -Tom's hangar list - -19/10/2023 - Create Query -23/01/2024 - update code to add changes made to the base Google sheet -20/05/2024 - amend to NOT filter out records with NO address (i.e. not matching the Party_ID) -30/05/2024 - rewrite to use the History data -13/06/2024 - slight aendments because of my cock-up! -14/06/2024 - make additional changes because the cycle hangar key_id & space have been swapped -19/06/2024 - trim the allocated space, there are leading spaces in the field!!! -29/07/2024 - change collection of tom's hangars to add an additional status -01/08/2024 - summerise ALL of the allocated PartyIDs (not just those that are active now). Use this list - to filter out 'allocated' PartyIDs from the Waiting List -19/08/2024 - add additional order by for cycle hangar allocation -20/08/2024 - additional checks for rubbish data!!!! -23/08/2024 - look again at the waiting list! -28/04/2024 - refine the 'hangar_can_be_filled' field case statement -26/09/2024 - add hangar location. -04/10/2024 - add additional checks for the waiting list -04/12/2024 - collect the interim cycle hangar waiting list -06/01/2025 - Amend to remove records from the waiting list that have been unsuscribed -13/01/2025 - add opt-in data -******************************************************************************************************************/ -/******************************************************************************* -Create a comparison between Toms Hangar list and EStreet -*******************************************************************************/ -With TomHangar as ( - SELECT - asset_no, asset_type, street_or_estate, zone, status, key_number, fob, location_description, - road_name, postcode, date_installed, easting, northing, road_or_pavement, - case - When asset_no like '%Bikehangar_1577%' Then '1577' - When asset_no like '%Bikehangar_H1439%' Then 'H1439' - When asset_no like '%Bikehangar_H1440%' Then 'Hangar_H1440' - When asset_no like '%Bikehangar_1435%' Then 'Bikehangar_H1435' - ELSE replace(asset_no, ' ','_') - END as HangarID - from "parking-raw-zone".parking_parking_ops_cycle_hangar_list - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - AND asset_type = 'Hangar' AND lower(status) IN ('active', 'estate locked gate issue') - /** 20-08-2024 add check for blank records **/ - AND Length(trim(asset_no))> 2), - -/** 20-08-2024 get ALL of the data AND check for duplicates **/ -Hanger as ( - SELECT HangarID as hanger_id, *, - ROW_NUMBER() OVER ( PARTITION BY HangarID ORDER BY HangarID DESC) H1 - FROM TomHangar), - -/** 20/08/2024 I check for duplicates and dont use the data??? change to use**/ -Hangar_Comp as ( - SELECT - asset_no, HangarID, hanger_id - FROM Hanger as A - /**LEFT JOIN Hanger as B ON A.HangarID = B.hanger_id AND H1 = 1 remove **/ - WHERE H1 = 1 - UNION ALL - SELECT 'new_only','new_only','new_only'), - -/*** 08-01-2025 collect the unsubscribed_emails ***/ -unsubscribed_emails as ( - SELECT *, - ROW_NUMBER() OVER ( PARTITION BY email_address ORDER BY email_address DESC) row1 - FROM "parking-raw-zone".parking_parking_cycle_hangar_unsubscribed_emails - WHERE import_date = (select max(import_date) - from "parking-raw-zone".parking_parking_cycle_hangar_unsubscribed_emails)), - -/*** 13/01/2025 added ***/ -opt_in_emails as ( - SELECT - please_note_your_email_address_has_been_prefilled_based_on_your_account_registration_please_do_not_amend_this as email, - please_select_one_of_the_options_below, - ROW_NUMBER() OVER ( PARTITION BY please_note_your_email_address_has_been_prefilled_based_on_your_account_registration_please_do_not_amend_this - ORDER BY please_note_your_email_address_has_been_prefilled_based_on_your_account_registration_please_do_not_amend_this - DESC) row1 - FROM "parking-raw-zone".parking_parking_opt_in_form_responses - WHERE import_date = (select max(import_date) - from "parking-raw-zone".parking_parking_opt_in_form_responses) - AND please_select_one_of_the_options_below like 'No.%'), -/******************************************************************************* -Obtain the latest Waiting List History -*******************************************************************************/ -Wait_History as ( - SELECT A.*, - ROW_NUMBER() OVER ( PARTITION BY A.party_id,hanger_id ORDER BY A.party_id,hanger_id, updated DESC ) R1 - FROM "dataplatform-prod-liberator-raw-zone".liberator_hangar_waiting_list_history as A - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), -/******************************************************************************* -Create the Waiting list - unique "party_id" -*******************************************************************************/ -waiting_list as ( - SELECT A.party_id, A.registration_type, A.hanger_id, A.new_hangars_only, A.offer_made, - A.uprn, A.registration_date, - B.party_id as History_party_id, A.hanger_id as History_hanger_id, B.status_from, B.status_to, - B.date_from, B.date_to, B.updated, B.created_by, B.registation_date as History_registation_date, - ROW_NUMBER() OVER ( PARTITION BY A.party_id, A.hanger_id ORDER BY A.party_id, A.hanger_id DESC) row1 - FROM "dataplatform-prod-liberator-raw-zone".liberator_hangar_waiting_list as A - LEFT JOIN Wait_History as B ON A.party_id = B.party_id AND A.hanger_id = B.hanger_id AND B.R1 = 1 - WHERE A.import_Date = format_datetime(current_date, 'yyyyMMdd')), -/*** Party List ***/ -Licence_Party as ( - SELECT * from "dataplatform-prod-liberator-raw-zone".liberator_licence_party - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), -/*** STREET ***/ -LLPG as ( - SELECT * - FROM "dataplatform-prod-liberator-raw-zone".liberator_permit_llpg - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), -/******************************************************************************* -04/12/2024 - interim cycle hangar waiting list remove unsubscribed emails -13/01/2025 - add review of opt in(OUT) email addresses -*******************************************************************************/ -Interim_Wait as ( - SELECT A.*, - CASE - When length(E.email_address) > 1 Then E.email_address - When length(F.email)> 1 Then F.email - END as email_address - FROM "parking-raw-zone".interim_cycle_wait_list as A - LEFT JOIN unsubscribed_emails as E ON upper(ltrim(rtrim(A.email))) = - upper(ltrim(rtrim(E.email_address))) AND E.row1 = 1 - LEFT JOIN opt_in_emails as F ON upper(ltrim(rtrim(A.email))) = - upper(ltrim(rtrim(F.email))) AND F.row1 = 1 - WHERE A.import_date = (select max(import_date) - from "parking-raw-zone".interim_cycle_wait_list)), - -/*** count the number on the waiting list by hangar ***/ -Interim_Wait_summary as ( - SELECT hanger_id, count(*) as Interim_Wait_Total - FROM Interim_Wait - WHERE email_address is NULL - GROUP BY hanger_id), -/******************************************************************************* -Cycle Hangar allocation details -*******************************************************************************/ -Cycle_Hangar_allocation as ( - SELECT - *, - /* 13/06 change order by to include ID */ - /* 14/06 change again to hangar_id and space because there are some hangars with multiple spaces - allocated to the same Party_id. UPPER(space) because there are records with lowercase space field */ - /* 19/06 trim the space */ - ROW_NUMBER() OVER ( PARTITION BY hanger_id, trim(upper(space)) - ORDER BY hanger_id, trim(upper(space)), date_of_allocation DESC, fee_due_date DESC, id DESC) row_num - FROM "dataplatform-prod-liberator-raw-zone".liberator_hangar_allocations - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - /* 14/06 change to exclude those records where keys have not been issued AND - remove those records where the Space and Key ID fields have been switched??? also - remove those records with null in the space field????*/ - AND key_issued = 'Y' AND length(key_id) > 2 AND space != 'null'), - -/*** 23/07/2024 - obtain the last party id/hangar allocation details ***/ -Party_ID_Allocation as ( - SELECT - *, - ROW_NUMBER() OVER ( PARTITION BY party_id, hanger_id - ORDER BY party_id, hanger_id, id DESC) row_num - FROM "dataplatform-prod-liberator-raw-zone".liberator_hangar_allocations - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - AND key_issued = 'Y' AND length(key_id) > 2 AND space != 'null'), - -/** 13/06 total the alloctaion details obtain above */ -Alloc_Total as ( - SELECT hanger_id, count(*) as total_alloc - FROM Cycle_Hangar_allocation - WHERE row_num = 1 AND allocation_status not IN ('cancelled', 'key_returned','key_not_returned') - GROUP BY hanger_id), - -Street_Rec as ( - SELECT * - FROM "dataplatform-prod-liberator-raw-zone".liberator_permit_llpg - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - AND address1 = 'STREET RECORD'), - -/*** 01/08/2024 Summerise the allocatd Party IDs to prevent duplication below **/ -Summary_Alloca_PartyID as ( - SELECT party_id, - ROW_NUMBER() OVER ( PARTITION BY party_id - ORDER BY party_id DESC) R1 - FROM Party_ID_Allocation - WHERE row_num = 1), - -Cycle_Hangar_Wait_List as ( - SELECT - A.party_id, first_name, surname, B.uprn as USER_UPRN, - B.address1, B.address2, B.address3, B.postcode, B.telephone_number, D.Address2 as Street,registration_date - ,A.hanger_id, E.party_id Allocated_Party_ID, History_party_id, History_hanger_id, status_to, updated - FROM waiting_list as A - LEFT JOIN Licence_Party as B ON A.party_id = B.business_party_id - LEFT JOIN LLPG as C ON B.uprn = cast(C.UPRN as varchar) - LEFT JOIN Street_Rec as D ON C.USRN = D.USRN - /*LEFT JOIN Summary_Alloca_PartyID as E ON A.party_id = E.party_id AND R1 = 1*/ - LEFT JOIN Cycle_Hangar_allocation as E ON A.party_id = E.party_id AND row_num = 1 - WHERE row1= 1 AND E.party_id is NULL AND status_to not IN ('removed','rejected offer','offer accepted')), -/************************************************************ -Waiting List CREATED -************************************************************/ -/* Amend to use Toms Hangar List */ -Estreet_Hanger as ( - SELECT HangarID as hanger_id, 'A' as space, road_name as hangar_location, 1 as H1 - FROM TomHangar - UNION ALL - SELECT HangarID as hanger_id, 'B' as space, road_name as hangar_location, 1 as H1 - FROM TomHangar - UNION ALL - SELECT HangarID as hanger_id, 'C' as space, road_name as hangar_location, 1 as H1 - FROM TomHangar - UNION ALL - SELECT HangarID as hanger_id, 'D' as space, road_name as hangar_location, 1 as H1 - FROM TomHangar - UNION ALL - SELECT HangarID as hanger_id, 'E' as space, road_name as hangar_location, 1 as H1 - FROM TomHangar - UNION ALL - SELECT HangarID as hanger_id, 'F' as space, road_name as hangar_location, 1 as H1 - FROM TomHangar - UNION ALL - SELECT 'new_only', ' ', 'NEWONLY', 1), - -Wait_List_Hangar as ( - SELECT A.party_id, A.hanger_id, - ROW_NUMBER() OVER ( PARTITION BY A.party_id, A.hanger_id - ORDER BY A.party_id, A.hanger_id DESC) H2 - FROM "dataplatform-prod-liberator-raw-zone".liberator_hangar_waiting_list as A - INNER JOIN Cycle_Hangar_Wait_List as B ON A.party_id = B.party_id AND A.hanger_id = B.hanger_id - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), - -Wait_List_Earlist_Latest as ( - SELECT A.hanger_id, max(A.registration_date) as Max_Date, min(A.registration_date) as Min_Date - FROM "dataplatform-prod-liberator-raw-zone".liberator_hangar_waiting_list as A - INNER JOIN Cycle_Hangar_Wait_List as B ON A.party_id = B.party_id - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') - AND A.registration_date not - IN ('2000-01-01','1900-12-13','1000-04-02','1100-04-02', - '1200-04-02','1300-04-02','1400-04-02','2000-12-17','1200-03-24') - GROUP BY A.hanger_id), - -Wait_total as ( - SELECT hanger_id, count(*) as Wait_Total - FROM Wait_List_Hangar - WHERE H2 = 1 - GROUP BY hanger_id), - -/* 13/06 - correctly use the allocated total against the list of Hangars from Tom */ -allocated_Total as ( - SELECT DISTINCT A.hanger_id, A.hangar_location, total_alloc as Total_Allocated - FROM Estreet_Hanger as A - LEFT JOIN Alloc_Total as B ON A.hanger_id = B.hanger_id - WHERE H1 = 1), - -/*** 04/12/2024 - change to use the interim waiting list totals ***/ -Full_Hangar_Data as ( - SELECT - A.hanger_id, A.hangar_location, - CASE - When A.hanger_id = 'new_only' Then 0 - ELSE Total_Allocated - END as Total_Allocated, - Interim_Wait_Total as Wait_Total, /*** remove Wait_Total,***/ - CASE - When A.hanger_id = 'new_only' Then 0 - ELSE ( 6 - Total_Allocated) - END as free_spaces, - Min_Date as Earlist_Registration_Date, - Max_Date as Latest_Registration_Date - FROM allocated_Total as A - LEFT JOIN Wait_total as B ON A.hanger_id = B.hanger_id - LEFT JOIN Wait_List_Earlist_Latest as C ON A.hanger_id = C.hanger_id - LEFT JOIN Interim_Wait_summary as D ON A.hanger_id = D.hanger_id), - -/** 20-08-2024 Use the de-dupe data NOT TomHangar **/ -Hangar_WAit_List as ( - SELECT - A.asset_no as Tom_Asset_No, B.hanger_id as HangarID, street_or_estate, zone, location_description, - hangar_location, postcode, date_installed, - CASE - When Total_Allocated is NULL Then 0 - ELSE Total_Allocated - END as Total_Allocated, - CASE - When Wait_Total is NULL OR Wait_Total = 0 Then 0 - ELSE Wait_Total - END as Wait_Total, - CASE - When free_spaces is NULL Then 6 - ELSE free_spaces - END as free_spaces, - Earlist_Registration_Date, Latest_Registration_Date - FROM Hanger as A - LEFT JOIN Hangar_Comp as B ON A.asset_no = B.asset_no - LEFT JOIN Full_Hangar_Data as C ON B.hanger_id = C.hanger_id - WHERE H1 = 1), - -/*** Output the data ***/ -Output as ( - SELECT *, - CASE - When Total_Allocated = 6 Then 'N/A' - /*** 28/08 amend to close the loop of no Wait ttal and free spaces ***/ - When Wait_Total = 0 AND free_spaces > 0 Then 'Yes' - When Wait_Total >= free_spaces Then 'Yes' - Else 'No' - END as hangar_can_be_filled - FROM Hangar_WAit_List - WHERE HangarID is not NULL - UNION ALL - SELECT A.Tom_Asset_No, A.HangarID, A.street_or_estate, A.zone, A.location_description, - A.hangar_location, A.postcode, A.date_installed, A.Total_Allocated, TotalWatch, A.free_spaces, - C.Earlist_Registration_Date, C.Latest_Registration_Date, - CASE - When A.Total_Allocated = 6 Then 'N/A' - When TotalWatch >= A.free_spaces Then 'Yes' - Else 'No' - END as hangar_can_be_filled - FROM Hangar_WAit_List as A - LEFT JOIN (SELECT hanger_id, count(*) as TotalWatch FROM waiting_list - GROUP BY hanger_id) B ON replace(A.Tom_Asset_No,' ','_') = B.hanger_id - LEFT JOIN Full_Hangar_Data as C ON B.hanger_id = C.hanger_id - WHERE A.HangarID is NULL) - -SELECT *, - CASE - When hangar_can_be_filled = 'No' AND - wait_total is not NULL Then free_spaces - Wait_Total - Else 0 - END as Spaces_that_cannot_be_allocated, - - format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), - 'yyyy-MM-dd HH:mm:ss') AS import_date_timestamp, - - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date -FROM Output -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_cycle_hangar_allocation_wait.py b/scripts/jobs/parking/parking_cycle_hangar_allocation_wait.py deleted file mode 100644 index 60bbb70cd..000000000 --- a/scripts/jobs/parking/parking_cycle_hangar_allocation_wait.py +++ /dev/null @@ -1,332 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - create_pushdown_predicate, - get_glue_env_var, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 -AmazonS3_node1658997944648 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="parking_parking_ops_cycle_hangar_list", - transformation_ctx="AmazonS3_node1658997944648", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1697705005761 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_llpg", - transformation_ctx="AmazonS3_node1697705005761", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1697704537304 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_cycle_hangars_denormalisation", - transformation_ctx="AmazonS3_node1697704537304", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1697705499200 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_allocations", - transformation_ctx="AmazonS3_node1697705499200", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1697704672904 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_waiting_list", - transformation_ctx="AmazonS3_node1697704672904", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1697704891824 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_party", - transformation_ctx="AmazonS3_node1697704891824", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node SQL -SqlQuery3 = """ -/******************************************************************************************************************** -parking_cycle_hangar_allocation - -The SQL details the number of cycle spaces that are occupied -in each cycle hangar. It also identifies the number of Parties -that are on the waiting list. The code has been amended to use -Tom's hangar list - -19/10/2023 - Create Query -15/05/2024 - amend to NOT filter out records with NO address (i.e. not matching the Party_ID) -*******************************************************************************************************************/ -/************************************************************ -Create a comparison between Toms Hangar list and EStreet -************************************************************/ -With TomHangar as ( - SELECT - asset_no, asset_type, street_or_estate, zone, status, key_number, fob, location_description, - road_name, postcode, date_installed, easting, northing, road_or_pavement, - case - When asset_no like '%Bikehangar_1577%' Then '1577' - When asset_no like '%Bikehangar_H1439%' Then 'H1439' - When asset_no like '%Bikehangar_H1440%' Then 'Hangar_H1440' - When asset_no like '%Bikehangar_1435%' Then 'Bikehangar_H1435' - ELSE replace(asset_no, ' ','_') - END as HangarID - from parking_parking_ops_cycle_hangar_list - WHERE import_date = (Select MAX(import_date) - from parking_parking_ops_cycle_hangar_list) - AND asset_type = 'Hangar' AND lower(status) like '%active%'), - -Hanger as ( - SELECT hanger_id, - ROW_NUMBER() OVER ( PARTITION BY hanger_id ORDER BY hanger_id DESC) H1 - From parking_cycle_hangars_denormalisation - WHERE import_date = (Select MAX(import_date) - from parking_cycle_hangars_denormalisation)), - -Hangar_Comp as ( - SELECT - asset_no, HangarID, B.hanger_id - FROM TomHangar as A - LEFT JOIN Hanger as B ON A.HangarID = B.hanger_id AND H1 = 1 - UNION ALL - SELECT 'new_only','new_only','new_only'), -/************************************************************ -Create the Waiting list - unique "party_id" -************************************************************/ -waiting_list as ( - SELECT *, - ROW_NUMBER() OVER ( PARTITION BY party_id, hanger_id ORDER BY party_id, hanger_id DESC) row1 - FROM liberator_hangar_waiting_list - WHERE Import_Date = (Select MAX(Import_Date) from - liberator_hangar_waiting_list)), -/*** Party List ***/ -Licence_Party as ( - SELECT * from liberator_licence_party - WHERE Import_Date = (Select MAX(Import_Date) from - liberator_licence_party)), -/*** STREET ***/ -LLPG as ( - SELECT * - FROM liberator_permit_llpg - WHERE import_date = (Select MAX(import_date) from - liberator_permit_llpg)), -/******************************************************************************* -Cycle Hangar allocation details -*******************************************************************************/ -Cycle_Hangar_allocation as ( - SELECT - *, - ROW_NUMBER() OVER ( PARTITION BY hanger_id, space - ORDER BY hanger_id, space, date_of_allocation DESC, fee_due_date DESC) row_num - FROM liberator_hangar_allocations - WHERE Import_Date = (Select MAX(Import_Date) from - liberator_hangar_allocations)), - -Street_Rec as ( - SELECT * - FROM liberator_permit_llpg - WHERE import_date = (Select MAX(import_date) from - liberator_permit_llpg) - AND address1 = 'STREET RECORD'), - -Cycle_Hangar_Wait_List as ( - SELECT - A.party_id, first_name, surname, B.uprn as USER_UPRN, - B.address1, B.address2, B.address3, B.postcode, B.telephone_number, D.Address2 as Street,registration_date - ,A.hanger_id, E.party_id Allocated_Party_ID - FROM waiting_list as A - LEFT JOIN Licence_Party as B ON A.party_id = B.business_party_id - LEFT JOIN LLPG as C ON B.uprn = cast(C.UPRN as string) - LEFT JOIN Street_Rec as D ON C.USRN = D.USRN - LEFT JOIN Cycle_Hangar_allocation as E ON A.party_id = E.party_id AND row_num = 1 - WHERE row1= 1 AND E.party_id is NULL), - -/************************************************************ -Waiting List CREATED -************************************************************/ -Estreet_Hanger as ( - SELECT hanger_id, space, hangar_location, - ROW_NUMBER() OVER ( PARTITION BY hanger_id, space, hangar_location - ORDER BY hanger_id, space, hangar_location DESC) H1 - From parking_cycle_hangars_denormalisation - WHERE import_date = (Select MAX(import_date) from parking_cycle_hangars_denormalisation) and - key_issued = 'Y' - UNION ALL - SELECT 'new_only', ' ', 'NEWONLY', 1), - -Wait_List_Hangar as ( - SELECT A.party_id, A.hanger_id, - ROW_NUMBER() OVER ( PARTITION BY A.party_id, A.hanger_id - ORDER BY A.party_id, A.hanger_id DESC) H2 - FROM liberator_hangar_waiting_list as A - INNER JOIN Cycle_Hangar_Wait_List as B ON A.party_id = B.party_id - WHERE import_date = (Select MAX(import_date) - FROM liberator_hangar_waiting_list)), - -Wait_List_Earlist_Latest as ( - SELECT A.hanger_id, max(A.registration_date) as Max_Date, min(A.registration_date) as Min_Date - FROM liberator_hangar_waiting_list as A - INNER JOIN Cycle_Hangar_Wait_List as B ON A.party_id = B.party_id - WHERE import_date = (Select MAX(import_date) - FROM liberator_hangar_waiting_list) - AND A.registration_date not - IN ('2000-01-01','1900-12-13','1000-04-02','1100-04-02', - '1200-04-02','1300-04-02','1400-04-02','2000-12-17','1200-03-24') - GROUP BY A.hanger_id), - -Wait_total as ( - SELECT hanger_id, count(*) as Wait_Total - FROM Wait_List_Hangar - WHERE H2 = 1 - GROUP BY hanger_id), - -allocated_Total as ( - SELECT hanger_id, hangar_location, count(*) as Total_Allocated - FROM Estreet_Hanger - WHERE H1 = 1 - GROUP BY hanger_id,hangar_location), - -Full_Hangar_Data as ( - SELECT - A.hanger_id, A.hangar_location, - CASE - When A.hanger_id = 'new_only' Then 0 - ELSE Total_Allocated - END as Total_Allocated, - Wait_Total, - CASE - When A.hanger_id = 'new_only' Then 0 - ELSE ( 6 - Total_Allocated) - END as free_spaces, - Min_Date as Earlist_Registration_Date, - Max_Date as Latest_Registration_Date - FROM allocated_Total as A - LEFT JOIN Wait_total as B ON A.hanger_id = B.hanger_id - LEFT JOIN Wait_List_Earlist_Latest as C ON A.hanger_id = C.hanger_id), - -Hangar_WAit_List as ( - SELECT - A.asset_no as Tom_Asset_No, B.hanger_id as HangarID, street_or_estate, zone, location_description, - postcode, date_installed, - CASE - When Total_Allocated is NULL Then 0 - ELSE Total_Allocated - END as Total_Allocated, - CASE - When Wait_Total is NULL Then 0 - ELSE Wait_Total - END as Wait_Total, - CASE - When free_spaces is NULL Then 6 - ELSE free_spaces - END as free_spaces, - Earlist_Registration_Date, Latest_Registration_Date - FROM TomHangar as A - LEFT JOIN Hangar_Comp as B ON A.asset_no = B.asset_no - LEFT JOIN Full_Hangar_Data as C ON B.hanger_id = C.hanger_id), - -/*** Output the data ***/ -Output as ( -SELECT *, - CASE - When Total_Allocated = 6 Then 'N/A' - When Wait_Total >= free_spaces Then 'Yes' - Else 'No' - END as hangar_can_be_filled -FROM Hangar_WAit_List -WHERE HangarID is not NULL -UNION ALL -SELECT A.Tom_Asset_No, A.HangarID, A.street_or_estate, A.zone, A.location_description, - A.postcode, A.date_installed, A.Total_Allocated, TotalWatch, A.free_spaces, - C.Earlist_Registration_Date, C.Latest_Registration_Date, - CASE - When A.Total_Allocated = 6 Then 'N/A' - When TotalWatch >= A.free_spaces Then 'Yes' - Else 'No' - END as hangar_can_be_filled -FROM Hangar_WAit_List as A -LEFT JOIN (SELECT hanger_id, count(*) as TotalWatch FROM waiting_list - GROUP BY hanger_id) B ON replace(A.Tom_Asset_No,' ','_') = B.hanger_id -LEFT JOIN Full_Hangar_Data as C ON B.hanger_id = C.hanger_id -WHERE A.HangarID is NULL) - -SELECT *, - CASE - When hangar_can_be_filled = 'No' Then free_spaces - Wait_Total - Else 0 - END as Spaces_that_cannot_be_allocated, - - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM Output -""" -SQL_node1658765472050 = sparkSqlQuery( - glueContext, - query=SqlQuery3, - mapping={ - "parking_parking_ops_cycle_hangar_list": AmazonS3_node1658997944648, - "parking_cycle_hangars_denormalisation": AmazonS3_node1697704537304, - "liberator_hangar_waiting_list": AmazonS3_node1697704672904, - "liberator_licence_party": AmazonS3_node1697704891824, - "liberator_permit_llpg": AmazonS3_node1697705005761, - "liberator_hangar_allocations": AmazonS3_node1697705499200, - }, - transformation_ctx="SQL_node1658765472050", -) - -# Script generated for node Amazon S3 -AmazonS3_node1658765590649 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/cyclehangarallocationwaitlist/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=PARTITION_KEYS, - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="AmazonS3_node1658765590649", -) -AmazonS3_node1658765590649.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="cyclehangarallocationwaitlist", -) -AmazonS3_node1658765590649.setFormat("glueparquet") -AmazonS3_node1658765590649.writeFrame(SQL_node1658765472050) -job.commit() diff --git a/scripts/jobs/parking/parking_cycle_hangar_interim_wait_list.py b/scripts/jobs/parking/parking_cycle_hangar_interim_wait_list.py deleted file mode 100644 index 8f9d2ab75..000000000 --- a/scripts/jobs/parking/parking_cycle_hangar_interim_wait_list.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "Parking_cycle_hangar_interim_wait_list" - -# The exact same query prototyped in pre-prod(stg) orprod Athena -query_on_athena = """ -/********************************************************************************* -Parking_cycle_hangar_interim_wait_list - -This SQL binds the interim waiting list to the spatial llpg data - -27/11/2024 - Create Query -*********************************************************************************/ -/* Obtain the latest llpg data */ -with llpg as ( - SELECT * FROM "parking-refined-zone".spatially_enriched_liberator_permit_llpg - WHERE import_date = (Select MAX(import_date) - from "parking-refined-zone".spatially_enriched_liberator_permit_llpg)) - -/* Collect and output the imterim Waiting list and link records to above */ -SELECT - forename, surname, email, party_id_to, party_id, A.uprn, A.address1, A.address2, A.postcode, - A.x, A.y, A.lat, A.long, housing_estate, - - format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS importdatetime, - - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date -FROM "parking-raw-zone".parking_parking_cycle_hangar_manual_waiting_list as A -LEFT JOIN llpg as B ON cast(A.uprn as decimal(20,0)) = B.uprn -WHERE A.import_Date = format_datetime(current_date, 'yyyyMMdd') -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_cycle_hangar_met_fail_monthly_format.py b/scripts/jobs/parking/parking_cycle_hangar_met_fail_monthly_format.py deleted file mode 100644 index e1d30e43f..000000000 --- a/scripts/jobs/parking/parking_cycle_hangar_met_fail_monthly_format.py +++ /dev/null @@ -1,369 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -Note: python file name should be the same as the table name -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "Parking_Cycle_Hangar_MET_FAIL_Monthly_Format" - -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ -/********************************************************************************* -Parking_Cycle_Hangar_MET_FAIL_Monthly_Format - -Temp SQL that formats the cycle hangar managment records For pivot report - -09/12/2022 - Create Query -05/07/2023 - rewrite to remove Calendar data -24/01/2024 - change date_fixed to repair_date because field names have been changed - in google -25/01/2024 - add checks for blank sign off month/year! -26/01/2024 - remove duplicated records -*********************************************************************************/ - -WITH - Defect_Basic as ( - SELECT - ref_no as reference_no, - hangar_corral_no, - CAST( - CASE - When length (reported_date) = 4 Then '2022' || '-0' || substr (reported_date, 4, 2) || '-' || substr (reported_date, 1, 2) - When length (reported_date) = 5 Then '2022' || '-' || substr (reported_date, 4, 2) || '-' || substr (reported_date, 1, 2) - When length (reported_date) = 8 Then '20' || substr (reported_date, 7, 2) || '-' || substr (reported_date, 4, 2) || '-' || substr (reported_date, 1, 2) - When reported_date like '%/%' Then substr (reported_date, 7, 4) || '-' || substr (reported_date, 4, 2) || '-' || substr (reported_date, 1, 2) - ELSE substr (cast(reported_date as varchar), 1, 10) - END as date - ) as reported_date, - CAST( - CASE - When repair_date like '%/%' Then substr (repair_date, 7, 4) || '-' || substr (repair_date, 4, 2) || '-' || substr (repair_date, 1, 2) - ELSE substr (cast(repair_date as varchar), 1, 10) - END as date - ) as repair_date, - CAST( - CASE - When repair_date like '%/%' Then substr (repair_date, 7, 4) || '-' || substr (repair_date, 4, 2) || '-01' - ELSE substr (cast(repair_date as varchar), 1, 8) || '01' - END as date - ) as Month_repair_date, - reported_issues_category, - permanent_action_taken, - turn_around_time_week_days, - target_turn_around, - met_not_met, - time_taken, - hangar_or_corral, - /* Add the collection of signoff month & year */ - CASE - When sign_off_year != '1899' - AND sign_off_month != '' Then sign_off_month - ELSE CASE - When repair_date like '%/%' Then substr (repair_date, 4, 2) - ELSE substr (cast(repair_date as varchar), 6, 2) - END - END as sign_off_month, - CASE - When sign_off_year != '1899' - AND sign_off_year != '' Then sign_off_year - ELSE CASE - When repair_date like '%/%' Then substr (repair_date, 7, 4) - ELSE substr (cast(repair_date as varchar), 1, 4) - END - END as sign_off_year - FROM - "parking-raw-zone".parking_parking_ops_cycle_hangar_mgt - WHERE - import_date = ( - Select - MAX(import_date) - FROM - "parking-raw-zone".parking_parking_ops_cycle_hangar_mgt - ) - AND length (ltrim (rtrim (reported_date))) > 0 - AND met_not_met not IN ('#VALUE!', '#N/A') - AND length (ltrim (rtrim (repair_date))) > 0 - ), - /*** Tom has informed me that I cannot use the Repair date, but use the sigh off month & year - USe these fields to recreate Month_repair_date ***/ - Defect as ( - SELECT - reference_no, - hangar_corral_no, - reported_date, - repair_date, - CAST( - sign_off_year || '-' || sign_off_month || '-01' as date - ) as Month_repair_date, - reported_issues_category, - permanent_action_taken, - turn_around_time_week_days, - target_turn_around, - met_not_met, - time_taken, - hangar_or_corral, - sign_off_month, - sign_off_year - FROM - Defect_Basic - ), - /******************************************************************************** - Obtain the category totals - ********************************************************************************/ - category_totals as ( - SELECT - Month_repair_date, - met_not_met, - count(*) as Total_Met_Fail - FROM - Defect - WHERE - repair_date >= - -- date_add(cast(substr(cast(current_date - -- as varchar), 1, 8)||'01' as date), -365) - date_add ( - 'day', - -365, - cast( - substr (cast(current_date as varchar), 1, 8) || '01' as date - ) - ) - AND met_not_met = 'Fail' - GROUP BY - Month_repair_date, - met_not_met - UNION ALL - SELECT - Month_repair_date, - met_not_met, - count(*) as Total_Met_Fail - FROM - Defect - WHERE - repair_date >= - -- date_add(cast(substr(cast(current_date - -- as varchar), 1, 8)||'01' as date), -365) - date_add ( - 'day', - -365, - cast( - substr (cast(current_date as varchar), 1, 8) || '01' as date - ) - ) - AND met_not_met = 'Met' - GROUP BY - Month_repair_date, - met_not_met - UNION ALL - SELECT - Month_repair_date, - met_not_met, - count(*) as Total_Met_Fail - FROM - Defect - WHERE - repair_date >= - -- date_add(cast(substr(cast(current_date - -- as varchar), 1, 8)||'01' as date), -365) - date_add ( - 'day', - -365, - cast( - substr (cast(current_date as varchar), 1, 8) || '01' as date - ) - ) - AND met_not_met = 'N/A' - GROUP BY - Month_repair_date, - met_not_met - ), - Categories as ( - SELECT distinct - A.Month_repair_date - FROM - Defect as A - ), - /******************************************************************************** - Obtain and format the totals - ********************************************************************************/ - Category_Format as ( - SELECT - A.Month_repair_date, - CASE - When B.Total_Met_Fail is not NULL Then B.Total_Met_Fail - Else 0 - END as Total_Fail, - CASE - When C.Total_Met_Fail is not NULL Then C.Total_Met_Fail - Else 0 - END as Total_Met, - CASE - When D.Total_Met_Fail is not NULL Then D.Total_Met_Fail - Else 0 - END as Total_NA - FROM - category_totals as A - LEFT JOIN category_totals as B ON A.Month_repair_date = B.Month_repair_date - AND B.met_not_met = 'Fail' - LEFT JOIN category_totals as C ON A.Month_repair_date = C.Month_repair_date - AND C.met_not_met = 'Met' - LEFT JOIN category_totals as D ON A.Month_repair_date = D.Month_repair_date - AND D.met_not_met = 'N/A' - WHERE - A.Month_repair_date >= - -- date_add(cast(substr(cast(current_date - -- as varchar), 1, 8)||'01' as date), -365) - date_add ( - 'day', - -365, - cast( - substr (cast(current_date as varchar), 1, 8) || '01' as date - ) - ) - AND A.Month_repair_date <= current_date - ), - /** De-Dupe the above data **/ - Category_Format_deDupe as ( - SELECT - *, - ROW_NUMBER() OVER ( - PARTITION BY - Month_repair_date, - Total_Fail, - Total_Met, - Total_NA - ORDER BY - Month_repair_date, - Total_Fail, - Total_Met, - Total_NA DESC - ) row_num1 - FROM - Category_Format - ), - /******************************************************************************** - Calculate the monthly totals & % - ********************************************************************************/ - Total as ( - SELECT - Month_repair_date, - 'Total' as Category, - SUM(cast(Total_Met as double) + Total_Fail + Total_NA) as Grand_total - FROM - Category_Format_deDupe - Where - row_num1 = 1 - GROUP BY - Month_repair_date - ), - Total_Percentage as ( - SELECT - Month_repair_date, - 'KPI_%' as Category, - ( - CAST( - CASE - When Total_Met + Total_Fail = 0 - AND Total_Met != 0 Then 100 - When Total_Met + Total_Fail = 0 - AND Total_Met = 0 Then 0 - ELSE cast(Total_Met as double) / (Total_Met + Total_Fail) - END as decimal(8, 2) - ) * 100 - ) as KPI - FROM - Category_Format_deDupe - Where - row_num1 = 1 - ), - /******************************************************************************** - Format the output - ********************************************************************************/ - Format_Report as ( - SELECT - Month_repair_date, - '' as Category, - 'Fail' as Total_Met_Fail, - Total_Fail as Total - FROM - Category_Format_deDupe - UNION ALL - SELECT - Month_repair_date, - '' as Category, - 'Met' as Total_Met_Fail, - Total_Met as Total - FROM - Category_Format_deDupe - UNION ALL - SELECT - Month_repair_date, - '' as Category, - 'N/A' as Total_Met_Fail, - Total_NA as Total - FROM - Category_Format_deDupe - UNION ALL - SELECT - Month_repair_date, - Category, - '' as Total_Met_Fail, - Grand_total - FROM - Total - UNION ALL - SELECT - Month_repair_date, - Category, - '' as Total_Met_Fail, - KPI - FROM - Total_Percentage - ), - Data_Dedupe as ( - SELECT - *, - ROW_NUMBER() OVER ( - PARTITION BY - month_repair_date, - category, - total_met_fail - ORDER BY - month_repair_date, - category, - total_met_fail DESC - ) row_num - FROM - Format_Report - ) - /******************************************************************************** - Ouput data - ********************************************************************************/ -SELECT - Month_repair_date, - Category, - Total_Met_Fail, - Total, - format_datetime ( - CAST(CURRENT_TIMESTAMP AS timestamp), - 'yyyy-MM-dd HH:mm:ss' - ) AS ImportDateTime, - format_datetime (current_date, 'yyyy') AS import_year, - format_datetime (current_date, 'MM') AS import_month, - format_datetime (current_date, 'dd') AS import_day, - format_datetime (current_date, 'yyyyMMdd') AS import_date -FROM - Data_Dedupe -WHERE - row_num = 1 -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_cycle_hangars_denormalisation.py b/scripts/jobs/parking/parking_cycle_hangars_denormalisation.py deleted file mode 100644 index 186bd0ce3..000000000 --- a/scripts/jobs/parking/parking_cycle_hangars_denormalisation.py +++ /dev/null @@ -1,167 +0,0 @@ -import sys -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.dynamicframe import DynamicFrame - -from scripts.helpers.helpers import get_glue_env_var, create_pushdown_predicate_for_max_date_partition_value -environment = get_glue_env_var("environment") - -def spark_sql_query(glue_context, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glue_context, transformation_ctx) - -SqlQuery0 = ''' -WITH HangarTypes as ( - SELECT id, type_name, hanger_class, cost, cost_per,import_year, import_month, import_day - FROM liberator_hangar_types - WHERE import_date = (SELECT MAX(import_date) FROM liberator_hangar_types)), - - -HangarDetails AS ( - SELECT A.ID , - HANGAR_TYPE , - HANGAR_ID , - IN_SERVICE , - MAINTENANCE_KEY , - SPACES , - HANGAR_LOCATION , - USRN , - LATITUDE , - LONGITUDE , - START_OF_LIFE , - END_OF_LIFE, - import_date, import_year, import_month, import_day, - ROW_NUMBER() OVER (PARTITION BY HANGAR_TYPE, HANGAR_ID, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, - LONGITUDE, START_OF_LIFE, END_OF_LIFE - ORDER BY HANGAR_TYPE, HANGAR_ID, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, LONGITUDE, - START_OF_LIFE, END_OF_LIFE DESC) AS ROW - --B.* - FROM liberator_hangar_details as A - --INNER JOIN HangarTypes as B ON A.id = B.id AND lower(A.hangar_type) = lower(B.hanger_class) - WHERE import_date = (SELECT MAX(import_date) FROM liberator_hangar_details) - ORDER BY HANGAR_ID), - - -HangarAllocBefore AS - (SELECT ID - , HANGER_ID - , KEY_ID - , SPACE - , PARTY_ID - , KEY_ISSUED - , DATE_OF_ALLOCATION - , ALLOCATION_STATUS - , FEE_DUE_DATE - , CREATED_BY - , ROW_NUMBER() OVER (PARTITION BY HANGER_ID, KEY_ID, SPACE, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION, ALLOCATION_STATUS, - FEE_DUE_DATE, CREATED_BY - ORDER BY HANGER_ID, KEY_ID, SPACE, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION, ALLOCATION_STATUS, FEE_DUE_DATE, CREATED_BY - DESC) AS ROW - FROM liberator_hangar_allocations - WHERE import_date = (SELECT MAX(import_date) FROM liberator_hangar_allocations)), - -HangarAlloc AS - (SELECT *, - ROW_NUMBER() - OVER (PARTITION BY HANGER_ID, SPACE - ORDER BY DATE_OF_ALLOCATION DESC, FEE_DUE_DATE DESC, ID DESC) AS RW - FROM HangarAllocBefore - WHERE ROW =1), - -licence_party as ( - SELECT * - FROM liberator_licence_party - WHERE import_date = (SELECT MAX(import_date) FROM liberator_licence_party)), - -CycleHangarAllocation AS - (SELECT A.*, - B.TITLE , - B.FIRST_NAME , - B.SURNAME , - B.ADDRESS1 , - B.ADDRESS2 , - B.ADDRESS3 , - B.POSTCODE , - B.TELEPHONE_NUMBER, - B.EMAIL_ADDRESS - FROM HangarAlloc as A - LEFT JOIN licence_party as B ON A.party_id = B.business_party_id - WHERE RW = 1 AND Allocation_Status NOT IN ('cancelled', 'key_returned') - ORDER BY HANGER_ID, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION) - -/*** Output the data ***/ -SELECT A.ID, HANGER_ID, KEY_ID, SPACE, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION, ALLOCATION_STATUS, FEE_DUE_DATE, CREATED_BY, - TITLE, FIRST_NAME, SURNAME, ADDRESS1, ADDRESS2, ADDRESS3, POSTCODE, TELEPHONE_NUMBER, A.EMAIL_ADDRESS, - /** Hangar details **/ - HANGAR_TYPE, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, LONGITUDE, START_OF_LIFE, END_OF_LIFE, - import_year, import_month, import_day, import_date - -FROM CycleHangarAllocation as A -LEFT JOIN HangarDetails as B ON A.HANGER_ID = B.HANGAR_ID AND B.ROW = 1 -ORDER BY HANGER_ID, SPACE -''' - -## @params: [JOB_NAME] -args = getResolvedOptions(sys.argv, ['JOB_NAME']) - -sc = SparkContext() -glue_context = GlueContext(sc) -spark = glue_context.spark_session -job = Job(glue_context) -job.init(args['JOB_NAME'], args) -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_licence_party", transformation_ctx = "DataSource0"] -## @return: DataSource0 -## @inputs: [] -DataSource0 = glue_context.create_dynamic_frame.from_catalog( - database = "dataplatform-" + environment + "-liberator-raw-zone", - table_name = "liberator_licence_party", - transformation_ctx = "DataSource0", - push_down_predicate = create_pushdown_predicate_for_max_date_partition_value("dataplatform-" + environment + "-liberator-raw-zone", "liberator_licence_party", 'import_date'), - additional_options = {"mergeSchema": "true"}) -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_hangar_allocations", transformation_ctx = "DataSource3"] -## @return: DataSource3 -## @inputs: [] -DataSource3 = glue_context.create_dynamic_frame.from_catalog( - database = "dataplatform-" + environment + "-liberator-raw-zone", - table_name = "liberator_hangar_allocations", - push_down_predicate = create_pushdown_predicate_for_max_date_partition_value("dataplatform-" + environment + "-liberator-raw-zone", "liberator_hangar_allocations", 'import_date'), - transformation_ctx = "DataSource3") -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_hangar_types", transformation_ctx = "DataSource2"] -## @return: DataSource2 -## @inputs: [] -DataSource2 = glue_context.create_dynamic_frame.from_catalog( - database = "dataplatform-" + environment + "-liberator-raw-zone", - table_name = "liberator_hangar_types", - push_down_predicate = create_pushdown_predicate_for_max_date_partition_value("dataplatform-" + environment + "-liberator-raw-zone", "liberator_hangar_types", 'import_date'), - transformation_ctx = "DataSource2") -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_hangar_details", transformation_ctx = "DataSource1"] -## @return: DataSource1 -## @inputs: [] -DataSource1 = glue_context.create_dynamic_frame.from_catalog( - database = "dataplatform-" + environment + "-liberator-raw-zone", - table_name = "liberator_hangar_details", - push_down_predicate = create_pushdown_predicate_for_max_date_partition_value("dataplatform-" + environment + "-liberator-raw-zone", "liberator_hangar_details", 'import_date'), - transformation_ctx = "DataSource1") -## @type: SqlCode -## @args: [sqlAliases = {"liberator_hangar_details": DataSource1, "liberator_hangar_types": DataSource2, "liberator_hangar_allocations": DataSource3, "liberator_licence_party": DataSource0}, sqlName = SqlQuery0, transformation_ctx = "Transform0"] -## @return: Transform0 -## @inputs: [dfc = DataSource1,DataSource2,DataSource3,DataSource0] -Transform0 = spark_sql_query(glue_context, query = SqlQuery0, mapping = {"liberator_hangar_details": DataSource1, "liberator_hangar_types": DataSource2, "liberator_hangar_allocations": DataSource3, "liberator_licence_party": DataSource0}, transformation_ctx = "Transform0") -## @type: DataSink -## @args: [connection_type = "s3", catalog_database_name = "dataplatform-" + environment + "-liberator-refined-zone", format = "glueparquet", connection_options = {"path": "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/parking_cycle_hangars_denormalisation/", "partitionKeys": ["import_year" ,"import_month" ,"import_day" ,"import_date"], "enableUpdateCatalog":true, "updateBehavior":"UPDATE_IN_DATABASE"}, catalog_table_name = "parking_cycle_hangars_denormalisation", transformation_ctx = "DataSink0"] -## @return: DataSink0 -## @inputs: [frame = Transform0] -DataSink0 = glue_context.getSink(path = "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/parking_cycle_hangars_denormalisation/", connection_type = "s3", updateBehavior = "UPDATE_IN_DATABASE", partitionKeys = ["import_year","import_month","import_day","import_date"], enableUpdateCatalog = True, transformation_ctx = "DataSink0") -DataSink0.setCatalogInfo(catalogDatabase = "dataplatform-" + environment + "-liberator-refined-zone",catalogTableName = "parking_cycle_hangars_denormalisation") -DataSink0.setFormat("glueparquet") -DataSink0.writeFrame(Transform0) - -job.commit() diff --git a/scripts/jobs/parking/parking_cycle_hangars_denormalisation_update.py b/scripts/jobs/parking/parking_cycle_hangars_denormalisation_update.py deleted file mode 100644 index 1f4a4191e..000000000 --- a/scripts/jobs/parking/parking_cycle_hangars_denormalisation_update.py +++ /dev/null @@ -1,208 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - create_pushdown_predicate, - create_pushdown_predicate_for_max_date_partition_value, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -environment = get_glue_env_var("environment") - -# Script generated for node liberator_license_party -liberator_license_party_node1628255463015 = ( - glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_party", - transformation_ctx="liberator_license_party_node1628255463015", - push_down_predicate=create_pushdown_predicate("import_date", 7), - ) -) - -# Script generated for node liberator_hangar_details -liberator_hangar_details_node1628254576316 = ( - glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_details", - transformation_ctx="liberator_hangar_details_node1628254576316", - push_down_predicate=create_pushdown_predicate("import_date", 7), - ) -) - -# Script generated for node liberator_hangar_types -liberator_hangar_types_node1628254679074 = ( - glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_types", - transformation_ctx="liberator_hangar_types_node1628254679074", - push_down_predicate=create_pushdown_predicate("import_date", 7), - ) -) - -# Script generated for node liberator_hangar_allocations -liberator_hangar_allocations_node1628254680802 = ( - glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_allocations", - transformation_ctx="liberator_hangar_allocations_node1628254680802", - push_down_predicate=create_pushdown_predicate("import_date", 7), - ) -) - -# Script generated for node ApplyMapping -SqlQuery52 = """ -/*************************************************************************************************************************** -Parking_cycle_hangar_denormalisation_update - -The SQL creates the denormalised cycle hangar data (remove the duplicates). This is an updated code from Phil - -07/12/2023 - create SQL -26/04/2024 - change the de-dupe from party_id to space -10/05/2024 - change the HangarAlloc ORDER BY from ID to DATE_OF_ALLOCATION DESC, FEE_DUE_DATE DESC, ID DESC -***************************************************************************************************************************/ -WITH HangarTypes as ( - SELECT id, type_name, hanger_class, cost, cost_per,import_year, import_month, import_day - FROM liberator_hangar_types - WHERE import_date = (SELECT MAX(import_date) FROM liberator_hangar_types)), - - -HangarDetails AS ( - SELECT A.ID , - HANGAR_TYPE , - HANGAR_ID , - IN_SERVICE , - MAINTENANCE_KEY , - SPACES , - HANGAR_LOCATION , - USRN , - LATITUDE , - LONGITUDE , - START_OF_LIFE , - END_OF_LIFE, - import_date, import_year, import_month, import_day, - ROW_NUMBER() OVER (PARTITION BY HANGAR_TYPE, HANGAR_ID, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, - LONGITUDE, START_OF_LIFE, END_OF_LIFE - ORDER BY HANGAR_TYPE, HANGAR_ID, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, LONGITUDE, - START_OF_LIFE, END_OF_LIFE DESC) AS ROW - FROM liberator_hangar_details as A - WHERE import_date = (SELECT MAX(import_date) FROM liberator_hangar_details) - ORDER BY HANGAR_ID), - - -HangarAllocBefore AS - (SELECT ID - , HANGER_ID - , KEY_ID - , SPACE - , PARTY_ID - , KEY_ISSUED - , DATE_OF_ALLOCATION - , ALLOCATION_STATUS - , FEE_DUE_DATE - , CREATED_BY - , ROW_NUMBER() OVER (PARTITION BY HANGER_ID, KEY_ID, SPACE, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION, ALLOCATION_STATUS, - FEE_DUE_DATE, CREATED_BY - ORDER BY HANGER_ID, KEY_ID, SPACE, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION, ALLOCATION_STATUS, FEE_DUE_DATE, CREATED_BY - DESC) AS ROW - FROM liberator_hangar_allocations - WHERE import_date = (SELECT MAX(import_date) FROM liberator_hangar_allocations)), - -HangarAlloc AS - (SELECT *, - ROW_NUMBER() - OVER (PARTITION BY HANGER_ID, SPACE - ORDER BY DATE_OF_ALLOCATION DESC, FEE_DUE_DATE DESC, ID DESC) AS RW - FROM HangarAllocBefore - WHERE ROW =1), - -licence_party as ( - SELECT party_id, business_party_id, title, first_name, surname, - address1, address2, address3, postcode, uprn, - telephone_number - FROM liberator_licence_party - WHERE import_date = (SELECT MAX(import_date) FROM liberator_licence_party)), - -CycleHangarAllocation AS - (SELECT A.*, - B.TITLE , - B.FIRST_NAME , - B.SURNAME , - B.ADDRESS1 , - B.ADDRESS2 , - B.ADDRESS3 , - B.POSTCODE , - B.TELEPHONE_NUMBER - FROM HangarAlloc as A - LEFT JOIN licence_party as B ON A.party_id = B.business_party_id - WHERE RW = 1 AND Allocation_Status NOT IN ('cancelled', 'key_returned') - ORDER BY HANGER_ID, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION) - -/*** Output the data ***/ -SELECT - A.ID, HANGER_ID, KEY_ID, SPACE, PARTY_ID, KEY_ISSUED, DATE_OF_ALLOCATION, ALLOCATION_STATUS, FEE_DUE_DATE, CREATED_BY, - TITLE, FIRST_NAME, SURNAME, ADDRESS1, ADDRESS2, ADDRESS3, POSTCODE, TELEPHONE_NUMBER, - /** Hangar details **/ - HANGAR_TYPE, IN_SERVICE, MAINTENANCE_KEY, SPACES, HANGAR_LOCATION, USRN, LATITUDE, LONGITUDE, START_OF_LIFE, END_OF_LIFE, - - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM CycleHangarAllocation as A -LEFT JOIN HangarDetails as B ON A.HANGER_ID = B.HANGAR_ID AND B.ROW = 1 -ORDER BY HANGER_ID, SPACE -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery52, - mapping={ - "liberator_hangar_details": liberator_hangar_details_node1628254576316, - "liberator_hangar_types": liberator_hangar_types_node1628254679074, - "liberator_hangar_allocations": liberator_hangar_allocations_node1628254680802, - "liberator_licence_party": liberator_license_party_node1628255463015, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/parking_cycle_hangars_denormalisation_update/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_cycle_hangars_denormalisation_update", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_cycle_hangars_denormalised_inservice_live_gds.py b/scripts/jobs/parking/parking_cycle_hangars_denormalised_inservice_live_gds.py deleted file mode 100644 index 3c8d7fc19..000000000 --- a/scripts/jobs/parking/parking_cycle_hangars_denormalised_inservice_live_gds.py +++ /dev/null @@ -1,278 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 -parking_cycle_hangars_denormalisation -AmazonS3parking_cycle_hangars_denormalisation_node1705433167883 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_cycle_hangars_denormalisation", - transformation_ctx="AmazonS3parking_cycle_hangars_denormalisation_node1705433167883", -) - -# Script generated for node Parking raw zone manual - parking_cycle_hangar_denormalised_backfill -Parkingrawzonemanualparking_cycle_hangar_denormalised_backfill_node1711467992130 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="cycle_hangar_denormalised___backfill", - transformation_ctx="Parkingrawzonemanualparking_cycle_hangar_denormalised_backfill_node1711467992130", -) - -# Script generated for node SQL Query -SqlQuery0 = """ -/* - Summary data from "dataplatform-prod-liberator-refined-zone".parking_cycle_hangars_denormalisation - Filtered by (import_day = '01') First of every month - and ( upper(in_service) like 'Y') Hangar is in service - and ( allocation_status like 'live') allocation is live - -26/03/2024 - added backfill data parking_cycle_hangar_denormalised_backfill - -*/ - -with cycle_den as ( -select distinct --* -parking_cycle_hangar_denormalised_backfill.id -,parking_cycle_hangar_denormalised_backfill.hanger_id -,parking_cycle_hangar_denormalised_backfill.key_id -,parking_cycle_hangar_denormalised_backfill.space -,parking_cycle_hangar_denormalised_backfill.party_id -,parking_cycle_hangar_denormalised_backfill.key_issued -,parking_cycle_hangar_denormalised_backfill.date_of_allocation -,parking_cycle_hangar_denormalised_backfill.allocation_status -,parking_cycle_hangar_denormalised_backfill.fee_due_date -,parking_cycle_hangar_denormalised_backfill.created_by -,parking_cycle_hangar_denormalised_backfill.title -,parking_cycle_hangar_denormalised_backfill.first_name -,parking_cycle_hangar_denormalised_backfill.surname -,parking_cycle_hangar_denormalised_backfill.address1 -,parking_cycle_hangar_denormalised_backfill.address2 -,parking_cycle_hangar_denormalised_backfill.address3 -,parking_cycle_hangar_denormalised_backfill.postcode -,parking_cycle_hangar_denormalised_backfill.telephone_number -,parking_cycle_hangar_denormalised_backfill.email_address -,parking_cycle_hangar_denormalised_backfill.hangar_type -,parking_cycle_hangar_denormalised_backfill.in_service -,parking_cycle_hangar_denormalised_backfill.maintenance_key -,parking_cycle_hangar_denormalised_backfill.spaces -,parking_cycle_hangar_denormalised_backfill.hangar_location -,parking_cycle_hangar_denormalised_backfill.usrn -,parking_cycle_hangar_denormalised_backfill.latitude -,parking_cycle_hangar_denormalised_backfill.longitude -,parking_cycle_hangar_denormalised_backfill.start_of_life -,parking_cycle_hangar_denormalised_backfill.end_of_life -,parking_cycle_hangar_denormalised_backfill.import_year_original as import_year -,parking_cycle_hangar_denormalised_backfill.import_month_original as import_month -,case when parking_cycle_hangar_denormalised_backfill.import_day_original like '1' then '01' else parking_cycle_hangar_denormalised_backfill.import_day_original end as import_day -,parking_cycle_hangar_denormalised_backfill.import_date_original as import_date ---parking_cycle_hangar_denormalised_backfill.import_datetime ---parking_cycle_hangar_denormalised_backfill.import_timestamp ---parking_cycle_hangar_denormalised_backfill.import_year as import_year_backfill ---parking_cycle_hangar_denormalised_backfill.import_month as import_month_backfill ---parking_cycle_hangar_denormalised_backfill.import_day as import_day_backfill ---parking_cycle_hangar_denormalised_backfill.import_date as import_date_backfill -from parking_cycle_hangar_denormalised_backfill --where import_date = (select max(import_date) from "parking-raw-zone-manual".parking_cycle_hangar_denormalised_backfill) --78,148 --83,742 --5,594 before distinct --78,340 --83,742 83,934 5,594 - with import day filter 60,681 --72,394 - where parking_cycle_hangar_denormalised_backfill.import_day_original = '1' - and upper(parking_cycle_hangar_denormalised_backfill.in_service) like 'Y' - and parking_cycle_hangar_denormalised_backfill.allocation_status like 'live' - -union all - -select --* -cast(id as string) as id -,hanger_id -,key_id -,space -,party_id -,key_issued -,date_of_allocation -,allocation_status -,fee_due_date -,created_by -,title -,first_name -,surname -,address1 -,address2 -,address3 -,postcode -,telephone_number -,email_address -,hangar_type -,in_service -,maintenance_key -,spaces -,hangar_location -,usrn -,latitude -,longitude -,start_of_life -,end_of_life -,import_year -,import_month -,import_day -,import_date - -from parking_cycle_hangars_denormalisation - where parking_cycle_hangars_denormalisation .import_day = '01' - and upper(parking_cycle_hangars_denormalisation.in_service) like 'Y' - and parking_cycle_hangars_denormalisation.allocation_status like 'live' - - ) - - -select -to_date(cycle_den.import_date, 'yyyyMMdd') as importdate -,cycle_den.import_date as original_import_date -,cycle_den.hanger_id -,cycle_den.hangar_type -,cycle_den.hangar_location - -/*Days to renew*/ -,sum(case when (datediff( cast(substr(cycle_den.fee_due_date, 1, 10) as date) , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) )) < 366 then 1 end) as allocate_due_within_one_year -,sum(case when (datediff( cast(substr(cycle_den.fee_due_date, 1, 10) as date) , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) )) > 365 then 1 end) as allocate_due_more_than_one_year -,sum(case when (datediff( cast(substr(cycle_den.fee_due_date, 1, 10) as date) , current_date )) < 366 then 1 end) as due_within_one_year -,sum(case when (datediff( cast(substr(cycle_den.fee_due_date, 1, 10) as date) , current_date )) > 365 then 1 end) as due_more_than_one_year - -,sum(case when cast(substr(cycle_den.fee_due_date, 1, 10) as date) > current_date then 1 end) as number_due_after_current_date -,sum(case when cast(substr(cycle_den.fee_due_date, 1, 10) as date) < current_date then 1 end) as number_due_before_current_date - -/*period hangar held breakdown years*/ - -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > -0 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 366 then 1 end) as held_1_less_one_year -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 365 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 731 then 1 end) as held_2_two_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 730 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 1096 then 1 end) as held_3_three_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 1095 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 1461 then 1 end) as held_4_four_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 1460 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 1826 then 1 end) as held_5_five_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 1825 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 2191 then 1 end) as held_6_six_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 2190 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 2556 then 1 end) as held_7_seven_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 2555 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 2921 then 1 end) as held_8_eight_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 2920 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 3286 then 1 end) as held_9_nine_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 3285 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 3651 then 1 end) as held_10_Ten_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 3650 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 4016 then 1 end) as held_11_eleven_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 4015 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 4381 then 1 end) as held_12_twelve_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 4380 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 4746 then 1 end) as held_13_thirteen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 4745 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 5111 then 1 end) as held_14_fourteen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 5110 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 5476 then 1 end) as held_15_fifteen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 5475 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 5841 then 1 end) as held_16_sixteen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 5840 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 6206 then 1 end) as held_17_seventeen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 6205 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 6571 then 1 end) as held_18_eighteen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 6570 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 6936 then 1 end) as held_19_nineteen_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 6935 and datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) < 7301 then 1 end) as held_20_twenty_years -,sum(case when datediff( current_date , cast(substr(cycle_den.date_of_allocation, 1, 10) as date) ) > 7300 then 1 end) as held_21_twentyOnePlus_years - -/*number of records*/ -,count(cycle_den.space) as num_space, count(*) as num_recs -- spaces, -,COUNT(distinct cycle_den.hanger_id) as number_hangars -,COUNT(distinct cycle_den.hanger_id)*6 as hangar_capacity - -/*In Service*/ -,sum(case when upper(cycle_den.in_service) like 'Y' then 1 end) as flag_in_service - - -/*key issued*/ -,sum(case when upper(cycle_den.key_issued) like 'Y' then 1 end) as number_key_issued -,sum(case when upper(cycle_den.key_issued) like 'N' then 1 end) as number_Key_not_issued - -/*Allocation Status*/ -,sum(case when cycle_den.allocation_status like 'live' then 1 end) as number_allocation_status_live - -/*Renters*/ -,count(distinct cycle_den.party_id) as number_party_ids - -,count(distinct cycle_den.email_address) as number_email_address - - -/*Key IDs*/ -,count(distinct cycle_den.key_id) as number_diff_key_ids -,count(cycle_den.key_id) as number_key_ids - - -/*spaces*/ -,count(cycle_den.space) as number_space -,count(distinct cycle_den.space) as number_diff_space_letters - -/*locations*/ -,count(distinct cycle_den.hangar_location) as number_diff_hangar_locations -,count(cycle_den.hangar_location) as number_hangar_locations - -/*hangar types*/ -,count(distinct cycle_den.hangar_type) as number_diff_hangar_type -,count(cycle_den.hangar_type) as number_hangar_type - -/*for partiion*/ - -,date_format(current_date, 'yyyy') AS import_year -,date_format(current_date, 'MM') AS import_month -,date_format(current_date, 'dd') AS import_day -,date_format(current_date, 'yyyyMMdd') AS import_date - - from cycle_den - where cycle_den.import_day = '01' or cycle_den.import_day = '1' - and upper(cycle_den.in_service) like 'Y' - and cycle_den.allocation_status like 'live' - -group by -cycle_den.import_date -,cycle_den.hanger_id -,cycle_den.hangar_type -,cycle_den.hangar_location -,date_format(current_date, 'yyyyMMdd') -""" -SQLQuery_node1705433170442 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_cycle_hangars_denormalisation": AmazonS3parking_cycle_hangars_denormalisation_node1705433167883, - "parking_cycle_hangar_denormalised_backfill": Parkingrawzonemanualparking_cycle_hangar_denormalised_backfill_node1711467992130, - }, - transformation_ctx="SQLQuery_node1705433170442", -) - -# Script generated for node Amazon S3 - parking_cycle_hangars_denormalised_inservice_live_gds -AmazonS3parking_cycle_hangars_denormalised_inservice_live_gds_node1705433173405 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/parking_cycle_hangars_denormalised_inservice_live_gds/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="AmazonS3parking_cycle_hangars_denormalised_inservice_live_gds_node1705433173405", -) -AmazonS3parking_cycle_hangars_denormalised_inservice_live_gds_node1705433173405.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_cycle_hangars_denormalised_inservice_live_gds", -) -AmazonS3parking_cycle_hangars_denormalised_inservice_live_gds_node1705433173405.setFormat( - "glueparquet", compression="snappy" -) -AmazonS3parking_cycle_hangars_denormalised_inservice_live_gds_node1705433173405.writeFrame( - SQLQuery_node1705433170442 -) -job.commit() diff --git a/scripts/jobs/parking/parking_cycle_hangars_waiting_list.py b/scripts/jobs/parking/parking_cycle_hangars_waiting_list.py deleted file mode 100644 index bde8d2cb5..000000000 --- a/scripts/jobs/parking/parking_cycle_hangars_waiting_list.py +++ /dev/null @@ -1,205 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - create_pushdown_predicate_for_max_date_partition_value, - get_glue_env_var, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 -AmazonS3_node1658997944648 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_waiting_list", - transformation_ctx="AmazonS3_node1658997944648", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value( - "dataplatform-" + environment + "-liberator-raw-zone", - "liberator_hangar_waiting_list", - "import_date", - ), -) - -# Script generated for node Amazon S3 -AmazonS3_node1685714415298 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_llpg", - transformation_ctx="AmazonS3_node1685714415298", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value( - "dataplatform-" + environment + "-liberator-raw-zone", - "liberator_permit_llpg", - "import_date", - ), -) - -# Script generated for node Amazon S3 -AmazonS3_node1685714634970 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_hangar_allocations", - transformation_ctx="AmazonS3_node1685714634970", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value( - "dataplatform-" + environment + "-liberator-raw-zone", - "liberator_hangar_allocations", - "import_date", - ), -) - -# Script generated for node Amazon S3 -AmazonS3_node1685714303815 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_party", - transformation_ctx="AmazonS3_node1685714303815", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value( - "dataplatform-" + environment + "-liberator-raw-zone", - "liberator_licence_party", - "import_date", - ), - additional_options={"mergeSchema": "true"}, -) - -# Script generated for node SQL -SqlQuery67 = """ -/******************************************************************************************************************** -Parking_Cycle_Hangar_Wait_List - -The SQL tidies the Cycle Hangar waiting list, It will create a single record for each Party ID on the list - -02/06/2023 - Create Query -09/06/2023 - add X & Y from LLPG -12/10/2023 - Add the different types of hangars requested by the party -********************************************************************************************************************/ -/*** Waiting List ***/ -WITH waiting_list as ( - SELECT *, - ROW_NUMBER() OVER ( PARTITION BY party_id - ORDER BY party_id DESC) row1 - FROM liberator_hangar_waiting_list - WHERE Import_Date = (Select MAX(Import_Date) from - liberator_hangar_waiting_list)), -/*** Party List ***/ -Licence_Party as ( - SELECT * - FROM liberator_licence_party - WHERE Import_Date = (Select MAX(Import_Date) from - liberator_licence_party)), -/*** STREET ***/ -LLPG as ( - SELECT * - FROM liberator_permit_llpg - WHERE import_date = (Select MAX(import_date) from - liberator_permit_llpg)), -/******************************************************************************* -Cycle Hangar allocation details -*******************************************************************************/ -Cycle_Hangar_allocation as ( - SELECT - *, - ROW_NUMBER() OVER ( PARTITION BY party_id - ORDER BY party_id, date_of_allocation DESC) row_num - FROM liberator_hangar_allocations - WHERE Import_Date = (Select MAX(Import_Date) from - liberator_hangar_allocations) - AND allocation_status IN ('live')), - -Street_Rec as ( - SELECT * - FROM liberator_permit_llpg - WHERE import_date = (Select MAX(import_date) from - liberator_permit_llpg) - AND address1 = 'STREET RECORD'), - -/** Obtain the unique Party IDs **/ -Party_ID_List as ( - SELECT - A.party_id, first_name, surname, B.uprn as USER_UPRN, - B.address1, B.address2, B.address3, B.postcode, - B.telephone_number, D.Address2 as Street, email_address, - B.record_created as Date_Registered, - C.x, C.y - FROM waiting_list as A - LEFT JOIN Licence_Party as B ON A.party_id = B.business_party_id - LEFT JOIN LLPG as C ON B.uprn = cast(C.UPRN as string) - LEFT JOIN Street_Rec as D ON C.USRN = D.USRN - LEFT JOIN Cycle_Hangar_allocation as E ON A.party_id = E.party_id AND row_num = 1 - WHERE row1= 1 AND E.party_id is NULL and D.Address2 is not NULL), - -Count_Hangar_Details as ( - SELECT party_id, - SUM(CASE - When registration_type = 'ESTATE' Then 1 - ELSE 0 - END) as Estate_Count, - SUM(CASE - When registration_type = 'NEWONLY' Then 1 - ELSE 0 - END) as Newonly_Count, - SUM(CASE - When registration_type = 'RESIDENT' Then 1 - ELSE 0 - END) as Resident_Count - FROM waiting_list - GROUP BY party_id) - -SELECT A.*, B.Estate_Count, B.Newonly_Count, B.Resident_Count, - - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS importdatetime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM Party_ID_List as A -LEFT JOIN Count_Hangar_Details as B ON A.party_id = B.party_id -""" -SQL_node1658765472050 = sparkSqlQuery( - glueContext, - query=SqlQuery67, - mapping={ - "liberator_hangar_waiting_list": AmazonS3_node1658997944648, - "liberator_licence_party": AmazonS3_node1685714303815, - "liberator_permit_llpg": AmazonS3_node1685714415298, - "liberator_hangar_allocations": AmazonS3_node1685714634970, - }, - transformation_ctx="SQL_node1658765472050", -) - -# Script generated for node Amazon S3 -AmazonS3_node1658765590649 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/parking_cycle_hangars_waiting_list/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=PARTITION_KEYS, - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="AmazonS3_node1658765590649", -) -AmazonS3_node1658765590649.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_cycle_hangars_waiting_list", -) -AmazonS3_node1658765590649.setFormat("glueparquet") -AmazonS3_node1658765590649.writeFrame(SQL_node1658765472050) -job.commit() diff --git a/scripts/jobs/parking/parking_dc_liberator_latest_permit_status.py b/scripts/jobs/parking/parking_dc_liberator_latest_permit_status.py deleted file mode 100644 index aa8ce0bfb..000000000 --- a/scripts/jobs/parking/parking_dc_liberator_latest_permit_status.py +++ /dev/null @@ -1,105 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import ( - get_glue_env_var, - get_latest_partitions, - PARTITION_KEYS, - create_pushdown_predicate, -) - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node S3 bucket -S3bucket_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - table_name="liberator_permit_status", - transformation_ctx="S3bucket_node1", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ ---Latest permit status by status ID in latest partition import_date -Select s.*,concat(CAST(s.status_id AS STRING),CAST(s.permit_referece AS STRING),CAST(s.status_date AS STRING)) as latest_permitstatus_id -,substr(s.permit_referece,1,3) as permit_type_prefix, -if(substr(s.permit_referece,1,3) = 'HYA','All zone', -if(substr(s.permit_referece,1,3) = 'HYG','Companion Badge', -if(substr(s.permit_referece,1,3) = 'HYB','Business', -if(substr(s.permit_referece,1,3) = 'HYD','Doctor', -if(substr(s.permit_referece,1,3) = 'HYP','Estate resident', -if(substr(s.permit_referece,1,3) = 'HYH','Health and social care', -if(substr(s.permit_referece,1,3) = 'HYR','Residents', -if(substr(s.permit_referece,1,3) = 'HYL','Leisure Centre Permit', -if(substr(s.permit_referece,1,3) = 'HYE','All Zone Business Voucher', -if(substr(s.permit_referece,1,3) = 'HYF','Film Voucher', -if(substr(s.permit_referece,1,3) = 'HYJ','Health and Social Care Voucher', -if(substr(s.permit_referece,1,3) = 'HYQ','Estate Visitor Voucher', -if(substr(s.permit_referece,1,3) = 'HYV','Resident Visitor Voucher', -if(substr(s.permit_referece,1,3)='HYN','Dispensation', -if(s.permit_referece ='HY50300','Dispensation', -substr(s.permit_referece,1,3) -))))))))))))))) as permit_type_category, - -if(substr(s.permit_referece,1,3) ='HYE','Voucher', -if(substr(s.permit_referece,1,3)='HYF','Voucher', -if(substr(s.permit_referece,1,3)='HYQ','Voucher', -if(substr(s.permit_referece,1,3)='HYV','Voucher', -if(substr(s.permit_referece,1,3)='HYJ','Voucher', -if(substr(s.permit_referece,1,3)='HYN','Dispensation', -if(s.permit_referece ='HY50300','Dispensation', -if(substr(s.permit_referece,1,3)='HYS','Suspension','Permit' -)))))))) as product_category - - -FROM liberator_permit_status s - -left Join -(select permit_referece, max(status_id) as max_status_id from liberator_permit_status Group by permit_referece) m on m.max_status_id = s.status_id - -where m.max_status_id = s.status_id and s.import_date = (select max(import_date) from liberator_permit_status) - -order by s.status_date desc, s.permit_referece, s.status_id desc - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={"liberator_permit_status": S3bucket_node1}, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/dc_liberator_latest_permit_status/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="dc_liberator_latest_permit_status", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_dc_liberator_permit_llpg_street_records.py b/scripts/jobs/parking/parking_dc_liberator_permit_llpg_street_records.py deleted file mode 100644 index d6722ada2..000000000 --- a/scripts/jobs/parking/parking_dc_liberator_permit_llpg_street_records.py +++ /dev/null @@ -1,137 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node S3 bucket -S3bucket_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - table_name="liberator_permit_llpg", - transformation_ctx="S3bucket_node1", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -with sr as (--llpg_streets_records: -select - UPRN as SR_UPRN, - ADDRESS1 as SR_ADDRESS1, - ADDRESS2 as SR_ADDRESS2, - ADDRESS3 as SR_ADDRESS3, - ADDRESS4 as SR_ADDRESS4, - ADDRESS5 as SR_ADDRESS5, - POST_CODE as SR_POST_CODE, - BPLU_CLASS as SR_BPLU_CLASS, - X as SR_X, - Y as SR_Y, - LATITUDE as SR_LATITUDE, - LONGITUDE as SR_LONGITUDE, - IN_CONGESTION_CHARGE_ZONE as SR_IN_CONGESTION_CHARGE_ZONE, - CPZ_CODE as SR_CPZ_CODE, - CPZ_NAME as SR_CPZ_NAME, - LAST_UPDATED_STAMP as SR_LAST_UPDATED_STAMP, - LAST_UPDATED_TX_STAMP as SR_LAST_UPDATED_TX_STAMP, - CREATED_STAMP as SR_CREATED_STAMP, - CREATED_TX_STAMP as SR_CREATED_TX_STAMP, - HOUSE_NAME as SR_HOUSE_NAME, - POST_CODE_PACKED as SR_POST_CODE_PACKED, - STREET_START_X as SR_STREET_START_X, - STREET_START_Y as SR_STREET_START_Y, - STREET_END_X as SR_STREET_END_X, - STREET_END_Y as SR_STREET_END_Y, - WARD_CODE as SR_WARD_CODE, -if(WARD_CODE = 'E05009367', 'BROWNSWOOD WARD', -if(WARD_CODE = 'E05009368', 'CAZENOVE WARD', -if(WARD_CODE = 'E05009369', 'CLISSOLD WARD', -if(WARD_CODE = 'E05009370', 'DALSTON WARD', -if(WARD_CODE = 'E05009371', 'DE BEAUVOIR WARD', -if(WARD_CODE = 'E05009372', 'HACKNEY CENTRAL WARD', -if(WARD_CODE = 'E05009373', 'HACKNEY DOWNS WARD', -if(WARD_CODE = 'E05009374', 'HACKNEY WICK WARD', -if(WARD_CODE = 'E05009375', 'HAGGERSTON WARD', -if(WARD_CODE = 'E05009376', 'HOMERTON WARD', -if(WARD_CODE = 'E05009377', 'HOXTON EAST AND SHOREDITCH WARD', -if(WARD_CODE = 'E05009378', 'HOXTON WEST WARD', -if(WARD_CODE = 'E05009379', 'KINGS PARK WARD', -if(WARD_CODE = 'E05009380', 'LEA BRIDGE WARD', -if(WARD_CODE = 'E05009381', 'LONDON FIELDS WARD', -if(WARD_CODE = 'E05009382', 'SHACKLEWELL WARD', -if(WARD_CODE = 'E05009383', 'SPRINGFIELD WARD', -if(WARD_CODE = 'E05009384', 'STAMFORD HILL WEST', -if(WARD_CODE = 'E05009385', 'STOKE NEWINGTON WARD', -if(WARD_CODE = 'E05009386', 'VICTORIA WARD', -if(WARD_CODE = 'E05009387', 'WOODBERRY DOWN WARD',WARD_CODE))))))))))))))))))))) as SR_ward_name, - PARISH_CODE as SR_PARISH_CODE, - PARENT_UPRN as SR_PARENT_UPRN, - PAO_START as SR_PAO_START, - PAO_START_SUFFIX as SR_PAO_START_SUFFIX, - PAO_END as SR_PAO_END, - PAO_END_SUFFIX as SR_PAO_END_SUFFIX, - PAO_TEXT as SR_PAO_TEXT, - SAO_START as SR_SAO_START, - SAO_START_SUFFIX as SR_SAO_START_SUFFIX, - SAO_END as SR_SAO_END, - SAO_END_SUFFIX as SR_SAO_END_SUFFIX, - SAO_TEXT as SR_SAO_TEXT, - DERIVED_BLPU as SR_DERIVED_BLPU, - USRN -,import_year -,import_month -,import_day -,import_date -FROM liberator_permit_llpg -where (ADDRESS1 like 'Street Record' or ADDRESS1 like 'STREET RECORD') and import_date = (SELECT MAX(import_date) FROM liberator_permit_llpg) -) -Select liberator_permit_llpg.uprn, liberator_permit_llpg.usrn, SR_ADDRESS1, SR_ADDRESS2, SR_ADDRESS3, SR_ADDRESS4, SR_ADDRESS5, SR_POST_CODE, SR_BPLU_CLASS, SR_X, SR_Y, SR_LATITUDE, SR_LONGITUDE, SR_IN_CONGESTION_CHARGE_ZONE, SR_CPZ_CODE, SR_CPZ_NAME, SR_LAST_UPDATED_STAMP, SR_LAST_UPDATED_TX_STAMP, SR_CREATED_STAMP, SR_CREATED_TX_STAMP, SR_HOUSE_NAME, SR_POST_CODE_PACKED, SR_STREET_START_X, SR_STREET_START_Y, SR_STREET_END_X, SR_STREET_END_Y, SR_WARD_CODE, SR_ward_name, SR_PARISH_CODE, SR_PARENT_UPRN, SR_PAO_START, SR_PAO_START_SUFFIX, SR_PAO_END, SR_PAO_END_SUFFIX, SR_PAO_TEXT, SR_SAO_START, SR_SAO_START_SUFFIX, SR_SAO_END, SR_SAO_END_SUFFIX, SR_SAO_TEXT, SR_DERIVED_BLPU -,liberator_permit_llpg.import_year -,liberator_permit_llpg.import_month -,liberator_permit_llpg.import_day -,liberator_permit_llpg.import_date - -FROM liberator_permit_llpg -Left join sr on sr.usrn = liberator_permit_llpg.usrn -where liberator_permit_llpg.import_date = (SELECT MAX(liberator_permit_llpg.import_date) FROM liberator_permit_llpg) -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={"liberator_permit_llpg": S3bucket_node1}, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/dc_liberator_permit_llpg_street_records/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="dc_liberator_permit_llpg_street_records", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_defect_met_fail.py b/scripts/jobs/parking/parking_defect_met_fail.py deleted file mode 100644 index cb04362c7..000000000 --- a/scripts/jobs/parking/parking_defect_met_fail.py +++ /dev/null @@ -1,127 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - create_pushdown_predicate_for_max_date_partition_value, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 -AmazonS3_node1658997944648 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="parking_parking_ops_db_defects_mgt", - transformation_ctx="AmazonS3_node1658997944648", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value( - "parking-raw-zone", "parking_parking_ops_db_defects_mgt", "import_date" - ), -) - -# Script generated for node SQL -SqlQuery33 = """ -/********************************************************************************************* -Parking_Defect_MET_FAIL - -Temp SQL that formats the defcet managment records for Fail/Met - -16/11/2022 - Create Query -15/01/2025 - found AND WHERE repair_date >= (??) removed AND -20/01/2025 - length(ltrim(rtrim(reported_date))) > 0 to length(ltrim(rtrim(reported_date))) > 5 -**********************************************************************************************/ -With Defect as ( -SELECT - reference_no, - CAST(CASE - When reported_date like '%0222%'Then '2022-'|| - substr(reported_date, 7, 2)||'-'|| - substr(reported_date, 9, 2) - - When reported_date like '%/%'Then substr(reported_date, 7, 4)||'-'|| - substr(reported_date, 4, 2)||'-'|| - substr(reported_date, 1, 2) - ELSE substr(cast(reported_date as string),1, 10) - END as date) as reported_date, - CAST(CASE - When repair_date like '%0222%'Then '2022-'|| - substr(repair_date, 7, 2)||'-'|| - substr(repair_date, 9, 2) - - When repair_date like '%/%'Then substr(repair_date, 7, 4)||'-'|| - substr(repair_date, 4, 2)||'-'|| - substr(repair_date, 1, 2) - ELSE substr(cast(repair_date as string),1, 10) - END as date) as repair_date, - category, - date_wo_sent, - expected_wo_completion_date, target_turn_around, met_not_met, - full_repair_category, - issue, engineer - -FROM parking_parking_ops_db_defects_mgt -WHERE import_date = (Select MAX(import_date) from parking_parking_ops_db_defects_mgt) -AND length(ltrim(rtrim(reported_date))) > 5 AND length(ltrim(rtrim(repair_date))) > 5 -AND met_not_met not IN ('#VALUE!','#N/A') /*('N/A','#N/A','#VALUE!')*/) - -SELECT - *, - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM Defect -WHERE repair_date >= - date_add(cast(substr(cast(current_date as string), 1, 8)||'01' as date), -365) - - -""" -SQL_node1658765472050 = sparkSqlQuery( - glueContext, - query=SqlQuery33, - mapping={"parking_parking_ops_db_defects_mgt": AmazonS3_node1658997944648}, - transformation_ctx="SQL_node1658765472050", -) - -# Script generated for node Amazon S3 -AmazonS3_node1658765590649 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Defect_MET_FAIL/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=PARTITION_KEYS, - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="AmazonS3_node1658765590649", -) -AmazonS3_node1658765590649.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Defect_MET_FAIL", -) -AmazonS3_node1658765590649.setFormat("glueparquet") -AmazonS3_node1658765590649.writeFrame(SQL_node1658765472050) -job.commit() diff --git a/scripts/jobs/parking/parking_defect_met_fail_monthly_format.py b/scripts/jobs/parking/parking_defect_met_fail_monthly_format.py deleted file mode 100644 index f8aad3308..000000000 --- a/scripts/jobs/parking/parking_defect_met_fail_monthly_format.py +++ /dev/null @@ -1,281 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node parking_raw_zone - parking_parking_ops_db_defects_mgt -parking_raw_zoneparking_parking_ops_db_defects_mgt_node1658997944648 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate=" to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="parking_parking_ops_db_defects_mgt", - transformation_ctx="parking_raw_zoneparking_parking_ops_db_defects_mgt_node1658997944648", -) - -# Script generated for node SQL -SqlQuery0 = """ -/********************************************************************************* -Parking_Defect_MET_FAIL_Monthly_Format - -Temp SQL that formats the defect managment records For pivot report - -18/11/2022 - Create Query -21/11/2022 - add sub/grand total(s) and calculate KPI percentage -22/11/2022 - change % calc to make it a decimal -28/07/2023 - add N/A records. Change closed date to a date made up -of sign off month and year NOT repair_date -*********************************************************************************/ -With Defect_Basic as ( -SELECT - reference_no, - CAST(CASE - When reported_date like '%0222%'Then '2022-'|| - substr(reported_date, 7, 2)||'-'|| - substr(reported_date, 9, 2) - - When reported_date like '%/%'Then substr(reported_date, 7, 4)||'-'|| - substr(reported_date, 4, 2)||'-'|| - substr(reported_date, 1, 2) - ELSE substr(cast(reported_date as string),1, 10) - END as date) as reported_date, - CAST(CASE - When repair_date like '%/%'Then substr(repair_date, 7, 4)||'-'|| - substr(repair_date, 4, 2)||'-'|| - substr(repair_date, 1, 2) - ELSE substr(cast(repair_date as string),1, 10) - END as date) as repair_date, - CAST(CASE - When repair_date like '%/%'Then substr(repair_date, 7, 4)||'-'|| - substr(repair_date, 4, 2)||'-01' - ELSE substr(cast(repair_date as string),1, 8)||'01' - END as date) as Month_repair_date, - CASE - When category = 'post' Then 'Post' - Else category - End as category, - date_wo_sent, - expected_wo_completion_date, target_turn_around, met_not_met, - full_repair_category, - issue, engineer, - CASE - When sign_off_year != '1899' Then sign_off_month - ELSE CASE - When repair_date like '%/%'Then substr(repair_date, 4, 2) - ELSE substr(cast(repair_date as string),6, 2) - END - END as sign_off_month, - CASE - When sign_off_year != '1899' Then sign_off_year - ELSE CASE - When repair_date like '%/%'Then substr(repair_date, 7, 4) - ELSE substr(cast(repair_date as string),1, 4) - END - END as sign_off_year -FROM parking_parking_ops_db_defects_mgt -WHERE import_date = (Select MAX(import_date) - FROM parking_parking_ops_db_defects_mgt) -AND length(ltrim(rtrim(reported_date))) > 0 -AND met_not_met not IN ('#VALUE!','#N/A') -AND length(ltrim(rtrim(repair_date)))> 0), - -/*** Tom has informed me that I cannot use the Repair date, but use the sigh off month & year -USe these fields to recreate Month_repair_date ***/ -Defect as ( - SELECT - reference_no, reported_date, repair_date, - CAST( sign_off_year||'-'||sign_off_month||'-01' as date) as Month_repair_date, - category, date_wo_sent, expected_wo_completion_date, target_turn_around, met_not_met, - full_repair_category, issue, engineer, sign_off_month, sign_off_year - FROM Defect_Basic), - -/******************************************************************************** -Obtain the category totals -********************************************************************************/ -category_totals as ( -SELECT - Month_repair_date, category, met_not_met, count(*) as Total_Met_Fail -FROM Defect -WHERE repair_date >= - date_add(cast(substr(cast(current_date as string), 1, 8)||'01' as date),-365) AND - met_not_met = 'Fail' -GROUP BY Month_repair_date, category, met_not_met -UNION ALL -SELECT - Month_repair_date, category, met_not_met, count(*) as Total_Met_Fail -FROM Defect -WHERE repair_date >= - date_add(cast(substr(cast(current_date as string), 1, 8)||'01' as date),-365) AND - met_not_met = 'Met' -GROUP BY Month_repair_date, category, met_not_met -UNION ALL -SELECT - Month_repair_date, category, met_not_met, count(*) as Total_Met_Fail -FROM Defect -WHERE repair_date >= - date_add(cast(substr(cast(current_date as string), 1, 8)||'01' as date), -365) AND - met_not_met = 'N/A' -GROUP BY Month_repair_date, category, met_not_met), -/******************************************************************************** -Obtain the categories -********************************************************************************/ -Categories as ( - SELECT distinct - A.Month_repair_date, B.category - FROM Defect as A - CROSS JOIN (Select distinct category from Defect) as B - Where length(B.category) > 0), - -/******************************************************************************** -Obtain and format the totals -********************************************************************************/ -Category_Format as ( -SELECT - A.Month_repair_date, A.category, - CASE When B.Total_Met_Fail is not NULL Then B.Total_Met_Fail Else 0 END as Total_Fail, - CASE When C.Total_Met_Fail is not NULL Then C.Total_Met_Fail Else 0 END as Total_Met, - CASE When D.Total_Met_Fail is not NULL Then D.Total_Met_Fail Else 0 END as Total_NA -FROM Categories as A -LEFT JOIN category_totals as B ON A.Month_repair_date = B.Month_repair_date AND - A.category = B.category AND B.met_not_met = 'Fail' -LEFT JOIN category_totals as C ON A.Month_repair_date = C.Month_repair_date AND - A.category = C.category AND C.met_not_met = 'Met' -LEFT JOIN category_totals as D ON A.Month_repair_date = D.Month_repair_date AND - A.category = D.category AND D.met_not_met = 'N/A' -WHERE A.Month_repair_date >= - date_add(cast(substr(cast(current_date as string), 1, 8)||'01' as date),-365)), - -/******************************************************************************** -Calculate the KPI % -********************************************************************************/ -PDKPI as ( - SELECT - Month_repair_date, - 'Total_P&D_KPI_%' as Category, - (cast((SUM(cast(Total_Met as double)) / - SUM((Total_Met + Total_Fail))) as decimal(8,2))*100) as PD_KPI - FROM Category_Format -WHERE category = 'P&D' -GROUP BY Month_repair_date), - -SLPKPI as ( - SELECT - Month_repair_date, - 'Total_Signs_Lines_Posts_KPI_%' as Category, - (cast((SUM(cast(Total_Met as double)) / - SUM((Total_Met + Total_Fail))) as decimal(8,2))*100) as Signs_Lines_Posts_KPI - FROM Category_Format -WHERE category in ('Lines','Post','Signs') -GROUP BY Month_repair_date), -/******************************************************************************** -Obtain the category totals & Grand totals -********************************************************************************/ -catgory_totals as ( - SELECT - Month_repair_date, - category||' Total' as category, - SUM(total_met_fail) as Total - FROM category_totals - GROUP BY Month_repair_date, category), - -grand_totals as ( - SELECT - Month_repair_date, - SUM(total_met_fail) as Total - FROM category_totals - GROUP BY Month_repair_date), -/******************************************************************************** -Format the output -********************************************************************************/ -Format_Report as ( - SELECT - Month_repair_date, category, 'Fail' as Total_Met_Fail, Total_Fail as Total - FROM Category_Format - UNION ALL - SELECT - Month_repair_date, category, 'Met' as Total_Met_Fail, Total_Met as Total - FROM Category_Format - UNION ALL - SELECT - Month_repair_date, category, 'N/A' as Total_Met_Fail, Total_NA as Total - FROM Category_Format - UNION ALL - SELECT - Month_repair_date, Category, '' as Total_Met_Fail, PD_KPI - FROM PDKPI - UNION ALL - SELECT - Month_repair_date, Category, '' as Total_Met_Fail, Signs_Lines_Posts_KPI - FROM SLPKPI - UNION ALL - SELECT - Month_repair_date, category, '' as Total_Met_Fail, Total - FROM catgory_totals - UNION ALL - SELECT - Month_repair_date, 'Total Grand' as category, '' as Total_Met_Fail, Total - FROM grand_totals) -/******************************************************************************** -Ouput data -********************************************************************************/ -SELECT - *, - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM Format_Report -""" -SQL_node1658765472050 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_parking_ops_db_defects_mgt": parking_raw_zoneparking_parking_ops_db_defects_mgt_node1658997944648 - }, - transformation_ctx="SQL_node1658765472050", -) - -# Script generated for node Amazon S3 -AmazonS3_node1658765590649 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Defect_MET_FAIL_Monthly_Format/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="AmazonS3_node1658765590649", -) -AmazonS3_node1658765590649.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Defect_MET_FAIL_Monthly_Format", -) -AmazonS3_node1658765590649.setFormat("glueparquet") -AmazonS3_node1658765590649.writeFrame(SQL_node1658765472050) -job.commit() diff --git a/scripts/jobs/parking/parking_estate_waiting_list_live_permits_type_gds.py b/scripts/jobs/parking/parking_estate_waiting_list_live_permits_type_gds.py deleted file mode 100644 index dde60a7d3..000000000 --- a/scripts/jobs/parking/parking_estate_waiting_list_live_permits_type_gds.py +++ /dev/null @@ -1,141 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame - -from scripts.helpers.helpers import get_glue_env_var, create_pushdown_predicate -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node S3 bucket - raw_zone - liberator_permit_estate_wl -S3bucketraw_zoneliberator_permit_estate_wl_node1 = ( - glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_estate_wl", - transformation_ctx="S3bucketraw_zoneliberator_permit_estate_wl_node1", - push_down_predicate=create_pushdown_predicate("import_date", 7), - ) -) - -# Script generated for node Amazon S3 - parking_permit_denormalised_gds_street_llpg -AmazonS3parking_permit_denormalised_gds_street_llpg_node1640271444228 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_permit_denormalised_gds_street_llpg", - transformation_ctx="AmazonS3parking_permit_denormalised_gds_street_llpg_node1640271444228", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/*Distinct Estate Waiting List with all live permits by permit type - -query to calculate number of live permits by type -*/ -with lp_type as( - select -sum(Case when live_permit_flag = 1 and blue_badge_number !='' and permit_type like 'Estate Resident' and (amount like '0.00' or amount like '0.000' or amount like '') then 1 else 0 end) as flag_lp_estate_bb_zero -,sum(Case when live_permit_flag = 1 and blue_badge_number !='' and permit_type != 'Estate Resident' then 1 else 0 end) as flag_lp_bb_onstreet - -,sum(Case when live_permit_flag = 1 and permit_type like 'Estate Resident' then 1 else 0 end) as flag_lp_estate -,sum(Case when live_permit_flag = 1 and permit_type like 'Business' then 1 else 0 end) as flag_lp_business -,sum(Case when live_permit_flag = 1 and permit_type like 'Doctor' then 1 else 0 end) as flag_lp_doctor -,sum(Case when live_permit_flag = 1 and permit_type like 'Leisure centre permit' then 1 else 0 end) as flag_lp_leisure -,sum(Case when live_permit_flag = 1 and permit_type like 'Health and Social Care' then 1 else 0 end) as flag_lp_hsc -,sum(Case when live_permit_flag = 1 and permit_type like 'Residents' then 1 else 0 end) as flag_lp_residents -,sum(Case when live_permit_flag = 1 and permit_type like 'Dispensation' then 1 else 0 end) as flag_lp_dispensation -,sum(Case when live_permit_flag = 1 and permit_type not in ('Estate Resident','All Zone','Companion Badge','Dispensation','Residents','Health and Social Care','Leisure centre permit','Doctor','Business') then 1 else 0 end) as flag_lp_other -,sum(Case when live_permit_flag = 1 and permit_type like 'All Zone' then 1 else 0 end) as flag_lp_all_zone -,sum(Case when live_permit_flag = 1 and permit_type like 'Companion Badge' then 1 else 0 end) as flag_lp_companion_badge -, sum(live_permit_flag) as live_permit -,uprn, sr_usrn, import_date from parking_permit_denormalised_gds_street_llpg where import_date = (select max(import_date) from parking_permit_denormalised_gds_street_llpg) - -Group by uprn, sr_usrn, import_date - ) - -/*query to identify and filter max record_created date for each waiting list id*/ -, wlmxrec as (SELECT id, concat(substr(Cast(application_date as varchar(10)),1, 7), '-01'), application_date, max(record_created) as mxrec, import_date FROM liberator_permit_estate_wl - -where import_date = (SELECT max(import_date) FROM liberator_permit_estate_wl) - -group by id, application_date, concat(substr(Cast(application_date as varchar(10)),1, 7), '-01'), import_date - ) - -SELECT distinct concat(substr(Cast(liberator_permit_estate_wl.application_date as varchar(10)),1, 7), '-01') as MonthYear -,lp_type.live_permit -,lp_type.flag_lp_estate_bb_zero, lp_type.flag_lp_bb_onstreet, lp_type.flag_lp_estate, lp_type.flag_lp_business, lp_type.flag_lp_doctor, lp_type.flag_lp_leisure, lp_type.flag_lp_hsc, lp_type.flag_lp_residents, lp_type.flag_lp_dispensation, lp_type.flag_lp_other, lp_type.flag_lp_all_zone, lp_type.flag_lp_companion_badge -,case when z_code !='' and estate_name != '' then concat(z_code,' - ', estate_name) -when z_code !='' and estate_name = '' then concat(z_code) -when z_code ='' and estate_name != '' then concat(estate_name) -else 'NONE' -end as zone_name -,case -when address_line_1 ='' then concat(cast(liberator_permit_estate_wl.id as string),' - ',substr(cast(liberator_permit_estate_wl.application_date as string),1,19),' - ',email) -when address_line_2 ='' then concat(cast(liberator_permit_estate_wl.id as string),' - ',substr(cast(liberator_permit_estate_wl.application_date as string),1,19),' - ',address_line_1,', ',postcode,' - ',email) -when address_line_3 ='' then concat(cast(liberator_permit_estate_wl.id as string),' - ',substr(cast(liberator_permit_estate_wl.application_date as string),1,19),' - ',address_line_1,', ',address_line_2,', ',postcode,' - ',email) -else concat(cast(liberator_permit_estate_wl.id as string),' - ',substr(cast(liberator_permit_estate_wl.application_date as string),1,19),' - ',address_line_1,', ',address_line_2,', ',address_line_3,', ',postcode,' - ',email) -end as wl_summary - - -,case -when address_line_2 ='' then concat(cast(liberator_permit_estate_wl.id as string),' - ',address_line_1,', ',postcode) -when address_line_3 ='' then concat(cast(liberator_permit_estate_wl.id as string),' - ',address_line_1,', ',address_line_2,', ',postcode) -else concat(cast(liberator_permit_estate_wl.id as string),' - ',address_line_1,', ',address_line_2,', ',address_line_3,', ',postcode) -end as wl_id_full_address -,case -when address_line_2 ='' then concat(address_line_1,', ',postcode) -when address_line_3 ='' then concat(address_line_1,', ',address_line_2,', ',postcode) -else concat(address_line_1,', ',address_line_2,', ',address_line_3,', ',postcode) -end as full_address -,liberator_permit_estate_wl.* FROM liberator_permit_estate_wl - -left join lp_type on lp_type.uprn = liberator_permit_estate_wl.uprn and lp_type.import_date = liberator_permit_estate_wl.import_date - -left join wlmxrec on wlmxrec.id = liberator_permit_estate_wl.id and wlmxrec.mxrec = liberator_permit_estate_wl.record_created and wlmxrec.import_date = liberator_permit_estate_wl.import_date - - -where liberator_permit_estate_wl.import_date = (SELECT max(liberator_permit_estate_wl.import_date) FROM liberator_permit_estate_wl) and cast(wlmxrec.mxrec as string) !='' - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_permit_denormalised_gds_street_llpg": AmazonS3parking_permit_denormalised_gds_street_llpg_node1640271444228, - "liberator_permit_estate_wl": S3bucketraw_zoneliberator_permit_estate_wl_node1, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" + environment + "-refined-zone/parking/liberator/parking_estate_waiting_list_live_permits_type_gds/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_estate_waiting_list_live_permits_type_gds", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_gds_permit_change_comparison.py b/scripts/jobs/parking/parking_gds_permit_change_comparison.py deleted file mode 100644 index abfb0812d..000000000 --- a/scripts/jobs/parking/parking_gds_permit_change_comparison.py +++ /dev/null @@ -1,107 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame - -from scripts.helpers.helpers import get_glue_env_var, create_pushdown_predicate -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -spark.conf.set("spark.sql.broadcastTimeout", 7200) - -# Script generated for node S3 - liberator_refined - parking_permit_denormalised_gds_street_llpg -S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_permit_denormalised_gds_street_llpg", - transformation_ctx="S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/*Permit changes comparison - compare changes in permits from the parking_permit_denormalised_gds_street_llpg table to be used in Google Data Studio -Compares latest import to previous import in table -01/03/2022 - Created glue job - -*/ - -with previous as ( -SELECT distinct concat(permit_reference, cast(start_date as string)) as p_unique ,parking_permit_denormalised_gds_street_llpg.import_date as p_import_date ,vrm as p_latest_vrm ,permit_reference as p_permit_reference ,application_date as p_application_date ,blue_badge_number as p_blue_badge_number ,blue_badge_expiry as p_blue_badge_expiry ,start_date as p_start_date ,end_date as p_end_date ,permit_type as p_permit_type ,cpz as p_cpz ,cpz_name as p_cpz_name ,latest_permit_status as p_latest_permit_status ,make as p_make ,model as p_model ,fuel as p_fuel ,engine_capactiy as p_engine_capactiy ,co2_emission as p_co2_emission ,foreign as p_foreign ,lpg_conversion as p_lpg_conversion ,vrm_record_created as p_vrm_record_created ,fin_year_flag as p_fin_year_flag ,fin_year as p_fin_year ,sr_usrn as p_USRN ,live_permit_flag as p_live_permit_flag - FROM parking_permit_denormalised_gds_street_llpg where parking_permit_denormalised_gds_street_llpg.import_date = (SELECT parking_permit_denormalised_gds_street_llpg.import_date - FROM parking_permit_denormalised_gds_street_llpg - WHERE parking_permit_denormalised_gds_street_llpg.import_date NOT IN (SELECT MAX(parking_permit_denormalised_gds_street_llpg.import_date) FROM parking_permit_denormalised_gds_street_llpg ) -ORDER BY parking_permit_denormalised_gds_street_llpg.import_date DESC LIMIT 1) -) -, current as ( -SELECT distinct concat(permit_reference, cast(start_date as string)) as c_unique ,parking_permit_denormalised_gds_street_llpg.import_date as c_import_date ,vrm as c_latest_vrm ,permit_reference as c_permit_reference ,application_date as c_application_date ,blue_badge_number as c_blue_badge_number ,blue_badge_expiry as c_blue_badge_expiry ,start_date as c_start_date ,end_date as c_end_date ,permit_type as c_permit_type ,cpz as c_cpz ,cpz_name as c_cpz_name ,latest_permit_status as c_latest_permit_status ,make as c_make ,model as c_model ,fuel as c_fuel ,engine_capactiy as c_engine_capactiy ,co2_emission as c_co2_emission ,foreign as c_foreign ,lpg_conversion as c_lpg_conversion ,vrm_record_created as c_vrm_record_created ,fin_year_flag as c_fin_year_flag ,fin_year as c_fin_year ,sr_usrn as c_USRN ,live_permit_flag as c_live_permit_flag FROM parking_permit_denormalised_gds_street_llpg where parking_permit_denormalised_gds_street_llpg.import_date = (SELECT max(parking_permit_denormalised_gds_street_llpg.import_date) FROM parking_permit_denormalised_gds_street_llpg) -) -, permit_num as ( -SELECT permit_reference as pn_permit_reference ,count(*) as num_permit ,min(application_date) as min_application_date ,max(application_date) as max_application_date ,min(start_date) as min_start_date ,max(start_date) as max_start_date ,min(end_date) as min_end_date ,max(end_date) as max_end_date FROM parking_permit_denormalised_gds_street_llpg where import_date = (SELECT max(import_date) FROM parking_permit_denormalised_gds_street_llpg) group by permit_reference order by permit_reference -) -select -case when c_latest_permit_status = p_latest_permit_status then 0 else 1 end as diff_permit_status -,case when previous.p_unique is null then 1 else 0 end as new_permit_start -,case when previous.p_unique is null and permit_num.num_permit < 2 then 1 else 0 end as new_permit -,case when c_latest_vrm = p_latest_vrm then 0 else 1 end as diff_vrm -,case when p_start_date = c_start_date then 0 else 1 end as diff_start_date -,case when p_end_date = c_end_date then 0 else 1 end as diff_end_date -,case when p_vrm_record_created = c_vrm_record_created then 0 else 1 end as diff_vrm_rec_create -,case when p_live_permit_flag = c_live_permit_flag then 0 else 1 end as diff_live_permit_flag -,case when c_latest_permit_status in('Approved','Renewed','Created','ORDER_APPROVED','PENDING_VRM_CHANGE','RENEW_EVID','PENDING_ADDR_CHANGE') and c_live_permit_flag = 1 then 1 else 0 end as live_flag - -,case when ((case when c_latest_permit_status = p_latest_permit_status then 0 else 1 end)=1 or (case when c_latest_vrm = p_latest_vrm then 0 else 1 end)=1 or (case when p_start_date = c_start_date then 0 else 1 end)=1 or (case when p_end_date = c_end_date then 0 else 1 end)=1 or (case when p_vrm_record_created = c_vrm_record_created then 0 else 1 end)=1 or (case when p_live_permit_flag = c_live_permit_flag then 0 else 1 end)=1) and c_latest_permit_status in('Approved','Renewed','Created','ORDER_APPROVED','PENDING_VRM_CHANGE','RENEW_EVID','PENDING_ADDR_CHANGE') and c_live_permit_flag = 1 -then 1 else 0 end as live_permit_change -,case when (case when c_latest_permit_status = p_latest_permit_status then 0 else 1 end)=1 or (case when c_latest_vrm = p_latest_vrm then 0 else 1 end)=1 or (case when p_start_date = c_start_date then 0 else 1 end)=1 or (case when p_end_date = c_end_date then 0 else 1 end)=1 or (case when p_vrm_record_created = c_vrm_record_created then 0 else 1 end)=1 or (case when p_live_permit_flag = c_live_permit_flag then 0 else 1 end)=1 -then 1 else 0 end as permit_change -,* - /*** Control Dates ***/ - ,substr(Cast(current_date as varchar(10)),1, 4) as import_year, - substr(Cast(current_date as varchar(10)),6, 2) as import_month, - substr(Cast(current_date as varchar(10)),9, 4) as import_day, - Cast(current_date as varchar(10)) as import_date - -from current -left join previous on current.c_unique = previous.p_unique -left join permit_num on current.c_permit_reference = permit_num.pn_permit_reference -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_permit_denormalised_gds_street_llpg": S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1 - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" + environment + "-refined-zone/parking/liberator/parking_gds_permit_change_comparison/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_gds_permit_change_comparison", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_gds_permit_change_comparison_specific_dates.py b/scripts/jobs/parking/parking_gds_permit_change_comparison_specific_dates.py deleted file mode 100644 index bfdd18caf..000000000 --- a/scripts/jobs/parking/parking_gds_permit_change_comparison_specific_dates.py +++ /dev/null @@ -1,103 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node S3 - liberator_refined - parking_permit_denormalised_gds_street_llpg -S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - table_name="parking_permit_denormalised_gds_street_llpg", - transformation_ctx="S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/*Permit changes comparison - compare changes in permits from the parking_permit_denormalised_gds_street_llpg table to be used in Google Data Studio -Compares latest import to previous import in table -01/03/2022 - Created glue job - -*/ - -with previous as ( -SELECT distinct concat(permit_reference, cast(start_date as string)) as p_unique ,parking_permit_denormalised_gds_street_llpg.import_date as p_import_date ,vrm as p_latest_vrm ,permit_reference as p_permit_reference ,application_date as p_application_date ,blue_badge_number as p_blue_badge_number ,blue_badge_expiry as p_blue_badge_expiry ,start_date as p_start_date ,end_date as p_end_date ,permit_type as p_permit_type ,cpz as p_cpz ,cpz_name as p_cpz_name ,latest_permit_status as p_latest_permit_status ,make as p_make ,model as p_model ,fuel as p_fuel ,engine_capactiy as p_engine_capactiy ,co2_emission as p_co2_emission ,foreign as p_foreign ,lpg_conversion as p_lpg_conversion ,vrm_record_created as p_vrm_record_created ,fin_year_flag as p_fin_year_flag ,fin_year as p_fin_year ,sr_usrn as p_USRN ,live_permit_flag as p_live_permit_flag - FROM parking_permit_denormalised_gds_street_llpg - where parking_permit_denormalised_gds_street_llpg.import_date = '20220629' -) -, current as ( -SELECT distinct concat(permit_reference, cast(start_date as string)) as c_unique ,parking_permit_denormalised_gds_street_llpg.import_date as c_import_date ,vrm as c_latest_vrm ,permit_reference as c_permit_reference ,application_date as c_application_date ,blue_badge_number as c_blue_badge_number ,blue_badge_expiry as c_blue_badge_expiry ,start_date as c_start_date ,end_date as c_end_date ,permit_type as c_permit_type ,cpz as c_cpz ,cpz_name as c_cpz_name ,latest_permit_status as c_latest_permit_status ,make as c_make ,model as c_model ,fuel as c_fuel ,engine_capactiy as c_engine_capactiy ,co2_emission as c_co2_emission ,foreign as c_foreign ,lpg_conversion as c_lpg_conversion ,vrm_record_created as c_vrm_record_created ,fin_year_flag as c_fin_year_flag ,fin_year as c_fin_year ,sr_usrn as c_USRN ,live_permit_flag as c_live_permit_flag FROM parking_permit_denormalised_gds_street_llpg -where parking_permit_denormalised_gds_street_llpg.import_date = '20220714' -) -, permit_num as ( -SELECT permit_reference as pn_permit_reference ,count(*) as num_permit ,min(application_date) as min_application_date ,max(application_date) as max_application_date ,min(start_date) as min_start_date ,max(start_date) as max_start_date ,min(end_date) as min_end_date ,max(end_date) as max_end_date FROM parking_permit_denormalised_gds_street_llpg where import_date = (SELECT max(import_date) FROM parking_permit_denormalised_gds_street_llpg) group by permit_reference order by permit_reference -) -select -case when c_latest_permit_status = p_latest_permit_status then 0 else 1 end as diff_permit_status -,case when previous.p_unique is null then 1 else 0 end as new_permit_start -,case when previous.p_unique is null and permit_num.num_permit < 2 then 1 else 0 end as new_permit -,case when c_latest_vrm = p_latest_vrm then 0 else 1 end as diff_vrm -,case when p_start_date = c_start_date then 0 else 1 end as diff_start_date -,case when p_end_date = c_end_date then 0 else 1 end as diff_end_date -,case when p_vrm_record_created = c_vrm_record_created then 0 else 1 end as diff_vrm_rec_create -,case when p_live_permit_flag = c_live_permit_flag then 0 else 1 end as diff_live_permit_flag -,case when c_latest_permit_status in('Approved','Renewed','Created','ORDER_APPROVED','PENDING_VRM_CHANGE','RENEW_EVID','PENDING_ADDR_CHANGE') and c_live_permit_flag = 1 then 1 else 0 end as live_flag - -,case when ((case when c_latest_permit_status = p_latest_permit_status then 0 else 1 end)=1 or (case when c_latest_vrm = p_latest_vrm then 0 else 1 end)=1 or (case when p_start_date = c_start_date then 0 else 1 end)=1 or (case when p_end_date = c_end_date then 0 else 1 end)=1 or (case when p_vrm_record_created = c_vrm_record_created then 0 else 1 end)=1 or (case when p_live_permit_flag = c_live_permit_flag then 0 else 1 end)=1) and c_latest_permit_status in('Approved','Renewed','Created','ORDER_APPROVED','PENDING_VRM_CHANGE','RENEW_EVID','PENDING_ADDR_CHANGE') and c_live_permit_flag = 1 -then 1 else 0 end as live_permit_change -,case when (case when c_latest_permit_status = p_latest_permit_status then 0 else 1 end)=1 or (case when c_latest_vrm = p_latest_vrm then 0 else 1 end)=1 or (case when p_start_date = c_start_date then 0 else 1 end)=1 or (case when p_end_date = c_end_date then 0 else 1 end)=1 or (case when p_vrm_record_created = c_vrm_record_created then 0 else 1 end)=1 or (case when p_live_permit_flag = c_live_permit_flag then 0 else 1 end)=1 -then 1 else 0 end as permit_change -,* - /*** Control Dates ***/ - ,substr(Cast(current_date as varchar(10)),1, 4) as import_year, - substr(Cast(current_date as varchar(10)),6, 2) as import_month, - substr(Cast(current_date as varchar(10)),9, 4) as import_day, - Cast(current_date as varchar(10)) as import_date - -from current -left join previous on current.c_unique = previous.p_unique -left join permit_num on current.c_permit_reference = permit_num.pn_permit_reference -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_permit_denormalised_gds_street_llpg": S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1 - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_gds_permit_change_comparison/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_gds_permit_change_comparison", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_market_licence_totals.py b/scripts/jobs/parking/parking_market_licence_totals.py deleted file mode 100644 index bfe8886d0..000000000 --- a/scripts/jobs/parking/parking_market_licence_totals.py +++ /dev/null @@ -1,210 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1636704737623 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="calendar", - transformation_ctx="AmazonS3_node1636704737623", -) - -# Script generated for node Amazon S3 -AmazonS3_node1638358321513 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_licence_full", - transformation_ctx="AmazonS3_node1638358321513", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************************* -Parking_Market_Licence_Totals - -The SQL builds the total number of Market trader licences extant on the 28th of each month - -02/12/2021 - Create SQL. -*************************************************************************************************************************/ -/*** Collect the 28th day of each month ***/ -With Calendar_Data as ( - SELECT - date as Calendar_date, workingday, dow, holiday, - - ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date, import_date DESC) row_num - - FROM calendar - WHERE date like '%28%'), -/*** Format the calendar date ***/ -CalendarFormat as ( - SELECT - Calendar_date, - CAST(CASE - When Calendar_date like '%/%'Then substr(Calendar_date, 7, 4)||'-'|| - substr(Calendar_date, 4, 2)||'-'||substr(Calendar_date, 1, 2) - ELSE substr(Calendar_date, 1, 10) - END as date) as Format_date - FROM Calendar_Data - WHERE row_num = 1), - -/*** rework the Requested date as some oof the records are PANTS! ***/ -Format_Requested_Date as ( - SELECT - licence_ref, - - /*** rework the Requested date as some oof the records are PANTS! ***/ - CAST(CASE - When requested_start_date is NULL Then substr(cast(application_date as string), 1,10) - When length(requested_start_date) < 10 Then substr(cast(application_date as string), 1,10) - When requested_start_date like '%.%' Then substr(cast(application_date as string), 1,10) - When requested_start_date like '%/%'Then - CASE - When substr(requested_start_date, 7,4) = '1900' Then substr(cast(application_date as string), 1,10) - - When cast(substr(requested_start_date, 4,2) as int) > 12 Then substr(cast(application_date as string), 1,10) - - ELSE substr(requested_start_date, 7,4)||'-'|| - substr(requested_start_date, 4,2)||'-'|| - substr(requested_start_date, 1,2) - END - ELSE substr(requested_start_date, 1, 10) - END as date) as requested_start_date - - FROM liberator_licence_licence_FULL - WHERE import_Date = (Select MAX(import_date) from liberator_licence_licence_full) and licence_ref like 'MAR%'), - -/*** Get the Market data ***/ -Market_Before as ( - SELECT - A.licence_ref, licence_type, licence_address, application_date, - - CASE - When lower(licence_type) = lower('MARKET-newtemp') Then 'Temp' - When lower(licence_type) = lower('MARKET-renewperm') Then 'Perm' - When lower(licence_type) = lower('MARKET-renewtemp') Then 'Temp' - ELSE 'Temp' - END as Licence_Type_PermTemp, - - B.requested_start_date, - - /*** rework the Actual date as some oof the records are PANTS! ***/ - - CAST(CASE - When actual_start_date is NULL Then substr(cast(B.requested_start_date as string), 1,10) - When length(actual_start_date) < 10 Then substr(cast(B.requested_start_date as string), 1,10) - When actual_start_date like '%.%' Then substr(cast(B.requested_start_date as string), 1,10) - When actual_start_date like '%/%'Then - CASE - When substr(actual_start_date, 7,4) = '1900' Then substr(cast(B.requested_start_date as string), 1,10) - - When cast(substr(actual_start_date, 4,2) as int) > 12 Then substr(cast(B.requested_start_date as string), 1,10) - - ELSE substr(actual_start_date, 7,4)||'-'|| - substr(actual_start_date, 4,2)||'-'|| - substr(actual_start_date, 1,2) - END - ELSE substr(actual_start_date, 1, 10) - END as date) as actual_start_date - - FROM liberator_licence_licence_FULL as A - LEFT JOIN Format_Requested_Date as B ON A.licence_ref = B.licence_ref - WHERE import_Date = (Select MAX(import_date) from liberator_licence_licence_full) and A.licence_ref like 'MAR%'), - -Market as ( - SELECT - *, - CAST(CASE - When Licence_Type_PermTemp = 'Temp' Then date_add(actual_start_date, 183) - When Licence_Type_PermTemp = 'Perm' Then date_add(actual_start_date, 365) - END as date) as End_Date, - - cast(substr(cast(actual_start_date as string), 1, 8)||'01' as date) as MonthYear - - from Market_Before), - -/** total the number of 'temp' licences active on the 28th of each month **/ -Temp_Market as ( - Select - Format_date, - count(*) as No_Temp_Licences - - From CalendarFormat as A, Market as B - Where A.Format_date between B.actual_start_date and B.End_Date AND Licence_Type_PermTemp = 'Temp' - group by Format_date), - -/** total the number of 'temp' licences active on the 28th of each month **/ -Perm_Market as ( - Select - Format_date, - count(*) as No_Perm_Licences - - From CalendarFormat as A, Market as B - Where A.Format_date between B.actual_start_date and B.End_Date AND Licence_Type_PermTemp = 'Perm' - group by Format_date) - -/*** Output the results ***/ -SELECT - A.Format_date, - No_Temp_Licences, - No_Perm_Licences, - - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM Temp_Market as A -LEFT JOIN Perm_Market as B ON A.Format_date = B.Format_date -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "calendar": AmazonS3_node1636704737623, - "liberator_licence_licence_FULL": AmazonS3_node1638358321513, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Market_Licence_Totals/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Market_Licence_Totals", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_markets_denormalisation.py b/scripts/jobs/parking/parking_markets_denormalisation.py deleted file mode 100644 index 80700c7ab..000000000 --- a/scripts/jobs/parking/parking_markets_denormalisation.py +++ /dev/null @@ -1,268 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1625732038443 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_licence_full", - transformation_ctx="AmazonS3_node1625732038443", -) - -# Script generated for node Amazon S3 -AmazonS3_node1631704526786 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_renewal", - transformation_ctx="AmazonS3_node1631704526786", -) - -# Script generated for node Amazon S3 -AmazonS3_node1637693702237 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_status", - transformation_ctx="AmazonS3_node1637693702237", -) - -# Script generated for node Amazon S3 -AmazonS3_node1637693786853 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_inspections", - transformation_ctx="AmazonS3_node1637693786853", -) - -# Script generated for node Amazon S3 -AmazonS3_node1637693831333 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_llpg", - transformation_ctx="AmazonS3_node1637693831333", -) - -# Script generated for node Amazon S3 -AmazonS3_node1637693886525 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_payments", - transformation_ctx="AmazonS3_node1637693886525", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/************************************************************************************************************** -Parking_Markets_Denormalisation - -This query de-normalises the markets & shopfront data - -23/11/2021 - create the query -****************************************************************************************************************/ - -/************************************************************************************************************************** -Get the Licence (Market & Shopfront) data -**************************************************************************************************************************/ -With Latest_Business_Before as ( - SELECT - unique_id, licence_ref, licence_type, licence_address, licence_uprn, red_route, application_date, created_by, - requested_start_date, actual_start_date, - /*** Get a possible End Date ***/ - CASE - When licence_type IN ('SF-newperm','MARKET-renewperm') Then cast('9999-12-31' as date) - When actual_start_date != '' Then - CASE - When actual_start_date like '%/%'Then - date_add(CAST(substr(actual_start_date, 7,4)||'-'|| - substr(actual_start_date, 4,2)||'-'|| - substr(actual_start_date, 1,2) as date), 364) - - ELSE cast(actual_start_date as date) END - When requested_start_date != '' Then - CASE - When requested_start_date like '%/%'Then - date_add(CAST(substr(requested_start_date, 7,4)||'-'|| - substr(requested_start_date, 4,2)||'-'|| - substr(requested_start_date, 1,2) as date), 364) - ELSE cast(requested_start_date as date) END - - END as EndDate, - - application_type, requested_width_1, requested_depth_1, requested_width_2, - requested_depth_2, requested_width_3, requested_depth_3, actual_width_1, actual_depth_1, actual_width_2, actual_depth_2, - actual_width_3, actual_depth_3, requested_area, actual_area, applicant_name, applicant_address, applicant_uprn, - business_id, business_name, business_address, business_uprn, business_type, business_registration_number, - approved_by, rejected_by, - /*** Create a Unique_ID by stripping off first characters ***/ - CASE - When Licence_ref like 'SF%' Then Replace(Licence_ref, 'SF') - When Licence_Ref like 'MAR%' Then Replace (licence_ref, 'MAR') - END as Unique_Ref, - - /*** Remove carriage return like feed ***/ - REPLACE(REPLACE(approved_notes, '\r',''), '\n','') as approved_notes, - REPLACE(REPLACE(reason_for_rejection, '\r',''), '\n','') as reason_for_rejection, - REPLACE(REPLACE(rejection_notes, '\r',''), '\n','') as rejection_notes - - FROM liberator_licence_licence_full - WHERE import_Date = (Select MAX(import_date) from liberator_licence_licence_full) - and business_name not in ('IvieSCSC Ltd','IvieSCSC') - order by Business_ID, requested_start_date), - -/*** Now index the END date to find the latest record for a business ***/ -Latest_Business as ( - SELECT - *, - /*** Create a Row Number to identify the latest record ***/ - ROW_NUMBER() OVER ( PARTITION BY Business_ID - ORDER BY Business_ID, EndDate DESC) row_num - FROM Latest_Business_Before), - -/************************************************************************************************************************** -Get the renewal data -**************************************************************************************************************************/ -Renewal_ref as ( - SELECT - renewal_ref,original_ref, renewal_type, renewal_date - FROM liberator_licence_renewal - WHERE import_Date = (Select MAX(import_date) from liberator_licence_licence_full)), - -/************************************************************************************************************************** -Get the renewal data -**************************************************************************************************************************/ -Status_Data as ( - SELECT - licence_ref, status, status_start_date, status_end_date, - - /*** Create a Row Number to identify the latest record ***/ - ROW_NUMBER() OVER ( PARTITION BY licence_ref - ORDER BY licence_ref, status_start_date DESC) row_num - FROM liberator_licence_status - WHERE import_Date = (Select MAX(import_date) from liberator_licence_status)), - -/************************************************************************************************************************** -Get the inspections data -**************************************************************************************************************************/ -Inspections as ( - SELECT - licence_ref, inspection_date, - REPLACE(REPLACE(inspection_reason, '\r',''), '\n','') as inspection_reason, - REPLACE(REPLACE(internal_notes, '\r',''), '\n','') as INSP_internal_notes, - REPLACE(REPLACE(external_notes, '\r',''), '\n','') as INSP_external_notes, - REPLACE(REPLACE(recommendation, '\r',''), '\n','') as INSP_recommendation, - area_width_1 as INSP_area_width_1, area_depth_1 as INSP_area_depth_1, - area_width_2 as INSP_area_width_2, area_depth_2 as INSP_area_depth_2, - area_width_3 as INSP_area_width_3, area_depth_3 as INSP_area_depth_3, - total_area as INSP_total_area, drawing as INSP_drawing, - - /*** Create a Row Number to identify the latest record ***/ - ROW_NUMBER() OVER ( PARTITION BY licence_ref - ORDER BY licence_ref, inspection_date DESC) row_num - - FROM liberator_licence_inspections - WHERE import_Date = (Select MAX(import_date) from liberator_licence_inspections)), - -/************************************************************************************************************************** -Get the LLPG data -**************************************************************************************************************************/ -LLPG as ( - SELECT * - FROM liberator_permit_llpg - WHERE import_Date = (Select MAX(import_date) from liberator_licence_inspections)), - -/************************************************************************************************************************** -Get the payments -**************************************************************************************************************************/ -Payments as ( - SELECT - licence_ref, replace(licence_ref, 'LICO') as Unique_Ref, - CASE - When Payment_Type = 'REFUND' Then -(cast(amount as double)) - Else cast(amount as double) - END as Amount - - FROM liberator_licence_payments - WHERE import_date = (Select MAX(import_date) from liberator_licence_payments)), - -Total_Payments as ( - SELECT - Licence_Ref, Unique_ref, round(Sum(Amount),2) as Total_Payment - FROM Payments - GROUP BY Licence_Ref, Unique_ref) - -/************************************************************************************************************************** -Output the data -**************************************************************************************************************************/ -SELECT - A.*, - renewal_type, renewal_date, original_ref, - status, status_start_date, status_end_date - inspection_reason, INSP_internal_notes, INSP_external_notes, INSP_recommendation, - INSP_area_width_1,INSP_area_depth_1, - INSP_area_width_2,INSP_area_depth_2, - INSP_area_width_3,INSP_area_depth_3, - INSP_total_area, INSP_total_area, bplu_class, Total_Payment, - - current_timestamp() as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM Latest_Business as A -LEFT JOIN Renewal_ref as B ON A.licence_ref = B.renewal_ref -LEFT JOIN Status_Data as C ON A.licence_ref = C.licence_ref AND C.row_num = 1 -LEFT JOIN Inspections as D ON A.licence_ref = D.licence_ref AND D.row_num = 1 -LEFT JOIN LLPG as E ON A.licence_uprn = cast(E.uprn as string) -LEFT JOIN Total_Payments as F ON A.Unique_Ref = F.Unique_ref -WHERE A.row_num = 1 -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "liberator_licence_status": AmazonS3_node1637693702237, - "liberator_licence_licence_full": AmazonS3_node1625732038443, - "liberator_licence_inspections": AmazonS3_node1637693786853, - "liberator_licence_renewal": AmazonS3_node1631704526786, - "liberator_permit_llpg": AmazonS3_node1637693831333, - "liberator_licence_payments": AmazonS3_node1637693886525, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Markets_Denormalisation/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Markets_Denormalisation", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_match_pcn_permit_vrm_with_address_match_llpg_nlpg_postcodes.py b/scripts/jobs/parking/parking_match_pcn_permit_vrm_with_address_match_llpg_nlpg_postcodes.py deleted file mode 100644 index 24b50c9df..000000000 --- a/scripts/jobs/parking/parking_match_pcn_permit_vrm_with_address_match_llpg_nlpg_postcodes.py +++ /dev/null @@ -1,837 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS -from scripts.helpers.helpers import create_pushdown_predicate_for_max_date_partition_value - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 - parking_permit_denormalised_gds_street_llpg -AmazonS3parking_permit_denormalised_gds_street_llpg_node1650475071787 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - table_name="parking_permit_denormalised_gds_street_llpg", - transformation_ctx="AmazonS3parking_permit_denormalised_gds_street_llpg_node1650475071787", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value("dataplatform-"+environment+"-liberator-refined-zone", "parking_permit_denormalised_gds_street_llpg", "import_date") -) - -# Script generated for node S3 bucket - unrestricted_address_api_dbo_national_address -S3bucketunrestricted_address_api_dbo_national_address_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-raw-zone-unrestricted-address-api", - table_name="unrestricted_address_api_dbo_national_address", - transformation_ctx="S3bucketunrestricted_address_api_dbo_national_address_node1", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value("dataplatform-"+environment+"-raw-zone-unrestricted-address-api", "unrestricted_address_api_dbo_national_address", "import_date") -) - -# Script generated for node Amazon S3 - pcnfoidetails_pcn_foi_full -AmazonS3pcnfoidetails_pcn_foi_full_node1650477300290 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - table_name="pcnfoidetails_pcn_foi_full", - transformation_ctx="AmazonS3pcnfoidetails_pcn_foi_full_node1650477300290", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value("dataplatform-"+environment+"-liberator-refined-zone", "pcnfoidetails_pcn_foi_full", "import_date") -) - -# Script generated for node Amazon S3 - unrestricted_address_api_dbo_hackney_address -AmazonS3unrestricted_address_api_dbo_hackney_address_node1650475065696 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-raw-zone-unrestricted-address-api", - table_name="unrestricted_address_api_dbo_hackney_address", - transformation_ctx="AmazonS3unrestricted_address_api_dbo_hackney_address_node1650475065696", - push_down_predicate=create_pushdown_predicate_for_max_date_partition_value("dataplatform-"+environment+"-raw-zone-unrestricted-address-api", "unrestricted_address_api_dbo_hackney_address", "import_date") -) - -# Script generated for node ApplyMapping -SqlQuery13 = """ -/*PCNs VRM match to Permits VRM with match to LLPG and NLPG for Registered and Current addresses Post Code last 13 months of pcn issue date -tables: -unrestricted_address_api_dbo_national_address -unrestricted_address_api_dbo_hackney_address -"dataplatform-"+environment+"-liberator-refined-zone"."parking_permit_denormalised_gds_street_llpg" -pcnfoidetails_pcn_foi_full - -20/04/2022 - created glue job -27/04/2022 - updated to remove duplicate post code from llpg and nlpg lookup tables and pcn addresses with commas removed -08/12/2022 - simpler postcode extracter - regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') -09/12/2022 - updated postcode extracter to case statement to flag those with no addresses -19/12/2022 - changed no space from regexp_replace to replace for postcode joins and matching -24/02/2023 - optimised query to alias NLPG and LLPG in joins and add additional fields -*/ - -With nlpg_pc_summ as (/*Summary Post Codes National Gazetteer - NLPG*/ - select row_number() over (partition by postcode_nospace order by postcode_nospace desc ) as nlpg_rn ,gazetteer as nlpg_gazetteer /*,locality as nlpg_locality_reg ,ward as nlpg_ward_reg ,town as nlpg_town_reg*/ ,postcode as nlpg_postcode ,postcode_nospace as nlpg_postcode_nospace - ,case - when upper(town) like 'LONDON%' and upper(locality) ='HACKNEY' and ward !='' then concat(ward, ' - ',locality, ' - ',town) - when upper(town) like 'LONDON%' and locality !='' then concat(locality, ' - ',town) - when upper(town) like 'LONDON%' and locality ='' and ward !='' then concat(ward, ' - ',town) - else town end as nlpg_area - /*,count(*) as nlpg_num_records_reg*/ FROM unrestricted_address_api_dbo_national_address where lpi_logical_status like 'Approved Preferred' - group by gazetteer, locality ,ward ,town ,postcode ,postcode_nospace -) -, llpg_pc_summ as (/*Summary Post Codes Hackney Gazetteer - LLPG*/ - select row_number() over (partition by replace(postcode_nospace,' ','') order by replace(postcode_nospace,' ','') desc ) as llpg_rn ,gazetteer as llpg_gazetteer /*,locality as llpg_locality_reg ,ward as llpg_ward_reg ,town as llpg_town_reg*/ ,postcode as llpg_postcode ,replace(postcode_nospace,' ','') as llpg_postcode_nospace - ,case - when upper(town) like 'LONDON%' and upper(locality) ='HACKNEY' and ward !='' then concat(ward, ' - ',locality, ' - ',town) - when upper(town) like 'LONDON%' and locality !='' then concat(locality, ' - ',town) - when upper(town) like 'LONDON%' and locality ='' and ward !='' then concat(ward, ' - ',town) - else town end as llpg_area - /*,count(*) as llpg_num_records_reg*/ FROM unrestricted_address_api_dbo_hackney_address where lpi_logical_status like 'Approved Preferred' - group by gazetteer, locality ,ward ,town ,postcode ,replace(postcode_nospace,' ','') -) - - - -,permit as (/*all permit records with llpg*/ - select replace(postcode,' ','') as per_postcode_ns - ,sr_usrn as per_sr_usrn - ,replace(sr_address1, ',', '') as per_sr_address1 - ,replace(sr_address2, ',', '') as per_sr_address2 - ,replace(street_description, ',', '') as per_street_description - ,sr_ward_code as per_sr_ward_code - ,sr_ward_name as per_sr_ward_name - ,property_shell as per_property_shell - ,blpu_class as per_blpu_class - ,replace(usage_primary, ',', '') as per_usage_primary - ,replace(usage_description, ',', '') as per_usage_description - ,replace(street, ',', '') as per_street - ,replace(add_type, ',', '') as per_add_type - ,add_class as per_add_class - ,zone_name as per_zone_name - ,replace(case - -when address_line_2 ='' and business_name ='' and hasc_organisation_name ='' and doctors_surgery_name ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',address_line_1,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when business_name !='' and hasc_organisation_name !='' and address_line_2 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',business_name,' - ',hasc_organisation_name,' - ',address_line_1,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when business_name !='' and doctors_surgery_name !='' and address_line_2 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',business_name,' - ',doctors_surgery_name,' - ',address_line_1,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when business_name !='' and address_line_2 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',business_name,' - ',address_line_1,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when hasc_organisation_name !='' and address_line_2 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',hasc_organisation_name,' - ',address_line_1,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when doctors_surgery_name !='' and address_line_2 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',doctors_surgery_name,' - ',address_line_1,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) - -when address_line_3 =''and business_name ='' and hasc_organisation_name ='' and doctors_surgery_name ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',address_line_1,' ',address_line_2,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when business_name !='' and hasc_organisation_name !='' and address_line_3 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',business_name,' - ',hasc_organisation_name,' - ',address_line_1,', ',address_line_2,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when business_name !='' and doctors_surgery_name !='' and address_line_3 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',business_name,' - ',doctors_surgery_name,' - ',address_line_1,', ',address_line_2,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when business_name !='' and address_line_3 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',business_name,' - ',address_line_1,' ',address_line_2,', ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when hasc_organisation_name !='' and address_line_3 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant ,' - ',hasc_organisation_name,' - ',address_line_1,' ',address_line_2,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when doctors_surgery_name !='' and address_line_3 ='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',doctors_surgery_name,' - ',address_line_1,' ',address_line_2,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) - - -when business_name !='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',business_name,' - ',address_line_1,' ',address_line_2,' ',address_line_3,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when hasc_organisation_name !='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',hasc_organisation_name,' - ',address_line_1,' ',address_line_2,' ',address_line_3,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -when doctors_surgery_name !='' then concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',doctors_surgery_name,' - ',address_line_1,' ',address_line_2,' ',address_line_3,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) - -else concat(permit_reference,' - ',forename_of_applicant,' ',surname_of_applicant,' - ',address_line_1,' ',address_line_2,' ',address_line_3,' ',parking_permit_denormalised_gds_street_llpg.postcode,' - ',email_address_of_applicant) -end, ',', '') as per_permit_contact_summary - ,replace(permit_summary, ',', '') as per_permit_summary - ,replace(permit_full_address, ',', '') as per_permit_full_address - ,replace(full_address, ',', '') as per_full_address - ,replace(permit_full_address_type, ',', '') as per_permit_full_address_type - ,replace(full_address_type, ',', '') as per_full_address_type - ,live_flag as per_live_flag - ,flag_lp_est_bb_zero as per_flag_lp_est_bb_zero - ,flag_lp_bb_onst as per_flag_lp_bb_onst - ,flag_lp_est as per_flag_lp_est - ,flag_lp_bus as per_flag_lp_bus - ,flag_lp_doc as per_flag_lp_doc - ,flag_lp_lc as per_flag_lp_lc - ,flag_lp_hsc as per_flag_lp_hsc - ,flag_lp_res as per_flag_lp_res - ,flag_lp_disp as per_flag_lp_disp - ,flag_lp_othr as per_flag_lp_othr - ,flag_lp_al as per_flag_lp_al - ,flag_lp_cb as per_flag_lp_cb - ,flag_ltn_london_fields as per_flag_ltn_london_fields - ,flag_name_ltn_london_fields as per_flag_name_ltn_london_fields - ,permit_reference as per_permit_reference - ,application_date as per_application_date - ,forename_of_applicant as per_forename_of_applicant - ,surname_of_applicant as per_surname_of_applicant - ,email_address_of_applicant as per_email_address_of_applicant - ,blue_badge_number as per_blue_badge_number - ,blue_badge_expiry as per_blue_badge_expiry - ,start_date as per_start_date - ,end_date as per_end_date - ,approval_date as per_approval_date - ,approved_by as per_approved_by - ,approval_type as per_approval_type - ,amount as per_amount - ,payment_location as per_payment_location - ,permit_type as per_permit_type - ,business_name as per_business_name - ,replace(hasc_organisation_name, ',', '') as per_hasc_organisation_name - ,replace(doctors_surgery_name, ',', '') as per_doctors_surgery_name - ,uprn as per_uprn - ,replace(address_line_1, ',', '') as per_address_line_1 - ,replace(address_line_2, ',', '') as per_address_line_2 - ,replace(address_line_3, ',', '') as per_address_line_3 - ,postcode as per_postcode - ,cpz as per_cpz - ,cpz_name as per_cpz_name - ,status as per_status - ,quantity as per_quantity - ,vrm as per_vrm - ,live_permit_flag as per_live_permit_flag - ,permit_fta_renewal as per_permit_fta_renewal - ,latest_permit_status as per_latest_permit_status - ,rn as per_rn - ,make as per_make - ,model as per_model - ,fuel as per_fuel - ,engine_capactiy as per_engine_capactiy - ,co2_emission as per_co2_emission - ,foreign as per_foreign - ,lpg_conversion as per_lpg_conversion - ,vrm_record_created as per_vrm_record_created - ,import_year as per_import_year - ,import_month as per_import_month - ,import_day as per_import_day - ,import_date as per_import_date - FROM parking_permit_denormalised_gds_street_llpg -) -, pcn as ( -Select/*Registered extracted post codes*/ -case when length(regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') ) = 0 or regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') is null or regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like '' or regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like ' ' then 'No Address' else regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') end -as reg_add_extracted_post_code -,Case when /*000*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 0 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 0 - then 'No Address' -when (/*398*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and upper(substr(registered_keeper_address, -6, 6)) like 'LONDON%') - or (/*298*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like 'Hackney%' or substr(registered_keeper_address, -9, 9) like 'Limited')) - or /*198*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or (/*196*/position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -9, 9) like 'Essex%') - or /*099*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 0 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 9 - then 'No Post Code' -When /*196 and n16*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -9, 9) like ' N16 %' then /*-9 = 9*/ substr(registered_keeper_address, -9, 9) -when (/*496*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -10, 8) like 'MK43%') then substr(registered_keeper_address, -10, 8) -when /*395*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 5 and substr(registered_keeper_address, -11, 8) like 'MK14%' then /*-11 =8*/ substr(registered_keeper_address, -11, 8) -when /*395*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 5 and substr(registered_keeper_address, -10, 7) like 'N17%' then /*-10 =7*/ substr(registered_keeper_address, -10, 7) -when (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like ',CM%' or substr(registered_keeper_address, -9, 9) like ',IG%')) - or /*597*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 5 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or (/*496*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -10, 8) like 'MK43%') - then /*8*/ substr(registered_keeper_address, -8, 8) -when /*598*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 5 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 or (/*597*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 5 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 and substr(registered_keeper_address, -9, 9) like 'NDON%') then /*4*/ substr(registered_keeper_address, -4, 4) -when (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like 'ONDON%' or substr(registered_keeper_address, -9, 9) like 'MFORD %' or substr(registered_keeper_address, -9, 9) like 'Essex %')) then /*3*/ substr(registered_keeper_address, -3, 3) -when /*897*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 8 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like 'T,%' or substr(registered_keeper_address, -9, 9) like 't,%' or substr(registered_keeper_address, -9, 9) like 'n,%' or substr(registered_keeper_address, -9, 9) like 'N,%' or substr(registered_keeper_address, -9, 9) like 'd,%' or substr(registered_keeper_address, -9, 9) like 'm,%')) - or /*496*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - or /*298*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or /*297*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or /*196*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - or (/*498*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and substr(registered_keeper_address, -9, 9) like 'HA9%') - then /*7*/ substr(registered_keeper_address, -7, 7) -when /*798*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 7 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 then /*2*/ substr(registered_keeper_address, -2, 2) -when (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like ', %' or substr(registered_keeper_address, -9, 9) like 'ON,%' or substr(registered_keeper_address, -9, 9) like 'on,%')) - or /*498*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or /*497*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or /*398*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or /*397*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - then /*6*/ substr(registered_keeper_address, -6, 6) -when/*197*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - then /*8*/ substr(registered_keeper_address, -9, 9) -when /*396*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - then /*5*/ substr(registered_keeper_address, -5, 5) -when /*296*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - then /*-7 =6*/ substr(registered_keeper_address, -7, 6) -when /*193*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 3 - then /*-13 = 7*/ substr(registered_keeper_address, -13, 7) - -else substr(registered_keeper_address, -9, 9) end as reg_add_extracted_post_code_v1 - -,replace(/*extracted post codes no spaces*/ -case when length(regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') ) = 0 or regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') is null or regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like '' or regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like ' ' then 'No Address' else regexp_extract(registered_keeper_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') end -,' ','') as reg_add_extracted_post_code_no_space - -,replace(/*extracted post codes no spaces*/ -(Case -when /*000*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 0 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 0 - then 'No Address' -when (/*398*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and upper(substr(registered_keeper_address, -6, 6)) like 'LONDON%') - or (/*298*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like 'Hackney%' or substr(registered_keeper_address, -9, 9) like 'Limited')) - or /*198*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or (/*196*/position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -9, 9) like 'Essex%') - or /*099*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 0 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 9 - then 'No Post Code' -When /*196 and n16*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -9, 9) like ' N16 %' then /*-9 = 9*/ substr(registered_keeper_address, -9, 9) -when (/*496*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -10, 8) like 'MK43%') then substr(registered_keeper_address, -10, 8) -when /*395*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 5 and substr(registered_keeper_address, -11, 8) like 'MK14%' then /*-11 =8*/ substr(registered_keeper_address, -11, 8) -when /*395*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 5 and substr(registered_keeper_address, -10, 7) like 'N17%' then /*-10 =7*/ substr(registered_keeper_address, -10, 7) -when (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like ',CM%' or substr(registered_keeper_address, -9, 9) like ',IG%')) - or /*597*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 5 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or (/*496*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 and substr(registered_keeper_address, -10, 8) like 'MK43%') - then /*8*/ substr(registered_keeper_address, -8, 8) -when /*598*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 5 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 or (/*597*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 5 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 and substr(registered_keeper_address, -9, 9) like 'NDON%') then /*4*/ substr(registered_keeper_address, -4, 4) -when (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like 'ONDON%' or substr(registered_keeper_address, -9, 9) like 'MFORD %' or substr(registered_keeper_address, -9, 9) like 'Essex %')) then /*3*/ substr(registered_keeper_address, -3, 3) -when /*897*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 8 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like 'T,%' or substr(registered_keeper_address, -9, 9) like 't,%' or substr(registered_keeper_address, -9, 9) like 'n,%' or substr(registered_keeper_address, -9, 9) like 'N,%' or substr(registered_keeper_address, -9, 9) like 'd,%' or substr(registered_keeper_address, -9, 9) like 'm,%')) - or /*496*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - or /*298*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or /*297*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or /*196*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - or (/*498*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and substr(registered_keeper_address, -9, 9) like 'HA9%') - then /*7*/ substr(registered_keeper_address, -7, 7) -when /*798*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 7 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - then /*2*/ substr(registered_keeper_address, -2, 2) -when (/*698*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 6 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 and ( substr(registered_keeper_address, -9, 9) like ', %' or substr(registered_keeper_address, -9, 9) like 'ON,%' or substr(registered_keeper_address, -9, 9) like 'on,%')) - or /*498*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or /*497*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 4 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - or /*398*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 8 - or /*397*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - then /*6*/ substr(registered_keeper_address, -6, 6) -when/*197*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 7 - then /*8*/ substr(registered_keeper_address, -9, 9) -when /*396*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 3 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - then /*5*/ substr(registered_keeper_address, -5, 5) -when /*296*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 2 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 6 - then /*-7 =6*/ substr(registered_keeper_address, -7, 6) -when /*193*/ position(' ' in substr(registered_keeper_address, -9, 9)) = 1 and length(regexp_replace(substr(registered_keeper_address, -9, 9),'\s','')) = 3 - then /*-13 = 7*/ substr(registered_keeper_address, -13, 7) - -else substr(registered_keeper_address, -9, 9) end ) -,' ','') as reg_add_extracted_post_code_no_space_v1 - -/*Current*/ -,case when length(regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') ) = 0 or regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') is null or regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like '' or regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like ' ' then 'No Address' else regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') end -as curr_add_extracted_post_code - -,Case/*extracted post codes*/ -when /*000*/ position(' ' in substr(current_ticket_address, -9, 9)) = 0 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 0 - then 'No Address' -when (/*398*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and upper(substr(current_ticket_address, -6, 6)) like 'LONDON%') - or (/*298*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like 'Hackney%' or substr(current_ticket_address, -9, 9) like 'Limited')) - or /*198*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or (/*196*/position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -9, 9) like 'Essex%') - or /*099*/ position(' ' in substr(current_ticket_address, -9, 9)) = 0 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 9 - then 'No Post Code' -when (/*196 and DA CR IG HP2 N1*/position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and (substr(current_ticket_address, -9, 9) like ' HP2 %' or substr(current_ticket_address, -9, 9) like ' N1%' or substr(current_ticket_address, -9, 9) like ' DA%' or substr(current_ticket_address, -9, 9) like ' CR%' or substr(current_ticket_address, -9, 9) like ' IG%' or substr(current_ticket_address, -9, 9) like ' E12 %' or substr(current_ticket_address, -9, 9) like ' B90 %' )) - or (/*496 and SL2*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -9, 9) like 'SL2 %') then substr(current_ticket_address, -9, 9) -when (/*496*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -10, 8) like 'MK43%') then substr(current_ticket_address, -10, 8) -when /*395*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 5 and substr(current_ticket_address, -11, 8) like 'MK14%' then /*-11 =8*/ substr(current_ticket_address, -11, 8) -when /*395*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 5 and substr(current_ticket_address, -10, 7) like 'N17%' then /*-10 =7*/ substr(current_ticket_address, -10, 7) -when (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like ',CM%' or substr(current_ticket_address, -9, 9) like ',IG%')) - or /*597*/ position(' ' in substr(current_ticket_address, -9, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or (/*496*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -10, 8) like 'MK43%') - then /*8*/ substr(current_ticket_address, -8, 8) -when /*598*/ position(' ' in substr(current_ticket_address, -9, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 or (/*597*/ position(' ' in substr(current_ticket_address, -9, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 and substr(current_ticket_address, -9, 9) like 'NDON%') then /*4*/ substr(current_ticket_address, -4, 4) -when (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like 'ONDON%' or substr(current_ticket_address, -9, 9) like 'MFORD %' or substr(current_ticket_address, -9, 9) like 'Essex %')) then /*3*/ substr(current_ticket_address, -3, 3) -when /*897*/ position(' ' in substr(current_ticket_address, -9, 9)) = 8 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like 'T,%' or substr(current_ticket_address, -9, 9) like 't,%' or substr(current_ticket_address, -9, 9) like 'n,%' or substr(current_ticket_address, -9, 9) like 'N,%' or substr(current_ticket_address, -9, 9) like 'd,%' or substr(current_ticket_address, -9, 9) like 'm,%')) - or /*496*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 - or /*298*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or /*297*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or /*196*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 - or (/*498*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and substr(current_ticket_address, -9, 9) like 'HA9%') - then /*7*/ substr(current_ticket_address, -7, 7) -when /*798*/ position(' ' in substr(current_ticket_address, -9, 9)) = 7 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - then /*2*/ substr(current_ticket_address, -2, 2) -when (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like ', %' or substr(current_ticket_address, -9, 9) like 'ON,%' or substr(current_ticket_address, -9, 9) like 'on,%')) - or /*498*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or /*497*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or /*398*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or /*397*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - then /*6*/ substr(current_ticket_address, -6, 6) -when/*197*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - then /*8*/ substr(current_ticket_address, -9, 9) -when /*396*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 - then /*5*/ substr(current_ticket_address, -5, 5) -when /*296*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 - then /*-7 =7*/ substr(current_ticket_address, -7, 7) -when /*-99=193 -139=296 */ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 and position(' ' in substr(current_ticket_address, -13, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -13, 9),'\s','')) = 6 then /*-11 =6*/ substr(current_ticket_address, -11, 6) -when /*-99=193 -139=196 */ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 and position(' ' in substr(current_ticket_address, -13, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -13, 9),'\s','')) = 6 then /*-12 =7*/ substr(current_ticket_address, -12, 7) -when /*-99=193 -139=597 */ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 and position(' ' in substr(current_ticket_address, -13, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -13, 9),'\s','')) = 7 then /*-13 =8*/ substr(current_ticket_address, -13, 8) -when /*193*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 - then /*-13 = 7*/ substr(current_ticket_address, -13, 7) - -else substr(current_ticket_address, -9, 9) end as curr_add_extracted_post_code_v1 - -,replace(/*extracted post codes no spaces*/ -case when length(regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') ) = 0 or regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') is null or regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like '' or regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') like ' ' then 'No Address' else regexp_extract(current_ticket_address, '([A-Za-z][A-Ha-hJ-Yj-y]?[0-9][A-Za-z0-9]? ?[0-9][A-Za-z]{2}|[Gg][Ii][Rr] ?0[Aa]{2})') end -,' ','') as curr_add_extracted_post_code_no_space - -,replace(/*extracted post codes no spaces*/ -(Case -when /*000*/ position(' ' in substr(current_ticket_address, -9, 9)) = 0 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 0 - then 'No Address' -when (/*398*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and upper(substr(current_ticket_address, -6, 6)) like 'LONDON%') - or (/*298*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like 'Hackney%' or substr(current_ticket_address, -9, 9) like 'Limited')) - or /*198*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or (/*196*/position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -9, 9) like 'Essex%') - or /*099*/ position(' ' in substr(current_ticket_address, -9, 9)) = 0 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 9 - then 'No Post Code' -when (/*196 and DA CR IG HP2 N1*/position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and (substr(current_ticket_address, -9, 9) like ' HP2 %' or substr(current_ticket_address, -9, 9) like ' N1%' or substr(current_ticket_address, -9, 9) like ' DA%' or substr(current_ticket_address, -9, 9) like ' CR%' or substr(current_ticket_address, -9, 9) like ' IG%' or substr(current_ticket_address, -9, 9) like ' E12 %' or substr(current_ticket_address, -9, 9) like ' B90 %' )) - or (/*496 and SL2*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -9, 9) like 'SL2 %') then substr(current_ticket_address, -9, 9) -when (/*496*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -10, 8) like 'MK43%') then substr(current_ticket_address, -10, 8) -when /*395*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 5 and substr(current_ticket_address, -11, 8) like 'MK14%' then /*-11 =8*/ substr(current_ticket_address, -11, 8) -when /*395*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 5 and substr(current_ticket_address, -10, 7) like 'N17%' then /*-10 =7*/ substr(current_ticket_address, -10, 7) -when (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like ',CM%' or substr(current_ticket_address, -9, 9) like ',IG%')) - or /*597*/ position(' ' in substr(current_ticket_address, -9, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or (/*496*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 and substr(current_ticket_address, -10, 8) like 'MK43%') - then /*8*/ substr(current_ticket_address, -8, 8) -when /*598*/ position(' ' in substr(current_ticket_address, -9, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 or (/*597*/ position(' ' in substr(current_ticket_address, -9, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 and substr(current_ticket_address, -9, 9) like 'NDON%') then /*4*/ substr(current_ticket_address, -4, 4) -when (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like 'ONDON%' or substr(current_ticket_address, -9, 9) like 'MFORD %' or substr(current_ticket_address, -9, 9) like 'Essex %')) then /*3*/ substr(current_ticket_address, -3, 3) -when /*897*/ position(' ' in substr(current_ticket_address, -9, 9)) = 8 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like 'T,%' or substr(current_ticket_address, -9, 9) like 't,%' or substr(current_ticket_address, -9, 9) like 'n,%' or substr(current_ticket_address, -9, 9) like 'N,%' or substr(current_ticket_address, -9, 9) like 'd,%' or substr(current_ticket_address, -9, 9) like 'm,%')) - or /*496*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 - or /*298*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or /*297*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or /*196*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 - or (/*498*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and substr(current_ticket_address, -9, 9) like 'HA9%') - then /*7*/ substr(current_ticket_address, -7, 7) -when /*798*/ position(' ' in substr(current_ticket_address, -9, 9)) = 7 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - then /*2*/ substr(current_ticket_address, -2, 2) -when (/*698*/ position(' ' in substr(current_ticket_address, -9, 9)) = 6 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 and ( substr(current_ticket_address, -9, 9) like ', %' or substr(current_ticket_address, -9, 9) like 'ON,%' or substr(current_ticket_address, -9, 9) like 'on,%')) - or /*498*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or /*497*/ position(' ' in substr(current_ticket_address, -9, 9)) = 4 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - or /*398*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 8 - or /*397*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - then /*6*/ substr(current_ticket_address, -6, 6) -when/*197*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 7 - then /*8*/ substr(current_ticket_address, -9, 9) -when /*396*/ position(' ' in substr(current_ticket_address, -9, 9)) = 3 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 then /*5*/ substr(current_ticket_address, -5, 5) -when /*296*/ position(' ' in substr(current_ticket_address, -9, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 6 then /*-7 =7*/ substr(current_ticket_address, -7, 7) -when /*-99=193 -139=296 */ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 and position(' ' in substr(current_ticket_address, -13, 9)) = 2 and length(regexp_replace(substr(current_ticket_address, -13, 9),'\s','')) = 6 then /*-11 =6*/ substr(current_ticket_address, -11, 6) -when /*-99=193 -139=196 */ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 and position(' ' in substr(current_ticket_address, -13, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -13, 9),'\s','')) = 6 then /*-12 =7*/ substr(current_ticket_address, -12, 7) -when /*-99=193 -139=597 */ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 and position(' ' in substr(current_ticket_address, -13, 9)) = 5 and length(regexp_replace(substr(current_ticket_address, -13, 9),'\s','')) = 7 then /*-13 =8*/ substr(current_ticket_address, -13, 8) -when /*193*/ position(' ' in substr(current_ticket_address, -9, 9)) = 1 and length(regexp_replace(substr(current_ticket_address, -9, 9),'\s','')) = 3 then /*-13 = 7*/ substr(current_ticket_address, -13, 7) - - -else substr(current_ticket_address, -9, 9) end ) -,' ','') as curr_add_extracted_post_code_no_space_v1 -,* FROM pcnfoidetails_pcn_foi_full -) - -/*.....................................................................................................................*/ -select distinct -Case -when cast(pcnissuedate as varchar(10)) like '2028-03-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2028-02-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2028-01-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-12-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-11-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-10-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-09-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-08-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-07-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-06-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-05-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-04-%' THEN '2027' -when cast(pcnissuedate as varchar(10)) like '2027-03-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2027-02-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2027-01-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-12-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-11-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-10-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-09-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-08-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-07-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-06-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-05-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-04-%' THEN '2026' -when cast(pcnissuedate as varchar(10)) like '2026-03-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2026-02-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2026-01-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-12-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-11-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-10-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-09-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-08-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-07-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-06-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-05-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-04-%' THEN '2025' -when cast(pcnissuedate as varchar(10)) like '2025-03-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2025-02-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2025-01-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-12-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-11-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-10-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-09-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-08-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-07-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-06-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-05-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-04-%' THEN '2024' -when cast(pcnissuedate as varchar(10)) like '2024-03-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2024-02-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2024-01-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-12-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-11-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-10-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-09-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-08-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-07-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-06-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-05-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-04-%' THEN '2023' -when cast(pcnissuedate as varchar(10)) like '2023-03-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2023-02-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2023-01-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-12-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-11-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-10-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-09-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-08-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-07-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-06-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-05-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-04-%' THEN '2022' -when cast(pcnissuedate as varchar(10)) like '2022-03-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2022-02-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2022-01-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-12-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-11-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-10-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-09-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-08-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-07-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-06-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-05-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-04-%' THEN '2021' -when cast(pcnissuedate as varchar(10)) like '2021-03-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2021-02-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2021-01-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-12-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-11-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-10-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-09-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-08-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-07-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-06-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-05-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-04-%' THEN '2020' -when cast(pcnissuedate as varchar(10)) like '2020-03-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2020-02-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2020-01-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-12-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-11-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-10-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-09-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-08-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-07-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-06-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-05-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-04-%' THEN '2019' -when cast(pcnissuedate as varchar(10)) like '2019-03-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2019-02-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2019-01-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-12-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-11-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-10-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-09-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-08-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-07-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-06-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-05-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-04-%' THEN '2018' -when cast(pcnissuedate as varchar(10)) like '2018-03-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2018-02-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2018-01-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-12-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-11-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-10-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-09-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-08-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-07-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-06-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-05-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-04-%' THEN '2017' -when cast(pcnissuedate as varchar(10)) like '2017-03-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2017-02-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2017-01-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-12-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-11-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-10-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-09-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-08-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-07-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-06-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-05-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-04-%' THEN '2016' -when cast(pcnissuedate as varchar(10)) like '2016-03-%' THEN '2015' -when cast(pcnissuedate as varchar(10)) like '2016-02-%' THEN '2015' -else '1900' -end as fy, - -/*PCN with no address linked to Permit with address*/ -case when -(/*no address*/case when reg_add_extracted_post_code_no_space like 'NoAddress' and curr_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=1 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -then 1 end as pcn_no_add_link_permit - -/*PCN with no address linked to Permit with address not cancelled or closed*/ -,case when -(/*no address*/case when reg_add_extracted_post_code_no_space like 'NoAddress' and curr_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=1 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and pcn_canx_date is null -and pcn_casecloseddate is null -then 1 end as open_pcn_no_add_link_permit - -/*pcn registered address not same as linked permit address*/ -,case when -(/*not match reg add permit add*/case when reg_add_extracted_post_code_no_space = per_postcode_ns then 0 else 1 end)=1 -and (/*has reg address*/case when reg_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=0 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and (/*no permit postcode*/case when per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end) =0 -then 1 end as reg_add_link_permit_diff - -/*pcn registered address not same as linked permit address not cancelled or close*/ -,case when -(/*not match reg add permit add*/case when reg_add_extracted_post_code_no_space = per_postcode_ns then 0 else 1 end)=1 -and (/*has reg address*/case when reg_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=0 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and (/*no permit postcode*/case when per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end) =0 -and pcn_canx_date is null -and pcn_casecloseddate is null -then 1 end as open_reg_add_link_permit_diff - -/*PCN with non llpg registered address linked to Permit with address*/ -,case when -(/*reg add not in llpg*/case when llpg_reg.llpg_postcode_nospace is not null or llpg_reg.llpg_postcode_nospace not like '' or llpg_reg.llpg_postcode_nospace not like ' ' then 1 else 0 end)=0 -and (/*has reg address*/case when reg_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=0 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and (/*no permit postcode*/case when per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end) =0 -then 1 end as reg_add_not_llpg_link_permit - -/*PCN with non llpg registered address linked to Permit with address not cancelled or close*/ -,case when -(/*reg add not in llpg*/case when llpg_reg.llpg_postcode_nospace is not null or llpg_reg.llpg_postcode_nospace not like '' or llpg_reg.llpg_postcode_nospace not like ' ' then 1 else 0 end)=0 -and (/*has reg address*/case when reg_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=0 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and (/*no permit postcode*/case when per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end) =0 -and pcn_canx_date is null -and pcn_casecloseddate is null -then 1 end as open_reg_add_not_llpg_link_permit - -/*PCN with non llpg current address linked to Permit with address*/ -,case when -(/*curr add not in llpg*/case when llpg_curr.llpg_postcode_nospace is not null or llpg_curr.llpg_postcode_nospace not like '' or llpg_curr.llpg_postcode_nospace not like ' ' then 1 else 0 end)=0 -and (/*has curr address*/case when curr_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=0 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and (/*no permit postcode*/case when per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end) =0 -then 1 end as curr_add_not_llpg_link_permit - -/*PCN with non llpg current address linked to Permit with address not cancelled or close*/ -,case when -(/*curr add not in llpg*/case when llpg_curr.llpg_postcode_nospace is not null or llpg_curr.llpg_postcode_nospace not like '' or llpg_curr.llpg_postcode_nospace not like ' ' then 1 else 0 end)=0 -and (/*has curr address*/case when curr_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end)=0 -and (/*Has permit*/case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end)=1 -and (/*no permit postcode*/case when per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end) =0 -and pcn_canx_date is null -and pcn_casecloseddate is null -then 1 end as open_curr_add_not_llpg_link_permit - -/*In LLPG or not*/ -,case when llpg_reg.llpg_postcode_nospace is not null or llpg_reg.llpg_postcode_nospace not like '' or llpg_reg.llpg_postcode_nospace not like ' ' then 1 else 0 end as post_code_in_llpg_reg -,case when llpg_curr.llpg_postcode_nospace is not null or llpg_curr.llpg_postcode_nospace not like '' or llpg_curr.llpg_postcode_nospace not like ' ' then 1 else 0 end as post_code_in_llpg_curr -,case when llpg_per.llpg_postcode_nospace is not null or llpg_per.llpg_postcode_nospace not like '' or llpg_per.llpg_postcode_nospace not like ' ' then 1 else 0 end as post_code_in_llpg_per -/*pcn postcode different from permit postcode*/ -,case when (per_postcode_ns is not null or per_postcode_ns not like '' or per_postcode_ns not like ' ') and reg_add_extracted_post_code_no_space is not null and reg_add_extracted_post_code_no_space = per_postcode_ns then 0 else 1 end as diff_reg_pc_per_pc -,case when (per_postcode_ns is not null or per_postcode_ns not like '' or per_postcode_ns not like ' ') and curr_add_extracted_post_code_no_space is not null and curr_add_extracted_post_code_no_space = per_postcode_ns then 0 else 1 end as diff_curr_pc_per_pc -,case when - ((per_postcode_ns is not null or per_postcode_ns not like '' or per_postcode_ns not like ' ') and reg_add_extracted_post_code_no_space is not null and reg_add_extracted_post_code_no_space = per_postcode_ns) - or ((per_postcode_ns is not null or per_postcode_ns not like '' or per_postcode_ns not like ' ') and curr_add_extracted_post_code_no_space is not null and curr_add_extracted_post_code_no_space = per_postcode_ns) - then 0 else 1 end as diff_pcn_pc_per_pc -/*pcn registered different to current postcode*/ -,case when reg_add_extracted_post_code_no_space is not null and (reg_add_extracted_post_code_no_space = curr_add_extracted_post_code_no_space) then 0 else 1 end as diff_reg_pc_curr_pc - -/*PCN no address*/ -,case when reg_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end as reg_add_no_address -,case when curr_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end as curr_add_no_address -,case when reg_add_extracted_post_code_no_space like 'NoAddress' and curr_add_extracted_post_code_no_space like 'NoAddress' then 1 else 0 end as pcn_both_no_address -/*PCN no postcode*/ -,case when reg_add_extracted_post_code_no_space like 'NoPostCode' then 1 else 0 end as reg_add_no_postcode -,case when curr_add_extracted_post_code_no_space like 'NoPostCode' then 1 else 0 end as curr_add_no_postcode -,case when reg_add_extracted_post_code_no_space like 'NoPostCode' and curr_add_extracted_post_code_no_space like 'NoPostCode' then 1 else 0 end as pcn_both_no_postcode - -/*pcn postcodes different to permit llpg postcode*/ -,case when llpg_per.llpg_postcode_nospace is not null and reg_add_extracted_post_code_no_space is not null and per_permit_type not like 'All Zone' and (reg_add_extracted_post_code_no_space = llpg_per.llpg_postcode_nospace) then 0 else 1 end as diff_reg_pc_llpg_per_pc -,case when llpg_per.llpg_postcode_nospace is not null and curr_add_extracted_post_code_no_space is not null and per_permit_type not like 'All Zone' and (curr_add_extracted_post_code_no_space = llpg_per.llpg_postcode_nospace) then 0 else 1 end as diff_curr_pc_llpg_per_pc -,case when - (llpg_per.llpg_postcode_nospace is not null and reg_add_extracted_post_code_no_space is not null and per_permit_type not like 'All Zone' and (reg_add_extracted_post_code_no_space = llpg_per.llpg_postcode_nospace)) - or (llpg_per.llpg_postcode_nospace is not null and curr_add_extracted_post_code_no_space is not null and per_permit_type not like 'All Zone' and (curr_add_extracted_post_code_no_space = llpg_per.llpg_postcode_nospace)) - then 0 else 1 end as diff_pcn_pc_llpg_per_pc -/*Permit linked to pcn record*/ -,case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 1 else 0 end as has_permit -,case when per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/ then 0 else 1 end as no_permit -/*Permit has no address in data*/ -,case when (per_permit_reference is not null or per_permit_reference like '' or per_permit_reference like ' ' /*some permits no address*/) and per_postcode_ns is null or per_postcode_ns like '' or per_postcode_ns like ' ' then 1 else 0 end as permit_no_address -, replace(registered_keeper_address, ',', '') as reg_add_no_comma -, replace(current_ticket_address, ',', '') as curr_add_no_comma -,replace(whereonlocation, ',', '') as whereonlocation_no_comma - -,case when replace(registered_keeper_address, ',', '') != 'Redacted. Cancellation due to vehicle mismatch SA99' and ( replace(registered_keeper_address, ',', '') in ('ONTO HOLDINGS LIMITED 05012- UNIT 3 HERMES COURT HERMES CLOSE WARWICK CV34 6NJ', -'CONTRACT VEHICLESLTD 049293 PO BOX 875 LEEDS LS1 9RX', -'VOLKSWAGEN GROUP LEASING C/O 012683 C/O VWFS (UK) LTD BRUNSWICK COURT YEOMANS DRIVE BLAKELANDS MILTON KEYNES MK14 5LR', -'OGILVIE FLEET LTD 04197- C/O OGILVIE FLEET LTD OGILVIE HOUSE 200 GLASGOW RD STIRLING FK7 8ES', -'NATALIA MOSCHOU 05074- C/O LEX AUTOLEASE LTD HEATHSIDE PARK HEATHSIDE PARK ROAD STOCKPORT SK3 0RB', -'VOLKSWAGEN GROUP LEASING C/O 049566 C/O EVENTECH LTD UNIT A BULLS BRIDGE CENTRE NORTH HYDE HAYES UB3 4QR', -'LEX AUTOLEASE LIMITED 05074- C/O LEX AUTOLEASE LIMITED HEATHSIDE PARK HEATHSIDE PARK ROAD STOCKPORT SK3 0RB', -'MSL VEHICLE SOLUTIONS LTD 049797 1 LAKESIDE CHEADLE SK8 3GW', -'LAND ROVER CONTRACT HIRE 05074- C/O LEX AUTOLEASE LIMITED HEATHSIDE PARK HEATHSIDE PARK ROAD STOCKPORT SK3 0RB', -'PCO RENTALS LTD 050374 7TH FLOOR HYDE HOUSE EDGWARE ROAD LONDON NW9 6LH', -'ALPHABET (GB) LTD 037886 PO BOX 1295 BOONGATE PETERBOROUGH PETERBOROUGH PE1 9QW', -'LEX AUTOLEASE LIMITED 05074- HEATHSIDE PARK HEATHSIDE PARK ROAD STOCKPORT SK3 0RB', -'REFLEX VEHICLE HIRE LTD 045901 22 BELTON ROAD WEST LOUGHBOROUGH LE11 5TR', -'PSD VEHICLE RENTAL 555 PRESCOTT ROAD ST HELENS WA10 3BZ', -'LEASEPLAN UK LIMITED 021490 C/O LEASEPLAN UK LTD 165 BATH ROAD SLOUGH SL1 4AA', -'AMI VEHICLE LEASING LTD 04555- 16-20 MILLERS YARD PARLIAMENT SQUARE HERTFORD SG14 1EZ', -'PENDRAGON VEHICLE MANAGEMENT LTD 032177 PENDRAGON HOUSE SIR FRANK WHITTLE ROAD DERBY DE21 4AZ', -'LEASEPLAN UK LTD 021490 C/O LEASEPLAN UK LTD 165 BATH ROAD SLOUGH SL1 4AA', -'CAR CASTLE LTD 145A PLASHET ROAD LONDON E13 0RA', -'FREE2MOVE LEASE 048847 C/O FREE2MOVE LEASE PO BOX 17263 SOLIHULL B93 3JB', -'VOLKSWAGEN FINANCIAL SERVICES(UK)LTD 012683 C/O VWFS (UK) LTD BRUNSWICK COURT YEOMANS DRIVE BLAKELANDS MILTON KEYNES MK14 5LR', -'VITUS VEHICLE HIRE LTD FLAT 9 SEBASTIAN HOUSE HOXTON STREET LONDON N1 6QH', -'ZENITH VEHICLE CONTRACTS LTD 037783 C/O ZENITH VEH CONTRACTS LTD NUMBER ONE GT EXHIBITION WAY KIRKSTALL FO LEEDS LS5 3BF', -'ZENITH VEHICLE CONTRACTS LIMITED 037783 NUMBER ONE GREAT EXHIBITION WAY KIRKSTALL FORGE LEEDS LS5 3BF', -'MARSHALL LEASING LTD C/O FOXTONS GROUP PLC 97 VICTORIA ROAD LONDON NW10 6DJ', -'LEASEPLAN UK LTD 021490 165 BATH ROAD SLOUGH SL1 4AA', -'KJ PCO RENTALS LTD UNIT A4 BOUNDS GREEN INDUSTRIAL ESTATE RINGWAY LONDON N11 2UD', -'CONTRACT VEHICLES LTD 049293 PO BOX 875 LEEDS LS1 9RX', -'REFLEX VEHICLE HIRE LIMITED 22 BELTON ROAD WEST LOUGHBOROUGH LE11 5TR', -'LEX AUTOLEASE LTD 05074- HEATHSIDE PARK HEATHSIDE PARK ROAD STOCKPORT SK3 0RB', -'SPLEND LTD 050532 C/O SPLEND LTD 393 EDGWARE ROAD CRICKLEWOOD LONDON NW2 6LN', -'AVT LTD ALL VEHICLE TYPES 83 HAWTHORNE CLOSE LONDON N1 4AW', -'SAFE & SOUND VEHICLE SYSTEMS 41 KENT CLOSE PUDSEY LS28 9EY', -'L & F PLANT HIRE LTD 15 WHARF ROAD ENFIELD EN3 4TA', -'ENTERPRISE RENT A CAR UK LTD 046103 C/O ENTERPRISE RENTACAR UK LTD PO BOX 77 ALDERSHOT GU11 9DY', -'SPRINTSHIFT COMMERCIAL VEHICLE HIRE 17 OUTRAM ROAD BROADWAY INDUSTRIAL ESTATE DUKINFIELD SK16 4XE', -'WESTGATE VEHICLE HIRE LTD SUITE 40 UNIMIX HOUSE ABBEY ROAD LONDON NW10 7TR', -'SPARK VEHICLE REPAIR 118 WARHAM ROAD LONDON N4 1AU', -'TOOMEY LEASING GROUP LTD 045688 SERVICE HOUSE WEST MAYNE BASILDON SS15 6RW', -'MG VEHICLE HIRE LTD FLAT 2 79 PARK ROAD LONDON E10 7BZ', -'LEASYS UK LTD 045135 C/O HUDSON KAPEL LTD PO BOX 1413 COOMBELANDS PARK BEDFORD MK44 1ZY', -'OGILVIE FLEET LTD 04197- OGILVIE HOUSE 200 GLASGOW RD STIRLING FK7 8ES', -'MOBILITY VEHICLE HIRE 238 BIRMINGHAM ROAD GREAT BARR BIRMINGHAM B43 7AH', -'SUNBLET RENTALS LTD 049293 C/O CONTRACT VEHICLES LTD PO BOX 875 LEEDS LS1 9RX', -'MAXXIS CAR LTD 162 A CHIGWELL ROAD WOODFORD LONDON E18 1HA', -'ARVAL MID-TERM RENTAL 02644- C/O ARVAL UK LTD WINDMILL HILL SWINDON SN5 6PE', -'RAPID VEHICLE MANAGEMENT 160 LONDON ROAD SEVENOAKS TN13 1BT', -'GLYN HOPKIN VEHICLE RENTAL LTD 279-289 LONDON ROAD - - ROMFORD RM7 9NP', -'ISMAAEL CAR RENTAL LTD 36 CAVELL ROAD LONDON N17 7BJ', -'CONTRACT VEHICLES LTD 049293 C/O CONTRACT VEHICLES LTD PO BOX 875 LEEDS LS1 9RX', -'ENTERPRISE RENTACAR UK LTD 046103 C/O ENTERPRISE RENTACAR UK LTD PO BOX 77 ALDERSHOT GU11 9DY', -'PENDRAGON VEHICLE MANAGEMENT LIMITED 032177 PENDRAGON HOUSE SIR FRANK WHITTLE ROAD DERBY DE21 4AZ', -'CHS VEHICLES LTD 126 GELDERD CLOSE LEEDS LS12 6DS', -'AK VEHICLE MANAGEMENT LTD 41 BLOCK A THE VISTA CENTRE SALISBURY ROAD HOUNSLOW TW4 6JQ', -'IMPERIAL VEHICLE HIRE LTD 050465 GROUND FLOOR SUITE B PEACOCK HOUSE NORTHBRIDGE ROAD BERKHAMSTED HP4 1EH', -'MITSUBISHI HC CAPITAL UK PLC 048781 HAKUBA HOUSE WHITE HORSE BUSINESS PARK TROWBRIDGE BA14 0FL', -'GLYN HOPKIN VEHICLE RENTAL LTD 279-289 LONDON ROAD ROMFORD RM7 9NP', -'EXCLUSIVE VEHICLE CONTRACTS LT D 5 AMPTHILL BUSINESS PARK STATION ROAD AMPTHILL BEDFORD MK45 2QW', -'AMT VEHICLE RENTAL LTD ATHLON MOBILITY SVCS UK LTD C/O AMT VEHICLE RENTAL LTD 174 ARMLEY ROAD LEEDS LS12 2QH', -'SMART VEHICLE RENTAL LTD 366 LORDSHIP LANE LONDON N17 7QX', -'ZENITH VEHICLE CONTRACTS LTD 037783 NUMBER ONE GREAT EXHIBITION WA LEEDS LS5 3BF', -'SMART VEHICLE RENTAL LIMITED 366 LORDSHIP LANE LONDON N17 7QX', -'VAUXHALL FINANCE PLC C/O ABRIDGE VEHICLE MANAGEMENT LTD 9 BLENHEIM COURT BROOK WAY LEATHERHEAD KT22 7NA', -'RAPID VEHICLE MANAGEMENT LTD 160 LONDON ROAD SEVENOAKS TN13 1BT', -'ZENITH VEHICLE CONTRACTS 037783 C/O ZENITH VEH CONTRACTS LTD NUMBER ONE GT EXHIBITION WAY KIRKSTALL FO LEEDS LS5 3BF', -'PRIME ECO CARS LTD 9 MALDEN ROAD LONDON NW5 3HS', -'CLOSE BROTHERS VEHICLE HIRE LOWS LANE STANTON BY DALE ILKESTON DE7 4QU', -'MARSHALL LEASING LTD C/O FOXTONS LTD 97 VICTORIA ROAD LONDON NW10 6DJ', -'TRUCKPOINT VEHICLE HIRE LTD 69 BOROUGH ROAD LONDON SE1 1DN', -'MERIDIAN VEHICLE SOLUTIONS LTD 048288 UNIT 21 MCDONALD BUSINESS PARK MCDONALD WAY HEMEL HEMPSTEAD HP2 7EB', -'ANNIE MCDONAGH 044441 VEHICLE KEEPER 6 CENTENARY PARK OMAGH BT78 5HH', -'FIRST VEHICLE MANAGMENTLTD WOODVILLE SOUTH STREET BOUGHTON-UNDER-BLEAN FAVERSHAM ME13 9NS', -'BUSSEY VEHICLE LEASING FIRST FLOOR 95 WHIFFLER ROAD NORWICH NR3 2AW', -'ABRIDGE VEHICLE MANAGEMENT LTD 048641 C/O ABRIDGE VEH MANAGEMENT LTD 9 BLENHEIM COURT BROOK WAY LEATHERHEAD KT22 7NA', -'ACROBAT VEHICLE RENTAL LTD UNION LANE KINGSCLERE NEWBURY RG20 4ST', -'NORTHGATE VEHICLE HIRE LTD 046541 C/O GOODE DURRANT ADMIN LTD PO BOX 287 DARLINGTON DL1 9LW', -'ARI FLEET LEASING UK LTD 042560 C/O ARI FLEET UK LTD METHUEN PARK CHIPPENHAM SN14 0GX', -'GAP VEHICLE HIRE LTD 050271 C/O GAP VEHICLE HIRE LTD 13 SYMINGTON DRIVE CLYDEBANK BUSINESS PARK CLYDEBANK G81 2LD', -'ZENITH VEHICLE CONTRACTS LTD 037783 C/O ZENITH VEH CONTRACTS LTD NUMBER ONE GT EXHIBITION WAY KIRKSTALL FORGE LEEDS LS5 3BF', -'GAP VEHICLE HIRE LTD 050271 13 SYMINGTON DRIVE CLYDEBANK BUSINESS PARK CLYDEBANK G81 2LD', -'VOLKSWAGEN GROUP LEASING 012683 C/O VWFS (UK) LTD BRUNSWICK COURT YEOMANS DRIVE BLAKELANDS MILTON KEYNES MK14 5LR', -'RIVUS FLEET SOLUTIONS LTD 033029 2620 KINGS COURT THE CRESCENT BIRMINGHAM BUSINESS PARK BIRMINGHAM B37 7YE', -'ALD AUTOMOTIVE LTD 036705 C/O ALD AUTOMOTIVE LIMITED OAKWOOD DRIVE EMERSONS GREEN BRISTOL BS16 7LB', -'ALD AUTOMOTIVE LTD 036705 OAKWOOD DRIVE EMERSONS GREEN BRISTOL BS16 7LB', -'ENTERPRISE RENTACAR UK LTD 046103 PO BOX 77 ALDERSHOT GU11 9DY', -'LOOKERS LEASING LIMITED 04233- C/O LOOKERS LEASING LIMITED LOOKERS HOUSE CARDALE PARK BECKWITH HEAD ROA HARROGATE HG3 1RY', -'THE AUTOMOBILE ASSOCIATION DEVELOPMENTS LTD 033029 C/O RIVUS FLEET SOLUTIONS LTD 2620 KINGS COURT THE CRESCENT BIRMINGHAM BUSINESS PARK BIRMINGHAM B37 7YE', -'MARSHALL LEASING LTD 032931 C/O MARSHALL LEASING LTD BRIDGE HOUSE ORCHARD LANE HUNTINGDON PE29 3QT', -'EVENTECH LTD UNIT A BULLS BRIDGE CENTRE NORTH HYDE GARDENS HAYES UB3 4QR', -'VWFS (UK) LTD 012683 BRUNSWICK COURT YEOMANS DRIVE BLAKELANDS MILTON KEYNES MK14 5LR', -'ALD AUTOMOTIVE LIMITED 036705 OAKWOOD DRIVE EMERSONS GREEN BRISTOL BS16 7LB', -'ALD AUTOMOTIVE LTD 036705 OAKWOOD PARK LODGE CAUSEWAY BRISTOL BS16 3JA', -'MARHSALL LEASING LTD C/O FOXTONS GROUP PLC 97 VICTORIA ROAD LONDON NW10 6DJ', -'RA CAR RENTAL LTD 75 LEYTON PARK ROAD LONDON E10 5RL', -'SHAHI CARS LIMITED SUITE 114 116 BALLARDS LANE LONDON N3 2DN', -'ARROW CAR HIRE UK LONDON LTD 57 HALLSVILE ROAD LONDON E16 1EE' -) ) then 1 end as rental_lease_reg_add -,case -when replace(registered_keeper_address, ',', '') != 'Redacted. Cancellation due to vehicle mismatch SA99' -and (upper(replace(registered_keeper_address, ',', '')) like '%RENTAL%' -or upper(replace(registered_keeper_address, ',', '')) like '%LEASING%' -or upper(replace(registered_keeper_address, ',', '')) like '%LEASE%' -or upper(replace(registered_keeper_address, ',', '')) like '%HIRE%' -or upper(replace(registered_keeper_address, ',', '')) like '%RENTACAR%' -or upper(replace(registered_keeper_address, ',', '')) like '%VEHICLE%') then 1 end as flag_vehicle_rental_hire_lease_reg_add -,case -when replace(registered_keeper_address, ',', '') != 'Redacted. Cancellation due to vehicle mismatch SA99' -and (upper(replace(registered_keeper_address, ',', '')) like '%LIMITED%' -or upper(replace(registered_keeper_address, ',', '')) like '%LTD%' -or upper(replace(registered_keeper_address, ',', '')) like '%PO BOX%' -or upper(replace(registered_keeper_address, ',', '')) like '%C/O%' -) then 1 end as flag_vehicle_company_reg_add -,pcn.*, permit.*--, nlpg_reg.*, nlpg_curr.*,nlpg_per.*, llpg_reg.*, llpg_curr.*,llpg_per.* ---*, ---output from NLPG --nlpg_rn, nlpg_gazetteer, nlpg_postcode, nlpg_postcode_nospace, nlpg_area ---output from llpg --llpg_rn, llpg_gazetteer, llpg_postcode, llpg_postcode_nospace, llpg_area - -, nlpg_reg.nlpg_gazetteer as nlpg_gazetteer_reg ,nlpg_reg.nlpg_postcode as nlpg_postcode_reg ,nlpg_reg.nlpg_postcode_nospace as nlpg_postcode_nospace_reg ,nlpg_reg.nlpg_area as nlpg_area_reg -,nlpg_curr.nlpg_gazetteer as nlpg_gazetteer_curr ,nlpg_curr.nlpg_postcode as nlpg_postcode_curr ,nlpg_curr.nlpg_postcode_nospace as nlpg_postcode_nospace_curr ,nlpg_curr.nlpg_area as nlpg_area_curr -,nlpg_per.nlpg_gazetteer as nlpg_gazetteer_per ,nlpg_per.nlpg_postcode as nlpg_postcode_per ,nlpg_per.nlpg_postcode_nospace as nlpg_postcode_nospace_per ,nlpg_per.nlpg_area as nlpg_area_per -,llpg_reg.llpg_gazetteer as llpg_gazetteer_reg ,llpg_reg.llpg_postcode as llpg_postcode_reg ,llpg_reg.llpg_postcode_nospace as llpg_postcode_nospace_reg ,llpg_reg.llpg_area as llpg_area_reg -,llpg_curr.llpg_gazetteer as llpg_gazetteer_curr ,llpg_curr.llpg_postcode as llpg_postcode_curr ,llpg_curr.llpg_postcode_nospace as llpg_postcode_nospace_curr ,llpg_curr.llpg_area as llpg_area_curr -,llpg_per.llpg_gazetteer as llpg_gazetteer_per ,llpg_per.llpg_postcode as llpg_postcode_per ,llpg_per.llpg_postcode_nospace as llpg_postcode_nospace_per ,llpg_per.llpg_area as llpg_area_per - -FROM pcn -left join permit on permit.per_vrm = pcn.vrm and (case when pcn.pcnissuedate >= permit.per_start_date and pcn.pcnissuedate <= permit.per_end_date then 1 else 0 end)=1 -left join nlpg_pc_summ nlpg_reg on upper(replace(nlpg_reg.nlpg_postcode_nospace,' ','')) = upper(pcn.reg_add_extracted_post_code_no_space) and nlpg_reg.nlpg_rn = 1 -left join nlpg_pc_summ nlpg_curr on upper(replace(nlpg_curr.nlpg_postcode_nospace,' ','')) = upper(pcn.curr_add_extracted_post_code_no_space) and nlpg_curr.nlpg_rn = 1 -left join nlpg_pc_summ nlpg_per on upper(replace(nlpg_per.nlpg_postcode_nospace,' ','')) = upper(replace(permit.per_postcode,' ','')) and nlpg_per.nlpg_rn = 1 - -left join llpg_pc_summ llpg_reg on upper(replace(llpg_reg.llpg_postcode_nospace,' ','')) = upper(pcn.reg_add_extracted_post_code_no_space) and llpg_reg.llpg_rn = 1 -left join llpg_pc_summ llpg_curr on upper(replace(llpg_curr.llpg_postcode_nospace,' ','')) = upper(pcn.curr_add_extracted_post_code_no_space) and llpg_curr.llpg_rn = 1 -left join llpg_pc_summ llpg_per on upper(replace(llpg_per.llpg_postcode_nospace,' ','')) = upper(replace(permit.per_postcode,' ','')) and llpg_per.llpg_rn = 1 - -where pcn.pcnissuedate > current_date - interval '13' month --Last 13 months from todays date -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery13, - mapping={ - "unrestricted_address_api_dbo_national_address": S3bucketunrestricted_address_api_dbo_national_address_node1, - "unrestricted_address_api_dbo_hackney_address": AmazonS3unrestricted_address_api_dbo_hackney_address_node1650475065696, - "parking_permit_denormalised_gds_street_llpg": AmazonS3parking_permit_denormalised_gds_street_llpg_node1650475071787, - "pcnfoidetails_pcn_foi_full": AmazonS3pcnfoidetails_pcn_foi_full_node1650477300290, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/manual/parking_match_pcn_permit_vrm_with_address_match_llpg_nlpg_postcodes/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="parking-refined-zone", - catalogTableName="parking_match_pcn_permit_vrm_with_address_match_llpg_nlpg_postcodes", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() - diff --git a/scripts/jobs/parking/parking_motorcycle_monthly_ringgo_payments.py b/scripts/jobs/parking/parking_motorcycle_monthly_ringgo_payments.py deleted file mode 100644 index 92feecf1e..000000000 --- a/scripts/jobs/parking/parking_motorcycle_monthly_ringgo_payments.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "parking_motorcycle_monthly_ringgo_payments" - -# The exact same query prototyped in pre-prod(stg) orprod Athena -query_on_athena = """ -/*********************************************************************************** -Parking_motorcycle_monthly_ringgo_payments - -The SQL totals the motorcycle ringgo payments and session count, by month -(for selected locations) - -17/07/2024 - Create SQL -*********************************************************************************/ -/*** Collect the Motorcycle lookup list, from Carly ***/ -With MC_Locations as ( - SELECT * FROM "parking-raw-zone".ringgo_mc_locations), - -/*** Collect the Ringgo Motorcycle data, from June 2024 ***/ -Ringgo_MC as ( - SELECT - CASE - When session_start like '%/%'Then - CAST(substr(session_start, 7, 4)||'-'||substr(session_start, 4,2)||'-'||'01' as date) - ELSE cast(concat(substr(Cast(session_start as varchar(10)),1, 7), '-01') as date) - END as MonthYear, - A.*, B.ringgo_mc_code, B.location - FROM "parking-raw-zone".ringgo_daily as A - INNER JOIN MC_Locations as B ON A.parking_zone = B.ringgo_mc_code - WHERE A.vehicle_type = 'Motorcycle' AND - A.import_date > '20240710'), - -/*** Collect the distinct dates ***/ -Collect_Months as ( - SELECT Distinct MonthYear FROM Ringgo_MC), - -/*** The list of dates is mated with the list of Ringgo -M/C locations ***/ -AddDates as ( -Select ringgo_mc_code, location, B.MonthYear as MC_Date -From MC_Locations -CROSS JOIN Collect_Months as B) - -/*** Pivot the Ringgo data to total number of sessions & total paid ***/ -SELECT - ringgo_mc_code, location, MC_Date, - CASE - When SessionCount is NULL Then 0 - Else SessionCount - END as SessionCount, - CASE - When TotalPaid is NULL Then 0 - Else TotalPaid - END as TotalPaid, - - format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS import_date_timestamp, - - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date - -FROM AddDates as A -LEFT JOIN (Select parking_zone_name, parking_zone, - MonthYear, COUNT(MonthYear) As SessionCount, - SUM(cast(parking_fee as double)) as TotalPaid - FROM Ringgo_MC - GROUP BY parking_zone_name, parking_zone,MonthYear) - as B ON A.ringgo_mc_code = B.parking_zone AND - A.MC_Date = B.MonthYear -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_motorcycle_permits_480.py b/scripts/jobs/parking/parking_motorcycle_permits_480.py deleted file mode 100644 index 0ef723129..000000000 --- a/scripts/jobs/parking/parking_motorcycle_permits_480.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) -args = getResolvedOptions(sys.argv, ['JOB_NAME']) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args['JOB_NAME'], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 - liberator_permit_vrm_480 -AmazonS3liberator_permit_vrm_480_node1725467933869 = glueContext.create_dynamic_frame.from_catalog(database="dataplatform-"+environment+"-liberator-raw-zone", table_name="liberator_permit_vrm_480", transformation_ctx="AmazonS3liberator_permit_vrm_480_node1725467933869") - -# Script generated for node Amazon S3 - parking_permit_denormalised_gds_street_llpg -AmazonS3parking_permit_denormalised_gds_street_llpg_node1725467934601 = glueContext.create_dynamic_frame.from_catalog(database="dataplatform-"+environment+"-liberator-refined-zone", push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", table_name="parking_permit_denormalised_gds_street_llpg", transformation_ctx="AmazonS3parking_permit_denormalised_gds_street_llpg_node1725467934601") - -# Script generated for node Amazon S3 - liberator_permit_vrm_update_480 -AmazonS3liberator_permit_vrm_update_480_node1725468292784 = glueContext.create_dynamic_frame.from_catalog(database="dataplatform-"+environment+"-liberator-raw-zone", table_name="liberator_permit_vrm_update_480", transformation_ctx="AmazonS3liberator_permit_vrm_update_480_node1725468292784") - -# Script generated for node SQL Query - Pemit match to 480 Motorcycle flag -SqlQuery0 = ''' -/* -All Permits data in latest import_date matched to all is_motorcycle like 'Y' in the VRM_480 and VRM_update_480 tables -04/09/2024 - Created -*/ - -with lp as ( -SELECT distinct concat(permit_reference,vrm) as unique_perm_vrm,* FROM parking_permit_denormalised_gds_street_llpg -where import_date = (SELECT max(import_date) FROM parking_permit_denormalised_gds_street_llpg) -) -, vrm_mc_all as (select distinct --import_date as vrm_import_date, vrm, -concat(permit_reference,vrm) as unique_perm_vrm, -is_motorcycle from liberator_permit_vrm_480 where is_motorcycle like 'Y' --or is_motorcycle like 'N' --and import_date = (select max(import_date) from "dataplatform-prod-liberator-raw-zone".liberator_permit_vrm_480) - -Union All -select distinct concat(permit_reference,new_vrm) as unique_perm_vrm, is_motorcycle from liberator_permit_vrm_update_480 where is_motorcycle like 'Y' -- and import_date = (select max(import_date) from "dataplatform-prod-liberator-raw-zone".liberator_permit_vrm_update_480) -) - -Select distinct vrm_mc_all.is_motorcycle, lp.* from vrm_mc_all -left join lp on vrm_mc_all.unique_perm_vrm = lp.unique_perm_vrm -order by lp.application_date asc -''' -SQLQueryPemitmatchto480Motorcycleflag_node1725467937913 = sparkSqlQuery(glueContext, query = SqlQuery0, mapping = {"parking_permit_denormalised_gds_street_llpg":AmazonS3parking_permit_denormalised_gds_street_llpg_node1725467934601, "liberator_permit_vrm_480":AmazonS3liberator_permit_vrm_480_node1725467933869, "liberator_permit_vrm_update_480":AmazonS3liberator_permit_vrm_update_480_node1725468292784}, transformation_ctx = "SQLQueryPemitmatchto480Motorcycleflag_node1725467937913") - -# Script generated for node Amazon S3 - parking_motorcycle_permits_480 -AmazonS3parking_motorcycle_permits_480_node1725467946706 = glueContext.getSink(path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_motorcycle_permits_480/", connection_type="s3", updateBehavior="UPDATE_IN_DATABASE", partitionKeys=["import_year", "import_month", "import_day", "import_date"], enableUpdateCatalog=True, transformation_ctx="AmazonS3parking_motorcycle_permits_480_node1725467946706") -AmazonS3parking_motorcycle_permits_480_node1725467946706.setCatalogInfo(catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone",catalogTableName="parking_motorcycle_permits_480") -AmazonS3parking_motorcycle_permits_480_node1725467946706.setFormat("glueparquet", compression="snappy") -AmazonS3parking_motorcycle_permits_480_node1725467946706.writeFrame(SQLQueryPemitmatchto480Motorcycleflag_node1725467937913) -job.commit() diff --git a/scripts/jobs/parking/parking_pcn_daily_print_monitoring.py b/scripts/jobs/parking/parking_pcn_daily_print_monitoring.py deleted file mode 100644 index 589c41bf7..000000000 --- a/scripts/jobs/parking/parking_pcn_daily_print_monitoring.py +++ /dev/null @@ -1,198 +0,0 @@ -import sys -import time -from typing import Dict -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.dynamicframe import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, create_pushdown_predicate - -def sparkSqlQuery(glue_context: GlueContext, query: str, mapping: Dict[str, DynamicFrame], transformation_ctx: str) -> DynamicFrame: - """ - Executes a SQL query using Spark SQL and returns the result as a DynamicFrame. - """ - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = glue_context.spark_session.sql(query) - return DynamicFrame.fromDF(result, glue_context, transformation_ctx) - -start_time = time.time() -# Parsing script arguments -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) - -# Initialize Spark and Glue contexts -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Retrieve environment variable -environment = get_glue_env_var("environment") -print(f"preparing the environment variable {(time.time() - start_time)/60:.2f} minutes") - - -# Script generated for node raw_zone_liberator_pcn_tickets -# Load data from Glue Catalog tables -start_time = time.time() -raw_zone_liberator_pcn_tickets_node1666105465609 = ( - glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - table_name="liberator_pcn_tickets", - transformation_ctx="raw_zone_liberator_pcn_tickets_node1666105465609", - push_down_predicate=create_pushdown_predicate("import_date",7) - ) -) -print(f"loading raw_zone_liberator_pcn_tickets_node1666105465609 {(time.time() - start_time)/60:.2f} minutes") - - -# Script generated for node raw_zone_liberator_pcn_audit -# Load data from Glue Catalog tables -start_time = time.time() -raw_zone_liberator_pcn_audit_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - table_name="liberator_pcn_audit", - transformation_ctx="raw_zone_liberator_pcn_audit_node1", - push_down_predicate=create_pushdown_predicate("import_date",7) -) -print(f"loading raw_zone_liberator_pcn_audit_node1 {(time.time() - start_time)/60:.2f} minutes") - - -# Script generated for node ApplyMapping -# Define and execute SQL query on loaded data -start_time = time.time() -SqlQuery0 = """ -/* -Takes data from Liberator raw zone for PCN print monitoring for the previous day of the import_date - -18/10/2022 - Created query -*/ -with pcn_audit as (select -ticket_ref as pcna_ticket_ref -,audit_row as pcna_audit_row -,user_name as pcna_user_name -,audit_message as pcna_audit_message -,record_date_time as pcna_record_date_time -,event_date_time as pcna_event_date_time -,cast(event_date_time as date) as event_date -,record_created as pcna_record_created - -,import_timestamp as pcna_import_timestamp -,import_year as pcna_import_year -,import_month as pcna_import_month -,import_date as pcna_import_date -,cast(concat(import_year,'-',import_month,'-',import_day) as date) as pcna_importdate - -FROM liberator_pcn_audit where -import_date = (select max (import_date) FROM liberator_pcn_audit ) - -and cast(event_date_time as date) = cast(concat(import_year,'-',import_month,'-',import_day) as date) - interval '1' day - -and ( -upper(audit_message) LIKE 'WARNING NOTICE EXTRACTED%' or -upper(audit_message) LIKE 'PCN EXTRACTED FOR PRINT' or -upper(audit_message) LIKE 'HHTPCN PRINTED%' or -upper(audit_message) LIKE 'PCN PRINTED%' or - -upper(audit_message) LIKE 'BUSLANE PCN EXTRACTED FOR ENFORCEMENT NOTICE%' or -upper(audit_message) LIKE 'EN PRINTED%' or - -upper(audit_message) LIKE 'PCN EXTRACTED FOR NTO%' or -upper(audit_message) LIKE 'NTO PRINTED%' or - -upper(audit_message) LIKE 'PCN EXTRACTED FOR CC%' or -upper(audit_message) LIKE 'CC PRINTED%' or - -upper(audit_message) LIKE 'OFR PRINTED%' or -upper(audit_message) LIKE 'PCN EXTRACTED FOR OFR%' -) -) - -SELECT -case -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'CCTV Bus Lane' then 'BLPCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'BUSLANE PCN EXTRACTED FOR ENFORCEMENT NOTICE%' and tickettype like 'CCTV Bus Lane' then 'EN' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'CCTV Bus Lane' then 'BL CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'CCTV Bus Lane' then 'BL OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'CCTV Moving traffic' then 'MT PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'WARNING NOTICE EXTRACTED FOR PRINT%' and tickettype like 'CCTV Moving traffic' then 'MT W' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'CCTV Moving traffic' then 'MT CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'CCTV Moving traffic' then 'MT OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'CCTV static' then 'ST PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'CCTV static' then 'ST CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'CCTV static' then 'ST OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'WARNING NOTICE EXTRACTED FOR PRINT%' and tickettype like 'CCTV static' then 'ST W' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR NTO%' and tickettype like 'hht' then 'NTO' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'hht' then 'CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'hht' then 'OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'hht' then 'HHT PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN PRINTED%' and tickettype like 'CCTV Bus Lane' then 'BLPCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'EN PRINTED %' and tickettype like 'CCTV Bus Lane' then 'EN' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'CCTV Bus Lane' then 'BL CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'CCTV Bus Lane' then 'BL OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN PRINTED%' and tickettype like 'CCTV Moving traffic' then 'MT PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'CCTV Moving traffic' then 'MT CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'CCTV Moving traffic' then 'MT OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN PRINTED%' and tickettype like 'CCTV static' then 'ST PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'CCTV static' then 'ST CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'CCTV static' then 'ST OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'NTO PRINTED%' and tickettype like 'hht' then 'NTO' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'hht' then 'CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'hht' then 'OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'HHTPCN PRINTED%' and tickettype like 'hht' then 'HHT PCN' -else upper(pcn_audit.pcna_audit_message) end as print_file_type -,* -FROM pcn_audit -left join liberator_pcn_tickets on pcna_ticket_ref = ticketserialnumber and pcna_import_date = liberator_pcn_tickets.import_date - - -""" - -# Execute the SQL query to do the transformation -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "liberator_pcn_audit": raw_zone_liberator_pcn_audit_node1, - "liberator_pcn_tickets": raw_zone_liberator_pcn_tickets_node1666105465609, - }, - transformation_ctx="ApplyMapping_node2", -) -print(f"executing ApplyMapping_node2 {(time.time() - start_time)/60:.2f} minutes") - -# Script generated for node S3 bucket -# Instantiate a Glue sink for writing data to an S3 bucket -start_time = time.time() -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_pcn_daily_print_monitoring/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) - -# Set the catalog information for the data being written. This includes specifying the database and table in the Glue Data Catalog. -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_pcn_daily_print_monitoring", -) -# Set the format of the data files (Parquet) to be written to the S3 bucket. -S3bucket_node3.setFormat("glueparquet") - -# Write the data frame to the S3 bucket using the configurations defined above. -# Convert DynamicFrame to DataFrame and coalesce into a single partition -# based on my review the each day's data is less than 400kb with a single parquet file -coalescedDF = ApplyMapping_node2.toDF().coalesce(1) -# Convert back to DynamicFrame -coalescedDynamicFrame = DynamicFrame.fromDF(coalescedDF, glueContext, "coalescedDF") -# Write the coalesced DynamicFrame to the S3 bucket -S3bucket_node3.writeFrame(coalescedDynamicFrame) - -print(f"writing S3bucket_node3 {(time.time() - start_time)/60:.2f} minutes") -# Commit the job to finalize the write operation and ensure that all resources are properly released. -job.commit() - diff --git a/scripts/jobs/parking/parking_pcn_daily_print_monitoring_all.py b/scripts/jobs/parking/parking_pcn_daily_print_monitoring_all.py deleted file mode 100644 index 22f429aec..000000000 --- a/scripts/jobs/parking/parking_pcn_daily_print_monitoring_all.py +++ /dev/null @@ -1,158 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node raw_zone_liberator_pcn_tickets -raw_zone_liberator_pcn_tickets_node1666105465609 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="liberator_pcn_tickets", - transformation_ctx="raw_zone_liberator_pcn_tickets_node1666105465609", -) - -# Script generated for node raw_zone_liberator_pcn_audit -raw_zone_liberator_pcn_audit_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="liberator_pcn_audit", - transformation_ctx="raw_zone_liberator_pcn_audit_node1", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/* -Takes data from Liberator raw zone for PCN print monitoring for the previous day of the import_date - -18/10/2022 - Created query -16/11/2022 - added case statement for print file type -23/11/2022 - added print file type case statement prefix -18/11/2022 - to run for all events not just previous event date -20/04/2023 - converted to script and added predicate for source tables -*/ -with pcn_audit as (select -ticket_ref as pcna_ticket_ref -,audit_row as pcna_audit_row -,user_name as pcna_user_name -,audit_message as pcna_audit_message -,record_date_time as pcna_record_date_time -,event_date_time as pcna_event_date_time -,cast(event_date_time as date) as event_date -,record_created as pcna_record_created - -,import_timestamp as pcna_import_timestamp -,import_year as pcna_import_year -,import_month as pcna_import_month -,import_date as pcna_import_date -,cast(concat(import_year,'-',import_month,'-',import_day) as date) as pcna_importdate - -FROM liberator_pcn_audit where -import_date = (select max (import_date) FROM liberator_pcn_audit ) - ---and cast(event_date_time as date) = cast(concat(import_year,'-',import_month,'-',import_day) as date) - interval '1' day - -and ( -upper(audit_message) LIKE 'WARNING NOTICE EXTRACTED%' or -upper(audit_message) LIKE 'PCN EXTRACTED FOR PRINT' or -upper(audit_message) LIKE 'HHTPCN PRINTED%' or -upper(audit_message) LIKE 'PCN PRINTED%' or - -upper(audit_message) LIKE 'BUSLANE PCN EXTRACTED FOR ENFORCEMENT NOTICE%' or -upper(audit_message) LIKE 'EN PRINTED%' or - -upper(audit_message) LIKE 'PCN EXTRACTED FOR NTO%' or -upper(audit_message) LIKE 'NTO PRINTED%' or - -upper(audit_message) LIKE 'PCN EXTRACTED FOR CC%' or -upper(audit_message) LIKE 'CC PRINTED%' or - -upper(audit_message) LIKE 'OFR PRINTED%' or -upper(audit_message) LIKE 'PCN EXTRACTED FOR OFR%' -) -) - -SELECT -case -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'CCTV Bus Lane' then 'BLPCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'BUSLANE PCN EXTRACTED FOR ENFORCEMENT NOTICE%' and tickettype like 'CCTV Bus Lane' then 'EN' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'CCTV Bus Lane' then 'BL CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'CCTV Bus Lane' then 'BL OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'CCTV Moving traffic' then 'MT PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'WARNING NOTICE EXTRACTED FOR PRINT%' and tickettype like 'CCTV Moving traffic' then 'MT W' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'CCTV Moving traffic' then 'MT CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'CCTV Moving traffic' then 'MT OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'CCTV static' then 'ST PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'CCTV static' then 'ST CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'CCTV static' then 'ST OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'WARNING NOTICE EXTRACTED FOR PRINT%' and tickettype like 'CCTV static' then 'ST W' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR NTO%' and tickettype like 'hht' then 'NTO' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR CC%' and tickettype like 'hht' then 'CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR OFR%' and tickettype like 'hht' then 'OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN EXTRACTED FOR PRINT%' and tickettype like 'hht' then 'HHT PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN PRINTED%' and tickettype like 'CCTV Bus Lane' then 'BLPCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'EN PRINTED %' and tickettype like 'CCTV Bus Lane' then 'EN' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'CCTV Bus Lane' then 'BL CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'CCTV Bus Lane' then 'BL OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN PRINTED%' and tickettype like 'CCTV Moving traffic' then 'MT PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'CCTV Moving traffic' then 'MT CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'CCTV Moving traffic' then 'MT OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'PCN PRINTED%' and tickettype like 'CCTV static' then 'ST PCN' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'CCTV static' then 'ST CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'CCTV static' then 'ST OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'NTO PRINTED%' and tickettype like 'hht' then 'NTO' -when upper(pcn_audit.pcna_audit_message) LIKE 'CC PRINTED %' and tickettype like 'hht' then 'CC' -when upper(pcn_audit.pcna_audit_message) LIKE 'OFR PRINTED%' and tickettype like 'hht' then 'OFR' -when upper(pcn_audit.pcna_audit_message) LIKE 'HHTPCN PRINTED%' and tickettype like 'hht' then 'HHT PCN' -else upper(pcn_audit.pcna_audit_message) end as print_file_type -,* -FROM pcn_audit -left join liberator_pcn_tickets on pcna_ticket_ref = ticketserialnumber and pcna_import_date = liberator_pcn_tickets.import_date - - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "liberator_pcn_audit": raw_zone_liberator_pcn_audit_node1, - "liberator_pcn_tickets": raw_zone_liberator_pcn_tickets_node1666105465609, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_pcn_daily_print_monitoring_all/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_pcn_daily_print_monitoring_all", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_permit_denormalisation_mc.py b/scripts/jobs/parking/parking_permit_denormalisation_mc.py deleted file mode 100644 index 694968cec..000000000 --- a/scripts/jobs/parking/parking_permit_denormalisation_mc.py +++ /dev/null @@ -1,562 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "parking_permit_denormalisation_mc" - -# The exact same query prototyped in pre-prod(stg) orprod Athena -query_on_athena = """ -/************************************************************************************************************** -parking_permit_denormalisation_mc - -This query outputs a de-normalised Permits data with the addition of the Motorcycle Flag - -Code has been switched the use of the raw vrm and vrm update tables to the 480 raw tables - -02/09/2024 - Create query from 'parking_permit_denormalisation' -30/10/2024 - add code to remove the dupliacte records -****************************************************************************************************************/ -/*** Obtain the Permits start/finish for FTA and renewals ***/ -With PermitStartFinishBEFORE as ( - SELECT permit_reference, - cast(application_date as timestamp) as application_date, - cast(substr(start_date, 1, 10) as date) as start_date, - cast(substr(end_date, 1, 10) as date) as end_date - - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_fta - where import_Date = format_datetime(current_date, 'yyyyMMdd') AND - permit_reference not like 'HYV%' AND - permit_reference not like 'HYS%' AND - permit_reference not like 'HYQ%' AND - permit_reference not like 'HYF%' AND - permit_reference not like 'HYJ%' AND - permit_reference not like 'HYE%' AND - (start_date != '' OR end_date != '') AND Permit_reference not like '%BWSCO%' - UNION ALL - SELECT permit_reference, - cast(renewal_application_date as timestamp) as renewal_application_date, - cast(substr(renewal_start_date, 1, 10) as date), - cast(substr(renewal_end_date, 1, 10) as date) - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_renewals - where import_Date = format_datetime(current_date, 'yyyyMMdd') AND - permit_reference not like 'HYV%' AND - permit_reference not like 'HYS%' AND - permit_reference not like 'HYQ%' AND - permit_reference not like 'HYF%' AND - permit_reference not like 'HYJ%' AND - permit_reference not like 'HYE%' AND - (renewal_start_date != '' OR renewal_end_date != '')), - --- Index the data to identify the duplicates -PermitStartFinish as ( - SELECT - permit_reference, application_date,start_date,end_date, - ROW_NUMBER() OVER (PARTITION BY permit_reference, application_date - ORDER BY permit_reference, application_date DESC) as row_num -FROM PermitStartFinishBEFORE), -/***************************************************************************************************************/ -CalendarFormat as ( - SELECT - CAST(CASE - When calendar.date like '%/%'Then substr(calendar.date, 7, 4)||'-'||substr(calendar.date, 4,2)||'-'||substr(calendar.date, 1,2) - ELSE substr(calendar.date, 1, 10) - end as date) as date, - workingday, - holiday, - dow, - fin_year, - fin_year_startdate, - fin_year_enddate, - ROW_NUMBER() OVER ( PARTITION BY calendar.date - ORDER BY calendar.date, import_date DESC) row_num - FROM "parking-raw-zone".calendar), - -CalendarMAX as ( - Select MAX(fin_year) as Max_Fin_Year - FROM CalendarFormat), - -/***************************************************************************************************************/ -LLPG_USRN As ( - SELECT cast(uprn as varchar) as uprn, usrn, cpz_code - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_llpg - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), - - -/***************************************************************************************************************/ -/*** Select the Permit approvals ***/ -PermitApprovalBefore as ( - SELECT A.permit_reference, - cast(approval_date as timestamp) as approval_date, - approved_by, - approval_type, - ROW_NUMBER() OVER (PARTITION BY A.permit_reference, CAST(substr(approval_date, 1, 10) as date) - ORDER BY A.permit_reference, CAST(substr(approval_date, 1, 10) as date) DESC) as R1 - -FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_approval as A -WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') AND - approval_type IN ('Approved','Rejected','Renewal')), - -PermitApprovalinter as ( - SELECT A.permit_reference, approval_date,approved_by,approval_type,application_date,start_date,end_date, - /*** Index to Find the duplicates ***/ - ROW_NUMBER() OVER (PARTITION BY A.permit_reference, cast(approval_date as date) - ORDER BY A.permit_reference, approval_date, application_date DESC) as Rn - FROM PermitApprovalBefore as A - LEFT JOIN PermitStartFinish as B ON A.permit_reference = B.permit_reference AND - CAST(approval_date as date) between cast(application_date as date) AND cast(end_date as date) - AND B.row_num = 1 and R1 = 1), - -PermitApproval as ( - SELECT permit_reference, approval_date,approved_by,approval_type,application_date,start_date,end_date, - /*** Index to Find the duplicates ***/ - ROW_NUMBER() OVER (PARTITION BY permit_reference, application_date - ORDER BY permit_reference,application_date) as Rn1 - FROM PermitApprovalinter - WHERE Rn = 1), - -/***************************************************************************************************************/ -/*** Get the VRN updates and then try to link to a Permit instance ***/ -/* 02/09 - Amend to use the _480 table and to obtain the motorcycle flag*/ -VRMDetails as ( - SELECT permit_reference, - vrm, make, model, fuel, engine_capactiy, co2_emission, foreign, lpg_conversion, is_motorcycle, - record_created, - ROW_NUMBER() OVER (PARTITION BY permit_reference, vrm - ORDER BY permit_reference, vrm, co2_emission DESC) as row_num - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_vrm_480 - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') AND vrm != ''), - -/* 02/09 - Amend to use the _480 table and to obtain the motorcycle flag*/ -VRMUpdateBefore as ( - SELECT A.permit_reference, - new_vrm, - A.is_motorcycle, - CASE - When new_vrm_effective_from = '0000-00-00 00:00:00' Then cast(record_created as date) - When cast(substr(new_vrm_effective_from, 1, 10) as date) is NULL Then cast(record_created as date) - ELSE cast(substr(new_vrm_effective_from, 1,10) as date) - END as new_vrm_effective_from - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_vrm_update_480 as A - --LEFT JOIN "dataplatform-stg-liberator-raw-zone".liberator_permit_vrm as B - --ON A.permit_reference = B.permit_reference - WHERE A.import_Date = format_datetime(current_date, 'yyyyMMdd')), - -VRMUpdate as ( - SELECT A.permit_reference, - new_vrm, - A.is_motorcycle, - new_vrm_effective_from, - B.application_date as Permit_application_date, - ROW_NUMBER() OVER (PARTITION BY A.permit_reference, B.application_date - ORDER BY A.permit_reference, B.application_date, new_vrm_effective_from DESC) as row_num - FROM VRMUpdateBefore as A - LEFT JOIN PermitStartFinish as B ON A.permit_reference = B.permit_reference AND - new_vrm_effective_from between start_date AND end_date and row_num = 1), - -/*** Because Persto SQL and Spark SQL is not effective enough....Get the very latest VRM against Permit *********/ -VRMLatest as ( - SELECT permit_reference, - new_vrm, - is_motorcycle, - new_vrm_effective_from, - ROW_NUMBER() OVER (PARTITION BY permit_reference - ORDER BY permit_reference, new_vrm_effective_from DESC) as row_num - FROM VRMUpdate - WHERE row_num = 1), - -/***************************************************************************************************************/ -/*** Get the Address Updates ***/ -AddressUpdateBefore as ( - SELECT - A.permit_reference, - CAST(substr(new_address_effective_from,1, 10) as date) as new_address_effective_from, - new_uprn, new_address_line_1, new_address_line_2, new_address_line_3,new_postcode, - new_cpz, new_cpz_name, - - start_date, end_date,application_date, - - ROW_NUMBER() OVER (PARTITION BY A.permit_reference, CAST(substr(new_address_effective_from,1, 10) as date) - ORDER BY A.permit_reference, CAST(substr(address_change_order_date,1, 10) as timestamp) DESC) as row_num - - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_address_update as A - LEFT JOIN PermitStartFinish as B ON A.permit_reference = B.permit_reference AND - CAST(substr(new_address_effective_from,1, 10) as date) between application_date AND end_date AND row_num = 1 - - WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') AND - new_address_effective_from != ''), - -/*** Capture the address updates to the same permit instance ***/ -AddressUpdate as ( - SELECT permit_reference, new_address_effective_from, new_uprn,new_address_line_1, new_address_line_2, new_address_line_3, - new_postcode, new_cpz, new_cpz_name, start_date, end_date, application_date, - - ROW_NUMBER() OVER (PARTITION BY permit_reference, application_date - ORDER BY permit_reference, application_date, new_address_effective_from DESC) as row_num - FROM AddressUpdateBefore - WHERE row_num = 1), - -/***************************************************************************************************************/ -/*** Get the Address Updates ***/ -Permit_FTA as -(Select * - FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_fta as A - WHERE A.import_Date = format_datetime(current_date, 'yyyyMMdd') AND - A.permit_reference not like 'HYV%' AND - A.permit_reference not like 'HYS%' AND - A.permit_reference not like 'HYQ%' AND - A.permit_reference not like 'HYF%' AND - A.permit_reference not like 'HYJ%' AND - A.permit_reference not like 'HYE%' AND - (A.start_date != '' OR A.end_date != '') and - (email_address_of_applicant != 'Liberator.Testers@Hackney.gov.uk' OR surname_of_applicant != 'Tester' OR - surname_of_applicant != 'Test')), -/******************************************************************************************************************/ -/*** Change this to a CTE so I can add the vehicle details later **/ -Permits as ( -SELECT - A.permit_reference, - cast(A.application_date as timestamp) as application_date, - /*** Applicant details ***/ - forename_of_applicant,surname_of_applicant, - email_address_of_applicant, - primary_phone,secondary_phone, - date_of_birth_of_applicant, - /*** Blue Badge details ***/ - blue_badge_number, blue_badge_expiry , - /*** start/finish Dates ***/ - cast(substr(A.start_date, 1, 10) as date) as start_date, - cast(substr(A.end_date, 1, 10) as date) as end_date, - /*** Approval Details ***/ - B.approval_date, B.approved_by, B.approval_type, - - /*** Payment Details ***/ - amount, payment_date, payment_method, payment_location, payment_by, payment_received, - /*** Hackney internal ***/ - directorate_to_be_charged, authorising_officer, cost_code, subjective_code, - - permit_type, ordered_by, - /*** Business, etc. ***/ - business_name, hasc_organisation_name, doctors_surgery_name, - - /*** Bays, dispensation ***/ - number_of_bays, number_of_days, number_of_dispensation_vehicles, dispensation_reason, - - /*** Address / UPRN ***/ - CASE -- UPRN - When new_address_effective_from is not NULL Then new_uprn - ELSE uprn - END as uprn, - CASE -- Address Line 1 - When new_address_effective_from is not NULL Then new_address_line_1 - ELSE address_line_1 - END as address_line_1, - CASE -- Address Line 2 - When new_address_effective_from is not NULL Then new_address_line_2 - ELSE address_line_2 - END as address_line_2, - CASE -- Address Line 3 - When new_address_effective_from is not NULL Then new_address_line_3 - ELSE address_line_3 - END as address_line_3, - CASE -- Postcode - When new_address_effective_from is not NULL Then new_postcode - ELSE postcode - END as postcode, - /*** Zone ***/ - CASE -- CPZ - When new_address_effective_from is not NULL Then new_cpz - ELSE cpz - END as cpz, - CASE -- CPZ Name - When new_address_effective_from is not NULL Then new_cpz_name - ELSE cpz_name - END as cpz_name, - /*** Vehicle ***/ - status, quantity, - CASE - When D.new_vrm is not NULL Then D.new_vrm - ELSE vrm - END as vrm, - D.is_motorcycle, - associated_to_order, - /*** Flags ***/ - CASE - When current_date between cast(substr(A.start_date, 1, 10) as date) AND - cast(substr(A.end_date, 1, 10) as date) Then 1 - ELSE 0 - END as Live_Permit_Flag, - 0 as Permit_Fta_Renewal, - Status as Latest_Permit_Status, - /*** Try and trap problems with the data ***/ - ROW_NUMBER() OVER ( PARTITION BY A.permit_reference, A.start_date, A.end_date - ORDER BY A.permit_reference, A.application_date DESC) as Rn -FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_fta as A -LEFT JOIN PermitApproval as B ON A.permit_reference = B.permit_reference and - cast(A.application_date as timestamp) = B.application_date and B.Rn1 = 1 - -LEFT JOIN AddressUpdate as C ON A.permit_reference = C.permit_reference and - cast(A.application_date as timestamp) = C.application_date and C.row_num = 1 - -LEFT JOIN VRMUpdate as D ON A.permit_reference = D.permit_reference and - cast(A.application_date as timestamp) = D.permit_application_date and D.row_num = 1 - -WHERE A.import_Date = format_datetime(current_date, 'yyyyMMdd') AND - A.permit_reference not like 'HYV%' AND - A.permit_reference not like 'HYS%' AND - A.permit_reference not like 'HYQ%' AND - A.permit_reference not like 'HYF%' AND - A.permit_reference not like 'HYJ%' AND - A.permit_reference not like 'HYE%' AND - (A.start_date != '' OR A.end_date != '') and - (email_address_of_applicant != 'Liberator.Testers@Hackney.gov.uk' OR surname_of_applicant != 'Tester' OR - surname_of_applicant != 'Test') -/*** and Permit Renewals ***/ -UNION ALL -SELECT A.permit_reference, - CAST(renewal_application_date as timestamp), - /*** Applicant details ***/ - forename_of_applicant, surname_of_applicant, email_address_of_applicant, primary_phone, secondary_phone, - date_of_birth_of_applicant, - - /*** Blue Badge details, get tthis from the FTA record (where valid ***/ - CASE - When D.blue_badge_expiry != '' THEN - CASE - When cast(blue_badge_expiry as date) >= cast(substr(renewal_start_date, 1, 10) as date) Then - CASE - When cast(blue_badge_expiry as date) <= - cast(substr(renewal_end_date, 1, 10) as date) Then D.blue_badge_number - When cast(blue_badge_expiry as date) >= - cast(substr(renewal_end_date, 1, 10) as date) Then D.blue_badge_number - END - ELSE '' - END - END as blue_badge_number, - CASE - When D.blue_badge_expiry != '' THEN - CASE - When cast(blue_badge_expiry as date) >= - cast(substr(renewal_start_date, 1, 10) as date) Then - CASE - When cast(blue_badge_expiry as date) <= - cast(substr(renewal_end_date, 1, 10) as date) Then D.blue_badge_expiry - When cast(blue_badge_expiry as date) >= - cast(substr(renewal_end_date, 1, 10) as date) Then D.blue_badge_expiry - END - ELSE '' - END - END as blue_badge_expiry, - - /*** start/finish Dates ***/ - cast(substr(renewal_start_date, 1, 10) as date), - cast(substr(renewal_end_date, 1, 10) as date), - - /*** Approval Details ***/ - B.approval_date, B.approved_by, B.approval_type, - - /*** Payment Details ***/ - renewal_cost, renewal_payment_date, renewal_payment_method, renewal_payment_location, - renewal_payment_by,renewal_payment_status, - - /*** Hackney internal ***/ - A.directorate_to_be_charged, A.authorising_officer, A.cost_code, A.subjective_code, - D.permit_type, renewal_ordered_by, - - /*** Business, etc. ***/ - '', '', '', - /*** Bays, dispensation ***/ - '', '', '', '', - /*** Address / UPRN ***/ - CASE -- UPRN - When new_address_effective_from is not NULL Then new_uprn - ELSE - CASE - When renewal_new_uprn != '' Then renewal_new_uprn - ELSE uprn - END - END, - CASE -- Address Line 1 - When new_address_effective_from is not NULL Then new_address_line_1 - ELSE - CASE - When renewal_new_address_line_1 != '' Then renewal_new_address_line_1 - ELSE address_line_1 - END - END, - CASE -- Address Line 2 - When new_address_effective_from is not NULL Then new_address_line_2 - ELSE - CASE - When renewal_new_address_line_2 != '' Then renewal_new_address_line_2 - ELSE address_line_2 - END - END, - CASE -- Address Line 3 - When new_address_effective_from is not NULL Then new_address_line_3 - ELSE - CASE - When renewal_new_address_line_3 != '' Then renewal_new_address_line_3 - ELSE address_line_3 - END - END, - CASE -- Postcode - When new_address_effective_from is not NULL Then new_postcode - ELSE - CASE - When renewal_new_postcode != '' Then renewal_new_postcode - ELSE postcode - END - END, - - /*** Zone ***/ - CASE -- CPZ - When new_address_effective_from is not NULL Then new_cpz - ELSE - CASE - When renewal_new_cpz != '' Then renewal_new_cpz - ELSE cpz - END - END, - CASE -- CPZ Name - When new_address_effective_from is not NULL Then new_cpz_name - ELSE - CASE - When reneral_new_cpz_name != '' Then reneral_new_cpz_name - ELSE cpz_name - END - END, - - /*** Vehicle ***/ - approval_type, - '', - CASE - When E.new_vrm is not NULL Then E.new_vrm - When renewal_new_vrm != '' Then renewal_new_vrm - When F.new_vrm is not NULL Then F.new_vrm - ELSE D.vrm - END, - CASE - When (E.is_motorcycle like 'N' or E.is_motorcycle like 'Y') Then E.is_motorcycle - When (F.is_motorcycle like 'N' or F.is_motorcycle like 'Y') Then F.is_motorcycle - ELSE E.is_motorcycle - END, - '', - /*** Flags ***/ - CASE - When current_date between cast(substr(renewal_start_date, 1, 10) as date) AND - cast(substr(renewal_end_date, 1, 10) as date) Then 1 - ELSE 0 - END, - 1, - D.Status, - /*** Try and trap any duplicates records ***/ - ROW_NUMBER() OVER ( PARTITION BY A.permit_reference,renewal_start_date, renewal_end_date - ORDER BY A.permit_reference, renewal_application_date DESC) as Rn - -FROM "dataplatform-stg-liberator-raw-zone".liberator_permit_renewals as A - -/*LEFT JOIN PermitApproval as B ON A.permit_reference = B.permit_reference AND - approval_date between cast(substr(renewal_start_date, 1, 10) as date) AND - cast(substr(renewal_end_date, 1, 10) as date)*/ - -/*LEFT JOIN AddressUpdate as C ON A.permit_reference = C.permit_reference AND - new_address_effective_from between cast(substr(renewal_start_date, 1, 10) as date) AND - cast(substr(renewal_end_date, 1, 10) as date) AND - new_address_effective_from > CAST(substr(renewal_application_date,1,10) as date)*/ - -LEFT JOIN PermitApproval as B ON A.permit_reference = B.permit_reference and - CAST(A.renewal_application_date as timestamp) = B.application_date and B.Rn1 = 1 - -LEFT JOIN AddressUpdate as C ON A.permit_reference = C.permit_reference and - CAST(A.renewal_application_date as timestamp) = C.application_date and C.row_num = 1 - -LEFT JOIN Permit_FTA as D ON A.permit_reference = D.permit_reference - -LEFT JOIN VRMUpdate as E ON A.permit_reference = E.permit_reference and - cast(A.renewal_application_date as timestamp) = E.permit_application_date and E.row_num = 1 - -LEFT JOIN VRMLatest as F ON A.permit_reference = F.permit_reference AND F.row_num = 1 - -WHERE A.import_Date = format_datetime(current_date, 'yyyyMMdd') AND - A.permit_reference not like 'HYV%' AND - A.permit_reference not like 'HYS%' AND - A.permit_reference not like 'HYQ%' AND - A.permit_reference not like 'HYF%' AND - A.permit_reference not like 'HYJ%' AND - A.permit_reference not like 'HYE%' AND - (renewal_start_date != '' OR renewal_end_date != '')), - -/******************************************************************************************************************/ -/*** Finally output the data, WITH The VRM details **/ -/*** 30-10-2024 add code to remove the duplicates ***/ -Output as ( - SELECT - A.permit_reference, A.application_date, A.forename_of_applicant, A.surname_of_applicant, - A.email_address_of_applicant, A.primary_phone, A.secondary_phone, A.date_of_birth_of_applicant, - A.blue_badge_number, A.blue_badge_expiry, A.start_date, A.end_date, A.approval_date, - A.approved_by, A.approval_type, A.amount, A.payment_date, A.payment_method, A.payment_location, - A.payment_by, A.payment_received, A.directorate_to_be_charged, A.authorising_officer, - A.cost_code, A.subjective_code, A.permit_type, A.ordered_by, A.business_name, A.hasc_organisation_name, - A.doctors_surgery_name, A.number_of_bays, A.number_of_days, A.number_of_dispensation_vehicles, - A.dispensation_reason, A.uprn, A.address_line_1, A.address_line_2, A.address_line_3, A.postcode, - A.cpz, A.cpz_name, A.status, A.quantity, A.vrm, associated_to_order, A.Live_Permit_Flag, - A.Permit_Fta_Renewal, A.Latest_Permit_Status, G.cpz_code, - /** VRM Details ***/ - make, model, fuel, engine_capactiy, co2_emission, foreign, lpg_conversion, record_created as VRM_record_created, - CASE - When (B.is_motorcycle like 'N' or B.is_motorcycle like 'Y') Then B.is_motorcycle - When (A.is_motorcycle like 'N' or A.is_motorcycle like 'Y') Then A.is_motorcycle - ELSE B.is_motorcycle - END as is_motorcycle, - /*** identify the current year & previous year ***/ - CASE - When F.Fin_Year = (Select Max_Fin_Year From CalendarMAX) Then 'Current' - When F.Fin_Year = (Select CAST(Cast(Max_Fin_Year as int)-1 as varchar(4)) From CalendarMAX) Then 'Previous' - Else '' - END as Fin_Year_Flag, - /** Financial Year **/ - Fin_Year, usrn, - /*** 30-10-2024 - add code to enable dedupe ***/ - ROW_NUMBER() OVER ( PARTITION BY A.permit_reference, A.application_date, - A.forename_of_applicant, A.surname_of_applicant - ORDER BY A.permit_reference, A.application_date, - A.forename_of_applicant, A.surname_of_applicant DESC) as R1 - From Permits as A - LEFT JOIN VRMDetails as B ON A.permit_reference = B.permit_reference AND A.vrm = B.vrm and B.row_num = 1 - LEFT JOIN CalendarFormat as F ON CAST(A.application_date as date) = date and F.row_num = 1 - LEFT JOIN LLPG_USRN as G ON A.uprn = G.uprn - Where A.Rn = 1) - -/*** Output the deduped data ***/ -SELECT - permit_reference, application_date, forename_of_applicant, surname_of_applicant, - email_address_of_applicant, primary_phone, secondary_phone, date_of_birth_of_applicant, - blue_badge_number, blue_badge_expiry, start_date, end_date, approval_date, - approved_by, approval_type, amount, payment_date, payment_method, payment_location, - payment_by, payment_received, directorate_to_be_charged, authorising_officer, - cost_code, subjective_code, permit_type, ordered_by, business_name, hasc_organisation_name, - doctors_surgery_name, number_of_bays, number_of_days, number_of_dispensation_vehicles, - dispensation_reason, uprn, address_line_1, address_line_2, address_line_3, postcode, - cpz, cpz_name, status, quantity, vrm, associated_to_order, Live_Permit_Flag, - Permit_Fta_Renewal, Latest_Permit_Status, cpz_code, - make, model, fuel, engine_capactiy, co2_emission, foreign, lpg_conversion, VRM_record_created, - is_motorcycle, Fin_Year_Flag, Fin_Year, usrn, - - format_datetime(current_date, 'yyyy') AS import_year, - format_datetime(current_date, 'MM') AS import_month, - format_datetime(current_date, 'dd') AS import_day, - format_datetime(current_date, 'yyyyMMdd') AS import_date -FROM Output -WHERE R1 = 1 -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_permit_denormalised_gds_street_llpg.py b/scripts/jobs/parking/parking_permit_denormalised_gds_street_llpg.py deleted file mode 100644 index 45f1b2156..000000000 --- a/scripts/jobs/parking/parking_permit_denormalised_gds_street_llpg.py +++ /dev/null @@ -1,683 +0,0 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - -environment = get_glue_env_var("environment") - -# The target table in liberator refined zone -table_name = "parking_permit_denormalised_gds_street_llpg" - -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ --->> "dataplatform-prod-liberator-refined-zone"."parking_permit_denormalised_gds_street_llpg" --- < "dataplatform-prod-liberator-refined-zone"."parking_permit_denormalised_data" --- < "dataplatform-prod-liberator-raw-zone"."liberator_permit_llpg" --- < "unrestricted-raw-zone"."geolive_llpg_llpg_address" --- (replaces "dataplatform-prod-raw-zone-unrestricted-address-api"."unrestricted_address_api_dbo_hackney_address") --- < "parking-raw-zone"."ltn_london_fields" - --- 2025-01-29: "geolive_llpg_llpg_address" replaced "unrestricted_address_api_dbo_hackney_address") --- 2025-02-12 Reformatted SQL, replaced wildcards and ordered columns as per existing tables in S3 - -WITH -street AS ( - SELECT - UPRN AS SR_UPRN, - ADDRESS1 AS SR_ADDRESS1, - ADDRESS2 AS SR_ADDRESS2, - ADDRESS3 AS SR_ADDRESS3, - ADDRESS4 AS SR_ADDRESS4, - ADDRESS5 AS SR_ADDRESS5, - POST_CODE AS SR_POST_CODE, - BPLU_CLASS AS SR_BPLU_CLASS, - X AS SR_X, - Y AS SR_Y, - LATITUDE AS SR_LATITUDE, - LONGITUDE AS SR_LONGITUDE, - IN_CONGESTION_CHARGE_ZONE AS SR_IN_CONGESTION_CHARGE_ZONE, - CPZ_CODE AS SR_CPZ_CODE, - CPZ_NAME AS SR_CPZ_NAME, - LAST_UPDATED_STAMP AS SR_LAST_UPDATED_STAMP, - LAST_UPDATED_TX_STAMP AS SR_LAST_UPDATED_TX_STAMP, - CREATED_STAMP AS SR_CREATED_STAMP, - CREATED_TX_STAMP AS SR_CREATED_TX_STAMP, - HOUSE_NAME AS SR_HOUSE_NAME, - POST_CODE_PACKED AS SR_POST_CODE_PACKED, - STREET_START_X AS SR_STREET_START_X, - STREET_START_Y AS SR_STREET_START_Y, - STREET_END_X AS SR_STREET_END_X, - STREET_END_Y AS SR_STREET_END_Y, - WARD_CODE AS SR_WARD_CODE, - IF(WARD_CODE = 'E05009367', 'BROWNSWOOD WARD', - IF(WARD_CODE = 'E05009368', 'CAZENOVE WARD', - IF(WARD_CODE = 'E05009369', 'CLISSOLD WARD', - IF(WARD_CODE = 'E05009370', 'DALSTON WARD', - IF(WARD_CODE = 'E05009371', 'DE BEAUVOIR WARD', - IF(WARD_CODE = 'E05009372', 'HACKNEY CENTRAL WARD', - IF(WARD_CODE = 'E05009373', 'HACKNEY DOWNS WARD', - IF(WARD_CODE = 'E05009374', 'HACKNEY WICK WARD', - IF(WARD_CODE = 'E05009375', 'HAGGERSTON WARD', - IF(WARD_CODE = 'E05009376', 'HOMERTON WARD', - IF(WARD_CODE = 'E05009377', 'HOXTON EAST AND SHOREDITCH WARD', - IF(WARD_CODE = 'E05009378', 'HOXTON WEST WARD', - IF(WARD_CODE = 'E05009379', 'KINGS PARK WARD', - IF(WARD_CODE = 'E05009380', 'LEA BRIDGE WARD', - IF(WARD_CODE = 'E05009381', 'LONDON FIELDS WARD', - IF(WARD_CODE = 'E05009382', 'SHACKLEWELL WARD', - IF(WARD_CODE = 'E05009383', 'SPRINGFIELD WARD', - IF(WARD_CODE = 'E05009384', 'STAMFORD HILL WEST', - IF(WARD_CODE = 'E05009385', 'STOKE NEWINGTON WARD', - IF(WARD_CODE = 'E05009386', 'VICTORIA WARD', - IF(WARD_CODE = 'E05009387', 'WOODBERRY DOWN WARD', - WARD_CODE)))))))))))))))))))) - ) AS SR_ward_name, - PARISH_CODE AS SR_PARISH_CODE, - PARENT_UPRN AS SR_PARENT_UPRN, - PAO_START AS SR_PAO_START, - PAO_START_SUFFIX AS SR_PAO_START_SUFFIX, - PAO_END AS SR_PAO_END, - PAO_END_SUFFIX AS SR_PAO_END_SUFFIX, - PAO_TEXT AS SR_PAO_TEXT, - SAO_START AS SR_SAO_START, - SAO_START_SUFFIX AS SR_SAO_START_SUFFIX, - SAO_END AS SR_SAO_END, - SAO_END_SUFFIX AS SR_SAO_END_SUFFIX, - SAO_TEXT AS SR_SAO_TEXT, - DERIVED_BLPU AS SR_DERIVED_BLPU, - USRN, - import_year, - import_month, - import_day, - import_date - FROM - "dataplatform-prod-liberator-raw-zone"."liberator_permit_llpg" - WHERE import_date = ( - SELECT MAX(import_date) - FROM "dataplatform-prod-liberator-raw-zone"."liberator_permit_llpg" - ) - AND ( - ADDRESS1 LIKE 'Street Record' - OR ADDRESS1 LIKE 'STREET RECORD' - ) -), - -llpg AS ( - SELECT - -- conversion of new column names and data-types to support the original query... - uprn, -- bigint -- Confirmed: was previously converted from double to bigint to prevent subsequent varchar casts being broken. - usrn, -- int - street_description, -- string - isparent AS property_shell, -- boolean - blpu_class, -- string - usage_primary, -- string - usage_description, -- string - planning_use_class, -- string - CAST(longitude AS DOUBLE) AS longitude, -- converts string to double - CAST(latitude AS DOUBLE) AS latitude, -- converts string to double - lpi_logical_status_code, -- int BS7666 code - lpi_logical_status, --string logical description as per previous API version - import_date -- string - FROM - "unrestricted-raw-zone"."geolive_llpg_llpg_address" - -- replaces "dataplatform-prod-raw-zone-unrestricted-address-api"."unrestricted_address_api_dbo_hackney_address" - WHERE - import_date = ( - SELECT - MAX(import_date) - FROM - "unrestricted-raw-zone"."geolive_llpg_llpg_address" - ) - AND lpi_logical_status_code = 1 -- like 'Approved Preferred' -) - -SELECT - street.usrn AS sr_usrn, - SR_ADDRESS1, - SR_ADDRESS2, - llpg.street_description, - SR_WARD_CODE, - SR_ward_name, - llpg.property_shell, - llpg.blpu_class, - llpg.usage_primary, - llpg.usage_description, - concat (CAST(street.usrn AS VARCHAR), ' - ', llpg.street_description) AS street, - concat (CAST(llpg.uprn AS VARCHAR), ' - ', llpg.usage_description) AS add_type, - concat (llpg.blpu_class, ' - ', llpg.planning_use_class) AS add_class, - CASE WHEN cpz != '' - AND cpz_name != '' THEN concat (cpz, ' - ', cpz_name) - WHEN cpz != '' - AND cpz_name = '' THEN concat (cpz, '-') - WHEN cpz = '' - AND cpz_name != '' THEN concat (cpz_name, '-') - ELSE 'NONE' - END AS zone_name, - CASE WHEN address_line_2 = '' - AND business_name = '' - AND hasc_organisation_name = '' - AND doctors_surgery_name = '' THEN concat ( - permit_reference, - ' - ', - address_line_1, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' - AND hasc_organisation_name != '' - AND address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - hasc_organisation_name, - ' - ', - address_line_1, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' - AND doctors_surgery_name != '' - AND address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - doctors_surgery_name, - ' - ', - address_line_1, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' - AND address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - address_line_1, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN hasc_organisation_name != '' - AND address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - hasc_organisation_name, - ' - ', - address_line_1, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN doctors_surgery_name != '' - AND address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - doctors_surgery_name, - ' - ', - address_line_1, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN address_line_3 = '' - AND business_name = '' - AND hasc_organisation_name = '' - AND doctors_surgery_name = '' THEN concat ( - permit_reference, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' - AND hasc_organisation_name != '' - AND address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - hasc_organisation_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' - AND doctors_surgery_name != '' - AND address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - doctors_surgery_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' - AND address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN hasc_organisation_name != '' - AND address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - hasc_organisation_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN doctors_surgery_name != '' - AND address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - doctors_surgery_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN business_name != '' THEN concat ( - permit_reference, - ' - ', - business_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN hasc_organisation_name != '' THEN concat ( - permit_reference, - ' - ', - hasc_organisation_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - WHEN doctors_surgery_name != '' THEN concat ( - permit_reference, - ' - ', - doctors_surgery_name, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - ELSE concat ( - permit_reference, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode, - ' - ', - email_address_of_applicant - ) - END AS permit_summary, - CASE WHEN address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - address_line_1, - ', ', - p.postcode - ) - WHEN address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode - ) - ELSE concat ( - permit_reference, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode - ) - END AS permit_full_address, - CASE WHEN address_line_2 = '' THEN concat (address_line_1, ', ', p.postcode) - WHEN address_line_3 = '' THEN concat ( - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode - ) - ELSE concat ( - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode - ) - END AS full_address, - CASE WHEN address_line_2 = '' THEN concat ( - permit_reference, - ' - ', - usage_primary, - ' - ', - address_line_1, - ', ', - p.postcode - ) - WHEN address_line_3 = '' THEN concat ( - permit_reference, - ' - ', - usage_primary, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode - ) - ELSE concat ( - permit_reference, - ' - ', - usage_primary, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode - ) - END AS permit_full_address_type, - CASE WHEN address_line_2 = '' THEN concat ( - usage_primary, - ' - ', - address_line_1, - ', ', - p.postcode - ) - WHEN address_line_3 = '' THEN concat ( - usage_primary, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - p.postcode - ) - ELSE concat ( - usage_primary, - ' - ', - address_line_1, - ', ', - address_line_2, - ', ', - address_line_3, - ', ', - p.postcode - ) - END AS full_address_type, - CASE WHEN latest_permit_status IN ( - 'Approved', - 'Renewed', - 'Created', - 'ORDER_APPROVED', - 'PENDING_VRM_CHANGE', - 'RENEW_EVID', - 'PENDING_ADDR_CHANGE' - ) - AND live_permit_flag = 1 THEN 1 - ELSE 0 - END AS live_flag, - CASE WHEN live_permit_flag = 1 - AND blue_badge_number != '' - AND permit_type LIKE 'Estate Resident' - AND ( - amount LIKE '0.00' - OR amount LIKE '0.000' - OR amount LIKE '' - ) THEN 1 - ELSE 0 - END AS flag_lp_est_bb_zero, - CASE WHEN live_permit_flag = 1 - AND blue_badge_number != '' - AND permit_type != 'Estate Resident' THEN 1 - ELSE 0 - END AS flag_lp_bb_onst, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Estate Resident' THEN 1 - ELSE 0 - END AS flag_lp_est, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Business' THEN 1 - ELSE 0 - END AS flag_lp_bus, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Doctor' THEN 1 - ELSE 0 - END AS flag_lp_doc, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Leisure centre permit' THEN 1 - ELSE 0 - END AS flag_lp_lc, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Health and Social Care' THEN 1 - ELSE 0 - END AS flag_lp_hsc, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Residents' THEN 1 - ELSE 0 - END AS flag_lp_res, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Dispensation' THEN 1 - ELSE 0 - END AS flag_lp_disp, - CASE WHEN live_permit_flag = 1 - AND permit_type NOT IN ( - 'Estate Resident', - 'All Zone', - 'Companion Badge', - 'Dispensation', - 'Residents', - 'Health and Social Care', - 'Leisure centre permit', - 'Doctor', - 'Business' - ) THEN 1 - ELSE 0 - END AS flag_lp_othr, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'All Zone' THEN 1 - ELSE 0 - END AS flag_lp_al, - CASE WHEN live_permit_flag = 1 - AND permit_type LIKE 'Companion Badge' THEN 1 - ELSE 0 - END AS flag_lp_cb, - - /* fields in ltn table -- ltn.uprn, ltn.usrn, ltn.street_name*/ - CASE WHEN ltn.uprn != '' THEN 1 - ELSE 0 - END AS flag_ltn_london_fields, - CASE WHEN ltn.uprn != '' THEN 'LTN London Fields' - ELSE 'NOT LTN London Fields' - END AS flag_name_ltn_london_fields, - - /* parking_permit_denormalised_data.* */ - p.permit_reference, - p.application_date, - p.forename_of_applicant, - p.surname_of_applicant, - p.email_address_of_applicant, - p.primary_phone, - p.secondary_phone, - p.date_of_birth_of_applicant, - p.blue_badge_number, - p.blue_badge_expiry, - p.start_date, - p.end_date, - p.approval_date, - p.approved_by, - p.approval_type, - p.amount, - p.payment_date, - p.payment_method, - p.payment_location, - p.payment_by, - p.payment_received, - p.directorate_to_be_charged, - p.authorising_officer, - p.cost_code, - p.subjective_code, - p.permit_type, - p.ordered_by, - p.business_name, - p.hasc_organisation_name, - p.doctors_surgery_name, - p.number_of_bays, - p.number_of_days, - p.number_of_dispensation_vehicles, - p.dispensation_reason, - p.uprn, - p.address_line_1, - p.address_line_2, - p.address_line_3, - p.postcode, - p.cpz, - p.cpz_name, - p.status, - p.quantity, - p.vrm, - p.associated_to_order, - p.live_permit_flag, - p.permit_fta_renewal, - p.latest_permit_status, - p.rn, - p.make, - p.model, - p.fuel, - p.engine_capactiy, - p.co2_emission, - p.foreign, - p.lpg_conversion, - p.vrm_record_created, - p.fin_year_flag, - p.fin_year, - p.usrn, - p.importdatetime, - - /* LLPG enrichment */ - llpg.planning_use_class, - llpg.longitude, - llpg.latitude, - - /* Partition columns from source table go last */ - p.import_year, - p.import_month, - p.import_day, - p.import_date - -FROM ( - SELECT * - FROM "dataplatform-prod-liberator-refined-zone"."parking_permit_denormalised_data" - WHERE import_date = ( - SELECT MAX(import_date) - FROM "dataplatform-prod-liberator-refined-zone"."parking_permit_denormalised_data" - ) - ) p -LEFT JOIN llpg - ON CAST(llpg.uprn AS VARCHAR) = CAST(p.uprn AS VARCHAR) - /*and - cast(concat(p.import_year,p.import_month,p.import_day) as varchar - ) = llpg.import_date*/ -LEFT JOIN street - ON CAST(street.usrn AS VARCHAR) = CAST(llpg.usrn AS VARCHAR) -LEFT JOIN ( - SELECT * - FROM "parking-raw-zone"."ltn_london_fields" - WHERE import_date = ( - SELECT MAX(import_date) - FROM "parking-raw-zone"."ltn_london_fields" - ) - ) ltn - ON ltn.uprn = p.uprn -; -""" - -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name -) diff --git a/scripts/jobs/parking/parking_reps_and_appeals_correspondance.py b/scripts/jobs/parking/parking_reps_and_appeals_correspondance.py deleted file mode 100644 index f13071448..000000000 --- a/scripts/jobs/parking/parking_reps_and_appeals_correspondance.py +++ /dev/null @@ -1,121 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.dynamicframe import DynamicFrame - -from scripts.helpers.helpers import get_glue_env_var -environment = get_glue_env_var("environment") - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - -SqlQuery0 = ''' -/************************************************************************************************************** -Correspondance Reps & Appeals Response period - -Percentage response to parking representations - -12/08/2021 - create the query -****************************************************************************************************************/ -WITH Postal_Response as ( -SELECT - ticketserialnumber, - cast(substr(date_received, 1, 10) as date) as date_received, - cast(substr(response_generated_at, 1, 10) as date) as response_generated_at, - concat(substr(Cast(date_received as varchar(10)),1, 7), '-01') as MonthYear, - Datediff(cast(substr(response_generated_at, 1, 10) as date), - cast(substr(date_received, 1, 10) as date)) as ResponseDays -from liberator_pcn_ic -where import_Date = (Select MAX(import_date) from liberator_pcn_ic) -AND date_received != '' AND response_generated_at != '' -AND length(ticketserialnumber) = 10 AND -Serviceable IN ('Challenges','Key worker','Removals','TOL','Charge certificate','Representations')), - -ResponseTotals as( -SELECT MonthYear, cast(count(*) as decimal) as TotalResponded, - SUM(Case - When ResponseDays <= 5 Then 1 - ELSE 0 - END) as LessThanEqual5, - SUM(Case - When ResponseDays >= 6 AND ResponseDays <=14 Then 1 - ELSE 0 - END) as SixToFourteen, - SUM(Case - When ResponseDays >= 15 AND ResponseDays <=47 Then 1 - ELSE 0 - END) as FifteenToFortySeven, - SUM(Case - When ResponseDays >= 48 AND ResponseDays <=56 Then 1 - ELSE 0 - END) as FortyEightToFiftySix, - SUM(Case - When ResponseDays > 56 Then 1 - ELSE 0 - END) as GTFiftySix - -FROM Postal_Response -GROUP BY MonthYear) -/*** Output report data ***/ -SELECT - MonthYear, - -- <= 5 - CAST((LessThanEqual5/cast(TotalResponded as double))*100 as decimal(10,2)) as Period1, - LessThanEqual5 as Period1Total, - -- >= 6 to <= 14 - CAST((SixToFourteen/cast(TotalResponded as double))*100 as decimal(10,2)) as Period2, - SixToFourteen as Period2Total, - -- >=15 to <= 47 - CAST((FifteenToFortySeven/cast(TotalResponded as double))*100 as decimal(10,2)) as Period3, - FifteenToFortySeven as Period3Total, - -- >=48 to <= 56 - CAST((FortyEightToFiftySix/cast(TotalResponded as double))*100 as decimal(10,2)) as Period4, - FortyEightToFiftySix as Period4Total, - -- >= 57 - CAST((GTFiftySix/cast(TotalResponded as double))*100 as decimal(10,2)) as Period5, - GTFiftySix as Period5Total, - - /*** Control Dates ***/ - substr(Cast(current_date as varchar(10)),1, 4) as import_year, - substr(Cast(current_date as varchar(10)),6, 2) as import_month, - substr(Cast(current_date as varchar(10)),9, 4) as import_day, - Cast(current_date as varchar(10)) as import_date - -FROM ResponseTotals -order by MonthYear -''' - -## @params: [JOB_NAME] -args = getResolvedOptions(sys.argv, ['JOB_NAME']) - -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args['JOB_NAME'], args) -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_pcn_ic", transformation_ctx = "DataSource0"] -## @return: DataSource0 -## @inputs: [] -DataSource0 = glueContext.create_dynamic_frame.from_catalog(database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_pcn_ic", transformation_ctx = "DataSource0") -## @type: SqlCode -## @args: [sqlAliases = {"liberator_pcn_ic": DataSource0}, sqlName = SqlQuery0, transformation_ctx = "Transform0"] -## @return: Transform0 -## @inputs: [dfc = DataSource0] -Transform0 = sparkSqlQuery(glueContext, query = SqlQuery0, mapping = {"liberator_pcn_ic": DataSource0}, transformation_ctx = "Transform0") -## @type: DataSink -## @args: [connection_type = "s3", catalog_database_name = "dataplatform-" + environment + "-liberator-refined-zone", format = "glueparquet", connection_options = {"path": "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Rep_and_Appeals_Corresp/", "partitionKeys": ["import_year" ,"import_month" ,"import_day" ,"import_date"], "enableUpdateCatalog":true, "updateBehavior":"UPDATE_IN_DATABASE"}, catalog_table_name = "Parking_Reps_and_Appeals_Correspondance_KPI", transformation_ctx = "DataSink0"] -## @return: DataSink0 -## @inputs: [frame = Transform0] -DataSink0 = glueContext.getSink(path = "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Rep_and_Appeals_Corresp/", connection_type = "s3", updateBehavior = "UPDATE_IN_DATABASE", partitionKeys = ["import_year","import_month","import_day","import_date"], enableUpdateCatalog = True, transformation_ctx = "DataSink0") -DataSink0.setCatalogInfo(catalogDatabase = "dataplatform-" + environment + "-liberator-refined-zone",catalogTableName = "Parking_Reps_and_Appeals_Correspondance_KPI") -DataSink0.setFormat("glueparquet") -DataSink0.writeFrame(Transform0) - -job.commit() \ No newline at end of file diff --git a/scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary.py b/scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary.py deleted file mode 100644 index 692e8caec..000000000 --- a/scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary.py +++ /dev/null @@ -1,114 +0,0 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.dynamicframe import DynamicFrame - -from scripts.helpers.helpers import get_glue_env_var, create_pushdown_predicate -environment = get_glue_env_var("environment") - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - -SqlQuery0 = ''' -/************************************************************************************************************** -Correspondance Reps & Appeals Response period - -Number Summary response to parking representations for GDS - -12/08/2021 - create the query -****************************************************************************************************************/ -WITH Postal_Response as ( -SELECT - ticketserialnumber, - cast(substr(date_received, 1, 10) as date) as date_received, - cast(substr(response_generated_at, 1, 10) as date) as response_generated_at, - concat(substr(Cast(date_received as varchar(10)),1, 7), '-01') as MonthYear, - Datediff(cast(substr(response_generated_at, 1, 10) as date), - cast(substr(date_received, 1, 10) as date)) as ResponseDays -from liberator_pcn_ic -where import_Date = (Select MAX(import_date) from liberator_pcn_ic) -AND date_received != '' AND response_generated_at != '' -AND length(ticketserialnumber) = 10 AND -Serviceable IN ('Challenges','Key worker','Removals','TOL','Charge certificate','Representations')) - -SELECT MonthYear, cast(count(*) as decimal) as TotalResponded, - SUM(Case - When ResponseDays <= 5 Then 1 - ELSE 0 - END) as LessThanEqual5, - SUM(Case - When ResponseDays >= 6 AND ResponseDays <=14 Then 1 - ELSE 0 - END) as SixToFourteen, - SUM(Case - When ResponseDays >= 15 AND ResponseDays <=47 Then 1 - ELSE 0 - END) as FifteenToFortySeven, - SUM(Case - When ResponseDays >= 48 AND ResponseDays <=56 Then 1 - ELSE 0 - END) as FortyEightToFiftySix, - SUM(Case - When ResponseDays > 56 Then 1 - ELSE 0 - END) as GTFiftySix, - SUM(Case - When ResponseDays <= 56 Then 1 - ELSE 0 - END) as kpiTotFiftySixLess, - SUM(Case - When ResponseDays <= 14 Then 1 - ELSE 0 - END) as kpiTotFourteenLess, - - /*** Control Dates ***/ - substr(Cast(current_date as varchar(10)),1, 4) as import_year, - substr(Cast(current_date as varchar(10)),6, 2) as import_month, - substr(Cast(current_date as varchar(10)),9, 4) as import_day, - Cast(current_date as varchar(10)) as import_date - -FROM Postal_Response -GROUP BY MonthYear -Order by MonthYear Desc - -''' - -## @params: [JOB_NAME] -args = getResolvedOptions(sys.argv, ['JOB_NAME']) - -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args['JOB_NAME'], args) -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_pcn_ic", transformation_ctx = "DataSource0"] -## @return: DataSource0 -## @inputs: [] -DataSource0 = glueContext.create_dynamic_frame.from_catalog( - database = "dataplatform-" + environment + "-liberator-raw-zone", - table_name = "liberator_pcn_ic", - transformation_ctx = "DataSource0", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) -## @type: SqlCode -## @args: [sqlAliases = {"liberator_pcn_ic": DataSource0}, sqlName = SqlQuery0, transformation_ctx = "Transform0"] -## @return: Transform0 -## @inputs: [dfc = DataSource0] -Transform0 = sparkSqlQuery(glueContext, query = SqlQuery0, mapping = {"liberator_pcn_ic": DataSource0}, transformation_ctx = "Transform0") -## @type: DataSink -## @args: [connection_type = "s3", catalog_database_name = "dataplatform-" + environment + "-liberator-refined-zone", format = "glueparquet", connection_options = {"path": "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Rep_and_Appeals_Corresp_KPI_GDS_Summary/", "partitionKeys": ["import_year" ,"import_month" ,"import_day" ,"import_date"], "enableUpdateCatalog":true, "updateBehavior":"UPDATE_IN_DATABASE"}, catalog_table_name = "parking_reps_and_appeals_correspondance_kpi_gds_summary", transformation_ctx = "DataSink0"] -## @return: DataSink0 -## @inputs: [frame = Transform0] -DataSink0 = glueContext.getSink(path = "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Rep_and_Appeals_Corresp_KPI_GDS_Summary/", connection_type = "s3", updateBehavior = "UPDATE_IN_DATABASE", partitionKeys = ["import_year","import_month","import_day","import_date"], enableUpdateCatalog = True, transformation_ctx = "DataSink0") -DataSink0.setCatalogInfo(catalogDatabase = "dataplatform-" + environment + "-liberator-refined-zone",catalogTableName = "parking_reps_and_appeals_correspondance_kpi_gds_summary") -DataSink0.setFormat("glueparquet") -DataSink0.writeFrame(Transform0) - -job.commit() \ No newline at end of file diff --git a/scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary_qtr.py b/scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary_qtr.py deleted file mode 100644 index 6d10263ff..000000000 --- a/scripts/jobs/parking/parking_reps_and_appeals_correspondance_kpi_gds_summary_qtr.py +++ /dev/null @@ -1,118 +0,0 @@ -import sys - -from awsglue.context import GlueContext -from awsglue.dynamicframe import DynamicFrame -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var - -environment = get_glue_env_var("environment") - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - -SqlQuery0 = ''' -/************************************************************************************************************** -Correspondance Reps & Appeals Response period - -Number Summary response to parking representations for GDS by Quarters - -12/08/2021 - create the query -****************************************************************************************************************/ -WITH Postal_Response as ( -SELECT - ticketserialnumber, - cast(substr(date_received, 1, 10) as date) as date_received, - cast(substr(response_generated_at, 1, 10) as date) as response_generated_at, - -- concat(substr(Cast(date_received as varchar(10)),1, 7), '-01') as MonthYear, - concat('Q',Cast(extract(quarter from date_received) as varchar(2)),' ',cast(extract(year from date_received)as varchar(4))) AS date_received_quarter, - concat(extract(year from date_received),extract(quarter from date_received)) AS quarter_sort, - Datediff(cast(substr(response_generated_at, 1, 10) as date), - cast(substr(date_received, 1, 10) as date)) as ResponseDays -from liberator_pcn_ic -where import_Date = (Select MAX(import_date) from liberator_pcn_ic) -AND date_received != '' AND response_generated_at != '' -AND length(ticketserialnumber) = 10 AND -Serviceable IN ('Challenges','Key worker','Removals','TOL','Charge certificate','Representations')) - -SELECT date_received_quarter, quarter_sort, cast(count(*) as decimal) as TotalResponded, - SUM(Case - When ResponseDays <= 5 Then 1 - ELSE 0 - END) as LessThanEqual5, - SUM(Case - When ResponseDays >= 6 AND ResponseDays <=14 Then 1 - ELSE 0 - END) as SixToFourteen, - SUM(Case - When ResponseDays >= 15 AND ResponseDays <=47 Then 1 - ELSE 0 - END) as FifteenToFortySeven, - SUM(Case - When ResponseDays >= 48 AND ResponseDays <=56 Then 1 - ELSE 0 - END) as FortyEightToFiftySix, - SUM(Case - When ResponseDays > 56 Then 1 - ELSE 0 - END) as GTFiftySix, - SUM(Case - When ResponseDays <= 56 Then 1 - ELSE 0 - END) as kpiTotFiftySixLess, - SUM(Case - When ResponseDays <= 14 Then 1 - ELSE 0 - END) as kpiTotFourteenLess, - - /*** Control Dates ***/ - substr(Cast(current_date as varchar(10)),1, 4) as import_year, - substr(Cast(current_date as varchar(10)),6, 2) as import_month, - substr(Cast(current_date as varchar(10)),9, 4) as import_day, - Cast(current_date as varchar(10)) as import_date - -FROM Postal_Response -GROUP BY date_received_quarter,quarter_sort -Order by quarter_sort Desc - -''' - -## @params: [JOB_NAME] -args = getResolvedOptions(sys.argv, ['JOB_NAME']) - -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args['JOB_NAME'], args) -## @type: DataSource -## @args: [database = "dataplatform-" + environment + "-liberator-raw-zone", table_name = "liberator_pcn_ic", transformation_ctx = "DataSource0"] -## @return: DataSource0 -## @inputs: [] -DataSource0 = glueContext.create_dynamic_frame.from_catalog( - database = "dataplatform-" + environment + "-liberator-raw-zone", - table_name = "liberator_pcn_ic", - transformation_ctx = "DataSource0", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) -## @type: SqlCode -## @args: [sqlAliases = {"liberator_pcn_ic": DataSource0}, sqlName = SqlQuery0, transformation_ctx = "Transform0"] -## @return: Transform0 -## @inputs: [dfc = DataSource0] -Transform0 = sparkSqlQuery(glueContext, query = SqlQuery0, mapping = {"liberator_pcn_ic": DataSource0}, transformation_ctx = "Transform0") -## @type: DataSink -## @args: [connection_type = "s3", catalog_database_name = "dataplatform-" + environment + "-liberator-refined-zone", format = "glueparquet", connection_options = {"path": "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Rep_and_Appeals_Corresp_KPI_GDS_Summary_QTR/", "partitionKeys": ["import_year" ,"import_month" ,"import_day" ,"import_date"], "enableUpdateCatalog":true, "updateBehavior":"UPDATE_IN_DATABASE"}, catalog_table_name = "parking_reps_and_appeals_correspondance_kpi_gds_summary_qtr", transformation_ctx = "DataSink0"] -## @return: DataSink0 -## @inputs: [frame = Transform0] -DataSink0 = glueContext.getSink(path = "s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Rep_and_Appeals_Corresp_KPI_GDS_Summary_QTR/", connection_type = "s3", updateBehavior = "UPDATE_IN_DATABASE", partitionKeys = ["import_year","import_month","import_day","import_date"], enableUpdateCatalog = True, transformation_ctx = "DataSink0") -DataSink0.setCatalogInfo(catalogDatabase = "dataplatform-" + environment + "-liberator-refined-zone",catalogTableName = "parking_reps_and_appeals_correspondance_kpi_gds_summary_qtr") -DataSink0.setFormat("glueparquet") -DataSink0.writeFrame(Transform0) - -job.commit() \ No newline at end of file diff --git a/scripts/jobs/parking/parking_school_street_vrms.py b/scripts/jobs/parking/parking_school_street_vrms.py deleted file mode 100644 index a52bd069b..000000000 --- a/scripts/jobs/parking/parking_school_street_vrms.py +++ /dev/null @@ -1,109 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node S3 - liberator_refined - parking_permit_denormalised_gds_street_llpg -S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_permit_denormalised_gds_street_llpg", - transformation_ctx="S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1647531223393 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="school_street_uprn", - transformation_ctx="AmazonS3_node1647531223393", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/********************************************************************************* -Parking_School_Street_VRMs - -This SQL creates the list of VRMs against the current valid Permits - -02/03/2022 - Create Query -*********************************************************************************/ -With Excluded_UPRNs as ( - SELECT uprn as excluded_uprns, excluded_location - FROM school_street_uprn) - -Select - permit_reference, - cast(application_date as timestamp) as application_date, - cast(substr(cast(start_date as varchar(10)), 1, 10) as date) as start_date, - cast(substr(cast(end_date as varchar(10)), 1, 10) as date) as end_date, - A.uprn, - vrm, - make, model, fuel, engine_capactiy, co2_emission, - - excluded_location as SS_Location, status, - - current_timestamp as ImportDateTime, - - import_date, - import_year, - import_month, - import_day - - FROM parking_permit_denormalised_gds_street_llpg as A - INNER JOIN Excluded_UPRNs as B ON A.uprn = B.excluded_uprns - Where Import_date = - (Select MAX(Import_date) FROM parking_permit_denormalised_gds_street_llpg) - AND substr(permit_reference,1,3) IN ('HYR', 'HYG','HYP') - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_permit_denormalised_gds_street_llpg": S3liberator_refinedparking_permit_denormalised_gds_street_llpg_node1, - "school_street_uprn": AmazonS3_node1647531223393, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_School_Street_VRMs/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_School_Street_VRMs", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_shop_front_licence_totals.py b/scripts/jobs/parking/parking_shop_front_licence_totals.py deleted file mode 100644 index 18837dcec..000000000 --- a/scripts/jobs/parking/parking_shop_front_licence_totals.py +++ /dev/null @@ -1,189 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - create_pushdown_predicate, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 -AmazonS3_node1636704737623 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="calendar", - transformation_ctx="AmazonS3_node1636704737623", -) - -# Script generated for node Amazon S3 -AmazonS3_node1638358321513 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_licence_licence_full", - transformation_ctx="AmazonS3_node1638358321513", - push_down_predicate=create_pushdown_predicate("import_date", 1), -) - -# Script generated for node ApplyMapping -SqlQuery28 = """ -/************************************************************************************************************************* -Parking_Shop_Front_Licence_Totals - -The SQL builds the total number of SF licences extant on the 28th of each month - -01/12/2021 - Create SQL. -*************************************************************************************************************************/ -/*** Collect the 28th day of each month ***/ -With Calendar_Data as ( - SELECT - date as Calendar_date, workingday, dow, holiday, - - ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date, import_date DESC) row_num - - FROM calendar - WHERE date like '%28%'), -/*** Format the calendar date ***/ -CalendarFormat as ( - SELECT - Calendar_date, - CAST(CASE - When Calendar_date like '%/%'Then substr(Calendar_date, 7, 4)||'-'|| - substr(Calendar_date, 4, 2)||'-'||substr(Calendar_date, 1, 2) - ELSE substr(Calendar_date, 1, 10) - END as date) as Format_date - FROM Calendar_Data - WHERE row_num = 1), - -/*** Get the Shop front data ***/ -ShopFront_Before as ( - SELECT - licence_ref, licence_type, licence_address, application_date, - - CASE - When lower(licence_type) = 'sf-newtemp' Then 'Temp' - When lower(licence_type) = 'sf-newperm' Then 'Perm' - ELSE 'Temp' - END as Licence_Type_PermTemp, - - CAST(CASE - When requested_start_date like '%/%'Then substr(requested_start_date, 7,4)||'-'|| - substr(requested_start_date, 4,2)||'-'|| - substr(requested_start_date, 1,2) - ELSE substr(requested_start_date, 1, 10) - end as date) as requested_start_date, - - CAST(CASE - WHEN length(actual_start_date) is NULL Then - CASE - When requested_start_date like '%/%'Then substr(requested_start_date, 7,4)||'-'|| - substr(requested_start_date, 4,2)||'-'|| - substr(requested_start_date, 1,2) - ELSE substr(requested_start_date, 1, 10) - END - WHEN requested_start_date like '%/%'Then substr(actual_start_date, 7,4)||'-'|| - substr(actual_start_date, 4,2)||'-'|| - substr(actual_start_date, 1,2) - ELSE substr(actual_start_date, 1, 10) - END as date) as actual_start_date - - FROM liberator_licence_licence_FULL - WHERE import_Date = (Select MAX(import_date) from liberator_licence_licence_full) and licence_ref like 'SF%' - AND licence_type != 'SF-application'), - -ShopFront as ( - SELECT - *, - CAST(CASE - When Licence_Type_PermTemp = 'Temp' Then date_add(actual_start_date, 183) - When Licence_Type_PermTemp = 'Perm' Then date_add(actual_start_date, 365) - END as date) as End_Date, - - cast(substr(cast(actual_start_date as string), 1, 8)||'01' as date) as MonthYear - - from ShopFront_Before), - -/** total the number of 'temp' licences active on the 28th of each month **/ -Temp_Shopfront as ( - Select - Format_date, - count(*) as No_Temp_Licences - - From CalendarFormat as A, ShopFront as B - Where A.Format_date between B.actual_start_date and B.End_Date AND Licence_Type_PermTemp = 'Temp' - group by Format_date), - -/** total the number of 'temp' licences active on the 28th of each month **/ -Perm_Shopfront as ( - Select - Format_date, - count(*) as No_Perm_Licences - - From CalendarFormat as A, ShopFront as B - Where A.Format_date between B.actual_start_date and B.End_Date AND Licence_Type_PermTemp = 'Perm' - group by Format_date) - -/*** Output the results ***/ -SELECT - A.Format_date, - No_Temp_Licences, - No_Perm_Licences, - - current_timestamp as ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM Temp_Shopfront as A -LEFT JOIN Perm_Shopfront as B ON A.Format_date = B.Format_date -WHERE A.Format_date <= current_date -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery28, - mapping={ - "calendar": AmazonS3_node1636704737623, - "liberator_licence_licence_FULL": AmazonS3_node1638358321513, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Shop_Front_Licence_Totals/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Shop_Front_Licence_Totals", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_visitor_voucher_qtrly_review.py b/scripts/jobs/parking/parking_visitor_voucher_qtrly_review.py deleted file mode 100644 index da66ffc21..000000000 --- a/scripts/jobs/parking/parking_visitor_voucher_qtrly_review.py +++ /dev/null @@ -1,323 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -# add variable -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 -AmazonS3_node1648207907397 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="calendar", - transformation_ctx="AmazonS3_node1648207907397", -) - -# Script generated for node Amazon S3 -AmazonS3_node1648208237020 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - table_name="parking_voucher_de_normalised", - transformation_ctx="AmazonS3_node1648208237020", -) - -# Script generated for node Amazon S3 -AmazonS3_node1652190326034 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - table_name="visitor_voucher_forecast", - transformation_ctx="AmazonS3_node1652190326034", -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/********************************************************************************* -Parking_Visitor_Voucher_Review - -This SQL totals the number of Visitor Voucher books (for Estate, 1 Day & 2 Hour Vouchers) -by Voucher Type and Quarter - -22/03/2022 - Create Query -10/05/2022 - rewrite query to meet Michaels requirements -*********************************************************************************/ -With Calendar_Data as ( - SELECT - date as Calendar_date, workingday, dow, holiday,fin_year, - CASE - When cast(substr(date, 6,2) as int) = 1 Then 'Q4' -- Jan - When cast(substr(date, 6,2) as int) = 2 Then 'Q4' -- Feb - When cast(substr(date, 6,2) as int) = 3 Then 'Q4' -- March - When cast(substr(date, 6,2) as int) = 4 Then 'Q1' -- Apr - When cast(substr(date, 6,2) as int) = 5 Then 'Q1' -- May - When cast(substr(date, 6,2) as int) = 6 Then 'Q1' -- June - When cast(substr(date, 6,2) as int) = 7 Then 'Q2' -- Jul - When cast(substr(date, 6,2) as int) = 8 Then 'Q2' -- Aug - When cast(substr(date, 6,2) as int) = 9 Then 'Q2' -- Sept - When cast(substr(date, 6,2) as int) = 10 Then 'Q3' -- Oct - When cast(substr(date, 6,2) as int) = 11 Then 'Q3' -- Nov - When cast(substr(date, 6,2) as int) = 12 Then 'Q3' -- Dec - Else '' - END as QTR, - - ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date, import_date DESC) row_num - FROM calendar), - -CalendarFormat as ( - SELECT - CAST(CASE - When Calendar_date like '%/%'Then substr(Calendar_date, 7, 4)||'-'|| - substr(Calendar_date, 4, 2)||'-'||substr(Calendar_date, 1, 2) - ELSE substr(Calendar_date, 1, 10) - END as date) as Format_date, fin_year, QTR - FROM Calendar_Data - WHERE row_num = 1), - -Latest_Report_Year as ( - SELECT - /**** Find the Latest Financial Year ***/ - CASE - /** Q1 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 4 Then cast(cast(fin_year as int) - 1 as string) - When cast(substr(cast(Format_date as string),6,2) as int) = 5 Then cast(cast(fin_year as int) - 1 as string) - When cast(substr(cast(Format_date as string),6,2) as int) = 6 Then cast(cast(fin_year as int) - 1 as string) - /** Q2 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 7 Then fin_year - When cast(substr(cast(Format_date as string),6,2) as int) = 8 Then fin_year - When cast(substr(cast(Format_date as string),6,2) as int) = 9 Then fin_year - /** Q3 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 10 Then fin_year - When cast(substr(cast(Format_date as string),6,2) as int) = 11 Then fin_year - When cast(substr(cast(Format_date as string),6,2) as int) = 12 Then fin_year - /** Q4 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 1 Then fin_year - When cast(substr(cast(Format_date as string),6,2) as int) = 2 Then fin_year - When cast(substr(cast(Format_date as string),6,2) as int) = 3 Then fin_year - END as LatestReportYear, - /**** Find the Latest Quarter ***/ - CASE - /** Q1 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 4 Then 'Q4' - When cast(substr(cast(Format_date as string),6,2) as int) = 5 Then 'Q4' - When cast(substr(cast(Format_date as string),6,2) as int) = 6 Then 'Q4' - /** Q2 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 7 Then 'Q1' - When cast(substr(cast(Format_date as string),6,2) as int) = 8 Then 'Q1' - When cast(substr(cast(Format_date as string),6,2) as int) = 9 Then 'Q1' - /** Q3 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 10 Then 'Q2' - When cast(substr(cast(Format_date as string),6,2) as int) = 11 Then 'Q2' - When cast(substr(cast(Format_date as string),6,2) as int) = 12 Then 'Q2' - /** Q4 **/ - When cast(substr(cast(Format_date as string),6,2) as int) = 1 Then 'Q3' - When cast(substr(cast(Format_date as string),6,2) as int) = 2 Then 'Q3' - When cast(substr(cast(Format_date as string),6,2) as int) = 3 Then 'Q3' - END as LatestQTR - From CalendarFormat - Where format_date = current_date), -/******************************************************************************************** -Obtain the Voucher forecast figures -********************************************************************************************/ - Voucher_Forecast as ( - Select monthyear, one_day, two_hour, estate, - CASE - When cast(substr(monthyear, 4,2) as int) = 1 Then 'Q4' -- Jan - When cast(substr(monthyear, 4,2) as int) = 2 Then 'Q4' -- Feb - When cast(substr(monthyear, 4,2) as int) = 3 Then 'Q4' -- March - When cast(substr(monthyear, 4,2) as int) = 4 Then 'Q1' -- Apr - When cast(substr(monthyear, 4,2) as int) = 5 Then 'Q1' -- May - When cast(substr(monthyear, 4,2) as int) = 6 Then 'Q1' -- June - When cast(substr(monthyear, 4,2) as int) = 7 Then 'Q2' -- Jul - When cast(substr(monthyear, 4,2) as int) = 8 Then 'Q2' -- Aug - When cast(substr(monthyear, 4,2) as int) = 9 Then 'Q2' -- Sept - When cast(substr(monthyear, 4,2) as int) = 10 Then 'Q3' -- Oct - When cast(substr(monthyear, 4,2) as int) = 11 Then 'Q3' -- Nov - When cast(substr(monthyear, 4,2) as int) = 12 Then 'Q3' -- Dec - Else '' - END as QTR, - CASE - When cast(substr(monthyear, 4,2) as int) = 1 Then cast(cast(substr(monthyear, 7,4) as int)-1 as string)--Jan - When cast(substr(monthyear, 4,2) as int) = 2 Then cast(cast(substr(monthyear, 7,4) as int)-1 as string)--Feb - When cast(substr(monthyear, 4,2) as int) = 3 Then cast(cast(substr(monthyear, 7,4) as int)-1 as string)--Mar - When cast(substr(monthyear, 4,2) as int) = 4 Then substr(monthyear, 7,4) -- Apr - When cast(substr(monthyear, 4,2) as int) = 5 Then substr(monthyear, 7,4) -- May - When cast(substr(monthyear, 4,2) as int) = 6 Then substr(monthyear, 7,4) -- June - When cast(substr(monthyear, 4,2) as int) = 7 Then substr(monthyear, 7,4) -- Jul - When cast(substr(monthyear, 4,2) as int) = 8 Then substr(monthyear, 7,4) -- Aug - When cast(substr(monthyear, 4,2) as int) = 9 Then substr(monthyear, 7,4) -- Sept - When cast(substr(monthyear, 4,2) as int) = 10 Then substr(monthyear, 7,4) -- Oct - When cast(substr(monthyear, 4,2) as int) = 11 Then substr(monthyear, 7,4) -- Nov - When cast(substr(monthyear, 4,2) as int) = 12 Then substr(monthyear, 7,4) -- Dec - Else '' - END as FinYear - From visitor_voucher_forecast), -/******************************************************************************************** -Obtain the Voucher de-normalised data. Format the Start and End number, remove rubbish -********************************************************************************************/ -Vouchers as ( - SELECT voucher_ref, street, permit_type, cpz, quantity, e_voucher, - /** Format the application Date **/ - CAST(CASE - When application_date like '%/%'Then substr(application_date, 7, 4)||'-'|| - substr(application_date, 4, 2)||'-'||substr(application_date, 1, 2) - ELSE substr(application_date, 1, 10) - END as date) as application_date, Status, fin_year, QTR, - - CASE - When Permit_Type = 'Estate Visitor Vouchers' Then cast(quantity as decimal) * 10 - When Permit_Type = '1 day visitors vouchers' Then cast(quantity as decimal) * 5 - When Permit_Type = '2 hour visitors vouchers' Then cast(quantity as decimal) * 20 - Else 0 - END as NoOFBooks - - from parking_voucher_de_normalised as A - LEFT JOIN CalendarFormat as B ON cast(substr(A.application_date, 1, 10) as date) = B.Format_date - Where Import_Date = (Select MAX(Import_Date) from parking_voucher_de_normalised) - AND Permit_type IN ('Estate Visitor Vouchers','2 hour visitors vouchers','1 day visitors vouchers') - AND cast(substr(application_date, 1, 10) as date) >= cast('2019-04-01' as date) - AND Status != 'Rejected'), -/******************************************************************************************** -Continue to format the data, obtain the no of Vouchers - 2019 data -********************************************************************************************/ -Voucher_Tidy_2019 as ( - SELECT voucher_ref, permit_type,quantity, cpz, NoOFBooks, QTR - from Vouchers - WHERE Fin_Year = '2019'), -/******************************************************************************************** -Summerise the 2019 data -********************************************************************************************/ -Voucher_Baseline_2019 as ( - SELECT permit_type, SUM(NoOFBooks) as TotalNo, SUM(NoOFBooks)/4 as QTRTotal - From Voucher_Tidy_2019 - WHERE Permit_Type = 'Estate Visitor Vouchers' OR cpz IN ('A', 'D','F','G','G2','H','K','L','N','P','Q','S','U','T') - GROUP BY permit_type), -/******************************************************************************************** -Continue to format the data, obtain the no of Vouchers - Last FULL FY Year -********************************************************************************************/ -Voucher_Tidy_Latest as ( - SELECT voucher_ref, permit_type,quantity, cpz, application_date,NoOFBooks, QTR, fin_year - from Vouchers - WHERE Fin_Year = (Select LatestReportYear from Latest_Report_Year)), - -Voucher_Baseline_Latest as ( - SELECT permit_type, - QTR, fin_year, - SUM(NoOFBooks) as TotalNo - - From Voucher_Tidy_Latest - WHERE Permit_Type = 'Estate Visitor Vouchers' OR cpz IN ('A', 'D','F','G','G2','H','K','L','N','P','Q','S','U','T') - GROUP BY permit_type, QTR, fin_year), -/******************************************************************************************** -Continue to format the data, obtain & Qtrly total the forecast figures -********************************************************************************************/ -Forecast_Qtrly as ( - Select - FinYear, QTR, SUM(one_day) as One_Day, - SUM(two_hour) as two_hour, - SUM(estate) as estate - From Voucher_Forecast - Group By FinYear, QTR), - -/*** Split for forecats data into each Permit Type ***/ -Format_Forecast as ( - Select - FinYear, QTR, - '1 day visitors vouchers' as Permit_Type, - One_Day as ForecastTotal , QtrTotal as Baseline_Qtr - From Forecast_Qtrly as A - inner join Voucher_Baseline_2019 as B ON '1 day visitors vouchers' = B.permit_type - UNION ALL - Select - FinYear, QTR, - '2 hour visitors vouchers' as Permit_Type, - two_hour as ForecastTotal, QtrTotal as Baseline_Qtr - From Forecast_Qtrly - inner join Voucher_Baseline_2019 as B ON '2 hour visitors vouchers' = B.permit_type - UNION ALL - Select - FinYear, QTR, - 'Estate Visitor Vouchers' as Permit_Type, - estate as ForecastTotal, QtrTotal as Baseline_Qtr - From Forecast_Qtrly - inner join Voucher_Baseline_2019 as B ON 'Estate Visitor Vouchers' = B.permit_type - UNION ALL - Select - '2019' as fin_year, - 'Q0' as QTR, - permit_type, - QtrTotal, 0 - From Voucher_Baseline_2019) -/******************************************************************************************** -Continue to format the data for printing -********************************************************************************************/ -SELECT - A.finyear, A.qtr, A.Permit_Type, - Case - When B.TotalNo is not NULL Then B.TotalNo - When A.finyear = '2019' Then ForecastTotal - Else 0 End as TotalNo, - Case - When A.finyear = '2019' Then 0 - Else ForecastTotal - End as ForecastTotal, Baseline_Qtr, - - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date -FROM Format_Forecast as A -LEFT JOIN Voucher_Baseline_Latest as B ON A.finyear = B.fin_year AND A.qtr = B.QTR AND A.Permit_Type = B.permit_type -Order by A.Permit_Type, A.finyear, A.qtr - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "calendar": AmazonS3_node1648207907397, - "parking_voucher_de_normalised": AmazonS3_node1648208237020, - "visitor_voucher_forecast": AmazonS3_node1652190326034, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Visitor_Voucher_Qtrly_Review/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=PARTITION_KEYS, - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Visitor_Voucher_Qtrly_Review", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_voucher_de_normalised.py b/scripts/jobs/parking/parking_voucher_de_normalised.py deleted file mode 100644 index 9f2bc04bd..000000000 --- a/scripts/jobs/parking/parking_voucher_de_normalised.py +++ /dev/null @@ -1,149 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var - -environment = get_glue_env_var("environment") - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) - -# Script generated for node Amazon S3 -AmazonS3_node1648207907397 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_voucher", - transformation_ctx="AmazonS3_node1648207907397", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1648208237020 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_fta", - transformation_ctx="AmazonS3_node1648208237020", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1648208633220 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_printing", - transformation_ctx="AmazonS3_node1648208633220", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node Amazon S3 -AmazonS3_node1649411092913 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - table_name="liberator_permit_evouchersession", - transformation_ctx="AmazonS3_node1649411092913", - push_down_predicate=create_pushdown_predicate("import_date", 7), -) - -# Script generated for node ApplyMapping -SqlQuery0 = """ -/********************************************************************************* -Parking_Voucher_De_Normalised - -This SQL creates a denormalised list of Vouchers - -23/03/2022 - Create Query -13/04/2022 - Add Voucher Start and End figures -26/09/2022 - Update to ensure that ONLY vouchers are obtained/displayed -*********************************************************************************/ -WITH Voucher_Data as ( - SELECT * FROM liberator_permit_voucher - WHERE import_Date = (Select MAX(import_date) from liberator_permit_voucher)), - -Permit_Voucher as ( - SELECT * FROM liberator_permit_fta - WHERE import_Date = (Select MAX(import_date) from liberator_permit_fta) - AND lower(permit_type) like '%voucher%'), - -E_Voucher_Sessions as ( - Select evoucherorderid, count(*) as NoofBookings, - sum(numberofevouchersinsession) as numberofevouchersinsession - FROM liberator_permit_evouchersession - WHERE import_Date = (Select MAX(import_date) from liberator_permit_evouchersession) - GROUP BY evoucherorderid), - -Permit_Print as ( - SELECT - Permit_reference, Issue_date as printed_date, printed_by, - ROW_NUMBER() OVER ( PARTITION BY Permit_reference - ORDER BY Permit_reference, Issue_date) R0 - FROM liberator_permit_printing - Where Import_Date = (Select MAX(Import_Date) from liberator_permit_printing)) - - -SELECT - A.permit_reference as Voucher_Ref, B.street,permit_type, amount, uprn, - status,application_date, cpz, cpz_name, - quantity , forename_of_applicant,surname_of_applicant, - email_address_of_applicant, A.associated_to_order, - printed_date, printed_by, e_voucher, - NoofBookings, numberofevouchersinsession, - voucher_start_number, voucher_end_number, - - date_format(CAST(CURRENT_TIMESTAMP AS timestamp), 'yyyy-MM-dd HH:mm:ss') AS ImportDateTime, - date_format(current_date, 'yyyy') AS import_year, - date_format(current_date, 'MM') AS import_month, - date_format(current_date, 'dd') AS import_day, - date_format(current_date, 'yyyyMMdd') AS import_date - -FROM Permit_Voucher as A -LEFT JOIN Voucher_Data as B ON A.permit_reference = B.permit_reference -LEFT JOIN Permit_Print as C ON A.permit_reference = C.permit_reference AND R0 = 1 -LEFT JOIN E_Voucher_Sessions as D ON A.permit_reference = D.evoucherorderid -WHERE surname_of_applicant != 'Test' - - -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "liberator_permit_voucher": AmazonS3_node1648207907397, - "liberator_permit_fta": AmazonS3_node1648208237020, - "liberator_permit_printing": AmazonS3_node1648208633220, - "liberator_permit_evouchersession": AmazonS3_node1649411092913, - }, - transformation_ctx="ApplyMapping_node2", -) - -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/Parking_Voucher_De_Normalised/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="Parking_Voucher_De_Normalised", -) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_vouchers_approved_summary_gds.py b/scripts/jobs/parking/parking_vouchers_approved_summary_gds.py deleted file mode 100644 index 8b2f31aaa..000000000 --- a/scripts/jobs/parking/parking_vouchers_approved_summary_gds.py +++ /dev/null @@ -1,571 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 liberator-raw-zone - liberator_permit_llpg -AmazonS3liberatorrawzoneliberator_permit_llpg_node1720617252559 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="liberator_permit_llpg", - transformation_ctx="AmazonS3liberatorrawzoneliberator_permit_llpg_node1720617252559", -) - -# Script generated for node Amazon S3 - liberator-refined-zone - parking_voucher_de_normalised -AmazonS3liberatorrefinedzoneparking_voucher_de_normalised_node1720617253376 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="parking_voucher_de_normalised", - transformation_ctx="AmazonS3liberatorrefinedzoneparking_voucher_de_normalised_node1720617253376", -) - -# Script generated for node Amazon S3 - raw-zone-unrestricted-address-api - unrestricted_address_api_dbo_hackney_address -AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)", - table_name="unrestricted_address_api_dbo_hackney_address", - transformation_ctx="AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032", -) - -# Script generated for node Amazon S3 - unrestricted_address_api_dbo_hackney_xref -AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724073104086 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)", - table_name="unrestricted_address_api_dbo_hackney_xref", - transformation_ctx="AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724073104086", -) - -# Script generated for node SQL Query -SqlQuery0 = """ -/* -03/07/2024 - updated vouchers approved version two -09/07/2024 - breakdown by household, street, zone, lbh versions. This is the CPZ version -19/08/2024 - added car free calculations - -Source tables: -"dataplatform-stg-liberator-refined-zone".parking_voucher_de_normalised -"dataplatform-stg-liberator-raw-zone".liberator_permit_llpg -"dataplatform-stg-raw-zone-unrestricted-address-api".unrestricted_address_api_dbo_hackney_address -"dataplatform-stg-raw-zone-unrestricted-address-api".unrestricted_address_api_dbo_hackney_xref - -*/ -with vou as ( -Select distinct -cast(concat(substr(Cast(parking_voucher_de_normalised.application_date as varchar(10)),1, 7), '-01') as date) as month_year_application_date --- cast(to_date(Substr("liberator_raw_zone".parking_voucher_de_normalised.application_date, 1,7), '%Y-%m') as date) as month_year_application_date - -,Case -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2031-03-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2031-02-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2031-01-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-12-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-11-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-10-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-09-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-08-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-07-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-06-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-05-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-04-%' THEN '2030'--2030 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-03-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-02-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-01-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-12-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-11-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-10-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-09-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-08-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-07-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-06-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-05-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-04-%' THEN '2029'--2029 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-03-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-02-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-01-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-12-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-11-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-10-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-09-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-08-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-07-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-06-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-05-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-04-%' THEN '2028'--2028 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-03-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-02-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-01-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-12-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-11-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-10-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-09-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-08-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-07-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-06-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-05-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-04-%' THEN '2027'--2027 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-03-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-02-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-01-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-12-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-11-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-10-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-09-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-08-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-07-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-06-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-05-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-04-%' THEN '2026'--2026 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-03-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-02-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-01-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-12-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-11-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-10-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-09-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-08-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-07-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-06-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-05-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-04-%' THEN '2025'--2025 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-03-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-02-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-01-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-12-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-11-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-10-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-09-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-08-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-07-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-06-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-05-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-04-%' THEN '2024'--2024 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-03-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-02-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-01-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-12-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-11-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-10-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-09-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-08-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-07-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-06-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-05-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-04-%' THEN '2023'--2023 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-03-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-02-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-01-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-12-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-11-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-10-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-09-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-08-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-07-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-06-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-05-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-04-%' THEN '2022'--2022 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-03-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-02-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-01-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-12-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-11-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-10-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-09-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-08-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-07-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-06-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-05-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-04-%' THEN '2021'--2021 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-03-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-02-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-01-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-12-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-11-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-10-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-09-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-08-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-07-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-06-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-05-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-04-%' THEN '2020'--2020 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-03-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-02-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-01-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-12-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-11-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-10-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-09-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-08-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-07-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-06-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-05-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-04-%' THEN '2019'--2019 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-03-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-02-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-01-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-12-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-11-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-10-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-09-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-08-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-07-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-06-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-05-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-04-%' THEN '2018'--2018 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-03-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-02-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-01-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-12-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-11-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-10-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-09-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-08-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-07-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-06-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-05-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-04-%' THEN '2017'--2017 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-03-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-02-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-01-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-12-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-11-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-10-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-09-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-08-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-07-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-06-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-05-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-04-%' THEN '2016'--2016 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-03-%' THEN '2015' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-02-%' THEN '2015'--2015 -else '1900' -end as fy - -,case when parking_voucher_de_normalised.cpz !='' and parking_voucher_de_normalised.cpz_name != '' then concat(parking_voucher_de_normalised.cpz,' - ', parking_voucher_de_normalised.cpz_name) -when parking_voucher_de_normalised.cpz !='' and parking_voucher_de_normalised.cpz_name = '' then parking_voucher_de_normalised.cpz -when parking_voucher_de_normalised.cpz ='' and parking_voucher_de_normalised.cpz_name != '' then parking_voucher_de_normalised.cpz_name -else 'NONE' -end as zone_name - -,to_date(cast(concat(substr(Cast(parking_voucher_de_normalised.application_date as varchar(10)),1, 7), '-01') as date), 'MMM YYYY') as mon_yr_app_date - -/*Approval status*/ -,case when upper(parking_voucher_de_normalised.status) in ('APPROVED','ORDER_APPROVED','PENDING_VRM_CHANGE','CREATED','PENDING_ADDR_CHANGE','PAYMENT_RECEIVED','RENEWED') THEN 1 ELSE 0 END as approved_status_flag - -/*Voucher Type and groupings*/ -,parking_voucher_de_normalised.permit_type as voucher_period -,substr(parking_voucher_de_normalised.voucher_ref,1,3) as PermitType -,case - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYA' then 'All zone' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYG' then 'Companion Badge' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYB' then 'Business' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYD' then 'Doctor' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYP' then 'Estate resident' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYH' then 'Health and social care' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYR' then 'Residents' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYL' then 'Leisure Centre Permit' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYE' then 'All Zone Business Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYF' then 'Film Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYJ' then 'Health and Social Care Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYQ' then 'Estate Visitor Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYV' then 'Resident Visitor Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYN' then 'Dispensation' - when parking_voucher_de_normalised.voucher_ref ='HY50300' then 'Dispensation' - else substr(parking_voucher_de_normalised.voucher_ref,1,3) - end as permit_type_category - -,Case when substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYE' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYF' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYQ' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYV' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYJ' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYN' then 'Dispensation' -when parking_voucher_de_normalised.voucher_ref ='HY50300' then 'Dispensation' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYS' then 'Suspension' else 'Permit' -end as product_category - -,case when (substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYE' or - substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYF'or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYQ' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYV' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYJ' ) and parking_voucher_de_normalised.e_voucher ='E' then 'eVoucher' -when (substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYE' or - substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYF'or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYQ' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYV' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYJ' ) and - parking_voucher_de_normalised.e_voucher <> 'E' then 'Paper Voucher' else 'Not Voucher' end as voucher_type_category - -/*Calculation of total vouchers in books*/ -,cast(case -when upper(permit_type) like '2 HOUR VISITORS VOUCHERS' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 20 * try_cast(quantity as double) --2 hour visitors vouchers x20 -when upper(permit_type) like '1 DAY VISITORS VOUCHERS' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 5 * try_cast(quantity as double) --1 day visitors vouchers x5 -when upper(permit_type) like 'FILM VOUCHER' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 10 * try_cast(quantity as double) --Film voucher x10 -when upper(permit_type) like 'ESTATE VISITOR VOUCHERS' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 10 * try_cast(quantity as double) --Estate Visitor Vouchers x10 -when upper(permit_type) like 'HEALTH AND SOCIAL CARE VOUCHER' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 5 * try_cast(quantity as double) --Health and Social Care Voucher x5 -when upper(permit_type) like 'ALL ZONE BUSINESS VOUCHER' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 10 * try_cast(quantity as double) --All Zone Business Voucher x10 -end as decimal (10,0)) as total_num_vouchers_in_book -,* -from parking_voucher_de_normalised where import_date = (Select max(import_date) from parking_voucher_de_normalised ) --544,689 - -) -, street as (select - UPRN as SR_UPRN, - ADDRESS1 as SR_ADDRESS1, - ADDRESS2 as SR_ADDRESS2, - ADDRESS3 as SR_ADDRESS3, - ADDRESS4 as SR_ADDRESS4, - ADDRESS5 as SR_ADDRESS5, - POST_CODE as SR_POST_CODE, - BPLU_CLASS as SR_BPLU_CLASS, - X as SR_X, - Y as SR_Y, - LATITUDE as SR_LATITUDE, - LONGITUDE as SR_LONGITUDE, - IN_CONGESTION_CHARGE_ZONE as SR_IN_CONGESTION_CHARGE_ZONE, - CPZ_CODE as SR_CPZ_CODE, - CPZ_NAME as SR_CPZ_NAME, - LAST_UPDATED_STAMP as SR_LAST_UPDATED_STAMP, - LAST_UPDATED_TX_STAMP as SR_LAST_UPDATED_TX_STAMP, - CREATED_STAMP as SR_CREATED_STAMP, - CREATED_TX_STAMP as SR_CREATED_TX_STAMP, - HOUSE_NAME as SR_HOUSE_NAME, - POST_CODE_PACKED as SR_POST_CODE_PACKED, - STREET_START_X as SR_STREET_START_X, - STREET_START_Y as SR_STREET_START_Y, - STREET_END_X as SR_STREET_END_X, - STREET_END_Y as SR_STREET_END_Y, - WARD_CODE as SR_WARD_CODE, -if(WARD_CODE = 'E05009367', 'BROWNSWOOD WARD', -if(WARD_CODE = 'E05009368', 'CAZENOVE WARD', -if(WARD_CODE = 'E05009369', 'CLISSOLD WARD', -if(WARD_CODE = 'E05009370', 'DALSTON WARD', -if(WARD_CODE = 'E05009371', 'DE BEAUVOIR WARD', -if(WARD_CODE = 'E05009372', 'HACKNEY CENTRAL WARD', -if(WARD_CODE = 'E05009373', 'HACKNEY DOWNS WARD', -if(WARD_CODE = 'E05009374', 'HACKNEY WICK WARD', -if(WARD_CODE = 'E05009375', 'HAGGERSTON WARD', -if(WARD_CODE = 'E05009376', 'HOMERTON WARD', -if(WARD_CODE = 'E05009377', 'HOXTON EAST AND SHOREDITCH WARD', -if(WARD_CODE = 'E05009378', 'HOXTON WEST WARD', -if(WARD_CODE = 'E05009379', 'KINGS PARK WARD', -if(WARD_CODE = 'E05009380', 'LEA BRIDGE WARD', -if(WARD_CODE = 'E05009381', 'LONDON FIELDS WARD', -if(WARD_CODE = 'E05009382', 'SHACKLEWELL WARD', -if(WARD_CODE = 'E05009383', 'SPRINGFIELD WARD', -if(WARD_CODE = 'E05009384', 'STAMFORD HILL WEST', -if(WARD_CODE = 'E05009385', 'STOKE NEWINGTON WARD', -if(WARD_CODE = 'E05009386', 'VICTORIA WARD', -if(WARD_CODE = 'E05009387', 'WOODBERRY DOWN WARD',WARD_CODE))))))))))))))))))))) as SR_ward_name, - PARISH_CODE as SR_PARISH_CODE, - PARENT_UPRN as SR_PARENT_UPRN, - PAO_START as SR_PAO_START, - PAO_START_SUFFIX as SR_PAO_START_SUFFIX, - PAO_END as SR_PAO_END, - PAO_END_SUFFIX as SR_PAO_END_SUFFIX, - PAO_TEXT as SR_PAO_TEXT, - SAO_START as SR_SAO_START, - SAO_START_SUFFIX as SR_SAO_START_SUFFIX, - SAO_END as SR_SAO_END, - SAO_END_SUFFIX as SR_SAO_END_SUFFIX, - SAO_TEXT as SR_SAO_TEXT, - DERIVED_BLPU as SR_DERIVED_BLPU, - USRN -,import_year -,import_month -,import_day -,import_date -FROM liberator_permit_llpg -where (ADDRESS1 like 'Street Record' or ADDRESS1 like 'STREET RECORD') and liberator_permit_llpg.import_date = (SELECT MAX(liberator_permit_llpg.import_date) FROM liberator_permit_llpg) -) -, llpg as ( - SELECT * FROM unrestricted_address_api_dbo_hackney_address where unrestricted_address_api_dbo_hackney_address.import_date = (SELECT max(unrestricted_address_api_dbo_hackney_address.import_date) FROM unrestricted_address_api_dbo_hackney_address) and lpi_logical_status like 'Approved Preferred' - ) -, cfxref as ( -select * from unrestricted_address_api_dbo_hackney_xref where import_date = (select max(import_date) from unrestricted_address_api_dbo_hackney_xref) and xref_name like 'Parking Blacklisted S106' -) - -select distinct - -vou.fy -,vou.cpz -,vou.cpz_name -,vou.zone_name -,vou.mon_yr_app_date -,vou.month_year_application_date -,vou.approved_status_flag -,vou.permit_type_category -,vou.voucher_period -,vou.PermitType--voucherType --as permittype -,vou.product_category -,vou.voucher_type_category - -/*Partitions*/ -, vou.import_year -, vou.import_month -, vou.import_day -, vou.import_date - - -/*Counters*/ - -/*Transactions*/ -,count(concat(vou.application_date,vou.email_address_of_applicant)) as num_unique_transaction_ref -,count(distinct concat(vou.application_date,vou.email_address_of_applicant)) as num_distinct_unique_transaction_ref - -/*Email addresses*/ -,count(vou.email_address_of_applicant) as num_email_address_ref -,count(distinct vou.email_address_of_applicant) as num_distinct_email_address_ref - -/*CPZ - Zone*/ -,count(distinct vou.cpz) as num_distinct_cpz - -,count(distinct vou.cpz_name) as num_distinct_cpz_name - -,count(distinct vou.zone_name) as num_distinct_zone_name - -/*Street(USRN), Ward and Household(UPRN)*/ -,count(distinct SR_ADDRESS2) as num_distinct_street_name -,count(distinct SR_WARD_CODE) as num_distinct_street_ward_name -,count(distinct vou.uprn) as num_distinct_uprn_households -,count(distinct street.usrn) as num_distinct_usrn - -/*Books, Bookings and Vouchers*/ -,cast(sum(try_cast(vou.quantity as double)) as decimal(15,0)) as total_quantity_books -,cast(sum(try_cast(vou.total_num_vouchers_in_book as double)) as decimal(15,0)) as total_num_vouchers_in_books -,cast(sum(try_cast(vou.noofbookings as double)) as decimal(15,0)) as total_num_ebookings -,cast(sum(try_cast(vou.numberofevouchersinsession as double)) as decimal(15,0)) as total_num_evouchers_in_session - -/*Income - Total Paid*/ -,cast(sum(try_cast(vou.amount as double)) as decimal(15,2)) as total_amount_paid - - -/*Car Free blacklisted properties counters*/ - -/*Transactions Car Free*/ -,count( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then concat(vou.application_date,vou.email_address_of_applicant) end) as num_unique_transaction_ref_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then concat(vou.application_date,vou.email_address_of_applicant) end) as num_distinct_unique_transaction_ref_cfxref - -/*Email addresses Car Free*/ -,count( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.email_address_of_applicant end) as num_email_address_ref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.email_address_of_applicant end) as num_distinct_email_address_ref_cfxref - -/*CPZ - Zone Car Free*/ -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.cpz end) as num_distinct_cpz_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.cpz_name end) as num_distinct_cpz_name_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.zone_name end) as num_distinct_zone_name_cfxref - -/*Street(USRN), Ward and Household(UPRN) Car Free*/ -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then SR_ADDRESS2 end) as num_distinct_street_name_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then SR_WARD_CODE end) as num_distinct_street_ward_name_cfxref ---,count(distinct vou.uprn) as num_distinct_uprn_households -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.uprn end) as num_distinct_uprn_households_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then street.usrn end) as num_distinct_usrn_cfxref - -/*Books, Bookings and Vouchers Car Free*/ -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.quantity as double) end ) as decimal(15,0)) as total_quantity_books_cfxref - -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.total_num_vouchers_in_book as double) end ) as decimal(15,0)) as total_num_vouchers_in_books_cfxref -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.noofbookings as double) end) as decimal(15,0)) as total_num_ebookings_cfxref -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.numberofevouchersinsession as double) end ) as decimal(15,0)) as total_num_evouchers_in_session_cfxref - -/*Income - Total Paid Car Free*/ -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.amount as double) end) as decimal(15,2)) as total_amount_paid_cfxref - - -from vou -left join llpg on cast(llpg.uprn as string) = cast(vou.uprn as string) -left join street on cast(street.usrn as string) = cast(llpg.usrn as string) -left join cfxref on cast(cfxref.uprn as string) = cast(vou.uprn as string) - -Where -vou.approved_status_flag = 1 - -group by -vou.fy -,vou.cpz -,vou.cpz_name -,vou.zone_name -,vou.mon_yr_app_date -,vou.month_year_application_date -,vou.approved_status_flag -,vou.permit_type_category -,vou.voucher_period -,vou.PermitType --voucherType -,vou.product_category -,vou.voucher_type_category - -/*Partitions*/ -, vou.import_year -, vou.import_month -, vou.import_day -, vou.import_date - -Order by -vou.month_year_application_date desc -,vou.PermitType asc --voucherType asc -,vou.voucher_period asc -,vou.voucher_type_category asc -,vou.cpz ---,vou.cpz_name ---,vou.zone_name -""" -SQLQuery_node1720617258366 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_voucher_de_normalised": AmazonS3liberatorrefinedzoneparking_voucher_de_normalised_node1720617253376, - "liberator_permit_llpg": AmazonS3liberatorrawzoneliberator_permit_llpg_node1720617252559, - "unrestricted_address_api_dbo_hackney_address": AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032, - "unrestricted_address_api_dbo_hackney_xref": AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724073104086, - }, - transformation_ctx="SQLQuery_node1720617258366", -) - -# Script generated for node Amazon S3 - parking_vouchers_approved_summary_gds -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/parking_vouchers_approved_summary_gds/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="AmazonS3parking_vouchers_approved_summary_gds_node1720617264938", -) -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_vouchers_approved_summary_gds", -) -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938.setFormat( - "glueparquet", compression="snappy" -) -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938.writeFrame( - SQLQuery_node1720617258366 -) -job.commit() diff --git a/scripts/jobs/parking/parking_vouchers_approved_summary_gds_lbh.py b/scripts/jobs/parking/parking_vouchers_approved_summary_gds_lbh.py deleted file mode 100644 index de2ca21b2..000000000 --- a/scripts/jobs/parking/parking_vouchers_approved_summary_gds_lbh.py +++ /dev/null @@ -1,566 +0,0 @@ -import sys - -from awsglue import DynamicFrame -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext - -from scripts.helpers.helpers import ( - PARTITION_KEYS, - get_glue_env_var, - get_latest_partitions, -) - - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node Amazon S3 - unrestricted_address_api_dbo_hackney_xref -AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)", - table_name="unrestricted_address_api_dbo_hackney_xref", - transformation_ctx="AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047", -) - -# Script generated for node Amazon S3 liberator-raw-zone - liberator_permit_llpg -AmazonS3liberatorrawzoneliberator_permit_llpg_node1720617252559 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="liberator_permit_llpg", - transformation_ctx="AmazonS3liberatorrawzoneliberator_permit_llpg_node1720617252559", -) - -# Script generated for node Amazon S3 - liberator-refined-zone - parking_voucher_de_normalised -AmazonS3liberatorrefinedzoneparking_voucher_de_normalised_node1720617253376 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-liberator-refined-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)", - table_name="parking_voucher_de_normalised", - transformation_ctx="AmazonS3liberatorrefinedzoneparking_voucher_de_normalised_node1720617253376", -) - -# Script generated for node Amazon S3 - raw-zone-unrestricted-address-api - unrestricted_address_api_dbo_hackney_address -AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)", - table_name="unrestricted_address_api_dbo_hackney_address", - transformation_ctx="AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032", -) - -# Script generated for node SQL Query -SqlQuery0 = """ -/* -03/07/2024 - updated vouchers approved version two -09/07/2024 - breakdown by household, street, zone, lbh versions. This is the LBH version -19/08/2024 - 19/08/2024 - added car free calculations - -Source tables: -"dataplatform-stg-liberator-refined-zone".parking_voucher_de_normalised -"dataplatform-stg-liberator-raw-zone".liberator_permit_llpg -"dataplatform-stg-raw-zone-unrestricted-address-api".unrestricted_address_api_dbo_hackney_address -"dataplatform-stg-raw-zone-unrestricted-address-api".unrestricted_address_api_dbo_hackney_xref - - -*/ -with vou as ( -Select distinct -cast(concat(substr(Cast(parking_voucher_de_normalised.application_date as varchar(10)),1, 7), '-01') as date) as month_year_application_date --- cast(to_date(Substr("liberator_raw_zone".parking_voucher_de_normalised.application_date, 1,7), '%Y-%m') as date) as month_year_application_date - -,Case -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2031-03-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2031-02-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2031-01-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-12-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-11-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-10-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-09-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-08-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-07-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-06-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-05-%' THEN '2030' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-04-%' THEN '2030'--2030 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-03-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-02-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2030-01-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-12-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-11-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-10-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-09-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-08-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-07-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-06-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-05-%' THEN '2029' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-04-%' THEN '2029'--2029 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-03-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-02-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2029-01-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-12-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-11-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-10-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-09-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-08-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-07-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-06-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-05-%' THEN '2028' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-04-%' THEN '2028'--2028 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-03-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-02-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2028-01-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-12-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-11-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-10-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-09-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-08-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-07-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-06-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-05-%' THEN '2027' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-04-%' THEN '2027'--2027 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-03-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-02-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2027-01-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-12-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-11-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-10-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-09-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-08-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-07-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-06-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-05-%' THEN '2026' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-04-%' THEN '2026'--2026 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-03-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-02-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2026-01-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-12-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-11-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-10-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-09-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-08-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-07-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-06-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-05-%' THEN '2025' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-04-%' THEN '2025'--2025 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-03-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-02-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2025-01-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-12-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-11-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-10-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-09-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-08-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-07-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-06-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-05-%' THEN '2024' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-04-%' THEN '2024'--2024 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-03-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-02-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2024-01-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-12-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-11-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-10-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-09-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-08-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-07-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-06-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-05-%' THEN '2023' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-04-%' THEN '2023'--2023 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-03-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-02-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2023-01-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-12-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-11-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-10-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-09-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-08-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-07-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-06-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-05-%' THEN '2022' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-04-%' THEN '2022'--2022 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-03-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-02-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2022-01-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-12-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-11-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-10-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-09-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-08-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-07-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-06-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-05-%' THEN '2021' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-04-%' THEN '2021'--2021 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-03-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-02-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2021-01-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-12-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-11-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-10-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-09-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-08-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-07-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-06-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-05-%' THEN '2020' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-04-%' THEN '2020'--2020 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-03-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-02-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2020-01-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-12-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-11-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-10-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-09-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-08-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-07-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-06-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-05-%' THEN '2019' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-04-%' THEN '2019'--2019 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-03-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-02-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2019-01-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-12-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-11-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-10-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-09-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-08-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-07-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-06-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-05-%' THEN '2018' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-04-%' THEN '2018'--2018 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-03-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-02-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2018-01-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-12-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-11-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-10-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-09-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-08-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-07-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-06-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-05-%' THEN '2017' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-04-%' THEN '2017'--2017 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-03-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-02-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2017-01-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-12-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-11-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-10-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-09-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-08-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-07-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-06-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-05-%' THEN '2016' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-04-%' THEN '2016'--2016 -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-03-%' THEN '2015' -when cast(parking_voucher_de_normalised.application_date as varchar(10)) like '2016-02-%' THEN '2015'--2015 -else '1900' -end as fy - -,case when parking_voucher_de_normalised.cpz !='' and parking_voucher_de_normalised.cpz_name != '' then concat(parking_voucher_de_normalised.cpz,' - ', parking_voucher_de_normalised.cpz_name) -when parking_voucher_de_normalised.cpz !='' and parking_voucher_de_normalised.cpz_name = '' then parking_voucher_de_normalised.cpz -when parking_voucher_de_normalised.cpz ='' and parking_voucher_de_normalised.cpz_name != '' then parking_voucher_de_normalised.cpz_name -else 'NONE' -end as zone_name - -,to_date(cast(concat(substr(Cast(parking_voucher_de_normalised.application_date as varchar(10)),1, 7), '-01') as date), 'MMM YYYY') as mon_yr_app_date - -/*Approval status*/ -,case when upper(parking_voucher_de_normalised.status) in ('APPROVED','ORDER_APPROVED','PENDING_VRM_CHANGE','CREATED','PENDING_ADDR_CHANGE','PAYMENT_RECEIVED','RENEWED') THEN 1 ELSE 0 END as approved_status_flag - -/*Voucher Type and groupings*/ -,parking_voucher_de_normalised.permit_type as voucher_period -,substr(parking_voucher_de_normalised.voucher_ref,1,3) as PermitType -,case - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYA' then 'All zone' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYG' then 'Companion Badge' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYB' then 'Business' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYD' then 'Doctor' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYP' then 'Estate resident' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYH' then 'Health and social care' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYR' then 'Residents' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYL' then 'Leisure Centre Permit' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYE' then 'All Zone Business Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYF' then 'Film Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYJ' then 'Health and Social Care Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYQ' then 'Estate Visitor Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) = 'HYV' then 'Resident Visitor Voucher' - when substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYN' then 'Dispensation' - when parking_voucher_de_normalised.voucher_ref ='HY50300' then 'Dispensation' - else substr(parking_voucher_de_normalised.voucher_ref,1,3) - end as permit_type_category - -,Case when substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYE' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYF' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYQ' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYV' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYJ' then 'Voucher' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYN' then 'Dispensation' -when parking_voucher_de_normalised.voucher_ref ='HY50300' then 'Dispensation' -when substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYS' then 'Suspension' else 'Permit' -end as product_category - -,case when (substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYE' or - substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYF'or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYQ' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYV' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYJ' ) and parking_voucher_de_normalised.e_voucher ='E' then 'eVoucher' -when (substr(parking_voucher_de_normalised.voucher_ref,1,3) ='HYE' or - substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYF'or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYQ' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYV' or -substr(parking_voucher_de_normalised.voucher_ref,1,3)='HYJ' ) and - parking_voucher_de_normalised.e_voucher <> 'E' then 'Paper Voucher' else 'Not Voucher' end as voucher_type_category - -/*Calculation of total vouchers in books*/ -,cast(case -when upper(permit_type) like '2 HOUR VISITORS VOUCHERS' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 20 * try_cast(quantity as double) --2 hour visitors vouchers x20 -when upper(permit_type) like '1 DAY VISITORS VOUCHERS' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 5 * try_cast(quantity as double) --1 day visitors vouchers x5 -when upper(permit_type) like 'FILM VOUCHER' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 10 * try_cast(quantity as double) --Film voucher x10 -when upper(permit_type) like 'ESTATE VISITOR VOUCHERS' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 10 * try_cast(quantity as double) --Estate Visitor Vouchers x10 -when upper(permit_type) like 'HEALTH AND SOCIAL CARE VOUCHER' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 5 * try_cast(quantity as double) --Health and Social Care Voucher x5 -when upper(permit_type) like 'ALL ZONE BUSINESS VOUCHER' and (quantity is not null or quantity not like '' or quantity not like ' ') THEN 10 * try_cast(quantity as double) --All Zone Business Voucher x10 -end as decimal (10,0)) as total_num_vouchers_in_book -,* -from parking_voucher_de_normalised where import_date = (Select max(import_date) from parking_voucher_de_normalised ) --544,689 - -) -, street as (select - UPRN as SR_UPRN, - ADDRESS1 as SR_ADDRESS1, - ADDRESS2 as SR_ADDRESS2, - ADDRESS3 as SR_ADDRESS3, - ADDRESS4 as SR_ADDRESS4, - ADDRESS5 as SR_ADDRESS5, - POST_CODE as SR_POST_CODE, - BPLU_CLASS as SR_BPLU_CLASS, - X as SR_X, - Y as SR_Y, - LATITUDE as SR_LATITUDE, - LONGITUDE as SR_LONGITUDE, - IN_CONGESTION_CHARGE_ZONE as SR_IN_CONGESTION_CHARGE_ZONE, - CPZ_CODE as SR_CPZ_CODE, - CPZ_NAME as SR_CPZ_NAME, - LAST_UPDATED_STAMP as SR_LAST_UPDATED_STAMP, - LAST_UPDATED_TX_STAMP as SR_LAST_UPDATED_TX_STAMP, - CREATED_STAMP as SR_CREATED_STAMP, - CREATED_TX_STAMP as SR_CREATED_TX_STAMP, - HOUSE_NAME as SR_HOUSE_NAME, - POST_CODE_PACKED as SR_POST_CODE_PACKED, - STREET_START_X as SR_STREET_START_X, - STREET_START_Y as SR_STREET_START_Y, - STREET_END_X as SR_STREET_END_X, - STREET_END_Y as SR_STREET_END_Y, - WARD_CODE as SR_WARD_CODE, -if(WARD_CODE = 'E05009367', 'BROWNSWOOD WARD', -if(WARD_CODE = 'E05009368', 'CAZENOVE WARD', -if(WARD_CODE = 'E05009369', 'CLISSOLD WARD', -if(WARD_CODE = 'E05009370', 'DALSTON WARD', -if(WARD_CODE = 'E05009371', 'DE BEAUVOIR WARD', -if(WARD_CODE = 'E05009372', 'HACKNEY CENTRAL WARD', -if(WARD_CODE = 'E05009373', 'HACKNEY DOWNS WARD', -if(WARD_CODE = 'E05009374', 'HACKNEY WICK WARD', -if(WARD_CODE = 'E05009375', 'HAGGERSTON WARD', -if(WARD_CODE = 'E05009376', 'HOMERTON WARD', -if(WARD_CODE = 'E05009377', 'HOXTON EAST AND SHOREDITCH WARD', -if(WARD_CODE = 'E05009378', 'HOXTON WEST WARD', -if(WARD_CODE = 'E05009379', 'KINGS PARK WARD', -if(WARD_CODE = 'E05009380', 'LEA BRIDGE WARD', -if(WARD_CODE = 'E05009381', 'LONDON FIELDS WARD', -if(WARD_CODE = 'E05009382', 'SHACKLEWELL WARD', -if(WARD_CODE = 'E05009383', 'SPRINGFIELD WARD', -if(WARD_CODE = 'E05009384', 'STAMFORD HILL WEST', -if(WARD_CODE = 'E05009385', 'STOKE NEWINGTON WARD', -if(WARD_CODE = 'E05009386', 'VICTORIA WARD', -if(WARD_CODE = 'E05009387', 'WOODBERRY DOWN WARD',WARD_CODE))))))))))))))))))))) as SR_ward_name, - PARISH_CODE as SR_PARISH_CODE, - PARENT_UPRN as SR_PARENT_UPRN, - PAO_START as SR_PAO_START, - PAO_START_SUFFIX as SR_PAO_START_SUFFIX, - PAO_END as SR_PAO_END, - PAO_END_SUFFIX as SR_PAO_END_SUFFIX, - PAO_TEXT as SR_PAO_TEXT, - SAO_START as SR_SAO_START, - SAO_START_SUFFIX as SR_SAO_START_SUFFIX, - SAO_END as SR_SAO_END, - SAO_END_SUFFIX as SR_SAO_END_SUFFIX, - SAO_TEXT as SR_SAO_TEXT, - DERIVED_BLPU as SR_DERIVED_BLPU, - USRN -,import_year -,import_month -,import_day -,import_date -FROM liberator_permit_llpg -where (ADDRESS1 like 'Street Record' or ADDRESS1 like 'STREET RECORD') and liberator_permit_llpg.import_date = (SELECT MAX(liberator_permit_llpg.import_date) FROM liberator_permit_llpg) -) -, llpg as ( - SELECT * FROM unrestricted_address_api_dbo_hackney_address where unrestricted_address_api_dbo_hackney_address.import_date = (SELECT max(unrestricted_address_api_dbo_hackney_address.import_date) FROM unrestricted_address_api_dbo_hackney_address) and lpi_logical_status like 'Approved Preferred' - ) -, cfxref as ( -select * from unrestricted_address_api_dbo_hackney_xref where import_date = (select max(import_date) from unrestricted_address_api_dbo_hackney_xref) and xref_name like 'Parking Blacklisted S106' -) - -select distinct - -vou.fy - -,vou.mon_yr_app_date -,vou.month_year_application_date -,vou.approved_status_flag -,vou.permit_type_category -,vou.voucher_period -,vou.PermitType--voucherType --as permittype -,vou.product_category -,vou.voucher_type_category - -/*Partitions*/ -, vou.import_year -, vou.import_month -, vou.import_day -, vou.import_date - - -/*Counters*/ - -/*Transactions*/ -,count(concat(vou.application_date,vou.email_address_of_applicant)) as num_unique_transaction_ref -,count(distinct concat(vou.application_date,vou.email_address_of_applicant)) as num_distinct_unique_transaction_ref - -/*Email addresses*/ -,count(vou.email_address_of_applicant) as num_email_address_ref -,count(distinct vou.email_address_of_applicant) as num_distinct_email_address_ref - -/*CPZ - Zone*/ -,count(distinct vou.cpz) as num_distinct_cpz - -,count(distinct vou.cpz_name) as num_distinct_cpz_name - -,count(distinct vou.zone_name) as num_distinct_zone_name - -/*Street(USRN), Ward and Household(UPRN)*/ -,count(distinct SR_ADDRESS2) as num_distinct_street_name -,count(distinct SR_WARD_CODE) as num_distinct_street_ward_name -,count(distinct vou.uprn) as num_distinct_uprn_households -,count(distinct street.usrn) as num_distinct_usrn - -/*Books, Bookings and Vouchers*/ -,cast(sum(try_cast(vou.quantity as double)) as decimal(15,0)) as total_quantity_books -,cast(sum(try_cast(vou.total_num_vouchers_in_book as double)) as decimal(15,0)) as total_num_vouchers_in_books -,cast(sum(try_cast(vou.noofbookings as double)) as decimal(15,0)) as total_num_ebookings -,cast(sum(try_cast(vou.numberofevouchersinsession as double)) as decimal(15,0)) as total_num_evouchers_in_session - -/*Income - Total Paid*/ -,cast(sum(try_cast(vou.amount as double)) as decimal(15,2)) as total_amount_paid - - -/*Car Free blacklisted properties counters*/ - -/*Transactions Car Free*/ -,count( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then concat(vou.application_date,vou.email_address_of_applicant) end) as num_unique_transaction_ref_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then concat(vou.application_date,vou.email_address_of_applicant) end) as num_distinct_unique_transaction_ref_cfxref - -/*Email addresses Car Free*/ -,count( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.email_address_of_applicant end) as num_email_address_ref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.email_address_of_applicant end) as num_distinct_email_address_ref_cfxref - -/*CPZ - Zone Car Free*/ -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.cpz end) as num_distinct_cpz_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.cpz_name end) as num_distinct_cpz_name_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.zone_name end) as num_distinct_zone_name_cfxref - -/*Street(USRN), Ward and Household(UPRN) Car Free*/ -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then SR_ADDRESS2 end) as num_distinct_street_name_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then SR_WARD_CODE end) as num_distinct_street_ward_name_cfxref ---,count(distinct vou.uprn) as num_distinct_uprn_households -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then vou.uprn end) as num_distinct_uprn_households_cfxref -,count(distinct case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then street.usrn end) as num_distinct_usrn_cfxref - -/*Books, Bookings and Vouchers Car Free*/ -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.quantity as double) end ) as decimal(15,0)) as total_quantity_books_cfxref - -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.total_num_vouchers_in_book as double) end ) as decimal(15,0)) as total_num_vouchers_in_books_cfxref -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.noofbookings as double) end) as decimal(15,0)) as total_num_ebookings_cfxref -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.numberofevouchersinsession as double) end ) as decimal(15,0)) as total_num_evouchers_in_session_cfxref - -/*Income - Total Paid Car Free*/ -,cast(sum( case when cfxref.uprn is not null or cast(cfxref.uprn as string) != '' then try_cast(vou.amount as double) end) as decimal(15,2)) as total_amount_paid_cfxref - - - -from vou -left join llpg on cast(llpg.uprn as string) = cast(vou.uprn as string) -left join street on cast(street.usrn as string) = cast(llpg.usrn as string) -left join cfxref on cast(cfxref.uprn as string) = cast(vou.uprn as string) - -Where -vou.approved_status_flag = 1 - -group by -vou.fy - -,vou.mon_yr_app_date -,vou.month_year_application_date -,vou.approved_status_flag -,vou.permit_type_category -,vou.voucher_period -,vou.PermitType --voucherType -,vou.product_category -,vou.voucher_type_category - -/*Partitions*/ -, vou.import_year -, vou.import_month -, vou.import_day -, vou.import_date - -Order by -vou.month_year_application_date desc -,vou.PermitType asc --voucherType asc -,vou.voucher_period asc -,vou.voucher_type_category asc -""" -SQLQuery_node1720617258366 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "parking_voucher_de_normalised": AmazonS3liberatorrefinedzoneparking_voucher_de_normalised_node1720617253376, - "liberator_permit_llpg": AmazonS3liberatorrawzoneliberator_permit_llpg_node1720617252559, - "unrestricted_address_api_dbo_hackney_address": AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032, - "unrestricted_address_api_dbo_hackney_xref": AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047, - }, - transformation_ctx="SQLQuery_node1720617258366", -) - -# Script generated for node Amazon S3 - parking_vouchers_approved_summary_gds -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938 = glueContext.getSink( - path="s3://dataplatform-" - + environment - + "-refined-zone/parking/liberator/parking_vouchers_approved_summary_gds_lbh/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - enableUpdateCatalog=True, - transformation_ctx="AmazonS3parking_vouchers_approved_summary_gds_node1720617264938", -) -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938.setCatalogInfo( - catalogDatabase="dataplatform-" + environment + "-liberator-refined-zone", - catalogTableName="parking_vouchers_approved_summary_gds_lbh", -) -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938.setFormat( - "glueparquet", compression="snappy" -) -AmazonS3parking_vouchers_approved_summary_gds_node1720617264938.writeFrame( - SQLQuery_node1720617258366 -) -job.commit() diff --git a/scripts/jobs/parking/spatial-enrichment-dictionary-preprod.json b/scripts/jobs/parking/spatial-enrichment-dictionary-preprod.json deleted file mode 100644 index 6940c7e0a..000000000 --- a/scripts/jobs/parking/spatial-enrichment-dictionary-preprod.json +++ /dev/null @@ -1,13 +0,0 @@ -[{ - "cycle_hangar_waiting_list": { - "database_name":"dataplatform-stg-liberator-refined-zone", - "table_name":"pparking_cycle_hangar_wait_list_with_coords", - "partition_keys":["import_year","import_month","import_day","import_date"], - "date_partition_name":"import_date", - "x_column":"x", - "y_column":"y", - "geom_format": "coords", - "source_crs": "epsg:27700", - "enrich_with":["housing_estate"] - } -}] diff --git a/scripts/jobs/parking/spatial-enrichment-dictionary.json b/scripts/jobs/parking/spatial-enrichment-dictionary.json deleted file mode 100644 index 83bdf0866..000000000 --- a/scripts/jobs/parking/spatial-enrichment-dictionary.json +++ /dev/null @@ -1,13 +0,0 @@ -[{ - "liberator_permit_llpg": { - "database_name":"dataplatform-prod-liberator-raw-zone", - "table_name":"liberator_permit_llpg", - "partition_keys":["import_year","import_month","import_day","import_date"], - "date_partition_name":"import_date", - "x_column":"x", - "y_column":"y", - "geom_format": "coords", - "source_crs": "epsg:27700", - "enrich_with":["housing_estate"] - } -}] diff --git a/terraform/etl/38-aws-glue-job-parking.tf b/terraform/etl/38-aws-glue-job-parking.tf deleted file mode 100644 index 9e8c58a49..000000000 --- a/terraform/etl/38-aws-glue-job-parking.tf +++ /dev/null @@ -1,85 +0,0 @@ -# Migrated job "parking_pcn_denormalisation" to dap-airflow om 30/05/2025 -# >> PCNFOIDetails_PCN_FOI_FULL - -# Migrated job "parking_persistent_evaders" to dap-airflow om 30/05/2025 - -# Migrated job "parking_school_street_vrms" to dap-airflow om 20/02/2025 - -# Migrated job "parking_estate_waiting_list_live_permits_type_gds" to dap-airflow om 20/02/2025 - -# Migrated job "parking_gds_permit_change_comparison" to dap-airflow om 20/02/2025 - -# Migrated job "parking_kpi_gds_summary" to dap-airflow om 30/05/2025 - -# Migrated job "parking_foi_pcn_gds_summary" to dap-airflow om 30/05/2025 - -# Migrated job "parking_permit_denormalised_gds_street_llpg" to dap-airflow om 20/02/2025 - - -# Migrated job "parking_pcn_create_event_log" to dap-airflow om 30/05/2025 -# >> PCNFOIDetails_PCN_EVENT_LOG - -# Migrated job "parking_pcn_report_summary" to dap-airflow om 30/05/2025 - -# Migrated job "parking_pcn_ltn_report_summary" to dap-airflow om 30/05/2025 - -# migrated Parking_Suspension_DeNormalised_Data to airflow on 19/05/2025 - -# migrated Parking_Permit_DeNormalised_Data to airflow on 20/05/2025 - -# The airflow has the latest version of these 7 tables -# removed Parking_Deployment_Target_Details -# removed parking_ceo_average_on_street -# removed parking_ceo_on_street -# removed parking_ceo_summary -# removed parking_ceo_average_on_street_hrs_mins_secs -# removed parking_percent_street_coverage -# removed parking_percent_street_coverage_cpz - -# Migrated "parking_foreign_vrm_pcns" to dap-airflow on 30/05/2025 - -# Migrated "parking_correspondence_performance_records_with_pcn" to dap-airflow on 30/05/2025 - -# parking_Disputes_KPI_GDS_Summary not in use anymore - confirmed with Davina - -# Migrated "parking_foi_pcn_gds_daily_summary" to dap-airflow on 30/05/2025 - -# Migrated "parking_eta_decision_records_pcn_dispute_gds" to dap-airflow on 30/05/2025 - -# migrated Parking_Permit_diesel_Tends_Bought_in_Month to airflow on 20/05/2025 - -# Migrated "parking_correspondence_performance_records_with_pcn_gds" to dap-airflow on 30/05/2025 - -# Migrated "parking_foi_pcn_gds_daily_summary_records" to dap-airflow on 30/05/2025 - -# Migrated job "parking_correspondence_performance_qa_with_totals_gds" to dap-airflow om 30/05/2025 - -# Migrated job "parking_defect_met_fail" to dap-airflow om 25/02/2025 - -# Migrated job "parking_match_pcn_permit_vrm_llpg_nlpg_postcodes" to dap-airflow om 20/02/2025 - -# Migrated job "parking_defect_met_fail_monthly_format" to dap-airflow om 25/02/2025 - -# Migrated job parking_permit_street_stress to airflow on 20/05/2025 - -# migrated parking_permit_street_stress_with_cpz to airflow on 20/05/2025 - -# Migrated "parking_correspondence_performance_records_with_pcn_downtime" to dap-airflow on 30/05/2025 -# Migrated "parking_correspondence_performance_records_with_pcn_downtime_gds" to dap-airflow on 30/05/2025 - -# Migrated "parking_open_pcns_vrms_linked_cancelled_ringer" to dap-airflow on 30/05/2025 - -# MRB 15-02-2024 job created -# migrated job "Parking_Suspensions_Processed" to dap-airflow on 19/05/2025 -# parking_suspensions_processed_with_finyear migrated to dap-airflow on 19/05/2025 - -# Migrated "parking_pcn_dvla_response_no_address" to dap-airflow on 30/05/2025 - -# Migrated job "parking_motorcycle_permits_480" to dap-airflow om 20/02/2025 -# Migrated job "parking_permit_street_cpz_stress_mc" to dap-airflow on 21/05/2025 - -# MRB 18-08-2024 job created -# Migrated job "parking_permit_denormalisation_mc" to dap-airflow om 01/05/2025 - -# MRB 20-08-2024 job created -# Migrated job "parking_all_suspensions_processed_review" to dap-airflow om 19/05/2025