From db33490ba468d52f5c90da382aa3ae0682b93b90 Mon Sep 17 00:00:00 2001 From: Tian Chen <38001883+Tian-2017@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:54:29 +0100 Subject: [PATCH] revert part of group 10 changes based on steve suggestion --- ...espondence_performance_records_with_pcn.py | 781 +++++--- ...e_performance_records_with_pcn_downtime.py | 1625 +++++++++------ ...rformance_records_with_pcn_downtime_gds.py | 1780 +++++++++++------ ...ndence_performance_records_with_pcn_gds.py | 958 +++++---- terraform/etl/38-aws-glue-job-parking.tf | 97 +- 5 files changed, 3347 insertions(+), 1894 deletions(-) diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py index b6e9f8c73..bb40f03db 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py @@ -1,312 +1,499 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") +""" +Only need to change the table name and the query prototyped on the Athena UI +by replacing table_name and query_on_athena +""" -# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full -AmazonS3refinedpcnfoidetails_pcn_foi_full_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="pcnfoidetails_pcn_foi_full", - transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1", -) +from scripts.helpers.athena_helpers import create_update_table_with_partition +from scripts.helpers.helpers import get_glue_env_var -# Script generated for node S3 bucket - Raw - liberator_pcn_ic -S3bucketRawliberator_pcn_ic_node1682353070282 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="liberator_pcn_ic", - transformation_ctx="S3bucketRawliberator_pcn_ic_node1682353070282", -) +environment = get_glue_env_var("environment") -# Script generated for node parking_raw_zone - parking_correspondence_performance_teams -parking_raw_zoneparking_correspondence_performance_teams_node1682353072411 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="parking_correspondence_performance_teams", - transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1682353072411", -) +# The target table in liberator refined zone +table_name = "parking_correspondence_performance_records_with_pcn" -# Script generated for node ApplyMapping -SqlQuery0 = """ -/* +# The exact same query prototyped in pre-prod(stg) or prod Athena +query_on_athena = """ +/* +parking_correspondence_performance_records_with_pcn.sql Correspondence Performance records last 13 months with PCN FOI records -16/06/2022 - Created + +-->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn" +--<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" +-- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" +-- < "parking-raw-zone"."parking_correspondence_performance_teams" + +16/06/2022 - Created 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing +27/02/2025 - Refactored SQL for migration back to AthenaSQL for dap-airflow +14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. +16/04/2025 - Workaround to orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready +*/ +WITH +team AS ( + SELECT DISTINCT + "start_date" AS t_start_date, + end_date AS t_end_date, + team AS t_team, + team_name AS t_team_name, + "role" AS t_role, + forename AS t_forename, + surname AS t_surname, + full_name AS t_full_name, + qa_doc_created_by AS t_qa_doc_created_by, + qa_doc_full_name AS t_qa_doc_full_name, + post_title AS t_post_title, + notes AS t_notes, + import_date AS t_import_date + FROM "parking-raw-zone"."parking_correspondence_performance_teams" + WHERE import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "parking-raw-zone"."parking_correspondence_performance_teams" g + ) +), +liberator AS ( + SELECT *, + + CAST(CURRENT_TIMESTAMP AS TIMESTAMP(3)) AS current_utc_timestamp, + TRY(CAST(date_received AS TIMESTAMP(3))) AS date_received_timestamp, -- pre-filtered by WHERE clause + + IF(whenassigned <> '', -- contains probable valid timestamp + TRY(CAST(whenassigned AS TIMESTAMP(3))), + CAST(NULL AS TIMESTAMP(3)) -- edge case + ) AS whenassigned_timestamp, + + IF(Response_generated_at <> '', -- contains probable valid timestamp + TRY(CAST(Response_generated_at AS TIMESTAMP(3))), + CAST(NULL AS TIMESTAMP(3)) -- edge case + ) AS Response_generated_at_timestamp, + CAST(CASE WHEN date_received <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' + CURRENT_DATE + ) + --ELSE NULL + END AS INTEGER) AS days_since_date_received, + + CAST(CASE WHEN date_received <> '' AND whenassigned <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' + TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE) + ) + --ELSE NULL + END AS INTEGER) AS days_since_date_received_whenassigned, + + CAST(CASE WHEN whenassigned <> '' AND response_generated_at = '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), + CURRENT_DATE + ) + --ELSE NULL -- edge case + END AS INTEGER) AS days_since_whenassigned, + + CAST(CASE WHEN whenassigned <> '' AND response_generated_at <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), + TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) + ) + --ELSE NULL -- edge case + END AS INTEGER) AS days_since_whenassigned_response_generated_at, + + CAST(CASE WHEN date_received <> '' AND response_generated_at <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), + TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) + ) + --ELSE NULL -- edge case + END AS INTEGER) AS days_since_date_received_response_generated_at + + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" + WHERE import_Date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g + ) + AND LENGTH(ticketserialnumber) = 10 -- ticket filter + AND date_received <> '' -- is the overriding condition for "13 months from todays date"! + AND TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE) > CURRENT_DATE - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + AND DATE_DIFF( + 'month', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), + CURRENT_DATE + ) <= 13 */ -With team as ( -select distinct start_date as t_start_date -,end_date as t_end_date -,team as t_team -,team_name as t_team_name -,role as t_role -,forename as t_forename -,surname as t_surname -,full_name as t_full_name -,qa_doc_created_by as t_qa_doc_created_by -,qa_doc_full_name as t_qa_doc_full_name -,post_title as t_post_title -,notes as t_notes -,import_date as t_import_date--* -from parking_correspondence_performance_teams where import_date = (select max(import_date) from parking_correspondence_performance_teams ) ) +SELECT + CAST(CASE + WHEN l.date_received <> '' AND l.whenassigned = '' + THEN 'Unassigned' + WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at = '' + THEN 'Assigned' + WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at <> '' + THEN 'Responded' + --ELSE NULL + END AS VARCHAR) AS response_status, + + CAST(l.current_utc_timestamp AS VARCHAR) AS current_time_stamp, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS unassigned_time, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS to_assigned_time, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_time, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_response_time, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS response_time, + + /*unassigned days*/ + CAST(CASE WHEN l.whenassigned = '' + THEN l.days_since_date_received + --ELSE NULL + END AS VARCHAR) AS unassigned_days, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + THEN + CASE + WHEN l.days_since_date_received <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received > 56 THEN '56 plus days' + --WHEN l.days_since_date_received IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS unassigned_days_group, + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + AND l.days_since_date_received <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS unassigned_days_kpiTotFiftySixLess, -Select -case -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' -end as response_status, -cast(current_timestamp as string) as Current_time_stamp, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, - -/*unassigned days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' -end as unassigned_days_group -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as unassigned_days_kpiTotFiftySixLess -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as unassigned_days_kpiTotFourteenLess, - -/*Days to assign*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as Days_to_assign_group -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as Days_to_assign_kpiTotFiftySixLess -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as Days_to_assign_kpiTotFourteenLess, - -/*assigned in progress days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' -end as assigned_in_progress_days_group -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFiftySixLess -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFourteenLess, - -/*assigned response days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' -end as assignedResponseDays_group -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFiftySixLess -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFourteenLess, - -/*Response days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as ResponseDays_group -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as ResponseDays_kpiTotFiftySixLess -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as ResponseDays_kpiTotFourteenLess, - -Response_generated_at -,Date_Received -,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear -,liberator_pcn_ic.Type -,Serviceable -,Service_category -,Response_written_by -,Letter_template -,Action_taken -,Related_to_PCN -,Cancellation_group -,Cancellation_reason -,whenassigned -,ticketserialnumber -,noderef -,liberator_pcn_ic.record_created -,liberator_pcn_ic.import_timestamp -,liberator_pcn_ic.import_year -,liberator_pcn_ic.import_month -,liberator_pcn_ic.import_day -,liberator_pcn_ic.import_date - -/*pcn data*/ -,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn -,pcnfoidetails_pcn_foi_full.pcnissuedate as pcn_pcnissuedate -,pcnfoidetails_pcn_foi_full.pcnissuedatetime as pcn_pcnissuedatetime -,pcnfoidetails_pcn_foi_full.pcn_canx_date as pcn_pcn_canx_date -,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup -,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason -,pcnfoidetails_pcn_foi_full.pcn_casecloseddate as pcn_pcn_casecloseddate -,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location -,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation -,pcnfoidetails_pcn_foi_full.zone as pcn_zone -,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn -,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode -,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix -,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype -,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm -,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake -,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel -,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour -,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo -,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice -,pcnfoidetails_pcn_foi_full.current_30_day_flag as pcn_current_30_day_flag -,pcnfoidetails_pcn_foi_full.isvda as pcn_isvda -,pcnfoidetails_pcn_foi_full.isvoid as pcn_isvoid -,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval -,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen -,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows -,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway -,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor -,pcnfoidetails_pcn_foi_full.warningflag as pcn_warningflag -,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts -,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason -,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount -,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received -,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount -,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void -,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method -,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref -,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from -,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to -,pcnfoidetails_pcn_foi_full.bailiff_processedon as pcn_bailiff_processedon -,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason -,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff -,pcnfoidetails_pcn_foi_full.warrantissuedate as pcn_warrantissuedate -,pcnfoidetails_pcn_foi_full.allocation as pcn_allocation -,pcnfoidetails_pcn_foi_full.eta_datenotified as pcn_eta_datenotified -,pcnfoidetails_pcn_foi_full.eta_packsubmittedon as pcn_eta_packsubmittedon -,pcnfoidetails_pcn_foi_full.eta_evidencedate as pcn_eta_evidencedate -,pcnfoidetails_pcn_foi_full.eta_adjudicationdate as pcn_eta_adjudicationdate -,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds -,pcnfoidetails_pcn_foi_full.eta_decisionreceived as pcn_eta_decisionreceived -,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome -,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby -,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby -,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address -,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address -,pcnfoidetails_pcn_foi_full.corresp_dispute_flag as pcn_corresp_dispute_flag -,pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as pcn_keyworker_corresp_dispute_flag -,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag -,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year -,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref -,pcnfoidetails_pcn_foi_full.nto_printed as pcn_nto_printed -,pcnfoidetails_pcn_foi_full.appeal_accepted as pcn_appeal_accepted -,pcnfoidetails_pcn_foi_full.arrived_in_pound as pcn_arrived_in_pound -,pcnfoidetails_pcn_foi_full.cancellation_reversed as pcn_cancellation_reversed -,pcnfoidetails_pcn_foi_full.cc_printed as pcn_cc_printed -,pcnfoidetails_pcn_foi_full.drr as pcn_drr -,pcnfoidetails_pcn_foi_full.en_printed as pcn_en_printed -,pcnfoidetails_pcn_foi_full.hold_released as pcn_hold_released -,pcnfoidetails_pcn_foi_full.dvla_response as pcn_dvla_response -,pcnfoidetails_pcn_foi_full.dvla_request as pcn_dvla_request -,pcnfoidetails_pcn_foi_full.full_rate_uplift as pcn_full_rate_uplift -,pcnfoidetails_pcn_foi_full.hold_until as pcn_hold_until -,pcnfoidetails_pcn_foi_full.lifted_at as pcn_lifted_at -,pcnfoidetails_pcn_foi_full.lifted_by as pcn_lifted_by -,pcnfoidetails_pcn_foi_full.loaded as pcn_loaded -,pcnfoidetails_pcn_foi_full.nor_sent as pcn_nor_sent -,pcnfoidetails_pcn_foi_full.notice_held as pcn_notice_held -,pcnfoidetails_pcn_foi_full.ofr_printed as pcn_ofr_printed -,pcnfoidetails_pcn_foi_full.pcn_printed as pcn_pcn_printed -,pcnfoidetails_pcn_foi_full.reissue_nto_requested as pcn_reissue_nto_requested -,pcnfoidetails_pcn_foi_full.reissue_pcn as pcn_reissue_pcn -,pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as pcn_set_back_to_pre_cc_stage -,pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as pcn_vehicle_released_for_auction -,pcnfoidetails_pcn_foi_full.warrant_issued as pcn_warrant_issued -,pcnfoidetails_pcn_foi_full.warrant_redistributed as pcn_warrant_redistributed -,pcnfoidetails_pcn_foi_full.warrant_request_granted as pcn_warrant_request_granted -,pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as pcn_ad_hoc_vq4_request -,pcnfoidetails_pcn_foi_full.paper_vq5_received as pcn_paper_vq5_received -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as pcn_pcn_extracted_for_buslane -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as pcn_pcn_extracted_for_pre_debt -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as pcn_pcn_extracted_for_collection -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as pcn_pcn_extracted_for_drr -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as pcn_pcn_extracted_for_cc -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as pcn_pcn_extracted_for_nto -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as pcn_pcn_extracted_for_print -,pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as pcn_warning_notice_extracted_for_print -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as pcn_pcn_extracted_for_ofr -,pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as pcn_pcn_extracted_for_warrant_request -,pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as pcn_pre_debt_new_debtor_details -,pcnfoidetails_pcn_foi_full.importdattime as pcn_importdattime -,pcnfoidetails_pcn_foi_full.importdatetime as pcn_importdatetime -,pcnfoidetails_pcn_foi_full.import_year as pcn_import_year -,pcnfoidetails_pcn_foi_full.import_month as pcn_import_month -,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day -,pcnfoidetails_pcn_foi_full.import_date as pcn_import_date -,team.* - - -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date -left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) - -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + AND l.days_since_date_received <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS unassigned_days_kpiTotFourteenLess, + /*Days to assign*/ + CAST(l.days_since_date_received_whenassigned AS VARCHAR) AS days_to_assign, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + THEN + CASE + WHEN l.days_since_date_received_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received_whenassigned > 56 THEN '56 plus days' + --WHEN l.days_since_date_received_whenassigned IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS days_to_assign_group, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + AND l.days_since_date_received_whenassigned <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS Days_to_assign_kpiTotFiftySixLess, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + AND l.days_since_date_received_whenassigned <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS Days_to_assign_kpiTotFourteenLess, + + /*assigned in progress days*/ + CAST(l.days_since_whenassigned AS VARCHAR) AS assigned_in_progress_days, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned > 56 THEN '56 plus days' + --WHEN l.days_since_whenassigned IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_days_group, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS assigned_in_progress_days_kpiTotFiftySixLess, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS assigned_in_progress_days_kpiTotFourteenLess, + + /*assigned response days*/ + CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) AS assignedResponseDays, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + THEN + CASE + WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' + --WHEN l.days_since_whenassigned_response_generated_at IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assignedResponseDays_group, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + AND l.days_since_whenassigned_response_generated_at <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS assignedResponseDays_kpiTotFiftySixLess, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + AND l.days_since_whenassigned_response_generated_at <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS assignedResponseDays_kpiTotFourteenLess, + + /*Response days*/ + CAST(l.days_since_date_received_response_generated_at AS VARCHAR) AS ResponseDays, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + THEN + CASE + WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' + --WHEN l.days_since_date_received_response_generated_at IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS ResponseDays_group, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS ResponseDays_kpiTotFiftySixLess, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS ResponseDays_kpiTotFourteenLess, + + l.Response_generated_at, --AS VARCHAR + l.Date_Received, --AS VARCHAR + + CAST(SUBSTR(TRY_CAST(l.date_received AS VARCHAR(10)), 1, 7) || '-01' AS VARCHAR) AS MonthYear, + + l.Type AS "type", --AS VARCHAR + l.Serviceable, --AS VARCHAR + l.Service_category, --AS VARCHAR + l.Response_written_by, --AS VARCHAR + l.Letter_template, --AS VARCHAR + l.Action_taken, --AS VARCHAR + l.Related_to_PCN, --AS VARCHAR + l.Cancellation_group, --AS VARCHAR + l.Cancellation_reason, --AS VARCHAR + l.whenassigned, --AS VARCHAR + l.ticketserialnumber, --AS VARCHAR + l.noderef, --AS VARCHAR + TRY_CAST(l.record_created AS TIMESTAMP(3)) AS record_created, -- cast from VARCHAR + l.import_timestamp, --AS VARCHAR + + /*** pcn columns taken from "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ***/ + -- Data sourced from the refined zone needs to be handled carefully... + -- Do not assume the following output column datatypes will be the same as "pcnfoidetails_pcn_foi_full". + -- Being left-joined to "pcnfoidetails_pcn_foi_full" will cause NULLs to appear in these columns. + -- And similar transforms elsewhere require different translations to the ones below... + /* These are the current automated translations developed for this transform:- + TRY_CAST(p.[source_column] AS DATE) AS [target_column], --[position]-- try date to date + COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS DATE), '%Y-%m-%d'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try date to string + TRY_CAST(p.[source_column] AS INTEGER) AS [target_column], --[position]-- try int to int + TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try timestamp to string + TRY_CAST(p.[source_column] AS TIMESTAMP) AS [target_column], --[position]-- try timestamp to timestamp + */ + -- These translations try to deal with problems without resorting to throwing errors, tolerating a reasonable degree of schema evolution in the "pcnfoidetails_pcn_foi_full" source. + TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --45-- try string to string (default) + TRY_CAST(p.pcnissuedate AS DATE) AS pcn_pcnissuedate, --46-- try date to date + TRY_CAST(p.pcnissuedatetime AS TIMESTAMP) AS pcn_pcnissuedatetime, --47-- try timestamp to timestamp + TRY_CAST(p.pcn_canx_date AS DATE) AS pcn_pcn_canx_date, --48-- try date to date + TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --49-- try string to string (default) + TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --50-- try string to string (default) + TRY_CAST(p.pcn_casecloseddate AS DATE) AS pcn_pcn_casecloseddate, --51-- try date to date + TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --52-- try string to string (default) + TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --53-- try string to string (default) + TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --54-- try string to string (default) + TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --55-- try string to string (default) + TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --56-- try string to string (default) + TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --57-- try string to string (default) + TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --58-- try string to string (default) + TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --59-- try string to string (default) + TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --60-- try string to string (default) + TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --61-- try string to string (default) + TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --62-- try string to string (default) + TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --63-- try string to string (default) + TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --64-- try string to string (default) + TRY_CAST(p.current_30_day_flag AS INTEGER) AS pcn_current_30_day_flag, --65-- try int to int + TRY_CAST(p.isvda AS INTEGER) AS pcn_isvda, --66-- try int to int + TRY_CAST(p.isvoid AS INTEGER) AS pcn_isvoid, --67-- try int to int + TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --68-- try string to string (default) + TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --69-- try string to string (default) + TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --70-- try string to string (default) + TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --71-- try string to string (default) + TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --72-- try string to string (default) + TRY_CAST(p.warningflag AS INTEGER) AS pcn_warningflag, --73-- try int to int + TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --74-- try string to string (default) + TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --75-- try string to string (default) + TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --76-- try string to string (default) + TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --77-- try string to string (default) + TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --78-- try string to string (default) + TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --79-- try string to string (default) + TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --80-- try string to string (default) + TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --81-- try string to string (default) + TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --82-- try string to string (default) + TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --83-- try string to string (default) + TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --84-- try string to string (default) + TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --85-- try string to string (default) + TRY_CAST(p.bailiff_processedon AS TIMESTAMP) AS pcn_bailiff_processedon, --86-- try timestamp to timestamp + TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --87-- try string to string (default) + TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --88-- try string to string (default) + TRY_CAST(p.warrantissuedate AS TIMESTAMP) AS pcn_warrantissuedate, --89-- try timestamp to timestamp + TRY_CAST(p.allocation AS INTEGER) AS pcn_allocation, --90-- try int to int + TRY_CAST(p.eta_datenotified AS TIMESTAMP) AS pcn_eta_datenotified, --91-- try timestamp to timestamp + TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP) AS pcn_eta_packsubmittedon, --92-- try timestamp to timestamp + TRY_CAST(p.eta_evidencedate AS TIMESTAMP) AS pcn_eta_evidencedate, --93-- try timestamp to timestamp + TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP) AS pcn_eta_adjudicationdate, --94-- try timestamp to timestamp + TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --95-- try string to string (default) + TRY_CAST(p.eta_decisionreceived AS TIMESTAMP) AS pcn_eta_decisionreceived, --96-- try timestamp to timestamp + TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --97-- try string to string (default) + TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --98-- try string to string (default) + TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --99-- try string to string (default) + TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --100-- try string to string (default) + TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --101-- try string to string (default) + TRY_CAST(p.corresp_dispute_flag AS INTEGER) AS pcn_corresp_dispute_flag, --102-- try int to int + TRY_CAST(p.keyworker_corresp_dispute_flag AS INTEGER) AS pcn_keyworker_corresp_dispute_flag, --103-- try int to int + TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --104-- try string to string (default) + TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --105-- try string to string (default) + TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --106-- try string to string (default) + TRY_CAST(p.nto_printed AS TIMESTAMP) AS pcn_nto_printed, --107-- try timestamp to timestamp + TRY_CAST(p.appeal_accepted AS TIMESTAMP) AS pcn_appeal_accepted, --108-- try timestamp to timestamp + TRY_CAST(p.arrived_in_pound AS TIMESTAMP) AS pcn_arrived_in_pound, --109-- try timestamp to timestamp + TRY_CAST(p.cancellation_reversed AS TIMESTAMP) AS pcn_cancellation_reversed, --110-- try timestamp to timestamp + TRY_CAST(p.cc_printed AS TIMESTAMP) AS pcn_cc_printed, --111-- try timestamp to timestamp + TRY_CAST(p.drr AS TIMESTAMP) AS pcn_drr, --112-- try timestamp to timestamp + TRY_CAST(p.en_printed AS TIMESTAMP) AS pcn_en_printed, --113-- try timestamp to timestamp + TRY_CAST(p.hold_released AS TIMESTAMP) AS pcn_hold_released, --114-- try timestamp to timestamp + TRY_CAST(p.dvla_response AS TIMESTAMP) AS pcn_dvla_response, --115-- try timestamp to timestamp + TRY_CAST(p.dvla_request AS TIMESTAMP) AS pcn_dvla_request, --116-- try timestamp to timestamp + TRY_CAST(p.full_rate_uplift AS TIMESTAMP) AS pcn_full_rate_uplift, --117-- try timestamp to timestamp + TRY_CAST(p.hold_until AS TIMESTAMP) AS pcn_hold_until, --118-- try timestamp to timestamp + TRY_CAST(p.lifted_at AS TIMESTAMP) AS pcn_lifted_at, --119-- try timestamp to timestamp + TRY_CAST(p.lifted_by AS TIMESTAMP) AS pcn_lifted_by, --120-- try timestamp to timestamp + TRY_CAST(p.loaded AS TIMESTAMP) AS pcn_loaded, --121-- try timestamp to timestamp + TRY_CAST(p.nor_sent AS TIMESTAMP) AS pcn_nor_sent, --122-- try timestamp to timestamp + TRY_CAST(p.notice_held AS TIMESTAMP) AS pcn_notice_held, --123-- try timestamp to timestamp + TRY_CAST(p.ofr_printed AS TIMESTAMP) AS pcn_ofr_printed, --124-- try timestamp to timestamp + TRY_CAST(p.pcn_printed AS TIMESTAMP) AS pcn_pcn_printed, --125-- try timestamp to timestamp + TRY_CAST(p.reissue_nto_requested AS TIMESTAMP) AS pcn_reissue_nto_requested, --126-- try timestamp to timestamp + TRY_CAST(p.reissue_pcn AS TIMESTAMP) AS pcn_reissue_pcn, --127-- try timestamp to timestamp + TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP) AS pcn_set_back_to_pre_cc_stage, --128-- try timestamp to timestamp + TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP) AS pcn_vehicle_released_for_auction, --129-- try timestamp to timestamp + TRY_CAST(p.warrant_issued AS TIMESTAMP) AS pcn_warrant_issued, --130-- try timestamp to timestamp + TRY_CAST(p.warrant_redistributed AS TIMESTAMP) AS pcn_warrant_redistributed, --131-- try timestamp to timestamp + TRY_CAST(p.warrant_request_granted AS TIMESTAMP) AS pcn_warrant_request_granted, --132-- try timestamp to timestamp + TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP) AS pcn_ad_hoc_vq4_request, --133-- try timestamp to timestamp + TRY_CAST(p.paper_vq5_received AS TIMESTAMP) AS pcn_paper_vq5_received, --134-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP) AS pcn_pcn_extracted_for_buslane, --135-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP) AS pcn_pcn_extracted_for_pre_debt, --136-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP) AS pcn_pcn_extracted_for_collection, --137-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP) AS pcn_pcn_extracted_for_drr, --138-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP) AS pcn_pcn_extracted_for_cc, --139-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP) AS pcn_pcn_extracted_for_nto, --140-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP) AS pcn_pcn_extracted_for_print, --141-- try timestamp to timestamp + TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP) AS pcn_warning_notice_extracted_for_print, --142-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP) AS pcn_pcn_extracted_for_ofr, --143-- try timestamp to timestamp + TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP) AS pcn_pcn_extracted_for_warrant_request, --144-- try timestamp to timestamp + TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP) AS pcn_pre_debt_new_debtor_details, --145-- try timestamp to timestamp + TRY_CAST(p.importdattime AS TIMESTAMP) AS pcn_importdattime, --146-- try timestamp to timestamp + TRY_CAST(p.importdatetime AS TIMESTAMP) AS pcn_importdatetime, --147-- try timestamp to timestamp + TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --148-- try string to string (default) + TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --149-- try string to string (default) + TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --150-- try string to string (default) + TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --151-- try string to string (default) + + /* teams columns expanded by above subquery */ + team.*, + + /* Partition columns moved to the end to keep schema alligned */ + l.import_year, + l.import_month, + l.import_day, + l.import_date + +FROM liberator l -- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" +LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p +/* + ON p.import_date = l.import_date -- joined on import partition + Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... + ...resulting in empty pcn_ columns in the output. + Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... + ...but for now we'll just use the following simple workaround... +*/ + ON p.import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g + ) + AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid +LEFT JOIN team -- < "parking-raw-zone"."parking_correspondence_performance_teams" + ON UPPER(team.t_full_name) = UPPER(l.Response_written_by) +; """ -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1, - "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1682353070282, - "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1682353072411, - }, - transformation_ctx="ApplyMapping_node2", -) -# Script generated for node S3 bucket -S3bucket_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="S3bucket_node3", -) -S3bucket_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_correspondence_performance_records_with_pcn", +create_update_table_with_partition( + environment=environment, query_on_athena=query_on_athena, table_name=table_name ) -S3bucket_node3.setFormat("glueparquet") -S3bucket_node3.writeFrame(ApplyMapping_node2) -job.commit() diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py index e0f61029b..fd7b47a8b 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py @@ -1,592 +1,1067 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") - -# Script generated for node parking_raw_zone - parking_correspondence_performance_teams -parking_raw_zoneparking_correspondence_performance_teams_node1682353072411 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="parking_correspondence_performance_teams", - transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1682353072411", -) +""" +Only need to change the table name and the query prototyped on the Athena UI +by replacing table_name and query_on_athena +""" -# Script generated for node S3 bucket - Raw - liberator_pcn_ic -S3bucketRawliberator_pcn_ic_node1682353070282 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="liberator_pcn_ic", - transformation_ctx="S3bucketRawliberator_pcn_ic_node1682353070282", -) +from scripts.helpers.athena_helpers import create_update_table_with_partition +from scripts.helpers.helpers import get_glue_env_var -# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full -AmazonS3refinedpcnfoidetails_pcn_foi_full_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="pcnfoidetails_pcn_foi_full", - transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1", -) +environment = get_glue_env_var("environment") -# Script generated for node parking_raw_zone - parking_officer_downtime -parking_raw_zoneparking_officer_downtime_node1697211483070 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="parking_officer_downtime", - transformation_ctx="parking_raw_zoneparking_officer_downtime_node1697211483070", -) +# The target table in liberator refined zone +table_name = "parking_correspondence_performance_records_with_pcn_downtime" -# Script generated for node ApplyMapping -SqlQuery0 = """ +# The exact same query prototyped in pre-prod(stg) or prod Athena +query_on_athena = """ +-->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn_downtime" +--< "parking-raw-zone"."parking_correspondence_performance_teams" (data from google spreadsheet) +--< "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" +--<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" +--< "parking-raw-zone"."parking_officer_downtime" (data from google form) /* Correspondence Performance records last 13 months with PCN FOI records -16/06/2022 - Created +16/06/2022 - Created 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing 02/10/2023 - modified to union data into one output for Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 -06/11/2024 - updated date formats in downtime data as source google sheet changed to yyyy-mm-dd HH:MM:ss from d/m/y +22/10/2024 - translated into PrestoSQL to move away from Glue +01/11/2024 + Refactored SQL to separate the inward column conversions from onward business logic using CTE subqueries and allow improved fault handling. + + Downtime's Google Sheets alternate date formats properly handled 28/11/2024 - updated downtime records for t_team details to pull from parking_correspondence_performance_teams instead on leaving blank +12/12/2024 + Integrated the previous downtime update into the refactored version + + Commented out alternative last 13 month filter that caused unexpectedly higher row count +05/03/2025 - Replaced conditional '' empty string outputs with NULL defaults to be consistent with previous Glue/SparkSQL implementations. +14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. Fixed faulty arithmetic translation of Glue/SparkSQL to AthenaSQL. +16/04/2025 - Workaround orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready */ - /*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ -With team as ( -select distinct start_date as t_start_date -,end_date as t_end_date -,team as t_team -,team_name as t_team_name -,role as t_role -,forename as t_forename -,surname as t_surname -,full_name as t_full_name -,qa_doc_created_by as t_qa_doc_created_by -,qa_doc_full_name as t_qa_doc_full_name -,post_title as t_post_title -,notes as t_notes -,import_date as t_import_date--* -from parking_correspondence_performance_teams where import_date = (select max(parking_correspondence_performance_teams.import_date) from parking_correspondence_performance_teams ) -) +WITH +team AS ( + SELECT DISTINCT + start_date AS t_start_date, + end_date AS t_end_date, + team AS t_team, + team_name AS t_team_name, + ROLE AS t_role, + forename AS t_forename, + surname AS t_surname, + full_name AS t_full_name, + qa_doc_created_by AS t_qa_doc_created_by, + qa_doc_full_name AS t_qa_doc_full_name, + post_title AS t_post_title, + notes AS t_notes, + import_date AS t_import_date --* + FROM "parking-raw-zone"."parking_correspondence_performance_teams" + WHERE import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "parking-raw-zone"."parking_correspondence_performance_teams" g + ) +), +liberator_pcn_icdate_received_last13months AS ( + SELECT + *, + CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS current_utc_timestamp, + TRY(CAST(date_received AS TIMESTAMP)) AS date_received_timestamp, -- pre-filtered by WHERE clause -select - -/*Fields for union downtime data*/ -'' as downtime_timestamp -,'' as downtime_email_address -,'' as officer_s_first_name -,'' as officer_s_last_name -,'' as liberator_system_username -,'' as liberator_system_id -,'' as import_datetime -,'' as multiple_downtime_days_flag -,'' as response_secs -,'' as response_mins -,'' as response_hours -,'' as response_days_plus_one -,'' as downtime_total_non_working_mins_with_lunch -, '' as downtime_total_non_working_mins -,'' as downtime_total_working_mins_with_lunch -,'' as downtime_total_working_mins -,'' as downtime_total_working_mins_with_lunch_net -,'' as downtime_total_working_mins_net -, '' as start_date_time -,'' as startdate -,'' as start_date_datetime -,'' as end_date_time -,'' as enddate -,'' as end_date_datetime - -/*Liberator Incoming Correspondence Data*/ -,case -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' -end as response_status, -cast(current_timestamp as string) as Current_time_stamp, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, - -/*unassigned days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' -end as unassigned_days_group -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFourteenLess, - -/*Days to assign*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as Days_to_assign_group -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFourteenLess, - -/*assigned in progress days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' -end as assigned_in_progress_days_group -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFourteenLess, - -/*assigned response days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' -end as assignedResponseDays_group -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFourteenLess, - -/*Response days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as ResponseDays_group -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff(cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFourteenLess - -,Response_generated_at -,Date_Received -,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear -,liberator_pcn_ic.Type -,Serviceable -,Service_category -,Response_written_by -,Letter_template -,Action_taken -,Related_to_PCN -,Cancellation_group -,Cancellation_reason -,whenassigned -,ticketserialnumber -,noderef -,cast(liberator_pcn_ic.record_created as string) as record_created -,liberator_pcn_ic.import_timestamp -,liberator_pcn_ic.import_year -,liberator_pcn_ic.import_month -,liberator_pcn_ic.import_day -,liberator_pcn_ic.import_date - -/*pcn data*/ -,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn -,cast(pcnfoidetails_pcn_foi_full.pcnissuedate as string) as pcn_pcnissuedate -,cast(pcnfoidetails_pcn_foi_full.pcnissuedatetime as string) as pcn_pcnissuedatetime -,cast(pcnfoidetails_pcn_foi_full.pcn_canx_date as string) as pcn_pcn_canx_date -,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup -,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason -,cast(pcnfoidetails_pcn_foi_full.pcn_casecloseddate as string) as pcn_pcn_casecloseddate -,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location -,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation -,pcnfoidetails_pcn_foi_full.zone as pcn_zone -,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn -,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode -,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix -,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype -,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm -,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake -,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel -,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour -,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo -,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice -,cast(pcnfoidetails_pcn_foi_full.current_30_day_flag as string) as pcn_current_30_day_flag -,cast(pcnfoidetails_pcn_foi_full.isvda as string) as pcn_isvda -,cast(pcnfoidetails_pcn_foi_full.isvoid as string) as pcn_isvoid -,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval -,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen -,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows -,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway -,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor -,cast(pcnfoidetails_pcn_foi_full.warningflag as string) as pcn_warningflag -,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts -,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason -,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount -,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received -,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount -,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void -,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method -,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref -,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from -,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to -,cast(pcnfoidetails_pcn_foi_full.bailiff_processedon as string) as pcn_bailiff_processedon -,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason -,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff -,cast(pcnfoidetails_pcn_foi_full.warrantissuedate as string) as pcn_warrantissuedate -,cast(pcnfoidetails_pcn_foi_full.allocation as string) as pcn_allocation -,cast(pcnfoidetails_pcn_foi_full.eta_datenotified as string) as pcn_eta_datenotified -,cast(pcnfoidetails_pcn_foi_full.eta_packsubmittedon as string) as pcn_eta_packsubmittedon -,cast(pcnfoidetails_pcn_foi_full.eta_evidencedate as string) as pcn_eta_evidencedate -,cast(pcnfoidetails_pcn_foi_full.eta_adjudicationdate as string) as pcn_eta_adjudicationdate -,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds -,cast(pcnfoidetails_pcn_foi_full.eta_decisionreceived as string) as pcn_eta_decisionreceived -,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome -,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby -,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby -,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address -,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address -,cast(pcnfoidetails_pcn_foi_full.corresp_dispute_flag as string) as pcn_corresp_dispute_flag -,cast(pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as string) as pcn_keyworker_corresp_dispute_flag -,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag -,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year -,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref -,cast(pcnfoidetails_pcn_foi_full.nto_printed as string) as pcn_nto_printed -,cast(pcnfoidetails_pcn_foi_full.appeal_accepted as string) as pcn_appeal_accepted -,cast(pcnfoidetails_pcn_foi_full.arrived_in_pound as string) as pcn_arrived_in_pound -,cast(pcnfoidetails_pcn_foi_full.cancellation_reversed as string) as pcn_cancellation_reversed -,cast(pcnfoidetails_pcn_foi_full.cc_printed as string) as pcn_cc_printed -,cast(pcnfoidetails_pcn_foi_full.drr as string) as pcn_drr -,cast(pcnfoidetails_pcn_foi_full.en_printed as string) as pcn_en_printed -,cast(pcnfoidetails_pcn_foi_full.hold_released as string) as pcn_hold_released -,cast(pcnfoidetails_pcn_foi_full.dvla_response as string) as pcn_dvla_response -,cast(pcnfoidetails_pcn_foi_full.dvla_request as string) as pcn_dvla_request -,cast(pcnfoidetails_pcn_foi_full.full_rate_uplift as string) as pcn_full_rate_uplift -,cast(pcnfoidetails_pcn_foi_full.hold_until as string) as pcn_hold_until -,cast(pcnfoidetails_pcn_foi_full.lifted_at as string) as pcn_lifted_at -,cast(pcnfoidetails_pcn_foi_full.lifted_by as string) as pcn_lifted_by -,cast(pcnfoidetails_pcn_foi_full.loaded as string) as pcn_loaded -,cast(pcnfoidetails_pcn_foi_full.nor_sent as string) as pcn_nor_sent -,cast(pcnfoidetails_pcn_foi_full.notice_held as string) as pcn_notice_held -,cast(pcnfoidetails_pcn_foi_full.ofr_printed as string) as pcn_ofr_printed -,cast(pcnfoidetails_pcn_foi_full.pcn_printed as string) as pcn_pcn_printed -,cast(pcnfoidetails_pcn_foi_full.reissue_nto_requested as string) as pcn_reissue_nto_requested -,cast(pcnfoidetails_pcn_foi_full.reissue_pcn as string) as pcn_reissue_pcn -,cast(pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as string) as pcn_set_back_to_pre_cc_stage -,cast(pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as string) as pcn_vehicle_released_for_auction -,cast(pcnfoidetails_pcn_foi_full.warrant_issued as string) as pcn_warrant_issued -,cast(pcnfoidetails_pcn_foi_full.warrant_redistributed as string) as pcn_warrant_redistributed -,cast(pcnfoidetails_pcn_foi_full.warrant_request_granted as string) as pcn_warrant_request_granted -,cast(pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as string) as pcn_ad_hoc_vq4_request -,cast(pcnfoidetails_pcn_foi_full.paper_vq5_received as string) as pcn_paper_vq5_received -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as string) as pcn_pcn_extracted_for_buslane -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as string) as pcn_pcn_extracted_for_pre_debt -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as string) as pcn_pcn_extracted_for_collection -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as string) as pcn_pcn_extracted_for_drr -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as string) as pcn_pcn_extracted_for_cc -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as string) as pcn_pcn_extracted_for_nto -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as string) as pcn_pcn_extracted_for_print -,cast(pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as string) as pcn_warning_notice_extracted_for_print -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as string) as pcn_pcn_extracted_for_ofr -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as string) as pcn_pcn_extracted_for_warrant_request -,cast(pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as string) as pcn_pre_debt_new_debtor_details -,cast(pcnfoidetails_pcn_foi_full.importdattime as string) as pcn_importdattime -,cast(pcnfoidetails_pcn_foi_full.importdatetime as string) as pcn_importdatetime -- -,pcnfoidetails_pcn_foi_full.import_year as pcn_import_year -,pcnfoidetails_pcn_foi_full.import_month as pcn_import_month -,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day -,pcnfoidetails_pcn_foi_full.import_date as pcn_import_date - -/*Teams data*/ -,t_start_date -,t_end_date -,t_team -,t_team_name -,t_role -,t_forename -,t_surname -,t_full_name -,t_qa_doc_created_by -,t_qa_doc_full_name -,t_post_title -,t_notes -,t_import_date - -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date -left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) - -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + IF(whenassigned <> '', -- contains probable valid timestamp + TRY(CAST(whenassigned AS TIMESTAMP)), + CAST(NULL AS TIMESTAMP) -- edge case + ) AS whenassigned_timestamp, -UNION + IF(Response_generated_at <> '', -- contains probable valid timestamp + TRY(CAST(Response_generated_at AS TIMESTAMP)), + CAST(NULL AS TIMESTAMP) -- edge case + ) AS Response_generated_at_timestamp, ---Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 -Select -cast(parking_officer_downtime.timestamp as string) as downtime_timestamp -,email_address as downtime_email_address -,TRIM(officer_s_first_name) as officer_s_first_name -,TRIM(officer_s_last_name) as officer_s_last_name -,liberator_system_username -,liberator_system_id -,cast(import_datetime as string) import_datetime -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60) > 1440 then 1 else 0 end as string) as multiple_downtime_days_flag -,cast(unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')) as string) as response_secs -,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 end as string) as response_mins -,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/3600 end as string) as response_hours - -/*Downtime calendar days*/ -,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1 end as string) as response_days_plus_one -- downtime days plus one calendar day - -/*Downtime non working mins*/ -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 948 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 948 end as string) as downtime_total_non_working_mins_with_lunch --if greater than 1440 mins then (days + 1) * 948 " -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 1008 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 1008 end as string) as downtime_total_non_working_mins --if greater than 1440 mins then (days + 1) * 1008 " - -/*Downtime working mins*/ -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 end as string) as downtime_total_working_mins_with_lunch -- 492 mins = 8hrs 12 mins working day including lunch" -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 end as string) as downtime_total_working_mins --432 mins = 7hrs 12mins working hours" - -/*Downtime working mins net (less downtime mins)*/ -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 - /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_with_lunch_net -- 492 mins = 8hrs 12 mins working day including lunch less downtime" -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 - (/* downtime mins (response_mins)*/ (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_net --432 mins = 7hrs 12mins working hours" - -,cast(substr(cast(start_date as string), 12, 5) as string) as start_date_time -,cast(substr(cast(start_date as string), 1, 10) as string) as startdate -,cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as start_date_datetime -,cast(substr(cast(end_date as string), 12, 5) as string) as end_date_time -,cast(substr(cast(end_date as string), 1, 10) as string) as enddate -,cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as end_date_datetime -,'Downtime' as response_status -,cast(current_timestamp as string) as Current_time_stamp - -/*Liberator Incoming Correspondence Data*/ -,'' as unassigned_time -,'' as to_assigned_time -,'' as assigned_in_progress_time -,'' as assigned_response_time -,(cast (cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) - cast ( Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) as string) ) as response_time --as downtime_duration --line 30 -,'' as unassigned_days -,'' as unassigned_days_group -,'' as unassigned_days_kpiTotFiftySixLess -,'' as unassigned_days_kpiTotFourteenLess -,'' as Days_to_assign -,'' as Days_to_assign_group -,'' as Days_to_assign_kpiTotFiftySixLess -,'' as Days_to_assign_kpiTotFourteenLess -,'' as assigned_in_progress_days -,'' as assigned_in_progress_days_group -,'' as assigned_in_progress_days_kpiTotFiftySixLess -,'' as assigned_in_progress_days_kpiTotFourteenLess -,'' as assignedResponseDays -,'' as assignedResponseDays_group -,'' as assignedResponseDays_kpiTotFiftySixLess -,'' as assignedResponseDays_kpiTotFourteenLess -,cast( Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) ) end as string) as ResponseDays --as downtime_duration in days -,'' as ResponseDays_group -,'' as ResponseDays_kpiTotFiftySixLess -,'' as ResponseDays_kpiTotFourteenLess -, end_date as Response_generated_at -, start_date as Date_Received -,concat(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,7), '-01') as MonthYear -,'Downtime' as Type -,parking_officer_downtime.Downtime as Serviceable -- downtime -,'' as Service_category -,concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name)) as Response_written_by --downtime officer full name -,'' as Letter_template -,'' as Action_taken -,'' as Related_to_PCN -,'' as Cancellation_group -,'' as Cancellation_reason -,'' as whenassigned -,'' as ticketserialnumber -,'' as noderef -,'' as record_created -,import_timestamp -,import_year -,import_month -,import_day -,cast(import_date as string) as import_date - -/*pcn data*/ -,'' as pcn_pcn -,'' as pcn_pcnissuedate -,'' as pcn_pcnissuedatetime --line 74 -,'' as pcn_pcn_canx_date -,'' as pcn_cancellationgroup -,'' as pcn_cancellationreason -,'' as pcn_pcn_casecloseddate -,'' as pcn_street_location -,'' as pcn_whereonlocation -,'' as pcn_zone -,'' as pcn_usrn -,'' as pcn_contraventioncode -,'' as pcn_contraventionsuffix -,'' as pcn_debttype -,'' as pcn_vrm -,'' as pcn_vehiclemake -,'' as pcn_vehiclemodel -,'' as pcn_vehiclecolour -,'' as pcn_ceo -,'' as pcn_ceodevice -,'' as pcn_current_30_day_flag -,'' as pcn_isvda -,'' as pcn_isvoid -,'' as pcn_isremoval -,'' as pcn_driverseen -,'' as pcn_allwindows -,'' as pcn_parkedonfootway -,'' as pcn_doctor -,'' as pcn_warningflag -,'' as pcn_progressionstage -,'' as pcn_nextprogressionstage --100 -,'' as pcn_nextprogressionstagestarts -,'' as pcn_holdreason -,'' as pcn_lib_initial_debt_amount -,'' as pcn_lib_payment_received -,'' as pcn_lib_write_off_amount -,'' as pcn_lib_payment_void -,'' as pcn_lib_payment_method -,'' as pcn_lib_payment_ref -,'' as pcn_baliff_from -,'' as pcn_bailiff_to -,'' as pcn_bailiff_processedon -,'' as pcn_bailiff_redistributionreason -,'' as pcn_bailiff -,'' as pcn_warrantissuedate -,'' as pcn_allocation -,'' as pcn_eta_datenotified -,'' as pcn_eta_packsubmittedon -,'' as pcn_eta_evidencedate -,'' as pcn_eta_adjudicationdate -,'' as pcn_eta_appealgrounds -,'' as pcn_eta_decisionreceived -,'' as pcn_eta_outcome -,'' as pcn_eta_packsubmittedby -,'' as pcn_cancelledby -,'' as pcn_registered_keeper_address -,'' as pcn_current_ticket_address -,'' as pcn_corresp_dispute_flag -,'' as pcn_keyworker_corresp_dispute_flag -,'' as pcn_fin_year_flag -,'' as pcn_fin_year -,'' as pcn_ticket_ref -,'' as pcn_nto_printed -,'' as pcn_appeal_accepted -,'' as pcn_arrived_in_pound -,'' as pcn_cancellation_reversed -,'' as pcn_cc_printed -,'' as pcn_drr -,'' as pcn_en_printed -,'' as pcn_hold_released -,'' as pcn_dvla_response -,'' as pcn_dvla_request -,'' as pcn_full_rate_uplift -,'' as pcn_hold_until -,'' as pcn_lifted_at -,'' as pcn_lifted_by -,'' as pcn_loaded -,'' as pcn_nor_sent -,'' as pcn_notice_held -,'' as pcn_ofr_printed -,'' as pcn_pcn_printed -,'' as pcn_reissue_nto_requested -,'' as pcn_reissue_pcn -,'' as pcn_set_back_to_pre_cc_stage -,'' as pcn_vehicle_released_for_auction -,'' as pcn_warrant_issued -,'' as pcn_warrant_redistributed -,'' as pcn_warrant_request_granted -,'' as pcn_ad_hoc_vq4_request -,'' as pcn_paper_vq5_received -,'' as pcn_pcn_extracted_for_buslane -,'' as pcn_pcn_extracted_for_pre_debt -,'' as pcn_pcn_extracted_for_collection -,'' as pcn_pcn_extracted_for_drr -,'' as pcn_pcn_extracted_for_cc -,'' as pcn_pcn_extracted_for_nto -,'' as pcn_pcn_extracted_for_print -,'' as pcn_warning_notice_extracted_for_print -,'' as pcn_pcn_extracted_for_ofr -,'' as pcn_pcn_extracted_for_warrant_request -,'' as pcn_pre_debt_new_debtor_details -,'' as pcn_importdattime -,'' as pcn_importdatetime -,'' as pcn_import_year -,'' as pcn_import_month -,'' as pcn_import_day -,'' as pcn_import_date - -/*Teams data*/ -,t_start_date -,t_end_date -,t_team -,t_team_name -,t_role -,t_forename -,t_surname -,t_full_name -,t_qa_doc_created_by -,t_qa_doc_full_name -,t_post_title -,t_notes -,t_import_date - -from parking_officer_downtime -left join team on upper(team.t_full_name) = upper(concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name))) --as Response_written_by --downtime officer full name -where import_date = (select max(import_date) from parking_officer_downtime) and timestamp not like '' and timestamp not like '#REF!' -AND cast(Substr( cast(parking_officer_downtime.timestamp as string) ,1,10) as date) > current_date - interval '13' month --Last 13 months from todays date ---order by timestamp desc -""" -ApplyMapping_node2 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1, - "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1682353070282, - "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1682353072411, - "parking_officer_downtime": parking_raw_zoneparking_officer_downtime_node1697211483070, - }, - transformation_ctx="ApplyMapping_node2", -) + CAST(DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' + CAST(CURRENT_TIMESTAMP AS DATE) + ) AS BIGINT) AS days_since_date_received, -# Script generated for node Liberator Refined - parking_correspondence_performance_records_with_pcn -LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn_downtime/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3", -) -LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_correspondence_performance_records_with_pcn_downtime", -) -LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3.setFormat( - "glueparquet" + CAST(IF( + whenassigned <> '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' + TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_date_recieved_whenassigned, + + CAST(IF( + whenassigned <> '' AND response_generated_at = '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_whenassigned, + + CAST(IF( + whenassigned <> '' AND response_generated_at <> '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), + TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_whenassigned_response_generated_at, + + CAST(IF( + response_generated_at <> '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' + TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_date_received_response_generated_at--, + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" + WHERE import_Date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g + ) + AND LENGTH(ticketserialnumber) = 10 -- ticket filter + AND date_received <>'' -- is the overriding condition for "13 months from todays date"! + AND CAST(SUBSTR(date_received, 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + DATE_DIFF( + 'month', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ) <= 13 +*/ +), +liberator_with_team AS ( + SELECT + /*Fields for union downtime data*/ + CAST('' AS VARCHAR) AS downtime_timestamp, + CAST('' AS VARCHAR) AS downtime_email_address, + CAST('' AS VARCHAR) AS officer_s_first_name, + CAST('' AS VARCHAR) AS officer_s_last_name, + CAST('' AS VARCHAR) AS liberator_system_username, + CAST('' AS VARCHAR) AS liberator_system_id, + CAST('' AS VARCHAR) AS import_datetime, + CAST('' AS VARCHAR) AS multiple_downtime_days_flag, + CAST('' AS VARCHAR) AS response_secs, + CAST('' AS VARCHAR) AS response_mins, + CAST('' AS VARCHAR) AS response_hours, + CAST('' AS VARCHAR) AS response_days_plus_one, + CAST('' AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, + CAST('' AS VARCHAR) AS downtime_total_non_working_mins, + CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch, + CAST('' AS VARCHAR) AS downtime_total_working_mins, + CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, + CAST('' AS VARCHAR) AS downtime_total_working_mins_net, + CAST('' AS VARCHAR) AS start_date_time, + CAST('' AS VARCHAR) AS startdate, + CAST('' AS VARCHAR) AS start_date_datetime, + CAST('' AS VARCHAR) AS end_date_time, + CAST('' AS VARCHAR) AS enddate, + CAST('' AS VARCHAR) AS end_date_datetime, + + /*Liberator Incoming Correspondence Data*/ + CAST(CASE -- having already asserted date_received <>'' + WHEN l.whenassigned = '' THEN 'Unassigned' + WHEN l.whenassigned <> '' AND l.response_generated_at = '' THEN 'Assigned' + WHEN l.whenassigned <> '' AND l.response_generated_at <> '' THEN 'Responded' + --ELSE NULL + END AS VARCHAR) AS response_status, + + CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS unassigned_time, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS to_assigned_time, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_time, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_response_time, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS response_time, + + /*unassigned days*/ + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + THEN CAST(days_since_date_received AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS unassigned_days, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + THEN + CASE + WHEN l.days_since_date_received <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS unassigned_days_group, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + AND l.days_since_date_received <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + AND l.days_since_date_received <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, + + /*Days to assign*/ + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + THEN CAST(l.days_since_date_recieved_whenassigned AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS Days_to_assign, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + THEN + CASE + WHEN l.days_since_date_recieved_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_date_recieved_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_recieved_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_recieved_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_recieved_whenassigned > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS Days_to_assign_group, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + AND l.days_since_date_recieved_whenassigned <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + AND l.days_since_date_recieved_whenassigned <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, + + /*assigned in progress days*/ + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN CAST(l.days_since_whenassigned AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_days, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_days_group, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, + + /*assigned response days*/ + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS assignedResponseDays, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assignedResponseDays_group, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at <> '' + AND l.days_since_whenassigned_response_generated_at <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + AND l.days_since_whenassigned_response_generated_at <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, + + /*Response days*/ + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + THEN CAST(l.days_since_date_received_response_generated_at AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS ResponseDays, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS ResponseDays_group, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, + + Response_generated_at, + Date_Received, + + SUBSTR(CAST(l.date_received AS VARCHAR (10)), 1, 7) || '-01' AS MonthYear, + + l.Type AS "Type", + l.Serviceable, + l.Service_category, + l.Response_written_by, + l.Letter_template, + l.Action_taken, + l.Related_to_PCN, + l.Cancellation_group, + l.Cancellation_reason, + l.whenassigned, + l.ticketserialnumber, + l.noderef, + CAST(l.record_created AS VARCHAR) AS record_created, + + l.import_timestamp, + + /*pcn data*/ + /*** pcn columns taken from "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ***/ + -- Data sourced from the refined zone needs to be handled carefully... + -- Do not assume the following output column datatypes will be the same as "pcnfoidetails_pcn_foi_full". + -- Being left-joined to "pcnfoidetails_pcn_foi_full" will cause NULLs to appear in these columns. + -- And similar transforms elsewhere require different translations to the ones below... + /* These are the current automated translations developed for this transform:- + TRY_CAST(p.[source_column] AS DATE) AS [target_column], --[position]-- try date to date + TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try int to string + COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS DATE), '%Y-%m-%d'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try date to string + TRY_CAST(p.[source_column] AS INTEGER) AS [target_column], --[position]-- try int to int + TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try timestamp to string + TRY_CAST(p.[source_column] AS TIMESTAMP) AS [target_column], --[position]-- try timestamp to timestamp + */ + -- These translations try to deal with problems without resorting to throwing errors, tolerating a reasonable degree of schema evolution in the "pcnfoidetails_pcn_foi_full" source. + -- For example, date and timestamp string formatting works with either string, date and timestamp source columns and works like this... + /* If source can be cast to a date then it can be correctly date-formatted into a string, otherwise just cast whatever it was to a string. + COALESCE( + DATE_FORMAT(TRY_CAST(p.xxxx AS DATE), '%Y-%m-%d'), + CAST(p.xxxx AS VARCHAR) --...and can be extended to handle other date formats. + ) AS pcn_xxxx, -- from DATE + */ + /* If source can be cast to a timestamp then it can be correctly timestamp-formatted into a string, otherwise just cast whatever it was to a string. + COALESCE( + DATE_FORMAT(TRY_CAST(p.xxxx AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), + CAST(p.xxxx AS VARCHAR) -- Alternative for transforming a time-zoned timestamp... + --...COALESCE(DATE_FORMAT(TRY_CAST(TRY(From_iso8601_timestamp(p.xxxx)) AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.xxxx AS VARCHAR)) + --...and can be extended to handle other timestamp formats. + ) AS pcn_xxxx, -- from TIMESTAMP + */ + TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --69-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedate AS DATE), '%Y-%m-%d'), CAST(p.pcnissuedate AS VARCHAR)) AS pcn_pcnissuedate, --70-- try date to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcnissuedatetime AS VARCHAR)) AS pcn_pcnissuedatetime, --71-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_canx_date AS DATE), '%Y-%m-%d'), CAST(p.pcn_canx_date AS VARCHAR)) AS pcn_pcn_canx_date, --72-- try date to string + TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --73-- try string to string (default) + TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --74-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_casecloseddate AS DATE), '%Y-%m-%d'), CAST(p.pcn_casecloseddate AS VARCHAR)) AS pcn_pcn_casecloseddate, --75-- try date to string + TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --76-- try string to string (default) + TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --77-- try string to string (default) + TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --78-- try string to string (default) + TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --79-- try string to string (default) + TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --80-- try string to string (default) + TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --81-- try string to string (default) + TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --82-- try string to string (default) + TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --83-- try string to string (default) + TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --84-- try string to string (default) + TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --85-- try string to string (default) + TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --86-- try string to string (default) + TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --87-- try string to string (default) + TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --88-- try string to string (default) + TRY_CAST(p.current_30_day_flag AS VARCHAR) AS pcn_current_30_day_flag, --89-- try int to string + TRY_CAST(p.isvda AS VARCHAR) AS pcn_isvda, --90-- try int to string + TRY_CAST(p.isvoid AS VARCHAR) AS pcn_isvoid, --91-- try int to string + TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --92-- try string to string (default) + TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --93-- try string to string (default) + TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --94-- try string to string (default) + TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --95-- try string to string (default) + TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --96-- try string to string (default) + TRY_CAST(p.warningflag AS VARCHAR) AS pcn_warningflag, --97-- try int to string + TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --98-- try string to string (default) + TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --99-- try string to string (default) + TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- try string to string (default) + TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --101-- try string to string (default) + TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- try string to string (default) + TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --103-- try string to string (default) + TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --104-- try string to string (default) + TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --105-- try string to string (default) + TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --106-- try string to string (default) + TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --107-- try string to string (default) + TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --108-- try string to string (default) + TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --109-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.bailiff_processedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.bailiff_processedon AS VARCHAR)) AS pcn_bailiff_processedon, --110-- try timestamp to string + TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- try string to string (default) + TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --112-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.warrantissuedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrantissuedate AS VARCHAR)) AS pcn_warrantissuedate, --113-- try timestamp to string + TRY_CAST(p.allocation AS VARCHAR) AS pcn_allocation, --114-- try int to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_datenotified AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_datenotified AS VARCHAR)) AS pcn_eta_datenotified, --115-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_packsubmittedon AS VARCHAR)) AS pcn_eta_packsubmittedon, --116-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_evidencedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_evidencedate AS VARCHAR)) AS pcn_eta_evidencedate, --117-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_adjudicationdate AS VARCHAR)) AS pcn_eta_adjudicationdate, --118-- try timestamp to string + TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --119-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_decisionreceived AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_decisionreceived AS VARCHAR)) AS pcn_eta_decisionreceived, --120-- try timestamp to string + TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --121-- try string to string (default) + TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- try string to string (default) + TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --123-- try string to string (default) + TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --124-- try string to string (default) + TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --125-- try string to string (default) + TRY_CAST(p.corresp_dispute_flag AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- try int to string + TRY_CAST(p.keyworker_corresp_dispute_flag AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- try int to string + TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --128-- try string to string (default) + TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --129-- try string to string (default) + TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --130-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.nto_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nto_printed AS VARCHAR)) AS pcn_nto_printed, --131-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.appeal_accepted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.appeal_accepted AS VARCHAR)) AS pcn_appeal_accepted, --132-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.arrived_in_pound AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.arrived_in_pound AS VARCHAR)) AS pcn_arrived_in_pound, --133-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.cancellation_reversed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cancellation_reversed AS VARCHAR)) AS pcn_cancellation_reversed, --134-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.cc_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cc_printed AS VARCHAR)) AS pcn_cc_printed, --135-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.drr AS VARCHAR)) AS pcn_drr, --136-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.en_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.en_printed AS VARCHAR)) AS pcn_en_printed, --137-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.hold_released AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_released AS VARCHAR)) AS pcn_hold_released, --138-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_response AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_response AS VARCHAR)) AS pcn_dvla_response, --139-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_request AS VARCHAR)) AS pcn_dvla_request, --140-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.full_rate_uplift AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.full_rate_uplift AS VARCHAR)) AS pcn_full_rate_uplift, --141-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.hold_until AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_until AS VARCHAR)) AS pcn_hold_until, --142-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_at AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_at AS VARCHAR)) AS pcn_lifted_at, --143-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_by AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_by AS VARCHAR)) AS pcn_lifted_by, --144-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.loaded AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.loaded AS VARCHAR)) AS pcn_loaded, --145-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.nor_sent AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nor_sent AS VARCHAR)) AS pcn_nor_sent, --146-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.notice_held AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.notice_held AS VARCHAR)) AS pcn_notice_held, --147-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.ofr_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ofr_printed AS VARCHAR)) AS pcn_ofr_printed, --148-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_printed AS VARCHAR)) AS pcn_pcn_printed, --149-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_nto_requested AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_nto_requested AS VARCHAR)) AS pcn_reissue_nto_requested, --150-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_pcn AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_pcn AS VARCHAR)) AS pcn_reissue_pcn, --151-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.set_back_to_pre_cc_stage AS VARCHAR)) AS pcn_set_back_to_pre_cc_stage, --152-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.vehicle_released_for_auction AS VARCHAR)) AS pcn_vehicle_released_for_auction, --153-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_issued AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_issued AS VARCHAR)) AS pcn_warrant_issued, --154-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_redistributed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_redistributed AS VARCHAR)) AS pcn_warrant_redistributed, --155-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_request_granted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_request_granted AS VARCHAR)) AS pcn_warrant_request_granted, --156-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ad_hoc_vq4_request AS VARCHAR)) AS pcn_ad_hoc_vq4_request, --157-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.paper_vq5_received AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.paper_vq5_received AS VARCHAR)) AS pcn_paper_vq5_received, --158-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_buslane AS VARCHAR)) AS pcn_pcn_extracted_for_buslane, --159-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_pre_debt AS VARCHAR)) AS pcn_pcn_extracted_for_pre_debt, --160-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_collection AS VARCHAR)) AS pcn_pcn_extracted_for_collection, --161-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_drr AS VARCHAR)) AS pcn_pcn_extracted_for_drr, --162-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_cc AS VARCHAR)) AS pcn_pcn_extracted_for_cc, --163-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_nto AS VARCHAR)) AS pcn_pcn_extracted_for_nto, --164-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_print AS VARCHAR)) AS pcn_pcn_extracted_for_print, --165-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warning_notice_extracted_for_print AS VARCHAR)) AS pcn_warning_notice_extracted_for_print, --166-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_ofr AS VARCHAR)) AS pcn_pcn_extracted_for_ofr, --167-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_warrant_request AS VARCHAR)) AS pcn_pcn_extracted_for_warrant_request, --168-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pre_debt_new_debtor_details AS VARCHAR)) AS pcn_pre_debt_new_debtor_details, --169-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.importdattime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdattime AS VARCHAR)) AS pcn_importdattime, --170-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.importdatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdatetime AS VARCHAR)) AS pcn_importdatetime, --171-- try timestamp to string + TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --172-- try string to string (default) + TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --173-- try string to string (default) + TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --174-- try string to string (default) + TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --175-- try string to string (default) + + /* teams */ + t.t_start_date, + t.t_end_date, + t.t_team, + t.t_team_name, + t.t_role, + t.t_forename, + t.t_surname, + t.t_full_name, + t.t_qa_doc_created_by, + t.t_qa_doc_full_name, + t.t_post_title, + t.t_notes, + t.t_import_date, + + /* Partition columns moved to the end to keep schema alligned */ + l.import_year, + l.import_month, + l.import_day, + l.import_date + + FROM liberator_pcn_icdate_received_last13months l + LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p + /* + ON p.import_date = l.import_date -- joined on import partition + Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... + ...resulting in empty pcn_ columns in the output. + Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... + ...but for now we'll just use the following simple workaround... + */ + ON p.import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g + ) + AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid + LEFT JOIN team t + ON UPPER(t.t_full_name) = UPPER(l.Response_written_by) +), + +downtime_start_date_end_date_last13months AS ( + SELECT *, + + IF(start_date LIKE '%/%/%', -- contains possible valid timestamp + TRY(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone + IF(start_date LIKE '%-%-%', -- contains other possible valid timestamp + TRY(PARSE_DATETIME(start_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone + CAST(NULL AS TIMESTAMP) -- edge case + ) + ) AS start_timestampZ, + + IF(end_date LIKE '%/%/%', -- contains possible valid timestamp + TRY(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone + IF(end_date LIKE '%-%-%', -- contains other possible valid timestamp + TRY(PARSE_DATETIME(end_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone + CAST(NULL AS TIMESTAMP) -- edge case + ) + ) AS end_timestampZ--, + + FROM "parking-raw-zone"."parking_officer_downtime" + WHERE import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "parking-raw-zone"."parking_officer_downtime" g + ) + AND "timestamp" NOT LIKE '' + AND "timestamp" NOT LIKE '#REF!' + AND CAST(SUBSTR(CAST("timestamp" as VARCHAR(30)), 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + DATE_DIFF( + 'month', + TRY(CAST(SUBSTR("timestamp", 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ) <= 13 +*/ +), + +downtime AS ( + SELECT *, + /* + TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS start_end_unixtime_seconds, + TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) AS end_unixtime_seconds, + + TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) + - TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS downtime_secs, + + -- downtime_secs / 60 AS downtime_mins + -- downtime_secs / 3600 AS downtime_hours + */ + + DATE_DIFF( + 'second', + start_timestampZ, + end_timestampZ + ) AS downtime_secs, + + DATE_DIFF( + 'minute', + start_timestampZ, + end_timestampZ + ) AS downtime_mins, + + DATE_DIFF( + 'hour', + start_timestampZ, + end_timestampZ + ) AS downtime_hours, + + DATE_DIFF( + 'day', + start_timestampZ, + end_timestampZ + ) AS downtime_days, + + TRIM(officer_s_first_name) || ' ' || TRIM(officer_s_last_name) AS Response_written_by --downtime officer full name + + FROM downtime_start_date_end_date_last13months +), + +google_form AS ( + --Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 + SELECT + CAST(d."timestamp" AS VARCHAR) AS downtime_timestamp, + d.email_address AS downtime_email_address, + TRIM(d.officer_s_first_name) AS officer_s_first_name, + TRIM(d.officer_s_last_name) AS officer_s_last_name, + d.liberator_system_username, + d.liberator_system_id, + CAST(d.import_datetime AS VARCHAR) AS import_datetime, + + /* Throughout asserts... + "timestamp" NOT LIKE '' + AND "timestamp" NOT LIKE '#REF!' + */ + CAST(CASE + WHEN d.downtime_mins > 1440 /* downtime mins (response_mins) */ + THEN '1' + ELSE '0' + END AS VARCHAR) AS multiple_downtime_days_flag, + + -- Because the Glue job's response_secs string output was cast from a bigint returned from unix_timestamp... + -- ...there was no need to cast it to a DOUBLE here... + CAST(d.downtime_secs AS VARCHAR) AS response_secs, + -- ...unlike the products below where SparkSQL and AthenaSQL differed in their arithmetic outcomes... + + /* This version was rejected because the output differs wildly from the original Glue output... + -- Here all input value was cast beforehand to DOUBLE. + CAST(CAST(d.downtime_secs AS DOUBLE)/60 AS VARCHAR) AS REJECTED_response_mins, + */ + -- So we are keeping this version because it produces closest to the original Glue output... + -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... + CAST(CAST(d.downtime_mins AS DOUBLE) AS VARCHAR) AS response_mins, + /* But might the following version be the better solution? + -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... + CAST(d.downtime_mins AS VARCHAR) AS ALTERNATIVE_response_mins, + */ + + -- We are keeping this version because it produces closest to the original Glue output... + -- Though beware, several decimal places may be produced in the VARCHAR output... + CAST(CAST(d.downtime_secs AS DOUBLE)/3600 AS VARCHAR) AS response_hours, + /* But might the following version be the better solution? + -- This does not produce the fractional mantissa because it is an integer calculation from start to finish... + CAST(d.downtime_hours AS VARCHAR) AS ALTERNATIVE_response_hours, + */ + + /*Downtime calendar days*/ + -- Warning: A small number outlier cases, producing negative values in original Glue output are also produced here... + CAST(d.downtime_days + 1 AS VARCHAR) AS response_days_plus_one, -- downtime days plus one calendar day + + /*Downtime non working mins*/ + CAST(CASE + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1 ) * 948 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 948 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, --if greater than 1440 mins then (days + 1) * 948 " + + CAST(CASE /* downtime mins (response_mins)*/ + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 1008 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 1008 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_non_working_mins, --if greater than 1440 mins then (days + 1) * 1008 " + + /*Downtime working mins*/ + CAST(CASE /* downtime mins (response_mins)*/ + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 492 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_working_mins_with_lunch, -- 492 mins = 8hrs 12 mins working day including lunch" + + CAST(CASE /* downtime mins (response_mins)*/ + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 432 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_working_mins, -- 432 mins = 7hrs 12mins working hours" + + /* Downtime working mins net (less downtime mins)*/ + + /* This version was rejected because the output differs wildly from the original Glue output... + -- Here all input values were cast beforehand to DOUBLE... + CAST(CASE -- downtime mins (response_mins) + WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 492 -- response_days_plus_one + WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 492 - CAST(d.downtime_secs AS DOUBLE)/60 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS REJECTED_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" + */ + -- So we are keeping this version because it produces closest to the original Glue output... + -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... + CAST(CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 + WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" + /* So might the following version be the better solution? + -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... + CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 + WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" + */ + + /* This version was rejected because the output differs wildly from the original Glue output... + -- Here all input values were cast beforehand to DOUBLE... + CAST(CASE -- downtime mins (response_mins) + WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 432 -- response_days_plus_one + WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 432 - CAST(d.downtime_secs AS DOUBLE)/60 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS REJECTED_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" + */ + -- So we are keeping this version because it produces closest to the original Glue output... + -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... + CAST(CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one + WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" + /* So might the following version be the better solution? + -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... + CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one + WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" + */ + + CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 12, 5) AS VARCHAR) AS start_date_time, + CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 1, 10) AS VARCHAR) AS startdate, + + CAST(SUBSTR( + CAST(d.start_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS VARCHAR) AS start_date_datetime, + + CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 12, 5) AS VARCHAR) AS end_date_time, + CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 1, 10) AS VARCHAR) AS enddate, + + CAST(SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS VARCHAR) AS end_date_datetime, + + CAST('Downtime' AS VARCHAR) AS response_status, + CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, + + /*Liberator Incoming Correspondence Data*/ + CAST('' AS VARCHAR) AS unassigned_time, + CAST('' AS VARCHAR) AS to_assigned_time, + CAST('' AS VARCHAR) AS assigned_in_progress_time, + CAST('' AS VARCHAR) AS assigned_response_time, + + -- Is there a better way to do this? Substring captures only the minutes... + 'INTERVAL ''' + || REGEXP_REPLACE(CAST( + TRY(CAST(SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS TIMESTAMP)) + - TRY(CAST(SUBSTR( + CAST(d.start_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS TIMESTAMP)) + AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + AS response_time, --as downtime_duration --line 30 + + + CAST('' AS VARCHAR) AS unassigned_days, + CAST('' AS VARCHAR) AS unassigned_days_group, + CAST('' AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, + CAST('' AS VARCHAR) AS Days_to_assign, + CAST('' AS VARCHAR) AS Days_to_assign_group, + CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, + CAST('' AS VARCHAR) AS assigned_in_progress_days, + CAST('' AS VARCHAR) AS assigned_in_progress_days_group, + CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, + CAST('' AS VARCHAR) AS assignedResponseDays, + CAST('' AS VARCHAR) AS assignedResponseDays_group, + CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, + + CAST(DATE_DIFF( + 'day', + CAST(SUBSTR( + CAST(d.start_timestampZ AS VARCHAR(30)), + 1, + 10 + ) AS DATE), + CAST(SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 10 + ) AS DATE) + ) AS VARCHAR) AS ResponseDays, --as downtime_duration in days + + CAST('' AS VARCHAR) AS ResponseDays_group, + CAST('' AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, + + d.end_date AS Response_generated_at, + d.start_date AS Date_Received, + + SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 7 + ) || '-01' AS MonthYear, + + 'Downtime' AS "Type", + + d.downtime AS Serviceable, -- downtime --,'Downtime' as Serviceable -- downtime + + CAST('' AS VARCHAR) AS Service_category, + + d.Response_written_by, --downtime officer full name + + CAST('' AS VARCHAR) AS Letter_template, + CAST('' AS VARCHAR) AS Action_taken, + CAST('' AS VARCHAR) AS Related_to_PCN, + CAST('' AS VARCHAR) AS Cancellation_group, + CAST('' AS VARCHAR) AS Cancellation_reason, + CAST('' AS VARCHAR) AS whenassigned, + CAST('' AS VARCHAR) AS ticketserialnumber, + CAST('' AS VARCHAR) AS noderef, + CAST('' AS VARCHAR) AS record_created, + d.import_timestamp, + + /*pcn data*/ + -- The original Glue SparkSQL had all of these pcn columns output as '' empty strings which either... + -- ...conflicted with the output column datatypes configured by Glue + -- ...and/or conflicted with the column translations done earlier in the SQL script. + /* Instead here, the null or empty translations are deliberately set to depend on the target datatype... + CAST(NULL AS DATE) AS [target_column], --[position]-- null date + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS INTEGER) AS [target_column], --[position]-- null int + CAST('' AS VARCHAR) AS [target_column], --[position]-- empty string + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS TIMESTAMP) AS [target_column], --[position]-- null timestamp + */ + CAST('' AS VARCHAR) AS pcn_pcn, --69-- empty string + CAST(NULL AS VARCHAR) AS pcn_pcnissuedate, --70-- null string + CAST(NULL AS VARCHAR) AS pcn_pcnissuedatetime, --71-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_canx_date, --72-- null string + CAST('' AS VARCHAR) AS pcn_cancellationgroup, --73-- empty string + CAST('' AS VARCHAR) AS pcn_cancellationreason, --74-- empty string + CAST(NULL AS VARCHAR) AS pcn_pcn_casecloseddate, --75-- null string + CAST('' AS VARCHAR) AS pcn_street_location, --76-- empty string + CAST('' AS VARCHAR) AS pcn_whereonlocation, --77-- empty string + CAST('' AS VARCHAR) AS pcn_zone, --78-- empty string + CAST('' AS VARCHAR) AS pcn_usrn, --79-- empty string + CAST('' AS VARCHAR) AS pcn_contraventioncode, --80-- empty string + CAST('' AS VARCHAR) AS pcn_contraventionsuffix, --81-- empty string + CAST('' AS VARCHAR) AS pcn_debttype, --82-- empty string + CAST('' AS VARCHAR) AS pcn_vrm, --83-- empty string + CAST('' AS VARCHAR) AS pcn_vehiclemake, --84-- empty string + CAST('' AS VARCHAR) AS pcn_vehiclemodel, --85-- empty string + CAST('' AS VARCHAR) AS pcn_vehiclecolour, --86-- empty string + CAST('' AS VARCHAR) AS pcn_ceo, --87-- empty string + CAST('' AS VARCHAR) AS pcn_ceodevice, --88-- empty string + CAST(NULL AS VARCHAR) AS pcn_current_30_day_flag, --89-- null string + CAST(NULL AS VARCHAR) AS pcn_isvda, --90-- null string + CAST(NULL AS VARCHAR) AS pcn_isvoid, --91-- null string + CAST('' AS VARCHAR) AS pcn_isremoval, --92-- empty string + CAST('' AS VARCHAR) AS pcn_driverseen, --93-- empty string + CAST('' AS VARCHAR) AS pcn_allwindows, --94-- empty string + CAST('' AS VARCHAR) AS pcn_parkedonfootway, --95-- empty string + CAST('' AS VARCHAR) AS pcn_doctor, --96-- empty string + CAST(NULL AS VARCHAR) AS pcn_warningflag, --97-- null string + CAST('' AS VARCHAR) AS pcn_progressionstage, --98-- empty string + CAST('' AS VARCHAR) AS pcn_nextprogressionstage, --99-- empty string + CAST('' AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- empty string + CAST('' AS VARCHAR) AS pcn_holdreason, --101-- empty string + CAST('' AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_received, --103-- empty string + CAST('' AS VARCHAR) AS pcn_lib_write_off_amount, --104-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_void, --105-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_method, --106-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_ref, --107-- empty string + CAST('' AS VARCHAR) AS pcn_baliff_from, --108-- empty string + CAST('' AS VARCHAR) AS pcn_bailiff_to, --109-- empty string + CAST(NULL AS VARCHAR) AS pcn_bailiff_processedon, --110-- null string + CAST('' AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- empty string + CAST('' AS VARCHAR) AS pcn_bailiff, --112-- empty string + CAST(NULL AS VARCHAR) AS pcn_warrantissuedate, --113-- null string + CAST(NULL AS VARCHAR) AS pcn_allocation, --114-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_datenotified, --115-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_packsubmittedon, --116-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_evidencedate, --117-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_adjudicationdate, --118-- null string + CAST('' AS VARCHAR) AS pcn_eta_appealgrounds, --119-- empty string + CAST(NULL AS VARCHAR) AS pcn_eta_decisionreceived, --120-- null string + CAST('' AS VARCHAR) AS pcn_eta_outcome, --121-- empty string + CAST('' AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- empty string + CAST('' AS VARCHAR) AS pcn_cancelledby, --123-- empty string + CAST('' AS VARCHAR) AS pcn_registered_keeper_address, --124-- empty string + CAST('' AS VARCHAR) AS pcn_current_ticket_address, --125-- empty string + CAST(NULL AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- null string + CAST(NULL AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- null string + CAST('' AS VARCHAR) AS pcn_fin_year_flag, --128-- empty string + CAST('' AS VARCHAR) AS pcn_fin_year, --129-- empty string + CAST('' AS VARCHAR) AS pcn_ticket_ref, --130-- empty string + CAST(NULL AS VARCHAR) AS pcn_nto_printed, --131-- null string + CAST(NULL AS VARCHAR) AS pcn_appeal_accepted, --132-- null string + CAST(NULL AS VARCHAR) AS pcn_arrived_in_pound, --133-- null string + CAST(NULL AS VARCHAR) AS pcn_cancellation_reversed, --134-- null string + CAST(NULL AS VARCHAR) AS pcn_cc_printed, --135-- null string + CAST(NULL AS VARCHAR) AS pcn_drr, --136-- null string + CAST(NULL AS VARCHAR) AS pcn_en_printed, --137-- null string + CAST(NULL AS VARCHAR) AS pcn_hold_released, --138-- null string + CAST(NULL AS VARCHAR) AS pcn_dvla_response, --139-- null string + CAST(NULL AS VARCHAR) AS pcn_dvla_request, --140-- null string + CAST(NULL AS VARCHAR) AS pcn_full_rate_uplift, --141-- null string + CAST(NULL AS VARCHAR) AS pcn_hold_until, --142-- null string + CAST(NULL AS VARCHAR) AS pcn_lifted_at, --143-- null string + CAST(NULL AS VARCHAR) AS pcn_lifted_by, --144-- null string + CAST(NULL AS VARCHAR) AS pcn_loaded, --145-- null string + CAST(NULL AS VARCHAR) AS pcn_nor_sent, --146-- null string + CAST(NULL AS VARCHAR) AS pcn_notice_held, --147-- null string + CAST(NULL AS VARCHAR) AS pcn_ofr_printed, --148-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_printed, --149-- null string + CAST(NULL AS VARCHAR) AS pcn_reissue_nto_requested, --150-- null string + CAST(NULL AS VARCHAR) AS pcn_reissue_pcn, --151-- null string + CAST(NULL AS VARCHAR) AS pcn_set_back_to_pre_cc_stage, --152-- null string + CAST(NULL AS VARCHAR) AS pcn_vehicle_released_for_auction, --153-- null string + CAST(NULL AS VARCHAR) AS pcn_warrant_issued, --154-- null string + CAST(NULL AS VARCHAR) AS pcn_warrant_redistributed, --155-- null string + CAST(NULL AS VARCHAR) AS pcn_warrant_request_granted, --156-- null string + CAST(NULL AS VARCHAR) AS pcn_ad_hoc_vq4_request, --157-- null string + CAST(NULL AS VARCHAR) AS pcn_paper_vq5_received, --158-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_buslane, --159-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_pre_debt, --160-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_collection, --161-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_drr, --162-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_cc, --163-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_nto, --164-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_print, --165-- null string + CAST(NULL AS VARCHAR) AS pcn_warning_notice_extracted_for_print, --166-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_ofr, --167-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_warrant_request, --168-- null string + CAST(NULL AS VARCHAR) AS pcn_pre_debt_new_debtor_details, --169-- null string + CAST(NULL AS VARCHAR) AS pcn_importdattime, --170-- null string + CAST(NULL AS VARCHAR) AS pcn_importdatetime, --171-- null string + CAST('' AS VARCHAR) AS pcn_import_year, --172-- empty string + CAST('' AS VARCHAR) AS pcn_import_month, --173-- empty string + CAST('' AS VARCHAR) AS pcn_import_day, --174-- empty string + CAST('' AS VARCHAR) AS pcn_import_date, --175-- empty string + + /*Teams data*/ + t.t_start_date, + t.t_end_date, + t.t_team, + t.t_team_name, + t.t_role, + t.t_forename, + t.t_surname, + t.t_full_name, + t.t_qa_doc_created_by, + t.t_qa_doc_full_name, + t.t_post_title, + t.t_notes, + t.t_import_date, + + /* Partition columns moved to the end to keep schema alligned */ + d.import_year, + d.import_month, + d.import_day, + CAST(d.import_date AS VARCHAR) AS import_date + + FROM downtime d + LEFT JOIN team t + ON UPPER(t.t_full_name) = UPPER(d.Response_written_by) --downtime officer full name ) -LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3.writeFrame( - ApplyMapping_node2 +SELECT * FROM liberator_with_team +UNION +SELECT * FROM google_form +--order by downtime_timestamp desc +; +""" + +create_update_table_with_partition( + environment=environment, query_on_athena=query_on_athena, table_name=table_name ) -job.commit() diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py index bdfeb6aa3..aba4ecc73 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py @@ -1,683 +1,1167 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") +""" +Only need to change the table name and the query prototyped on the Athena UI +by replacing table_name and query_on_athena +""" -# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full -AmazonS3refinedpcnfoidetails_pcn_foi_full_node1697453826330 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="pcnfoidetails_pcn_foi_full", - transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1697453826330", -) +from scripts.helpers.athena_helpers import create_update_table_with_partition +from scripts.helpers.helpers import get_glue_env_var -# Script generated for node S3 bucket - Raw - liberator_pcn_ic -S3bucketRawliberator_pcn_ic_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="liberator_pcn_ic", - transformation_ctx="S3bucketRawliberator_pcn_ic_node1", -) +environment = get_glue_env_var("environment") -# Script generated for node parking_raw_zone - parking_officer_downtime -parking_raw_zoneparking_officer_downtime_node1697453968832 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="parking_officer_downtime", - transformation_ctx="parking_raw_zoneparking_officer_downtime_node1697453968832", -) +# The target table in liberator refined zone +table_name = "parking_correspondence_performance_records_with_pcn_downtime_gds" -# Script generated for node parking_raw_zone - parking_correspondence_performance_teams -parking_raw_zoneparking_correspondence_performance_teams_node1697453966201 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="parking_correspondence_performance_teams", - transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1697453966201", -) - -# Script generated for node SQL Query -SqlQuery0 = """ +# The exact same query prototyped in pre-prod(stg) or prod Athena +query_on_athena = """ +-->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn_downtime_gds" +--< "parking-raw-zone"."parking_correspondence_performance_teams" (data from google spreadsheet) +--< "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" +--<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" +--< "parking-raw-zone"."parking_officer_downtime" (data from google form) /* Correspondence Performance records last 13 months with PCN FOI records -16/06/2022 - Created +16/06/2022 - Created 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing 02/10/2023 - modified to union data into one output for Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 08/11/2023 - modified to add monthly officer totals from original gds job -06/11/2024 - updated date formats in downtime data as source google sheet changed to yyyy-mm-dd HH:MM:ss from d/m/y +22/10/2024 - translated into PrestoSQL to move away from Glue +05/11/2024 + Refactored SQL to separate the inward column conversions from onward business logic using CTE subqueries and allow improved fault handling. + + Downtime's Google Sheets alternate date formats properly handled 28/11/2024 - updated downtime records for t_team details to pull from parking_correspondence_performance_teams instead on leaving blank +12/12/2024 + Integrated the previous downtime update into the refactored version + + Commented out alternative last 13 month filter that caused unexpectedly higher row count +05/03/2025 - Replaced conditional '' empty string outputs with NULL defaults to be consistent with previous Glue/SparkSQL implementations. +05/03/2025 - Removed redundant left joins from subquery +14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. Fixed faulty arithmetic translation of Glue/SparkSQL to AthenaSQL. +16/04/2025 - Workaround orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready +*/ +/*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ +WITH +responses AS ( + SELECT + /* officer_total_rep_dates*/ + l.Response_written_by AS tot_rep_Response_written_by, + SUBSTR(l.response_generated_at, 1, 10) AS tot_rep_response_date, + SUBSTR(l.response_generated_at, 1, 7) || '-01' AS tot_rep_monthYear_response, + -- Empty "response_generated_at" outputs '-01' which would get counted! + -- Makes no difference: SUBSTR(CAST(liberator_pcn_ic.response_generated_at AS VARCHAR(10)), 1, 7) || '-01' AS tot_rep_monthYear_response, + l.Response_written_by || SUBSTR(l.response_generated_at, 1, 10) AS tot_rep_unique_link, + + /* total_response_month*/ + l.Response_written_by AS mth_tot_rep_Response_written_by, + SUBSTR(l.response_generated_at, 1, 10) AS mth_tot_rep_response_date, + SUBSTR(l.response_generated_at, 1, 7) || '-01' AS mth_tot_rep_monthYear_response, + l.Response_written_by || SUBSTR(l.response_generated_at, 1, 7) || '-01' AS mth_tot_rep_unique_link + -- Q. How do empty "response_generated_at" rows effect subsequent business logic? + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" l + -- The unreferenced p. caused SQL linter error ST11 + -- Then discoverd the row count was not expanded by the left join to p because p.pcn is unique within import_date +/* -- Therefore removed this redundant left join... + LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p + ON l.ticketserialnumber = p.pcn + AND l.import_date = p.import_date */ + WHERE l.import_Date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g + ) + AND l.response_generated_at <> '' -- Q. Should I remove empty "response_generated_at" to prevent side-effects? + AND LENGTH(l.ticketserialnumber) = 10 + AND CAST(SUBSTR(l.date_received, 1, 10) AS DATE) > CURRENT_DATE - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + DATE_DIFF( + 'month', + TRY(CAST(SUBSTR(l.date_received, 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ) <= 13 +*/ +), + +officer_total_rep_dates AS ( + SELECT + tot_rep_Response_written_by, + tot_rep_response_date, + tot_rep_monthYear_response, + tot_rep_unique_link, + + COUNT(DISTINCT tot_rep_response_date) AS tot_rep_total_dates, + -- COUNT(DISTINCT SUBSTR(liberator_pcn_ic.response_generated_at, 1, 10)) AS tot_rep_total_dates, + COUNT(*) AS tot_rep_records + FROM + responses + GROUP BY + tot_rep_Response_written_by, + tot_rep_response_date, + tot_rep_monthYear_response, + tot_rep_unique_link +), + +total_response_month AS ( + SELECT + mth_tot_rep_Response_written_by, + mth_tot_rep_monthYear_response, + mth_tot_rep_unique_link, + + COUNT(DISTINCT mth_tot_rep_response_date) AS mth_tot_rep_total_dates, + -- COUNT(DISTINCT SUBSTR(liberator_pcn_ic.response_generated_at, 1, 10)) AS mth_tot_rep_total_dates, + COUNT(*) AS mth_tot_rep_records + FROM + responses + GROUP BY + mth_tot_rep_Response_written_by, + mth_tot_rep_monthYear_response, + mth_tot_rep_unique_link +), /*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ -With officer_total_rep_dates as( -select -liberator_pcn_ic.Response_written_by as tot_rep_Response_written_by -,substr(liberator_pcn_ic.response_generated_at, 1, 10) as tot_rep_response_date -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as tot_rep_unique_link +team AS ( + SELECT DISTINCT + start_date AS t_start_date, + end_date AS t_end_date, + team AS t_team, + team_name AS t_team_name, + ROLE AS t_role, + forename AS t_forename, + surname AS t_surname, + full_name AS t_full_name, + qa_doc_created_by AS t_qa_doc_created_by, + qa_doc_full_name AS t_qa_doc_full_name, + post_title AS t_post_title, + notes AS t_notes, + import_date AS t_import_date --* + FROM "parking-raw-zone"."parking_correspondence_performance_teams" + WHERE import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "parking-raw-zone"."parking_correspondence_performance_teams" g + ) +), -,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as tot_rep_total_dates -,count (*) as tot_rep_records +liberator_pcn_icdate_received_last13months AS ( + SELECT + *, + CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS current_utc_timestamp, + TRY(CAST(date_received AS TIMESTAMP)) AS date_received_timestamp, -- pre-filtered by WHERE clause + IF(whenassigned <> '', -- contains probable valid timestamp + TRY(CAST(whenassigned AS TIMESTAMP)), + CAST(NULL AS TIMESTAMP) -- edge case + ) AS whenassigned_timestamp, -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date + IF(Response_generated_at <> '', -- contains probable valid timestamp + TRY(CAST(Response_generated_at AS TIMESTAMP)), + CAST(NULL AS TIMESTAMP) -- edge case + ) AS Response_generated_at_timestamp, -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + CAST(DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' + CAST(CURRENT_TIMESTAMP AS DATE) + ) AS BIGINT) AS days_since_date_received, -group by -liberator_pcn_ic.Response_written_by --as tot_rep_Response_written_by -,substr(liberator_pcn_ic.response_generated_at, 1, 10) --as tot_rep_response_date -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) --as tot_rep_unique_link + CAST(IF( + whenassigned <> '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' + TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_date_recieved_whenassigned, -) + CAST(IF( + whenassigned <> '' AND response_generated_at = '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_whenassigned, -, total_response_month as ( -select -liberator_pcn_ic.Response_written_by as mth_tot_rep_Response_written_by -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as mth_tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as mth_tot_rep_unique_link + CAST(IF( + whenassigned <> '' AND response_generated_at <> '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), + TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_whenassigned_response_generated_at, -,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as mth_tot_rep_total_dates -,count (*) as mth_tot_rep_records + CAST(IF( + response_generated_at <> '', + DATE_DIFF( + 'day', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' + TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) + ), + NULL -- edge case + ) AS BIGINT) AS days_since_date_received_response_generated_at, + /*links to officer monthly totals*/ + CAST(Response_written_by || SUBSTR(response_generated_at, 1, 10) AS VARCHAR) AS link_officer_total_rep_dates, + CAST(Response_written_by || IF(response_generated_at <> '', SUBSTR(response_generated_at, 1, 7) || '-01', '') AS VARCHAR) AS link_month_total--, -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" + WHERE import_Date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g + ) + AND LENGTH(ticketserialnumber) = 10 -- ticket filter + AND date_received <>'' -- is the overriding condition for "13 months from todays date"! + AND CAST(SUBSTR(date_received, 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + DATE_DIFF( + 'month', + TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ) <= 13 +*/ +), -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date +liberator_with_team_gds AS ( + SELECT + /*Fields for union downtime data*/ + CAST('' AS VARCHAR) AS downtime_timestamp, + CAST('' AS VARCHAR) AS downtime_email_address, + CAST('' AS VARCHAR) AS officer_s_first_name, + CAST('' AS VARCHAR) AS officer_s_last_name, + CAST('' AS VARCHAR) AS liberator_system_username, + CAST('' AS VARCHAR) AS liberator_system_id, + CAST('' AS VARCHAR) AS import_datetime, + CAST('' AS VARCHAR) AS multiple_downtime_days_flag, + CAST('' AS VARCHAR) AS response_secs, + CAST('' AS VARCHAR) AS response_mins, + CAST('' AS VARCHAR) AS response_hours, + CAST('' AS VARCHAR) AS response_days_plus_one, + CAST('' AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, + CAST('' AS VARCHAR) AS downtime_total_non_working_mins, + CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch, + CAST('' AS VARCHAR) AS downtime_total_working_mins, + CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, + CAST('' AS VARCHAR) AS downtime_total_working_mins_net, + CAST('' AS VARCHAR) AS start_date_time, + CAST('' AS VARCHAR) AS startdate, + CAST('' AS VARCHAR) AS start_date_datetime, + CAST('' AS VARCHAR) AS end_date_time, + CAST('' AS VARCHAR) AS enddate, + CAST('' AS VARCHAR) AS end_date_datetime, + /*Liberator Incoming Correspondence Data*/ + CAST(CASE -- having already asserted date_received <>'' + WHEN l.whenassigned = '' THEN 'Unassigned' + WHEN l.whenassigned <> '' AND l.response_generated_at = '' THEN 'Assigned' + WHEN l.whenassigned <> '' AND l.response_generated_at <> '' THEN 'Responded' + --ELSE NULL + END AS VARCHAR) AS response_status, -group by -liberator_pcn_ic.Response_written_by --as mth_tot_rep_Response_written_by -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as mth_tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) --as mth_tot_rep_unique_link -order by -concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') desc --as mth_tot_rep_monthYear_response + CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, -) + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS unassigned_time, -/*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ -,team as ( -select distinct start_date as t_start_date -,end_date as t_end_date -,team as t_team -,team_name as t_team_name -,role as t_role -,forename as t_forename -,surname as t_surname -,full_name as t_full_name -,qa_doc_created_by as t_qa_doc_created_by -,qa_doc_full_name as t_qa_doc_full_name -,post_title as t_post_title -,notes as t_notes -,import_date as t_import_date--* -from parking_correspondence_performance_teams where import_date = (select max(parking_correspondence_performance_teams.import_date) from parking_correspondence_performance_teams ) -) + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS to_assigned_time, -select - -/*Fields for union downtime data*/ -'' as downtime_timestamp -,'' as downtime_email_address -,'' as officer_s_first_name -,'' as officer_s_last_name -,'' as liberator_system_username -,'' as liberator_system_id -,'' as import_datetime -,'' as multiple_downtime_days_flag -,'' as response_secs -,'' as response_mins -,'' as response_hours -,'' as response_days_plus_one -,'' as downtime_total_non_working_mins_with_lunch -, '' as downtime_total_non_working_mins -,'' as downtime_total_working_mins_with_lunch -,'' as downtime_total_working_mins -,'' as downtime_total_working_mins_with_lunch_net -,'' as downtime_total_working_mins_net -, '' as start_date_time -,'' as startdate -,'' as start_date_datetime -,'' as end_date_time -,'' as enddate -,'' as end_date_datetime - -/*Liberator Incoming Correspondence Data*/ -,case -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' -end as response_status, -cast(current_timestamp as string) as Current_time_stamp, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, - -/*unassigned days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' -end as unassigned_days_group -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFourteenLess, - -/*Days to assign*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as Days_to_assign_group -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFourteenLess, - -/*assigned in progress days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' -end as assigned_in_progress_days_group -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFourteenLess, - -/*assigned response days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' -end as assignedResponseDays_group -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFourteenLess, - -/*Response days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as ResponseDays_group -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff(cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFiftySixLess -,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFourteenLess - -,Response_generated_at -,Date_Received -,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear -,liberator_pcn_ic.Type -,Serviceable -,Service_category -,Response_written_by -,Letter_template -,Action_taken -,Related_to_PCN -,Cancellation_group -,Cancellation_reason -,whenassigned -,ticketserialnumber -,noderef -,cast(liberator_pcn_ic.record_created as string) as record_created -,liberator_pcn_ic.import_timestamp -,liberator_pcn_ic.import_year -,liberator_pcn_ic.import_month -,liberator_pcn_ic.import_day -,liberator_pcn_ic.import_date - -/*pcn data*/ -,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn -,cast(pcnfoidetails_pcn_foi_full.pcnissuedate as string) as pcn_pcnissuedate -,cast(pcnfoidetails_pcn_foi_full.pcnissuedatetime as string) as pcn_pcnissuedatetime -,cast(pcnfoidetails_pcn_foi_full.pcn_canx_date as string) as pcn_pcn_canx_date -,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup -,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason -,cast(pcnfoidetails_pcn_foi_full.pcn_casecloseddate as string) as pcn_pcn_casecloseddate -,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location -,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation -,pcnfoidetails_pcn_foi_full.zone as pcn_zone -,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn -,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode -,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix -,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype -,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm -,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake -,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel -,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour -,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo -,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice -,cast(pcnfoidetails_pcn_foi_full.current_30_day_flag as string) as pcn_current_30_day_flag -,cast(pcnfoidetails_pcn_foi_full.isvda as string) as pcn_isvda -,cast(pcnfoidetails_pcn_foi_full.isvoid as string) as pcn_isvoid -,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval -,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen -,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows -,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway -,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor -,cast(pcnfoidetails_pcn_foi_full.warningflag as string) as pcn_warningflag -,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts -,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason -,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount -,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received -,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount -,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void -,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method -,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref -,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from -,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to -,cast(pcnfoidetails_pcn_foi_full.bailiff_processedon as string) as pcn_bailiff_processedon -,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason -,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff -,cast(pcnfoidetails_pcn_foi_full.warrantissuedate as string) as pcn_warrantissuedate -,cast(pcnfoidetails_pcn_foi_full.allocation as string) as pcn_allocation -,cast(pcnfoidetails_pcn_foi_full.eta_datenotified as string) as pcn_eta_datenotified -,cast(pcnfoidetails_pcn_foi_full.eta_packsubmittedon as string) as pcn_eta_packsubmittedon -,cast(pcnfoidetails_pcn_foi_full.eta_evidencedate as string) as pcn_eta_evidencedate -,cast(pcnfoidetails_pcn_foi_full.eta_adjudicationdate as string) as pcn_eta_adjudicationdate -,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds -,cast(pcnfoidetails_pcn_foi_full.eta_decisionreceived as string) as pcn_eta_decisionreceived -,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome -,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby -,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby -,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address -,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address -,cast(pcnfoidetails_pcn_foi_full.corresp_dispute_flag as string) as pcn_corresp_dispute_flag -,cast(pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as string) as pcn_keyworker_corresp_dispute_flag -,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag -,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year -,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref -,cast(pcnfoidetails_pcn_foi_full.nto_printed as string) as pcn_nto_printed -,cast(pcnfoidetails_pcn_foi_full.appeal_accepted as string) as pcn_appeal_accepted -,cast(pcnfoidetails_pcn_foi_full.arrived_in_pound as string) as pcn_arrived_in_pound -,cast(pcnfoidetails_pcn_foi_full.cancellation_reversed as string) as pcn_cancellation_reversed -,cast(pcnfoidetails_pcn_foi_full.cc_printed as string) as pcn_cc_printed -,cast(pcnfoidetails_pcn_foi_full.drr as string) as pcn_drr -,cast(pcnfoidetails_pcn_foi_full.en_printed as string) as pcn_en_printed -,cast(pcnfoidetails_pcn_foi_full.hold_released as string) as pcn_hold_released -,cast(pcnfoidetails_pcn_foi_full.dvla_response as string) as pcn_dvla_response -,cast(pcnfoidetails_pcn_foi_full.dvla_request as string) as pcn_dvla_request -,cast(pcnfoidetails_pcn_foi_full.full_rate_uplift as string) as pcn_full_rate_uplift -,cast(pcnfoidetails_pcn_foi_full.hold_until as string) as pcn_hold_until -,cast(pcnfoidetails_pcn_foi_full.lifted_at as string) as pcn_lifted_at -,cast(pcnfoidetails_pcn_foi_full.lifted_by as string) as pcn_lifted_by -,cast(pcnfoidetails_pcn_foi_full.loaded as string) as pcn_loaded -,cast(pcnfoidetails_pcn_foi_full.nor_sent as string) as pcn_nor_sent -,cast(pcnfoidetails_pcn_foi_full.notice_held as string) as pcn_notice_held -,cast(pcnfoidetails_pcn_foi_full.ofr_printed as string) as pcn_ofr_printed -,cast(pcnfoidetails_pcn_foi_full.pcn_printed as string) as pcn_pcn_printed -,cast(pcnfoidetails_pcn_foi_full.reissue_nto_requested as string) as pcn_reissue_nto_requested -,cast(pcnfoidetails_pcn_foi_full.reissue_pcn as string) as pcn_reissue_pcn -,cast(pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as string) as pcn_set_back_to_pre_cc_stage -,cast(pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as string) as pcn_vehicle_released_for_auction -,cast(pcnfoidetails_pcn_foi_full.warrant_issued as string) as pcn_warrant_issued -,cast(pcnfoidetails_pcn_foi_full.warrant_redistributed as string) as pcn_warrant_redistributed -,cast(pcnfoidetails_pcn_foi_full.warrant_request_granted as string) as pcn_warrant_request_granted -,cast(pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as string) as pcn_ad_hoc_vq4_request -,cast(pcnfoidetails_pcn_foi_full.paper_vq5_received as string) as pcn_paper_vq5_received -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as string) as pcn_pcn_extracted_for_buslane -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as string) as pcn_pcn_extracted_for_pre_debt -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as string) as pcn_pcn_extracted_for_collection -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as string) as pcn_pcn_extracted_for_drr -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as string) as pcn_pcn_extracted_for_cc -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as string) as pcn_pcn_extracted_for_nto -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as string) as pcn_pcn_extracted_for_print -,cast(pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as string) as pcn_warning_notice_extracted_for_print -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as string) as pcn_pcn_extracted_for_ofr -,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as string) as pcn_pcn_extracted_for_warrant_request -,cast(pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as string) as pcn_pre_debt_new_debtor_details -,cast(pcnfoidetails_pcn_foi_full.importdattime as string) as pcn_importdattime -,cast(pcnfoidetails_pcn_foi_full.importdatetime as string) as pcn_importdatetime -- -,pcnfoidetails_pcn_foi_full.import_year as pcn_import_year -,pcnfoidetails_pcn_foi_full.import_month as pcn_import_month -,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day -,pcnfoidetails_pcn_foi_full.import_date as pcn_import_date - - -/*officer monthly totals*/ -,cast(concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as string) as link_officer_total_rep_dates -,cast(concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as string) as link_month_total - -,cast(total_response_month.mth_tot_rep_monthYear_response as string) as mth_tot_rep_monthYear_response -,cast(total_response_month.mth_tot_rep_total_dates as string) as mth_tot_rep_total_dates -,cast(total_response_month.mth_tot_rep_records as string) as mth_tot_rep_records - - -,cast(officer_total_rep_dates.tot_rep_response_date as string) as tot_rep_response_date -,cast(officer_total_rep_dates.tot_rep_total_dates as string) as tot_rep_total_date -,cast(officer_total_rep_dates.tot_rep_records as string) as tot_rep_records - - -/*Teams data*/ -,t_start_date -,t_end_date -,t_team -,t_team_name -,t_role -,t_forename -,t_surname -,t_full_name -,t_qa_doc_created_by -,t_qa_doc_full_name -,t_post_title -,t_notes -,t_import_date - -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date - -left join officer_total_rep_dates on concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) = officer_total_rep_dates.tot_rep_unique_link - -left join total_response_month on concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) = total_response_month.mth_tot_rep_unique_link - - -left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) - -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_time, -UNION + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_response_time, ---Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 -Select -cast(parking_officer_downtime.timestamp as string) as downtime_timestamp -,email_address as downtime_email_address -,TRIM(officer_s_first_name) as officer_s_first_name -,TRIM(officer_s_last_name) as officer_s_last_name -,liberator_system_username -,liberator_system_id -,cast(import_datetime as string) import_datetime -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60) > 1440 then 1 else 0 end as string) as multiple_downtime_days_flag -,cast(unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')) as string) as response_secs -,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 end as string) as response_mins -,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/3600 end as string) as response_hours - -/*Downtime calendar days*/ -,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1 end as string) as response_days_plus_one -- downtime days plus one calendar day - -/*Downtime non working mins*/ -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 948 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 948 end as string) as downtime_total_non_working_mins_with_lunch --if greater than 1440 mins then (days + 1) * 948 " -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 1008 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 1008 end as string) as downtime_total_non_working_mins --if greater than 1440 mins then (days + 1) * 1008 " - -/*Downtime working mins*/ -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 end as string) as downtime_total_working_mins_with_lunch -- 492 mins = 8hrs 12 mins working day including lunch" -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 end as string) as downtime_total_working_mins --432 mins = 7hrs 12mins working hours" - -/*Downtime working mins net (less downtime mins)*/ -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 - /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_with_lunch_net -- 492 mins = 8hrs 12 mins working day including lunch less downtime" -,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 - When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 - (/* downtime mins (response_mins)*/ (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_net --432 mins = 7hrs 12mins working hours" - -,cast(substr(cast(start_date as string), 12, 5) as string) as start_date_time -,cast(substr(cast(start_date as string), 1, 10) as string) as startdate -,cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as start_date_datetime -,cast(substr(cast(end_date as string), 12, 5) as string) as end_date_time -,cast(substr(cast(end_date as string), 1, 10) as string) as enddate -,cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as end_date_datetime -,'Downtime' as response_status -,cast(current_timestamp as string) as Current_time_stamp - -/*Liberator Incoming Correspondence Data*/ -,'' as unassigned_time -,'' as to_assigned_time -,'' as assigned_in_progress_time -,'' as assigned_response_time -,(cast (cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) - cast ( Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) as string) ) as response_time --as downtime_duration --line 30 -,'' as unassigned_days -,'' as unassigned_days_group -,'' as unassigned_days_kpiTotFiftySixLess -,'' as unassigned_days_kpiTotFourteenLess -,'' as Days_to_assign -,'' as Days_to_assign_group -,'' as Days_to_assign_kpiTotFiftySixLess -,'' as Days_to_assign_kpiTotFourteenLess -,'' as assigned_in_progress_days -,'' as assigned_in_progress_days_group -,'' as assigned_in_progress_days_kpiTotFiftySixLess -,'' as assigned_in_progress_days_kpiTotFourteenLess -,'' as assignedResponseDays -,'' as assignedResponseDays_group -,'' as assignedResponseDays_kpiTotFiftySixLess -,'' as assignedResponseDays_kpiTotFourteenLess -,cast( Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) ) end as string) as ResponseDays --as downtime_duration in days -,'' as ResponseDays_group -,'' as ResponseDays_kpiTotFiftySixLess -,'' as ResponseDays_kpiTotFourteenLess -, end_date as Response_generated_at -, start_date as Date_Received -,concat(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,7), '-01') as MonthYear -,'Downtime' as Type -,parking_officer_downtime.Downtime as Serviceable -- downtime -,'' as Service_category -,concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name)) as Response_written_by --downtime officer full name -,'' as Letter_template -,'' as Action_taken -,'' as Related_to_PCN -,'' as Cancellation_group -,'' as Cancellation_reason -,'' as whenassigned -,'' as ticketserialnumber -,'' as noderef -,'' as record_created -,import_timestamp -,import_year -,import_month -,import_day -,cast(import_date as string) as import_date - -/*pcn data*/ -,'' as pcn_pcn -,'' as pcn_pcnissuedate -,'' as pcn_pcnissuedatetime --line 74 -,'' as pcn_pcn_canx_date -,'' as pcn_cancellationgroup -,'' as pcn_cancellationreason -,'' as pcn_pcn_casecloseddate -,'' as pcn_street_location -,'' as pcn_whereonlocation -,'' as pcn_zone -,'' as pcn_usrn -,'' as pcn_contraventioncode -,'' as pcn_contraventionsuffix -,'' as pcn_debttype -,'' as pcn_vrm -,'' as pcn_vehiclemake -,'' as pcn_vehiclemodel -,'' as pcn_vehiclecolour -,'' as pcn_ceo -,'' as pcn_ceodevice -,'' as pcn_current_30_day_flag -,'' as pcn_isvda -,'' as pcn_isvoid -,'' as pcn_isremoval -,'' as pcn_driverseen -,'' as pcn_allwindows -,'' as pcn_parkedonfootway -,'' as pcn_doctor -,'' as pcn_warningflag -,'' as pcn_progressionstage -,'' as pcn_nextprogressionstage --100 -,'' as pcn_nextprogressionstagestarts -,'' as pcn_holdreason -,'' as pcn_lib_initial_debt_amount -,'' as pcn_lib_payment_received -,'' as pcn_lib_write_off_amount -,'' as pcn_lib_payment_void -,'' as pcn_lib_payment_method -,'' as pcn_lib_payment_ref -,'' as pcn_baliff_from -,'' as pcn_bailiff_to -,'' as pcn_bailiff_processedon -,'' as pcn_bailiff_redistributionreason -,'' as pcn_bailiff -,'' as pcn_warrantissuedate -,'' as pcn_allocation -,'' as pcn_eta_datenotified -,'' as pcn_eta_packsubmittedon -,'' as pcn_eta_evidencedate -,'' as pcn_eta_adjudicationdate -,'' as pcn_eta_appealgrounds -,'' as pcn_eta_decisionreceived -,'' as pcn_eta_outcome -,'' as pcn_eta_packsubmittedby -,'' as pcn_cancelledby -,'' as pcn_registered_keeper_address -,'' as pcn_current_ticket_address -,'' as pcn_corresp_dispute_flag -,'' as pcn_keyworker_corresp_dispute_flag -,'' as pcn_fin_year_flag -,'' as pcn_fin_year -,'' as pcn_ticket_ref -,'' as pcn_nto_printed -,'' as pcn_appeal_accepted -,'' as pcn_arrived_in_pound -,'' as pcn_cancellation_reversed -,'' as pcn_cc_printed -,'' as pcn_drr -,'' as pcn_en_printed -,'' as pcn_hold_released -,'' as pcn_dvla_response -,'' as pcn_dvla_request -,'' as pcn_full_rate_uplift -,'' as pcn_hold_until -,'' as pcn_lifted_at -,'' as pcn_lifted_by -,'' as pcn_loaded -,'' as pcn_nor_sent -,'' as pcn_notice_held -,'' as pcn_ofr_printed -,'' as pcn_pcn_printed -,'' as pcn_reissue_nto_requested -,'' as pcn_reissue_pcn -,'' as pcn_set_back_to_pre_cc_stage -,'' as pcn_vehicle_released_for_auction -,'' as pcn_warrant_issued -,'' as pcn_warrant_redistributed -,'' as pcn_warrant_request_granted -,'' as pcn_ad_hoc_vq4_request -,'' as pcn_paper_vq5_received -,'' as pcn_pcn_extracted_for_buslane -,'' as pcn_pcn_extracted_for_pre_debt -,'' as pcn_pcn_extracted_for_collection -,'' as pcn_pcn_extracted_for_drr -,'' as pcn_pcn_extracted_for_cc -,'' as pcn_pcn_extracted_for_nto -,'' as pcn_pcn_extracted_for_print -,'' as pcn_warning_notice_extracted_for_print -,'' as pcn_pcn_extracted_for_ofr -,'' as pcn_pcn_extracted_for_warrant_request -,'' as pcn_pre_debt_new_debtor_details -,'' as pcn_importdattime -,'' as pcn_importdatetime -,'' as pcn_import_year -,'' as pcn_import_month -,'' as pcn_import_day -,'' as pcn_import_date - - -/*officer monthly totals*/ -,'' as link_officer_total_rep_dates -,'' as link_month_total - -,'' as mth_tot_rep_monthYear_response -,'' as mth_tot_rep_total_dates -,'' as mth_tot_rep_records - - -,'' as tot_rep_response_date -,'' as tot_rep_total_date -,'' as tot_rep_records - -/*Teams data*/ -,t_start_date -,t_end_date -,t_team -,t_team_name -,t_role -,t_forename -,t_surname -,t_full_name -,t_qa_doc_created_by -,t_qa_doc_full_name -,t_post_title -,t_notes -,t_import_date - -from parking_officer_downtime -left join team on upper(team.t_full_name) = upper(concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name))) --as Response_written_by --downtime officer full name -where import_date = (select max(import_date) from parking_officer_downtime) and timestamp not like '' and timestamp not like '#REF!' -AND cast(Substr( cast(parking_officer_downtime.timestamp as string) ,1,10) as date) > current_date - interval '13' month --Last 13 months from todays date ---order by timestamp desc -""" -SQLQuery_node1697453694651 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1, - "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1697453826330, - "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1697453966201, - "parking_officer_downtime": parking_raw_zoneparking_officer_downtime_node1697453968832, - }, - transformation_ctx="SQLQuery_node1697453694651", -) + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS response_time, -# Script generated for node Liberator Refined - parking_correspondence_performance_records_with_pcn_downtime_gds -LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn_downtime_gds/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2", -) -LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_correspondence_performance_records_with_pcn_downtime_gds", -) -LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2.setFormat( - "glueparquet" + /*unassigned days*/ + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + THEN CAST(days_since_date_received AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS unassigned_days, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + THEN + CASE + WHEN l.days_since_date_received <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS unassigned_days_group, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + AND l.days_since_date_received <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned = '' + AND l.days_since_date_received <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, + + /*Days to assign*/ + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + THEN CAST(l.days_since_date_recieved_whenassigned AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS Days_to_assign, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + THEN + CASE + WHEN l.days_since_date_recieved_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_date_recieved_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_recieved_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_recieved_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_recieved_whenassigned > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS Days_to_assign_group, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + AND l.days_since_date_recieved_whenassigned <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, + + CAST(CASE -- asserted date_received <>'' + WHEN l.whenassigned <> '' + AND l.days_since_date_recieved_whenassigned <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, + + /*assigned in progress days*/ + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN CAST(l.days_since_whenassigned AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_days, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_days_group, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, + + /*assigned response days*/ + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS assignedResponseDays, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assignedResponseDays_group, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at <> '' + AND l.days_since_whenassigned_response_generated_at <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, + + CAST(CASE + WHEN l.whenassigned <> '' + AND l.response_generated_at = '' + AND l.days_since_whenassigned_response_generated_at <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, + + /*Response days*/ + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + THEN CAST(l.days_since_date_received_response_generated_at AS VARCHAR) + --ELSE NULL + END AS VARCHAR) AS ResponseDays, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS ResponseDays_group, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 56 + THEN '1' + ELSE '0' + END AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, + + CAST(CASE -- asserted date_received <>'' + WHEN l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 14 + THEN '1' + ELSE '0' + END AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, + + Response_generated_at, + Date_Received, + + SUBSTR(CAST(l.date_received AS VARCHAR(10)), 1, 7) || '-01' AS MonthYear, -- asserted date_received <>'' + + l.Type AS "Type", + l.Serviceable, + l.Service_category, + l.Response_written_by, + l.Letter_template, + l.Action_taken, + l.Related_to_PCN, + l.Cancellation_group, + l.Cancellation_reason, + l.whenassigned, + l.ticketserialnumber, + l.noderef, + CAST(l.record_created AS VARCHAR) AS record_created, + + l.import_timestamp, + + /*pcn data*/ + -- The original Glue SparkSQL had all of these pcn columns output as '' empty strings which either... + -- ...conflicted with the output column datatypes configured by Glue + -- ...and/or conflicted with the column translations done earlier in the SQL script. + /* Instead here, the null or empty translations are deliberately set to depend on the target datatype... + CAST(NULL AS DATE) AS [target_column], --[position]-- null date + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS INTEGER) AS [target_column], --[position]-- null int + CAST('' AS VARCHAR) AS [target_column], --[position]-- empty string + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS TIMESTAMP) AS [target_column], --[position]-- null timestamp + */ + TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --69-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedate AS DATE), '%Y-%m-%d'), CAST(p.pcnissuedate AS VARCHAR)) AS pcn_pcnissuedate, --70-- try date to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcnissuedatetime AS VARCHAR)) AS pcn_pcnissuedatetime, --71-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_canx_date AS DATE), '%Y-%m-%d'), CAST(p.pcn_canx_date AS VARCHAR)) AS pcn_pcn_canx_date, --72-- try date to string + TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --73-- try string to string (default) + TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --74-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_casecloseddate AS DATE), '%Y-%m-%d'), CAST(p.pcn_casecloseddate AS VARCHAR)) AS pcn_pcn_casecloseddate, --75-- try date to string + TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --76-- try string to string (default) + TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --77-- try string to string (default) + TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --78-- try string to string (default) + TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --79-- try string to string (default) + TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --80-- try string to string (default) + TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --81-- try string to string (default) + TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --82-- try string to string (default) + TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --83-- try string to string (default) + TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --84-- try string to string (default) + TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --85-- try string to string (default) + TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --86-- try string to string (default) + TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --87-- try string to string (default) + TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --88-- try string to string (default) + TRY_CAST(p.current_30_day_flag AS VARCHAR) AS pcn_current_30_day_flag, --89-- try int to string + TRY_CAST(p.isvda AS VARCHAR) AS pcn_isvda, --90-- try int to string + TRY_CAST(p.isvoid AS VARCHAR) AS pcn_isvoid, --91-- try int to string + TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --92-- try string to string (default) + TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --93-- try string to string (default) + TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --94-- try string to string (default) + TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --95-- try string to string (default) + TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --96-- try string to string (default) + TRY_CAST(p.warningflag AS VARCHAR) AS pcn_warningflag, --97-- try int to string + TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --98-- try string to string (default) + TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --99-- try string to string (default) + TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- try string to string (default) + TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --101-- try string to string (default) + TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- try string to string (default) + TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --103-- try string to string (default) + TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --104-- try string to string (default) + TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --105-- try string to string (default) + TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --106-- try string to string (default) + TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --107-- try string to string (default) + TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --108-- try string to string (default) + TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --109-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.bailiff_processedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.bailiff_processedon AS VARCHAR)) AS pcn_bailiff_processedon, --110-- try timestamp to string + TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- try string to string (default) + TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --112-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.warrantissuedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrantissuedate AS VARCHAR)) AS pcn_warrantissuedate, --113-- try timestamp to string + TRY_CAST(p.allocation AS VARCHAR) AS pcn_allocation, --114-- try int to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_datenotified AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_datenotified AS VARCHAR)) AS pcn_eta_datenotified, --115-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_packsubmittedon AS VARCHAR)) AS pcn_eta_packsubmittedon, --116-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_evidencedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_evidencedate AS VARCHAR)) AS pcn_eta_evidencedate, --117-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_adjudicationdate AS VARCHAR)) AS pcn_eta_adjudicationdate, --118-- try timestamp to string + TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --119-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_decisionreceived AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_decisionreceived AS VARCHAR)) AS pcn_eta_decisionreceived, --120-- try timestamp to string + TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --121-- try string to string (default) + TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- try string to string (default) + TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --123-- try string to string (default) + TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --124-- try string to string (default) + TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --125-- try string to string (default) + TRY_CAST(p.corresp_dispute_flag AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- try int to string + TRY_CAST(p.keyworker_corresp_dispute_flag AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- try int to string + TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --128-- try string to string (default) + TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --129-- try string to string (default) + TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --130-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.nto_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nto_printed AS VARCHAR)) AS pcn_nto_printed, --131-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.appeal_accepted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.appeal_accepted AS VARCHAR)) AS pcn_appeal_accepted, --132-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.arrived_in_pound AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.arrived_in_pound AS VARCHAR)) AS pcn_arrived_in_pound, --133-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.cancellation_reversed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cancellation_reversed AS VARCHAR)) AS pcn_cancellation_reversed, --134-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.cc_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cc_printed AS VARCHAR)) AS pcn_cc_printed, --135-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.drr AS VARCHAR)) AS pcn_drr, --136-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.en_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.en_printed AS VARCHAR)) AS pcn_en_printed, --137-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.hold_released AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_released AS VARCHAR)) AS pcn_hold_released, --138-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_response AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_response AS VARCHAR)) AS pcn_dvla_response, --139-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_request AS VARCHAR)) AS pcn_dvla_request, --140-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.full_rate_uplift AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.full_rate_uplift AS VARCHAR)) AS pcn_full_rate_uplift, --141-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.hold_until AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_until AS VARCHAR)) AS pcn_hold_until, --142-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_at AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_at AS VARCHAR)) AS pcn_lifted_at, --143-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_by AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_by AS VARCHAR)) AS pcn_lifted_by, --144-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.loaded AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.loaded AS VARCHAR)) AS pcn_loaded, --145-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.nor_sent AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nor_sent AS VARCHAR)) AS pcn_nor_sent, --146-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.notice_held AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.notice_held AS VARCHAR)) AS pcn_notice_held, --147-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.ofr_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ofr_printed AS VARCHAR)) AS pcn_ofr_printed, --148-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_printed AS VARCHAR)) AS pcn_pcn_printed, --149-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_nto_requested AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_nto_requested AS VARCHAR)) AS pcn_reissue_nto_requested, --150-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_pcn AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_pcn AS VARCHAR)) AS pcn_reissue_pcn, --151-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.set_back_to_pre_cc_stage AS VARCHAR)) AS pcn_set_back_to_pre_cc_stage, --152-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.vehicle_released_for_auction AS VARCHAR)) AS pcn_vehicle_released_for_auction, --153-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_issued AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_issued AS VARCHAR)) AS pcn_warrant_issued, --154-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_redistributed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_redistributed AS VARCHAR)) AS pcn_warrant_redistributed, --155-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_request_granted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_request_granted AS VARCHAR)) AS pcn_warrant_request_granted, --156-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ad_hoc_vq4_request AS VARCHAR)) AS pcn_ad_hoc_vq4_request, --157-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.paper_vq5_received AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.paper_vq5_received AS VARCHAR)) AS pcn_paper_vq5_received, --158-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_buslane AS VARCHAR)) AS pcn_pcn_extracted_for_buslane, --159-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_pre_debt AS VARCHAR)) AS pcn_pcn_extracted_for_pre_debt, --160-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_collection AS VARCHAR)) AS pcn_pcn_extracted_for_collection, --161-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_drr AS VARCHAR)) AS pcn_pcn_extracted_for_drr, --162-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_cc AS VARCHAR)) AS pcn_pcn_extracted_for_cc, --163-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_nto AS VARCHAR)) AS pcn_pcn_extracted_for_nto, --164-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_print AS VARCHAR)) AS pcn_pcn_extracted_for_print, --165-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warning_notice_extracted_for_print AS VARCHAR)) AS pcn_warning_notice_extracted_for_print, --166-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_ofr AS VARCHAR)) AS pcn_pcn_extracted_for_ofr, --167-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_warrant_request AS VARCHAR)) AS pcn_pcn_extracted_for_warrant_request, --168-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pre_debt_new_debtor_details AS VARCHAR)) AS pcn_pre_debt_new_debtor_details, --169-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.importdattime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdattime AS VARCHAR)) AS pcn_importdattime, --170-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.importdatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdatetime AS VARCHAR)) AS pcn_importdatetime, --171-- try timestamp to string + TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --172-- try string to string (default) + TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --173-- try string to string (default) + TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --174-- try string to string (default) + TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --175-- try string to string (default) + + /*links to officer monthly totals*/ + l.link_officer_total_rep_dates, + l.link_month_total, + + /* officer monthly totals*/ + CAST(m.mth_tot_rep_monthYear_response AS VARCHAR) AS mth_tot_rep_monthYear_response, + CAST(m.mth_tot_rep_total_dates AS VARCHAR) AS mth_tot_rep_total_dates, + CAST(m.mth_tot_rep_records AS VARCHAR) AS mth_tot_rep_records, + CAST(o.tot_rep_response_date AS VARCHAR) AS tot_rep_response_date, + CAST(o.tot_rep_total_dates AS VARCHAR) AS tot_rep_total_date, + CAST(o.tot_rep_records AS VARCHAR) AS tot_rep_records, + + /* teams */ + t.t_start_date, + t.t_end_date, + t.t_team, + t.t_team_name, + t.t_role, + t.t_forename, + t.t_surname, + t.t_full_name, + t.t_qa_doc_created_by, + t.t_qa_doc_full_name, + t.t_post_title, + t.t_notes, + t.t_import_date, + + /* Partition columns moved to the end to keep schema alligned */ + l.import_year, + l.import_month, + l.import_day, + l.import_date + + FROM liberator_pcn_icdate_received_last13months l + + LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p + /* + ON p.import_date = l.import_date -- joined on import partition + Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... + ...resulting in empty pcn_ columns in the output. + Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... + ...but for now we'll just use the following simple workaround... + */ + ON p.import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g + ) + AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid + + LEFT JOIN team t + ON UPPER(t.t_full_name) = UPPER(l.Response_written_by) + + LEFT JOIN officer_total_rep_dates o + ON l.link_officer_total_rep_dates = o.tot_rep_unique_link + + LEFT JOIN total_response_month m + ON l.link_month_total = m.mth_tot_rep_unique_link +), + + +downtime_start_date_end_date_last13months AS ( + SELECT *, + + IF(start_date LIKE '%/%/%', -- contains possible valid timestamp + TRY(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone + IF(start_date LIKE '%-%-%', -- contains other possible valid timestamp + TRY(PARSE_DATETIME(start_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone + CAST(NULL AS TIMESTAMP) -- edge case + ) + ) AS start_timestampZ, + + IF(end_date LIKE '%/%/%', -- contains possible valid timestamp + TRY(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone + IF(end_date LIKE '%-%-%', -- contains other possible valid timestamp + TRY(PARSE_DATETIME(end_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone + CAST(NULL AS TIMESTAMP) -- edge case + ) + ) AS end_timestampZ--, + + FROM "parking-raw-zone"."parking_officer_downtime" + WHERE import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "parking-raw-zone"."parking_officer_downtime" g + ) + AND "timestamp" NOT LIKE '' + AND "timestamp" NOT LIKE '#REF!' + AND CAST(SUBSTR(CAST("timestamp" as VARCHAR(30)), 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + DATE_DIFF( + 'month', + TRY(CAST(SUBSTR("timestamp", 1, 10) AS DATE)), + CAST(CURRENT_TIMESTAMP AS DATE) + ) <= 13 +*/ +), + +downtime AS ( + SELECT *, + /* + TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS start_end_unixtime_seconds, + TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) AS end_unixtime_seconds, + + TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) + - TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS downtime_secs, + + -- downtime_secs / 60 AS downtime_mins + -- downtime_secs / 3600 AS downtime_hours + */ + + DATE_DIFF( + 'second', + start_timestampZ, + end_timestampZ + ) AS downtime_secs, + + DATE_DIFF( + 'minute', + start_timestampZ, + end_timestampZ + ) AS downtime_mins, + + DATE_DIFF( + 'hour', + start_timestampZ, + end_timestampZ + ) AS downtime_hours, + + DATE_DIFF( + 'day', + start_timestampZ, + end_timestampZ + ) AS downtime_days, + + TRIM(officer_s_first_name) || ' ' || TRIM(officer_s_last_name) AS Response_written_by --downtime officer full name + + FROM downtime_start_date_end_date_last13months +), + +google_form AS ( + --Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 + SELECT + CAST(d."timestamp" AS VARCHAR) AS downtime_timestamp, + d.email_address AS downtime_email_address, + TRIM(d.officer_s_first_name) AS officer_s_first_name, + TRIM(d.officer_s_last_name) AS officer_s_last_name, + d.liberator_system_username, + d.liberator_system_id, + CAST(d.import_datetime AS VARCHAR) AS import_datetime, + + /* Throughout asserts... + "timestamp" NOT LIKE '' + AND "timestamp" NOT LIKE '#REF!' + */ + CAST(CASE + WHEN d.downtime_mins > 1440 /* downtime mins (response_mins) */ + THEN '1' + ELSE '0' + END AS VARCHAR) AS multiple_downtime_days_flag, + + -- Because the Glue job's response_secs string output was cast from a bigint returned from unix_timestamp... + -- ...there was no need to cast it to a DOUBLE here... + CAST(d.downtime_secs AS VARCHAR) AS response_secs, + -- ...unlike the products below where SparkSQL and AthenaSQL differed in their arithmetic outcomes... + + /* This version was rejected because the output differs wildly from the original Glue output... + -- Here all input value was cast beforehand to DOUBLE. + CAST(CAST(d.downtime_secs AS DOUBLE)/60 AS VARCHAR) AS REJECTED_response_mins, + */ + -- So we are keeping this version because it produces closest to the original Glue output... + -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... + CAST(CAST(d.downtime_mins AS DOUBLE) AS VARCHAR) AS response_mins, + /* But might the following version be the better solution? + -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... + CAST(d.downtime_mins AS VARCHAR) AS ALTERNATIVE_response_mins, + */ + + -- We are keeping this version because it produces closest to the original Glue output... + -- Though beware, several decimal places may be produced in the VARCHAR output... + CAST(CAST(d.downtime_secs AS DOUBLE)/3600 AS VARCHAR) AS response_hours, + /* But might the following version be the better solution? + -- This does not produce the fractional mantissa because it is an integer calculation from start to finish... + CAST(d.downtime_hours AS VARCHAR) AS ALTERNATIVE_response_hours, + */ + + /*Downtime calendar days*/ + -- Warning: A small number outlier cases, producing negative values in the original Glue output are also produced here... + CAST(d.downtime_days + 1 AS VARCHAR) AS response_days_plus_one, -- downtime days plus one calendar day + + /*Downtime non working mins*/ + CAST(CASE + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1 ) * 948 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 948 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, --if greater than 1440 mins then (days + 1) * 948 " + + CAST(CASE /* downtime mins (response_mins)*/ + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 1008 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 1008 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_non_working_mins, --if greater than 1440 mins then (days + 1) * 1008 " + + /*Downtime working mins*/ + CAST(CASE /* downtime mins (response_mins)*/ + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 492 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_working_mins_with_lunch, -- 492 mins = 8hrs 12 mins working day including lunch" + + CAST(CASE /* downtime mins (response_mins)*/ + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 /*response_days_plus_one*/ + WHEN d.downtime_mins <= 1440 THEN 432 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS downtime_total_working_mins, -- 432 mins = 7hrs 12mins working hours" + + /* Downtime working mins net (less downtime mins)*/ + + /* This version was rejected because the output differs wildly from the original Glue output... + -- Here all input values were cast beforehand to DOUBLE... + CAST(CASE -- downtime mins (response_mins) + WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 492 -- response_days_plus_one + WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 492 - CAST(d.downtime_secs AS DOUBLE)/60 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS REJECTED_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" + */ + -- So we are keeping this version because it produces closest to the original Glue output... + -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... + CAST(CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 + WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" + /* So might the following version be the better solution? + -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... + CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 + WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" + */ + + /* This version was rejected because the output differs wildly from the original Glue output... + -- Here all input values were cast beforehand to DOUBLE... + CAST(CASE -- downtime mins (response_mins) + WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 432 -- response_days_plus_one + WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 432 - CAST(d.downtime_secs AS DOUBLE)/60 + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS REJECTED_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" + */ + -- So we are keeping this version because it produces closest to the original Glue output... + -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... + CAST(CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one + WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" + /* So might the following version be the better solution? + -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... + CAST(CASE -- downtime mins (response_mins) + WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one + WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins + --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table + END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" + */ + + CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 12, 5) AS VARCHAR) AS start_date_time, + CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 1, 10) AS VARCHAR) AS startdate, + + CAST(SUBSTR( + CAST(d.start_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS VARCHAR) AS start_date_datetime, + + CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 12, 5) AS VARCHAR) AS end_date_time, + CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 1, 10) AS VARCHAR) AS enddate, + + CAST(SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS VARCHAR) AS end_date_datetime, + + CAST('Downtime' AS VARCHAR) AS response_status, + CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, + + /*Liberator Incoming Correspondence Data*/ + CAST('' AS VARCHAR) AS unassigned_time, + CAST('' AS VARCHAR) AS to_assigned_time, + CAST('' AS VARCHAR) AS assigned_in_progress_time, + CAST('' AS VARCHAR) AS assigned_response_time, + + -- Is there a better way to do this? Substring captures only the minutes... + 'INTERVAL ''' + || REGEXP_REPLACE(CAST( + TRY(CAST(SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS TIMESTAMP)) + - TRY(CAST(SUBSTR( + CAST(d.start_timestampZ AS VARCHAR(30)), + 1, + 16 + ) AS TIMESTAMP)) + AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + AS response_time, --as downtime_duration --line 30 + + CAST('' AS VARCHAR) AS unassigned_days, + CAST('' AS VARCHAR) AS unassigned_days_group, + CAST('' AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, + CAST('' AS VARCHAR) AS Days_to_assign, + CAST('' AS VARCHAR) AS Days_to_assign_group, + CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, + CAST('' AS VARCHAR) AS assigned_in_progress_days, + CAST('' AS VARCHAR) AS assigned_in_progress_days_group, + CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, + CAST('' AS VARCHAR) AS assignedResponseDays, + CAST('' AS VARCHAR) AS assignedResponseDays_group, + CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, + + CAST(DATE_DIFF( + 'day', + CAST(SUBSTR( + CAST(d.start_timestampZ AS VARCHAR(30)), + 1, + 10 + ) AS DATE), + CAST(SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 10 + ) AS DATE) + ) AS VARCHAR) AS ResponseDays, --as downtime_duration in days + + CAST('' AS VARCHAR) AS ResponseDays_group, + CAST('' AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, + CAST('' AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, + + d.end_date AS Response_generated_at, + d.start_date AS Date_Received, + + SUBSTR( + CAST(d.end_timestampZ AS VARCHAR(30)), + 1, + 7 + ) || '-01' AS MonthYear, + + 'Downtime' AS "Type", + + d.downtime AS Serviceable, -- downtime --,'Downtime' as Serviceable -- downtime + + CAST('' AS VARCHAR) AS Service_category, + + d.Response_written_by, --downtime officer full name + + CAST('' AS VARCHAR) AS Letter_template, + CAST('' AS VARCHAR) AS Action_taken, + CAST('' AS VARCHAR) AS Related_to_PCN, + CAST('' AS VARCHAR) AS Cancellation_group, + CAST('' AS VARCHAR) AS Cancellation_reason, + CAST('' AS VARCHAR) AS whenassigned, + CAST('' AS VARCHAR) AS ticketserialnumber, + CAST('' AS VARCHAR) AS noderef, + CAST('' AS VARCHAR) AS record_created, + d.import_timestamp, + + /*pcn data*/ + -- The original Glue SparkSQL had all of these pcn columns output as '' empty strings which either... + -- ...conflicted with the output column datatypes configured by Glue + -- ...and/or conflicted with the column translations done earlier in the SQL script. + /* Instead here, the null or empty translations are deliberately dependent upon the target datatype... + CAST(NULL AS DATE) AS [target_column], --[position]-- null date + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS INTEGER) AS [target_column], --[position]-- null int + CAST('' AS VARCHAR) AS [target_column], --[position]-- empty string + CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string + CAST(NULL AS TIMESTAMP) AS [target_column], --[position]-- null timestamp + */ + CAST('' AS VARCHAR) AS pcn_pcn, --69-- empty string + CAST(NULL AS VARCHAR) AS pcn_pcnissuedate, --70-- null string + CAST(NULL AS VARCHAR) AS pcn_pcnissuedatetime, --71-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_canx_date, --72-- null string + CAST('' AS VARCHAR) AS pcn_cancellationgroup, --73-- empty string + CAST('' AS VARCHAR) AS pcn_cancellationreason, --74-- empty string + CAST(NULL AS VARCHAR) AS pcn_pcn_casecloseddate, --75-- null string + CAST('' AS VARCHAR) AS pcn_street_location, --76-- empty string + CAST('' AS VARCHAR) AS pcn_whereonlocation, --77-- empty string + CAST('' AS VARCHAR) AS pcn_zone, --78-- empty string + CAST('' AS VARCHAR) AS pcn_usrn, --79-- empty string + CAST('' AS VARCHAR) AS pcn_contraventioncode, --80-- empty string + CAST('' AS VARCHAR) AS pcn_contraventionsuffix, --81-- empty string + CAST('' AS VARCHAR) AS pcn_debttype, --82-- empty string + CAST('' AS VARCHAR) AS pcn_vrm, --83-- empty string + CAST('' AS VARCHAR) AS pcn_vehiclemake, --84-- empty string + CAST('' AS VARCHAR) AS pcn_vehiclemodel, --85-- empty string + CAST('' AS VARCHAR) AS pcn_vehiclecolour, --86-- empty string + CAST('' AS VARCHAR) AS pcn_ceo, --87-- empty string + CAST('' AS VARCHAR) AS pcn_ceodevice, --88-- empty string + CAST(NULL AS VARCHAR) AS pcn_current_30_day_flag, --89-- null string + CAST(NULL AS VARCHAR) AS pcn_isvda, --90-- null string + CAST(NULL AS VARCHAR) AS pcn_isvoid, --91-- null string + CAST('' AS VARCHAR) AS pcn_isremoval, --92-- empty string + CAST('' AS VARCHAR) AS pcn_driverseen, --93-- empty string + CAST('' AS VARCHAR) AS pcn_allwindows, --94-- empty string + CAST('' AS VARCHAR) AS pcn_parkedonfootway, --95-- empty string + CAST('' AS VARCHAR) AS pcn_doctor, --96-- empty string + CAST(NULL AS VARCHAR) AS pcn_warningflag, --97-- null string + CAST('' AS VARCHAR) AS pcn_progressionstage, --98-- empty string + CAST('' AS VARCHAR) AS pcn_nextprogressionstage, --99-- empty string + CAST('' AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- empty string + CAST('' AS VARCHAR) AS pcn_holdreason, --101-- empty string + CAST('' AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_received, --103-- empty string + CAST('' AS VARCHAR) AS pcn_lib_write_off_amount, --104-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_void, --105-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_method, --106-- empty string + CAST('' AS VARCHAR) AS pcn_lib_payment_ref, --107-- empty string + CAST('' AS VARCHAR) AS pcn_baliff_from, --108-- empty string + CAST('' AS VARCHAR) AS pcn_bailiff_to, --109-- empty string + CAST(NULL AS VARCHAR) AS pcn_bailiff_processedon, --110-- null string + CAST('' AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- empty string + CAST('' AS VARCHAR) AS pcn_bailiff, --112-- empty string + CAST(NULL AS VARCHAR) AS pcn_warrantissuedate, --113-- null string + CAST(NULL AS VARCHAR) AS pcn_allocation, --114-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_datenotified, --115-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_packsubmittedon, --116-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_evidencedate, --117-- null string + CAST(NULL AS VARCHAR) AS pcn_eta_adjudicationdate, --118-- null string + CAST('' AS VARCHAR) AS pcn_eta_appealgrounds, --119-- empty string + CAST(NULL AS VARCHAR) AS pcn_eta_decisionreceived, --120-- null string + CAST('' AS VARCHAR) AS pcn_eta_outcome, --121-- empty string + CAST('' AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- empty string + CAST('' AS VARCHAR) AS pcn_cancelledby, --123-- empty string + CAST('' AS VARCHAR) AS pcn_registered_keeper_address, --124-- empty string + CAST('' AS VARCHAR) AS pcn_current_ticket_address, --125-- empty string + CAST(NULL AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- null string + CAST(NULL AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- null string + CAST('' AS VARCHAR) AS pcn_fin_year_flag, --128-- empty string + CAST('' AS VARCHAR) AS pcn_fin_year, --129-- empty string + CAST('' AS VARCHAR) AS pcn_ticket_ref, --130-- empty string + CAST(NULL AS VARCHAR) AS pcn_nto_printed, --131-- null string + CAST(NULL AS VARCHAR) AS pcn_appeal_accepted, --132-- null string + CAST(NULL AS VARCHAR) AS pcn_arrived_in_pound, --133-- null string + CAST(NULL AS VARCHAR) AS pcn_cancellation_reversed, --134-- null string + CAST(NULL AS VARCHAR) AS pcn_cc_printed, --135-- null string + CAST(NULL AS VARCHAR) AS pcn_drr, --136-- null string + CAST(NULL AS VARCHAR) AS pcn_en_printed, --137-- null string + CAST(NULL AS VARCHAR) AS pcn_hold_released, --138-- null string + CAST(NULL AS VARCHAR) AS pcn_dvla_response, --139-- null string + CAST(NULL AS VARCHAR) AS pcn_dvla_request, --140-- null string + CAST(NULL AS VARCHAR) AS pcn_full_rate_uplift, --141-- null string + CAST(NULL AS VARCHAR) AS pcn_hold_until, --142-- null string + CAST(NULL AS VARCHAR) AS pcn_lifted_at, --143-- null string + CAST(NULL AS VARCHAR) AS pcn_lifted_by, --144-- null string + CAST(NULL AS VARCHAR) AS pcn_loaded, --145-- null string + CAST(NULL AS VARCHAR) AS pcn_nor_sent, --146-- null string + CAST(NULL AS VARCHAR) AS pcn_notice_held, --147-- null string + CAST(NULL AS VARCHAR) AS pcn_ofr_printed, --148-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_printed, --149-- null string + CAST(NULL AS VARCHAR) AS pcn_reissue_nto_requested, --150-- null string + CAST(NULL AS VARCHAR) AS pcn_reissue_pcn, --151-- null string + CAST(NULL AS VARCHAR) AS pcn_set_back_to_pre_cc_stage, --152-- null string + CAST(NULL AS VARCHAR) AS pcn_vehicle_released_for_auction, --153-- null string + CAST(NULL AS VARCHAR) AS pcn_warrant_issued, --154-- null string + CAST(NULL AS VARCHAR) AS pcn_warrant_redistributed, --155-- null string + CAST(NULL AS VARCHAR) AS pcn_warrant_request_granted, --156-- null string + CAST(NULL AS VARCHAR) AS pcn_ad_hoc_vq4_request, --157-- null string + CAST(NULL AS VARCHAR) AS pcn_paper_vq5_received, --158-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_buslane, --159-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_pre_debt, --160-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_collection, --161-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_drr, --162-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_cc, --163-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_nto, --164-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_print, --165-- null string + CAST(NULL AS VARCHAR) AS pcn_warning_notice_extracted_for_print, --166-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_ofr, --167-- null string + CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_warrant_request, --168-- null string + CAST(NULL AS VARCHAR) AS pcn_pre_debt_new_debtor_details, --169-- null string + CAST(NULL AS VARCHAR) AS pcn_importdattime, --170-- null string + CAST(NULL AS VARCHAR) AS pcn_importdatetime, --171-- null string + CAST('' AS VARCHAR) AS pcn_import_year, --172-- empty string + CAST('' AS VARCHAR) AS pcn_import_month, --173-- empty string + CAST('' AS VARCHAR) AS pcn_import_day, --174-- empty string + CAST('' AS VARCHAR) AS pcn_import_date, --175-- empty string + + /*officer monthly totals*/ + CAST('' AS VARCHAR) AS link_officer_total_rep_dates, + CAST('' AS VARCHAR) AS link_month_total, + CAST('' AS VARCHAR) AS mth_tot_rep_monthYear_response, + CAST('' AS VARCHAR) AS mth_tot_rep_total_dates, + CAST('' AS VARCHAR) AS mth_tot_rep_records, + CAST('' AS VARCHAR) AS tot_rep_response_date, + CAST('' AS VARCHAR) AS tot_rep_total_date, + CAST('' AS VARCHAR) AS tot_rep_records, + + /*Teams data*/ + t.t_start_date, + t.t_end_date, + t.t_team, + t.t_team_name, + t.t_role, + t.t_forename, + t.t_surname, + t.t_full_name, + t.t_qa_doc_created_by, + t.t_qa_doc_full_name, + t.t_post_title, + t.t_notes, + t.t_import_date, + + /* Partition columns moved to the end to keep schema alligned */ + d.import_year, + d.import_month, + d.import_day, + CAST(d.import_date AS VARCHAR) AS import_date + + FROM downtime d + LEFT JOIN team t + ON UPPER(t.t_full_name) = UPPER(d.Response_written_by) --downtime officer full name ) -LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2.writeFrame( - SQLQuery_node1697453694651 +SELECT * FROM liberator_with_team_gds +UNION +SELECT * FROM google_form +--order by downtime_timestamp desc +; +""" + +create_update_table_with_partition( + environment=environment, query_on_athena=query_on_athena, table_name=table_name ) -job.commit() diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py index bd861c5e6..9c23b41fe 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py @@ -1,383 +1,601 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS - -def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: - for alias, frame in mapping.items(): - frame.toDF().createOrReplaceTempView(alias) - result = spark.sql(query) - return DynamicFrame.fromDF(result, glueContext, transformation_ctx) - - -args = getResolvedOptions(sys.argv, ["JOB_NAME"]) -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session -job = Job(glueContext) -job.init(args["JOB_NAME"], args) -environment = get_glue_env_var("environment") +""" +Only need to change the table name and the query prototyped on the Athena UI +by replacing table_name and query_on_athena +""" -# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full -AmazonS3refinedpcnfoidetails_pcn_foi_full_node1657284942794 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-refined-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="pcnfoidetails_pcn_foi_full", - transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1657284942794", -) +from scripts.helpers.athena_helpers import create_update_table_with_partition +from scripts.helpers.helpers import get_glue_env_var -# Script generated for node S3 bucket - Raw - liberator_pcn_ic -S3bucketRawliberator_pcn_ic_node1 = glueContext.create_dynamic_frame.from_catalog( - database="dataplatform-"+environment+"-liberator-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="liberator_pcn_ic", - transformation_ctx="S3bucketRawliberator_pcn_ic_node1", -) +environment = get_glue_env_var("environment") -# Script generated for node parking_raw_zone - parking_correspondence_performance_teams -parking_raw_zoneparking_correspondence_performance_teams_node1682094127227 = glueContext.create_dynamic_frame.from_catalog( - database="parking-raw-zone", - push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", - table_name="parking_correspondence_performance_teams", - transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1682094127227", -) +# The target table in liberator refined zone +table_name = "parking_correspondence_performance_records_with_pcn_gds" -# Script generated for node SQL -SqlQuery0 = """ -/* +# The exact same query prototyped in pre-prod(stg) or prod Athena +query_on_athena = """ +/* Correspondence Performance records last 13 months with PCN FOI records -16/06/2022 - Created + +-->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn_gds" +--<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" +-- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" +-- < "parking-raw-zone"."parking_correspondence_performance_teams" + +16/06/2022 - Created 30/11/2022 - with totals for Month and response dates for each officer 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing - +04/03/2025 - Refactored SQL for migration back to AthenaSQL for dap-airflow +05/03/2025 - Removed redundant left joins from subqueries +14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. +16/04/2025 - Workaround to orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready */ -With officer_total_rep_dates as( -select -liberator_pcn_ic.Response_written_by as tot_rep_Response_written_by -,substr(liberator_pcn_ic.response_generated_at, 1, 10) as tot_rep_response_date -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as tot_rep_unique_link - -,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as tot_rep_total_dates -,count (*) as tot_rep_records - - -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date - -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - -group by -liberator_pcn_ic.Response_written_by --as tot_rep_Response_written_by -,substr(liberator_pcn_ic.response_generated_at, 1, 10) --as tot_rep_response_date -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) --as tot_rep_unique_link - -) -, total_response_month as ( -select -liberator_pcn_ic.Response_written_by as mth_tot_rep_Response_written_by -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as mth_tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as mth_tot_rep_unique_link - -,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as mth_tot_rep_total_dates -,count (*) as mth_tot_rep_records - - -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date - -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - - -group by -liberator_pcn_ic.Response_written_by --as mth_tot_rep_Response_written_by -,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as mth_tot_rep_monthYear_response -,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) --as mth_tot_rep_unique_link -order by -concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') desc --as mth_tot_rep_monthYear_response - -) -, team as ( -select distinct start_date as t_start_date -,end_date as t_end_date -,team as t_team -,team_name as t_team_name -,role as t_role -,forename as t_forename -,surname as t_surname -,full_name as t_full_name -,qa_doc_created_by as t_qa_doc_created_by -,qa_doc_full_name as t_qa_doc_full_name -,post_title as t_post_title -,notes as t_notes -,import_date as t_import_date--* -from parking_correspondence_performance_teams where import_date = (select max(import_date) from parking_correspondence_performance_teams ) +WITH + +liberator AS ( + + SELECT *, + + CAST(CURRENT_TIMESTAMP AS TIMESTAMP(3)) AS current_utc_timestamp, + TRY(CAST(date_received AS TIMESTAMP(3))) AS date_received_timestamp, -- pre-filtered by WHERE clause + + IF(whenassigned <> '', -- contains probable valid timestamp + TRY(CAST(whenassigned AS TIMESTAMP(3))), + CAST(NULL AS TIMESTAMP(3)) -- edge case + ) AS whenassigned_timestamp, + + IF(Response_generated_at <> '', -- contains probable valid timestamp + TRY(CAST(Response_generated_at AS TIMESTAMP(3))), + CAST(NULL AS TIMESTAMP(3)) -- edge case + ) AS Response_generated_at_timestamp, + + CAST(CASE WHEN date_received <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' + CURRENT_DATE + ) + --ELSE NULL + END AS INTEGER) AS days_since_date_received, + + CAST(CASE WHEN date_received <> '' AND whenassigned <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' + TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE) + ) + --ELSE NULL + END AS INTEGER) AS days_since_date_received_whenassigned, + + CAST(CASE WHEN whenassigned <> '' AND response_generated_at = '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), + CURRENT_DATE + ) + --ELSE NULL -- edge case + END AS INTEGER) AS days_since_whenassigned, + + CAST(CASE WHEN whenassigned <> '' AND response_generated_at <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), + TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) + ) + --ELSE NULL -- edge case + END AS INTEGER) AS days_since_whenassigned_response_generated_at, + + CAST(CASE WHEN date_received <> '' AND response_generated_at <> '' + THEN DATE_DIFF( + 'day', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), + TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) + ) + --ELSE NULL -- edge case + END AS INTEGER) AS days_since_date_received_response_generated_at, + + /* officer_total_rep_dates */ + Response_written_by AS tot_rep_Response_written_by, + SUBSTR(response_generated_at, 1, 10) AS tot_rep_response_date, + CONCAT(SUBSTR(CAST(response_generated_at AS VARCHAR(10)), 1, 7), '-01') AS tot_rep_monthYear_response, + CONCAT( + Response_written_by, + SUBSTR(response_generated_at, 1, 10) + ) AS tot_rep_unique_link, + + /* total_response_month */ + Response_written_by AS mth_tot_rep_Response_written_by, + SUBSTR(response_generated_at, 1, 10) AS mth_tot_rep_response_date, + CONCAT(SUBSTR(CAST(response_generated_at AS VARCHAR(10)), 1, 7), '-01') AS mth_tot_rep_monthYear_response, + CONCAT( + Response_written_by, + CONCAT(SUBSTR(CAST(response_generated_at AS VARCHAR(10)), 1, 7), '-01') + ) AS mth_tot_rep_unique_link + + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" + WHERE import_Date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g + ) + AND LENGTH(ticketserialnumber) = 10 -- ticket filter + AND date_received <> '' -- is the overriding condition for "13 months from todays date"! + AND TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE) > CURRENT_DATE - INTERVAL '13' MONTH + -- Last 13 months from todays date +/* -- This alternative method captures slightly more records... + AND DATE_DIFF( + 'month', + TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), + CURRENT_DATE + ) <= 13 +*/ +), + +officer_total_rep_dates AS ( + SELECT + l.tot_rep_Response_written_by, + l.tot_rep_response_date, + l.tot_rep_monthYear_response, + l.tot_rep_unique_link, + COUNT(DISTINCT l.tot_rep_response_date) AS tot_rep_total_dates, + COUNT(*) AS tot_rep_records + -- The unreferenced p. caused SQL linter error ST11 + -- Then discoverd COUNT(*) was not expanded by the left join to p because p.pcn is unique within import_date + FROM liberator l +/* -- Therefore removed this redundant left join... + LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p + ON l.ticketserialnumber = p.pcn + AND l.import_date = p.import_date +*/ + GROUP BY + l.tot_rep_Response_written_by, + l.tot_rep_response_date, + l.tot_rep_monthYear_response, + l.tot_rep_unique_link +), + +total_response_month AS ( + SELECT + l.mth_tot_rep_Response_written_by, + l.mth_tot_rep_monthYear_response, + l.mth_tot_rep_unique_link, + COUNT(DISTINCT l.mth_tot_rep_response_date) AS mth_tot_rep_total_dates, + COUNT(*) AS mth_tot_rep_records + -- The unreferenced p. caused SQL linter error ST11 + -- Then discoverd COUNT(*) was not expanded by the left join to p because p.pcn is unique within import_date + FROM liberator l +/* -- Therefore removed this redundant left join... + LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p + ON l.ticketserialnumber = p.pcn + AND l.import_date = p.import_date +*/ + GROUP BY + l.mth_tot_rep_Response_written_by, + l.mth_tot_rep_monthYear_response, + l.mth_tot_rep_unique_link + --ORDER BY + -- l.mth_rep_monthYear_response DESC +), + +team AS ( + SELECT DISTINCT + "start_date" AS t_start_date, + end_date AS t_end_date, + team AS t_team, + team_name AS t_team_name, + "role" AS t_role, + forename AS t_forename, + surname AS t_surname, + full_name AS t_full_name, + qa_doc_created_by AS t_qa_doc_created_by, + qa_doc_full_name AS t_qa_doc_full_name, + post_title AS t_post_title, + notes AS t_notes, + import_date AS t_import_date + FROM "parking-raw-zone"."parking_correspondence_performance_teams" + WHERE import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "parking-raw-zone"."parking_correspondence_performance_teams" g + ) ) - -Select -case -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' -when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' -end as response_status, -cast(current_timestamp as string) as Current_time_stamp, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, - -/*unassigned days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' -end as unassigned_days_group -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as unassigned_days_kpiTotFiftySixLess -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as unassigned_days_kpiTotFourteenLess, - -/*Days to assign*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as Days_to_assign_group -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as Days_to_assign_kpiTotFiftySixLess -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as Days_to_assign_kpiTotFourteenLess, - -/*assigned in progress days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' -end as assigned_in_progress_days_group -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFiftySixLess -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFourteenLess, - -/*assigned response days*/ -cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, -Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' -end as assignedResponseDays_group -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFiftySixLess -,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFourteenLess, - -/*Response days*/ -cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, -Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' -When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' -end as ResponseDays_group -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as ResponseDays_kpiTotFiftySixLess -,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as ResponseDays_kpiTotFourteenLess, - -Response_generated_at -,Date_Received -,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear -,liberator_pcn_ic.Type -,Serviceable -,Service_category -,Response_written_by -,Letter_template -,Action_taken -,Related_to_PCN -,Cancellation_group -,Cancellation_reason -,whenassigned -,ticketserialnumber -,noderef -,substr(cast(liberator_pcn_ic.record_created as string), 1, 19) as record_created -,substr(cast(liberator_pcn_ic.import_timestamp as string), 1, 19) as import_timestamp -,substr(cast(liberator_pcn_ic.import_year as string), 1, 04) as import_year -,substr(cast(liberator_pcn_ic.import_month as string), 1, 02) as import_month -,liberator_pcn_ic.import_day -,substr(cast(liberator_pcn_ic.import_date as string), 1, 08) as import_date - -/*pcn data*/ -,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn -,substr(cast(pcnfoidetails_pcn_foi_full.pcnissuedate as string), 1, 10) as pcn_pcnissuedate -,substr(cast(pcnfoidetails_pcn_foi_full.pcnissuedatetime as string), 1, 19) as pcn_pcnissuedatetime -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_canx_date as string), 1, 10) as pcn_pcn_canx_date -,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup -,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_casecloseddate as string), 1, 10) as pcn_pcn_casecloseddate -,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location -,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation -,pcnfoidetails_pcn_foi_full.zone as pcn_zone -,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn -,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode -,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix -,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype -,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm -,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake -,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel -,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour -,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo -,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice -,pcnfoidetails_pcn_foi_full.current_30_day_flag as pcn_current_30_day_flag -,pcnfoidetails_pcn_foi_full.isvda as pcn_isvda -,pcnfoidetails_pcn_foi_full.isvoid as pcn_isvoid -,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval -,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen -,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows -,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway -,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor -,pcnfoidetails_pcn_foi_full.warningflag as pcn_warningflag -,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage -,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts -,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason -,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount -,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received -,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount -,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void -,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method -,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref -,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from -,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to -,substr(cast(pcnfoidetails_pcn_foi_full.bailiff_processedon as string), 1, 19) as pcn_bailiff_processedon -,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason -,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff -,substr(cast(pcnfoidetails_pcn_foi_full.warrantissuedate as string), 1, 19) as pcn_warrantissuedate -,pcnfoidetails_pcn_foi_full.allocation as pcn_allocation -,substr(cast(pcnfoidetails_pcn_foi_full.eta_datenotified as string), 1, 19) as pcn_eta_datenotified -,substr(cast(pcnfoidetails_pcn_foi_full.eta_packsubmittedon as string), 1, 19) as pcn_eta_packsubmittedon -,substr(cast(pcnfoidetails_pcn_foi_full.eta_evidencedate as string), 1, 19) as pcn_eta_evidencedate -,substr(cast(pcnfoidetails_pcn_foi_full.eta_adjudicationdate as string), 1, 19) as pcn_eta_adjudicationdate -,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds -,pcnfoidetails_pcn_foi_full.eta_decisionreceived as pcn_eta_decisionreceived -,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome -,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby -,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby -,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address -,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address -,pcnfoidetails_pcn_foi_full.corresp_dispute_flag as pcn_corresp_dispute_flag -,pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as pcn_keyworker_corresp_dispute_flag -,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag -,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year -,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref -,substr(cast(pcnfoidetails_pcn_foi_full.nto_printed as string), 1, 19) as pcn_nto_printed -,substr(cast(pcnfoidetails_pcn_foi_full.appeal_accepted as string), 1, 19) as pcn_appeal_accepted -,substr(cast(pcnfoidetails_pcn_foi_full.arrived_in_pound as string), 1, 19) as pcn_arrived_in_pound -,substr(cast(pcnfoidetails_pcn_foi_full.cancellation_reversed as string), 1, 19) as pcn_cancellation_reversed -,substr(cast(pcnfoidetails_pcn_foi_full.cc_printed as string), 1, 19) as pcn_cc_printed -,substr(cast(pcnfoidetails_pcn_foi_full.drr as string), 1, 19) as pcn_drr -,substr(cast(pcnfoidetails_pcn_foi_full.en_printed as string), 1, 19) as pcn_en_printed -,substr(cast(pcnfoidetails_pcn_foi_full.hold_released as string), 1, 19) as pcn_hold_released -,substr(cast(pcnfoidetails_pcn_foi_full.dvla_response as string), 1, 19) as pcn_dvla_response -,substr(cast(pcnfoidetails_pcn_foi_full.dvla_request as string), 1, 19) as pcn_dvla_request -,substr(cast(pcnfoidetails_pcn_foi_full.full_rate_uplift as string), 1, 19) as pcn_full_rate_uplift -,substr(cast(pcnfoidetails_pcn_foi_full.hold_until as string), 1, 19) as pcn_hold_until -,substr(cast(pcnfoidetails_pcn_foi_full.lifted_at as string), 1, 19) as pcn_lifted_at -,substr(cast(pcnfoidetails_pcn_foi_full.lifted_by as string), 1, 19) as pcn_lifted_by -,substr(cast(pcnfoidetails_pcn_foi_full.loaded as string), 1, 19) as pcn_loaded -,substr(cast(pcnfoidetails_pcn_foi_full.nor_sent as string), 1, 19) as pcn_nor_sent -,substr(cast(pcnfoidetails_pcn_foi_full.notice_held as string), 1, 19) as pcn_notice_held -,substr(cast(pcnfoidetails_pcn_foi_full.ofr_printed as string), 1, 19) as pcn_ofr_printed -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_printed as string), 1, 19) as pcn_pcn_printed -,substr(cast(pcnfoidetails_pcn_foi_full.reissue_nto_requested as string), 1, 19) as pcn_reissue_nto_requested -,substr(cast(pcnfoidetails_pcn_foi_full.reissue_pcn as string), 1, 19) as pcn_reissue_pcn -,substr(cast(pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as string), 1, 19) as pcn_set_back_to_pre_cc_stage -,substr(cast(pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as string), 1, 19) as pcn_vehicle_released_for_auction -,substr(cast(pcnfoidetails_pcn_foi_full.warrant_issued as string), 1, 19) as pcn_warrant_issued -,substr(cast(pcnfoidetails_pcn_foi_full.warrant_redistributed as string), 1, 19) as pcn_warrant_redistributed -,substr(cast(pcnfoidetails_pcn_foi_full.warrant_request_granted as string), 1, 19) as pcn_warrant_request_granted -,substr(cast(pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as string), 1, 19) as pcn_ad_hoc_vq4_request -,substr(cast(pcnfoidetails_pcn_foi_full.paper_vq5_received as string), 1, 19) as pcn_paper_vq5_received -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as string), 1, 19) as pcn_pcn_extracted_for_buslane -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as string), 1, 19) as pcn_pcn_extracted_for_pre_debt -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as string), 1, 19) as pcn_pcn_extracted_for_collection -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as string), 1, 19) as pcn_pcn_extracted_for_drr -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as string), 1, 19) as pcn_pcn_extracted_for_cc -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as string), 1, 19) as pcn_pcn_extracted_for_nto -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as string), 1, 19) as pcn_pcn_extracted_for_print -,substr(cast(pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as string), 1, 19) as pcn_warning_notice_extracted_for_print -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as string), 1, 19) as pcn_pcn_extracted_for_ofr -,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as string), 1, 19) as pcn_pcn_extracted_for_warrant_request -,substr(cast(pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as string), 1, 19) as pcn_pre_debt_new_debtor_details -,substr(cast(pcnfoidetails_pcn_foi_full.importdattime as string), 1, 19) as pcn_importdattime -,substr(cast(pcnfoidetails_pcn_foi_full.importdatetime as string), 1, 19) as pcn_importdatetime -,substr(cast(pcnfoidetails_pcn_foi_full.import_year as string), 1, 04) as pcn_import_year -,substr(cast(pcnfoidetails_pcn_foi_full.import_month as string), 1, 02) as pcn_import_month -,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day -,substr(cast(pcnfoidetails_pcn_foi_full.import_date as string), 1, 08) as pcn_import_date - - -,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as link_officer_total_rep_dates -,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as link_month_total - -,total_response_month.mth_tot_rep_monthYear_response -,total_response_month.mth_tot_rep_total_dates -,total_response_month.mth_tot_rep_records - - -,officer_total_rep_dates.tot_rep_response_date -,officer_total_rep_dates.tot_rep_total_dates -,officer_total_rep_dates.tot_rep_records -,team.* - -from liberator_pcn_ic -left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date - -left join officer_total_rep_dates on concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) = officer_total_rep_dates.tot_rep_unique_link - -left join total_response_month on concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) = total_response_month.mth_tot_rep_unique_link - -left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) - -where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) -AND length(liberator_pcn_ic.ticketserialnumber) = 10 -AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - - +SELECT + CAST(CASE + WHEN l.date_received <> '' AND l.whenassigned = '' + THEN 'Unassigned' + WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at = '' + THEN 'Assigned' + WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at <> '' + THEN 'Responded' + --ELSE NULL + END AS VARCHAR) AS response_status, + + CAST(l.current_utc_timestamp AS VARCHAR) AS current_time_stamp, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS unassigned_time, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS to_assigned_time, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_time, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS assigned_response_time, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + THEN 'INTERVAL ''' + || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa + || ''' DAY TO SECOND' + --ELSE NULL + END AS VARCHAR) AS response_time, + + /*unassigned days*/ + CAST(CASE WHEN l.whenassigned = '' + THEN l.days_since_date_received + --ELSE NULL + END AS VARCHAR) AS unassigned_days, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + THEN + CASE + WHEN l.days_since_date_received <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received > 56 THEN '56 plus days' + --WHEN l.days_since_date_received IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS unassigned_days_group, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + AND l.days_since_date_received <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS unassigned_days_kpiTotFiftySixLess, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' + AND l.days_since_date_received <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS unassigned_days_kpiTotFourteenLess, + + /*Days to assign*/ + CAST(l.days_since_date_received_whenassigned AS VARCHAR) AS days_to_assign, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + THEN + CASE + WHEN l.days_since_date_received_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received_whenassigned > 56 THEN '56 plus days' + --WHEN l.days_since_date_received_whenassigned IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS days_to_assign_group, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + AND l.days_since_date_received_whenassigned <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS Days_to_assign_kpiTotFiftySixLess, + + CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' + AND l.days_since_date_received_whenassigned <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS Days_to_assign_kpiTotFourteenLess, + + /*assigned in progress days*/ + CAST(l.days_since_whenassigned AS VARCHAR) AS assigned_in_progress_days, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + THEN + CASE + WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned > 56 THEN '56 plus days' + --WHEN l.days_since_whenassigned IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assigned_in_progress_days_group, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS assigned_in_progress_days_kpiTotFiftySixLess, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' + AND l.days_since_whenassigned <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS assigned_in_progress_days_kpiTotFourteenLess, + + /*assigned response days*/ + CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) AS assignedResponseDays, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + THEN + CASE + WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' + --WHEN l.days_since_whenassigned_response_generated_at IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS assignedResponseDays_group, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + AND l.days_since_whenassigned_response_generated_at <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS assignedResponseDays_kpiTotFiftySixLess, + + CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' + AND l.days_since_whenassigned_response_generated_at <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS assignedResponseDays_kpiTotFourteenLess, + + /*Response days*/ + CAST(l.days_since_date_received_response_generated_at AS VARCHAR) AS ResponseDays, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + THEN + CASE + WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' + WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' + WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' + --WHEN l.days_since_date_received_response_generated_at IS NULL THEN NULL -- edge case examined + --ELSE NULL + END + --ELSE NULL + END AS VARCHAR) AS ResponseDays_group, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 56 + THEN 1 + ELSE 0 + END AS INTEGER) AS ResponseDays_kpiTotFiftySixLess, + + CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' + AND l.days_since_date_received_response_generated_at <= 14 + THEN 1 + ELSE 0 + END AS INTEGER) AS ResponseDays_kpiTotFourteenLess, + + l.Response_generated_at, --AS VARCHAR + l.Date_Received, --AS VARCHAR + + CAST(SUBSTR(TRY_CAST(l.date_received AS VARCHAR(10)), 1, 7) || '-01' AS VARCHAR) AS MonthYear, + + l.Type AS "type", --AS VARCHAR + l.Serviceable, --AS VARCHAR + l.Service_category, --AS VARCHAR + l.Response_written_by, --AS VARCHAR + l.Letter_template, --AS VARCHAR + l.Action_taken, --AS VARCHAR + l.Related_to_PCN, --AS VARCHAR + l.Cancellation_group, --AS VARCHAR + l.Cancellation_reason, --AS VARCHAR + l.whenassigned, --AS VARCHAR + l.ticketserialnumber, --AS VARCHAR + l.noderef, --AS VARCHAR + + CAST(l.record_created AS VARCHAR) AS record_created, -- cast from VARCHAR see **NOTE** below + l.import_timestamp, --AS VARCHAR + + /*** pcn columns taken from "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ***/ + -- Data sourced from the refined zone needs to be handled carefully... + -- Do not assume the following output column datatypes will be the same as "pcnfoidetails_pcn_foi_full". + -- Being left-joined to "pcnfoidetails_pcn_foi_full" will cause NULLs to appear in these columns. + -- And similar transforms elsewhere require different translations to the ones below... + /* These are the current automated translations developed for this transform:- + COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS DATE), '%Y-%m-%d'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try date to string + TRY_CAST(p.[source_column] AS INTEGER) AS [target_column], --[position]-- try int to int + TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try timestamp to string + TRY_CAST(p.[source_column] AS TIMESTAMP) AS [target_column], --[position]-- try timestamp to timestamp + */ + -- These translations try to deal with problems without resorting to throwing errors, tolerating a reasonable degree of schema evolution in the "pcnfoidetails_pcn_foi_full" source. + -- For example, date and timestamp string formatting works with either string, date and timestamp source columns and works like this... + /* If source can be cast to a date then it can be correctly date-formatted into a string, otherwise just cast whatever it was to a string. + COALESCE( + DATE_FORMAT(TRY_CAST(p.xxxx AS DATE), '%Y-%m-%d'), + CAST(p.xxxx AS VARCHAR) --...and can be extended to handle other date formats. + ) AS pcn_xxxx, -- from DATE + */ + /* If source can be cast to a timestamp then it can be correctly timestamp-formatted into a string, otherwise just cast whatever it was to a string. + COALESCE( + DATE_FORMAT(TRY_CAST(p.xxxx AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), + CAST(p.xxxx AS VARCHAR) -- Alternative for transforming a time-zoned timestamp... + --...COALESCE(DATE_FORMAT(TRY_CAST(TRY(From_iso8601_timestamp(p.xxxx)) AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.xxxx AS VARCHAR)) + --...and can be extended to handle other timestamp formats. + ) AS pcn_xxxx, -- from TIMESTAMP + */ + TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --45-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedate AS DATE), '%Y-%m-%d'), CAST(p.pcnissuedate AS VARCHAR)) AS pcn_pcnissuedate, --46-- try date to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcnissuedatetime AS VARCHAR)) AS pcn_pcnissuedatetime, --47-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_canx_date AS DATE), '%Y-%m-%d'), CAST(p.pcn_canx_date AS VARCHAR)) AS pcn_pcn_canx_date, --48-- try date to string + TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --49-- try string to string (default) + TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --50-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_casecloseddate AS DATE), '%Y-%m-%d'), CAST(p.pcn_casecloseddate AS VARCHAR)) AS pcn_pcn_casecloseddate, --51-- try date to string + TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --52-- try string to string (default) + TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --53-- try string to string (default) + TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --54-- try string to string (default) + TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --55-- try string to string (default) + TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --56-- try string to string (default) + TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --57-- try string to string (default) + TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --58-- try string to string (default) + TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --59-- try string to string (default) + TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --60-- try string to string (default) + TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --61-- try string to string (default) + TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --62-- try string to string (default) + TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --63-- try string to string (default) + TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --64-- try string to string (default) + TRY_CAST(p.current_30_day_flag AS INTEGER) AS pcn_current_30_day_flag, --65-- try int to int + TRY_CAST(p.isvda AS INTEGER) AS pcn_isvda, --66-- try int to int + TRY_CAST(p.isvoid AS INTEGER) AS pcn_isvoid, --67-- try int to int + TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --68-- try string to string (default) + TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --69-- try string to string (default) + TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --70-- try string to string (default) + TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --71-- try string to string (default) + TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --72-- try string to string (default) + TRY_CAST(p.warningflag AS INTEGER) AS pcn_warningflag, --73-- try int to int + TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --74-- try string to string (default) + TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --75-- try string to string (default) + TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --76-- try string to string (default) + TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --77-- try string to string (default) + TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --78-- try string to string (default) + TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --79-- try string to string (default) + TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --80-- try string to string (default) + TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --81-- try string to string (default) + TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --82-- try string to string (default) + TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --83-- try string to string (default) + TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --84-- try string to string (default) + TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --85-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.bailiff_processedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.bailiff_processedon AS VARCHAR)) AS pcn_bailiff_processedon, --86-- try timestamp to string + TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --87-- try string to string (default) + TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --88-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.warrantissuedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrantissuedate AS VARCHAR)) AS pcn_warrantissuedate, --89-- try timestamp to string + TRY_CAST(p.allocation AS INTEGER) AS pcn_allocation, --90-- try int to int + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_datenotified AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_datenotified AS VARCHAR)) AS pcn_eta_datenotified, --91-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_packsubmittedon AS VARCHAR)) AS pcn_eta_packsubmittedon, --92-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_evidencedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_evidencedate AS VARCHAR)) AS pcn_eta_evidencedate, --93-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_adjudicationdate AS VARCHAR)) AS pcn_eta_adjudicationdate, --94-- try timestamp to string + TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --95-- try string to string (default) + TRY_CAST(p.eta_decisionreceived AS TIMESTAMP) AS pcn_eta_decisionreceived, --96-- try timestamp to timestamp + TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --97-- try string to string (default) + TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --98-- try string to string (default) + TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --99-- try string to string (default) + TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --100-- try string to string (default) + TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --101-- try string to string (default) + TRY_CAST(p.corresp_dispute_flag AS INTEGER) AS pcn_corresp_dispute_flag, --102-- try int to int + TRY_CAST(p.keyworker_corresp_dispute_flag AS INTEGER) AS pcn_keyworker_corresp_dispute_flag, --103-- try int to int + TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --104-- try string to string (default) + TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --105-- try string to string (default) + TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --106-- try string to string (default) + COALESCE(DATE_FORMAT(TRY_CAST(p.nto_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nto_printed AS VARCHAR)) AS pcn_nto_printed, --107-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.appeal_accepted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.appeal_accepted AS VARCHAR)) AS pcn_appeal_accepted, --108-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.arrived_in_pound AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.arrived_in_pound AS VARCHAR)) AS pcn_arrived_in_pound, --109-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.cancellation_reversed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cancellation_reversed AS VARCHAR)) AS pcn_cancellation_reversed, --110-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.cc_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cc_printed AS VARCHAR)) AS pcn_cc_printed, --111-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.drr AS VARCHAR)) AS pcn_drr, --112-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.en_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.en_printed AS VARCHAR)) AS pcn_en_printed, --113-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.hold_released AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_released AS VARCHAR)) AS pcn_hold_released, --114-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_response AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_response AS VARCHAR)) AS pcn_dvla_response, --115-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_request AS VARCHAR)) AS pcn_dvla_request, --116-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.full_rate_uplift AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.full_rate_uplift AS VARCHAR)) AS pcn_full_rate_uplift, --117-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.hold_until AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_until AS VARCHAR)) AS pcn_hold_until, --118-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_at AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_at AS VARCHAR)) AS pcn_lifted_at, --119-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_by AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_by AS VARCHAR)) AS pcn_lifted_by, --120-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.loaded AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.loaded AS VARCHAR)) AS pcn_loaded, --121-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.nor_sent AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nor_sent AS VARCHAR)) AS pcn_nor_sent, --122-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.notice_held AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.notice_held AS VARCHAR)) AS pcn_notice_held, --123-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.ofr_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ofr_printed AS VARCHAR)) AS pcn_ofr_printed, --124-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_printed AS VARCHAR)) AS pcn_pcn_printed, --125-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_nto_requested AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_nto_requested AS VARCHAR)) AS pcn_reissue_nto_requested, --126-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_pcn AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_pcn AS VARCHAR)) AS pcn_reissue_pcn, --127-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.set_back_to_pre_cc_stage AS VARCHAR)) AS pcn_set_back_to_pre_cc_stage, --128-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.vehicle_released_for_auction AS VARCHAR)) AS pcn_vehicle_released_for_auction, --129-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_issued AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_issued AS VARCHAR)) AS pcn_warrant_issued, --130-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_redistributed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_redistributed AS VARCHAR)) AS pcn_warrant_redistributed, --131-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_request_granted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_request_granted AS VARCHAR)) AS pcn_warrant_request_granted, --132-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ad_hoc_vq4_request AS VARCHAR)) AS pcn_ad_hoc_vq4_request, --133-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.paper_vq5_received AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.paper_vq5_received AS VARCHAR)) AS pcn_paper_vq5_received, --134-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_buslane AS VARCHAR)) AS pcn_pcn_extracted_for_buslane, --135-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_pre_debt AS VARCHAR)) AS pcn_pcn_extracted_for_pre_debt, --136-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_collection AS VARCHAR)) AS pcn_pcn_extracted_for_collection, --137-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_drr AS VARCHAR)) AS pcn_pcn_extracted_for_drr, --138-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_cc AS VARCHAR)) AS pcn_pcn_extracted_for_cc, --139-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_nto AS VARCHAR)) AS pcn_pcn_extracted_for_nto, --140-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_print AS VARCHAR)) AS pcn_pcn_extracted_for_print, --141-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warning_notice_extracted_for_print AS VARCHAR)) AS pcn_warning_notice_extracted_for_print, --142-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_ofr AS VARCHAR)) AS pcn_pcn_extracted_for_ofr, --143-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_warrant_request AS VARCHAR)) AS pcn_pcn_extracted_for_warrant_request, --144-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pre_debt_new_debtor_details AS VARCHAR)) AS pcn_pre_debt_new_debtor_details, --145-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.importdattime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdattime AS VARCHAR)) AS pcn_importdattime, --146-- try timestamp to string + COALESCE(DATE_FORMAT(TRY_CAST(p.importdatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdatetime AS VARCHAR)) AS pcn_importdatetime, --147-- try timestamp to string + TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --148-- try string to string (default) + TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --149-- try string to string (default) + TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --150-- try string to string (default) + TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --151-- try string to string (default) + + /* Links to aggregations */ + CAST(l.tot_rep_unique_link AS VARCHAR) AS link_officer_total_rep_dates, + CAST(l.mth_tot_rep_unique_link AS VARCHAR) AS link_month_total, + + /* Monthly aggregations */ + CAST(m.mth_tot_rep_monthYear_response AS VARCHAR) AS mth_tot_rep_monthYear_response, + TRY_CAST(m.mth_tot_rep_total_dates AS BIGINT) AS mth_tot_rep_total_dates, + TRY_CAST(m.mth_tot_rep_records AS BIGINT) AS mth_tot_rep_records, + + /* Daily Aggregations */ + CAST(d.tot_rep_response_date AS VARCHAR) AS tot_rep_response_date, + TRY_CAST(d.tot_rep_total_dates AS BIGINT) AS tot_rep_total_dates, + TRY_CAST(d.tot_rep_records AS BIGINT) AS tot_rep_records, + + /* teams columns expanded by above subquery */ + team.*, + + /* Partition columns moved to the end to keep schema alligned */ + l.import_year, + l.import_month, + l.import_day, + l.import_date + +FROM liberator l -- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" +LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p +/* + ON p.import_date = l.import_date -- joined on import partition + Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... + ...resulting in empty pcn_ columns in the output. + Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... + ...but for now we'll just use the following simple workaround... +*/ + ON p.import_date IN ( + SELECT MAX(g.import_date) AS import_date + FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g + ) + AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid +LEFT JOIN officer_total_rep_dates d -- < Daily Aggregations of (l LEFT JOIN p) + ON l.tot_rep_unique_link = d.tot_rep_unique_link +LEFT JOIN total_response_month m -- < Monthly Aggregations of (l LEFT JOIN p) + ON l.mth_tot_rep_unique_link = m.mth_tot_rep_unique_link + LEFT JOIN team -- < "parking-raw-zone"."parking_correspondence_performance_teams" + ON UPPER(team.t_full_name) = UPPER(l.Response_written_by) -- Can this name match be improved? +; """ -SQL_node1658765472050 = sparkSqlQuery( - glueContext, - query=SqlQuery0, - mapping={ - "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1657284942794, - "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1, - "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1682094127227, - }, - transformation_ctx="SQL_node1658765472050", -) -# Script generated for node Amazon S3 -AmazonS3_node1658765590649 = glueContext.getSink( - path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn_gds/", - connection_type="s3", - updateBehavior="UPDATE_IN_DATABASE", - partitionKeys=["import_year", "import_month", "import_day", "import_date"], - compression="snappy", - enableUpdateCatalog=True, - transformation_ctx="AmazonS3_node1658765590649", +create_update_table_with_partition( + environment=environment, query_on_athena=query_on_athena, table_name=table_name ) -AmazonS3_node1658765590649.setCatalogInfo( - catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", - catalogTableName="parking_correspondence_performance_records_with_pcn_gds", -) -AmazonS3_node1658765590649.setFormat("glueparquet") -AmazonS3_node1658765590649.writeFrame(SQL_node1658765472050) -job.commit() - diff --git a/terraform/etl/38-aws-glue-job-parking.tf b/terraform/etl/38-aws-glue-job-parking.tf index a1a23126a..caf98dce3 100644 --- a/terraform/etl/38-aws-glue-job-parking.tf +++ b/terraform/etl/38-aws-glue-job-parking.tf @@ -392,7 +392,30 @@ module "parking_foreign_vrm_pcns" { } } -# Migrated job "parking_correspondence_performance_records_with_pcn" to dap-airflow om 14/04/2025 + +module "parking_correspondence_performance_records_with_pcn" { + source = "../modules/aws-glue-job" + is_live_environment = local.is_live_environment + is_production_environment = local.is_production_environment + department = module.department_parking_data_source + job_name = "${local.short_identifier_prefix}parking_correspondence_performance_records_with_pcn" + helper_module_key = data.aws_s3_object.helpers.key + pydeequ_zip_key = data.aws_s3_object.pydeequ.key + spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id + script_name = "parking_correspondence_performance_records_with_pcn" + triggered_by_job = module.parking_pcn_denormalisation.job_name + job_description = "correspondence performance records with pcn" + workflow_name = "${local.short_identifier_prefix}parking-liberator-data-workflow" + trigger_enabled = local.is_production_environment + number_of_workers_for_glue_job = 10 + glue_job_worker_type = "G.1X" + glue_version = "4.0" + job_parameters = { + "--job-bookmark-option" = "job-bookmark-disable" + "--environment" = var.environment + "--conf" = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY" + } +} module "parking_disputes_kpi_gds_summary" { source = "../modules/aws-glue-job" @@ -483,8 +506,30 @@ module "Parking_Permit_Diesel_Trends_Bought_in_Month" { "--environment" = var.environment } } +module "parking_correspondence_performance_records_with_pcn_gds" { + source = "../modules/aws-glue-job" + is_live_environment = local.is_live_environment + is_production_environment = local.is_production_environment + department = module.department_parking_data_source + job_name = "${local.short_identifier_prefix}parking_correspondence_performance_records_with_pcn_gds" + helper_module_key = data.aws_s3_object.helpers.key + pydeequ_zip_key = data.aws_s3_object.pydeequ.key + spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id + script_name = "parking_correspondence_performance_records_with_pcn_gds" + triggered_by_job = module.parking_pcn_denormalisation.job_name + job_description = "parking_correspondence_performance_records_with_pcn with no timestamp for GDS" + workflow_name = "${local.short_identifier_prefix}parking-liberator-data-workflow" + trigger_enabled = local.is_production_environment + number_of_workers_for_glue_job = 10 + glue_job_worker_type = "G.1X" + glue_version = "4.0" + job_parameters = { + "--job-bookmark-option" = "job-bookmark-disable" + "--environment" = var.environment + "--conf" = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY" + } +} -# Migrated job "parking_correspondence_performance_records_with_pcn_gds" to dap-airflow om 14/04/2025 module "parking_foi_pcn_gds_daily_summary_records" { source = "../modules/aws-glue-job" @@ -589,9 +634,53 @@ module "parking_permit_street_stress_with_cpz" { "--environment" = var.environment } } +module "parking_correspondence_performance_records_with_pcn_downtime" { + source = "../modules/aws-glue-job" + is_live_environment = local.is_live_environment + is_production_environment = local.is_production_environment + department = module.department_parking_data_source + job_name = "${local.short_identifier_prefix}parking_correspondence_performance_records_with_pcn_downtime" + helper_module_key = data.aws_s3_object.helpers.key + pydeequ_zip_key = data.aws_s3_object.pydeequ.key + spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id + script_name = "parking_correspondence_performance_records_with_pcn_downtime" + triggered_by_job = module.parking_pcn_denormalisation.job_name + job_description = "correspondence performance records with pcn FOI records Team details and Downtime data" + workflow_name = "${local.short_identifier_prefix}parking-liberator-data-workflow" + trigger_enabled = local.is_production_environment + number_of_workers_for_glue_job = 10 + glue_job_worker_type = "G.1X" + glue_version = "4.0" + job_parameters = { + "--job-bookmark-option" = "job-bookmark-disable" + "--environment" = var.environment + "--conf" = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY" + } +} -# Migrated job "parking_correspondence_performance_records_with_pcn_downtime" to dap-airflow om 14/04/2025 -# Migrated job "parking_correspondence_performance_records_with_pcn_downtime_gds" to dap-airflow om 14/04/2025 +module "parking_correspondence_performance_records_with_pcn_downtime_gds" { + source = "../modules/aws-glue-job" + is_live_environment = local.is_live_environment + is_production_environment = local.is_production_environment + department = module.department_parking_data_source + job_name = "${local.short_identifier_prefix}parking_correspondence_performance_records_with_pcn_downtime_gds" + helper_module_key = data.aws_s3_object.helpers.key + pydeequ_zip_key = data.aws_s3_object.pydeequ.key + spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id + script_name = "parking_correspondence_performance_records_with_pcn_downtime_gds" + triggered_by_job = module.parking_pcn_denormalisation.job_name + job_description = "correspondence performance records with pcn FOI records Team details and Downtime data for Google Studio - gds" + workflow_name = "${local.short_identifier_prefix}parking-liberator-data-workflow" + trigger_enabled = local.is_production_environment + number_of_workers_for_glue_job = 10 + glue_job_worker_type = "G.1X" + glue_version = "4.0" + job_parameters = { + "--job-bookmark-option" = "job-bookmark-disable" + "--environment" = var.environment + "--conf" = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY" + } +} module "parking_open_pcns_vrms_linked_cancelled_ringer" { source = "../modules/aws-glue-job"