From 89d108f55bd5456a0b72c473698c19e467981d98 Mon Sep 17 00:00:00 2001 From: Steve Farr <36167148+stevefarrhackneygovuk@users.noreply.github.com> Date: Thu, 24 Apr 2025 09:36:55 +0100 Subject: [PATCH] Revert 4 glue jobs back to their original SparkSQL versions --- ...espondence_performance_records_with_pcn.py | 781 +++----- ...e_performance_records_with_pcn_downtime.py | 1625 ++++++--------- ...rformance_records_with_pcn_downtime_gds.py | 1780 ++++++----------- ...ndence_performance_records_with_pcn_gds.py | 958 ++++----- 4 files changed, 1890 insertions(+), 3254 deletions(-) diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py index bb40f03db..b6e9f8c73 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn.py @@ -1,499 +1,312 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job +from awsglue import DynamicFrame +from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS + +def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: + for alias, frame in mapping.items(): + frame.toDF().createOrReplaceTempView(alias) + result = spark.sql(query) + return DynamicFrame.fromDF(result, glueContext, transformation_ctx) + + +args = getResolvedOptions(sys.argv, ["JOB_NAME"]) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) environment = get_glue_env_var("environment") -# The target table in liberator refined zone -table_name = "parking_correspondence_performance_records_with_pcn" +# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full +AmazonS3refinedpcnfoidetails_pcn_foi_full_node1 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-refined-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="pcnfoidetails_pcn_foi_full", + transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1", +) -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ -/* -parking_correspondence_performance_records_with_pcn.sql -Correspondence Performance records last 13 months with PCN FOI records +# Script generated for node S3 bucket - Raw - liberator_pcn_ic +S3bucketRawliberator_pcn_ic_node1682353070282 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="liberator_pcn_ic", + transformation_ctx="S3bucketRawliberator_pcn_ic_node1682353070282", +) --->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn" ---<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" --- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" --- < "parking-raw-zone"."parking_correspondence_performance_teams" +# Script generated for node parking_raw_zone - parking_correspondence_performance_teams +parking_raw_zoneparking_correspondence_performance_teams_node1682353072411 = glueContext.create_dynamic_frame.from_catalog( + database="parking-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="parking_correspondence_performance_teams", + transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1682353072411", +) -16/06/2022 - Created +# Script generated for node ApplyMapping +SqlQuery0 = """ +/* +Correspondence Performance records last 13 months with PCN FOI records +16/06/2022 - Created 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing -27/02/2025 - Refactored SQL for migration back to AthenaSQL for dap-airflow -14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. -16/04/2025 - Workaround to orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready -*/ -WITH -team AS ( - SELECT DISTINCT - "start_date" AS t_start_date, - end_date AS t_end_date, - team AS t_team, - team_name AS t_team_name, - "role" AS t_role, - forename AS t_forename, - surname AS t_surname, - full_name AS t_full_name, - qa_doc_created_by AS t_qa_doc_created_by, - qa_doc_full_name AS t_qa_doc_full_name, - post_title AS t_post_title, - notes AS t_notes, - import_date AS t_import_date - FROM "parking-raw-zone"."parking_correspondence_performance_teams" - WHERE import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "parking-raw-zone"."parking_correspondence_performance_teams" g - ) -), -liberator AS ( - SELECT *, - - CAST(CURRENT_TIMESTAMP AS TIMESTAMP(3)) AS current_utc_timestamp, - TRY(CAST(date_received AS TIMESTAMP(3))) AS date_received_timestamp, -- pre-filtered by WHERE clause - - IF(whenassigned <> '', -- contains probable valid timestamp - TRY(CAST(whenassigned AS TIMESTAMP(3))), - CAST(NULL AS TIMESTAMP(3)) -- edge case - ) AS whenassigned_timestamp, - - IF(Response_generated_at <> '', -- contains probable valid timestamp - TRY(CAST(Response_generated_at AS TIMESTAMP(3))), - CAST(NULL AS TIMESTAMP(3)) -- edge case - ) AS Response_generated_at_timestamp, - CAST(CASE WHEN date_received <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' - CURRENT_DATE - ) - --ELSE NULL - END AS INTEGER) AS days_since_date_received, - - CAST(CASE WHEN date_received <> '' AND whenassigned <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' - TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE) - ) - --ELSE NULL - END AS INTEGER) AS days_since_date_received_whenassigned, - - CAST(CASE WHEN whenassigned <> '' AND response_generated_at = '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), - CURRENT_DATE - ) - --ELSE NULL -- edge case - END AS INTEGER) AS days_since_whenassigned, - - CAST(CASE WHEN whenassigned <> '' AND response_generated_at <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), - TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) - ) - --ELSE NULL -- edge case - END AS INTEGER) AS days_since_whenassigned_response_generated_at, - - CAST(CASE WHEN date_received <> '' AND response_generated_at <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), - TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) - ) - --ELSE NULL -- edge case - END AS INTEGER) AS days_since_date_received_response_generated_at - - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" - WHERE import_Date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g - ) - AND LENGTH(ticketserialnumber) = 10 -- ticket filter - AND date_received <> '' -- is the overriding condition for "13 months from todays date"! - AND TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE) > CURRENT_DATE - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - AND DATE_DIFF( - 'month', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), - CURRENT_DATE - ) <= 13 */ +With team as ( +select distinct start_date as t_start_date +,end_date as t_end_date +,team as t_team +,team_name as t_team_name +,role as t_role +,forename as t_forename +,surname as t_surname +,full_name as t_full_name +,qa_doc_created_by as t_qa_doc_created_by +,qa_doc_full_name as t_qa_doc_full_name +,post_title as t_post_title +,notes as t_notes +,import_date as t_import_date--* +from parking_correspondence_performance_teams where import_date = (select max(import_date) from parking_correspondence_performance_teams ) ) -SELECT - CAST(CASE - WHEN l.date_received <> '' AND l.whenassigned = '' - THEN 'Unassigned' - WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at = '' - THEN 'Assigned' - WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at <> '' - THEN 'Responded' - --ELSE NULL - END AS VARCHAR) AS response_status, - - CAST(l.current_utc_timestamp AS VARCHAR) AS current_time_stamp, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS unassigned_time, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS to_assigned_time, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_time, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_response_time, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS response_time, - - /*unassigned days*/ - CAST(CASE WHEN l.whenassigned = '' - THEN l.days_since_date_received - --ELSE NULL - END AS VARCHAR) AS unassigned_days, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - THEN - CASE - WHEN l.days_since_date_received <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received > 56 THEN '56 plus days' - --WHEN l.days_since_date_received IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS unassigned_days_group, - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - AND l.days_since_date_received <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS unassigned_days_kpiTotFiftySixLess, - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - AND l.days_since_date_received <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS unassigned_days_kpiTotFourteenLess, +Select +case +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' +end as response_status, +cast(current_timestamp as string) as Current_time_stamp, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, + +/*unassigned days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' +end as unassigned_days_group +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as unassigned_days_kpiTotFiftySixLess +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as unassigned_days_kpiTotFourteenLess, + +/*Days to assign*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as Days_to_assign_group +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as Days_to_assign_kpiTotFiftySixLess +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as Days_to_assign_kpiTotFourteenLess, + +/*assigned in progress days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' +end as assigned_in_progress_days_group +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFiftySixLess +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFourteenLess, + +/*assigned response days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' +end as assignedResponseDays_group +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFiftySixLess +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFourteenLess, + +/*Response days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as ResponseDays_group +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as ResponseDays_kpiTotFiftySixLess +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as ResponseDays_kpiTotFourteenLess, + +Response_generated_at +,Date_Received +,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear +,liberator_pcn_ic.Type +,Serviceable +,Service_category +,Response_written_by +,Letter_template +,Action_taken +,Related_to_PCN +,Cancellation_group +,Cancellation_reason +,whenassigned +,ticketserialnumber +,noderef +,liberator_pcn_ic.record_created +,liberator_pcn_ic.import_timestamp +,liberator_pcn_ic.import_year +,liberator_pcn_ic.import_month +,liberator_pcn_ic.import_day +,liberator_pcn_ic.import_date + +/*pcn data*/ +,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn +,pcnfoidetails_pcn_foi_full.pcnissuedate as pcn_pcnissuedate +,pcnfoidetails_pcn_foi_full.pcnissuedatetime as pcn_pcnissuedatetime +,pcnfoidetails_pcn_foi_full.pcn_canx_date as pcn_pcn_canx_date +,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup +,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason +,pcnfoidetails_pcn_foi_full.pcn_casecloseddate as pcn_pcn_casecloseddate +,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location +,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation +,pcnfoidetails_pcn_foi_full.zone as pcn_zone +,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn +,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode +,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix +,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype +,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm +,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake +,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel +,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour +,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo +,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice +,pcnfoidetails_pcn_foi_full.current_30_day_flag as pcn_current_30_day_flag +,pcnfoidetails_pcn_foi_full.isvda as pcn_isvda +,pcnfoidetails_pcn_foi_full.isvoid as pcn_isvoid +,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval +,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen +,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows +,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway +,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor +,pcnfoidetails_pcn_foi_full.warningflag as pcn_warningflag +,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts +,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason +,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount +,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received +,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount +,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void +,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method +,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref +,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from +,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to +,pcnfoidetails_pcn_foi_full.bailiff_processedon as pcn_bailiff_processedon +,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason +,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff +,pcnfoidetails_pcn_foi_full.warrantissuedate as pcn_warrantissuedate +,pcnfoidetails_pcn_foi_full.allocation as pcn_allocation +,pcnfoidetails_pcn_foi_full.eta_datenotified as pcn_eta_datenotified +,pcnfoidetails_pcn_foi_full.eta_packsubmittedon as pcn_eta_packsubmittedon +,pcnfoidetails_pcn_foi_full.eta_evidencedate as pcn_eta_evidencedate +,pcnfoidetails_pcn_foi_full.eta_adjudicationdate as pcn_eta_adjudicationdate +,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds +,pcnfoidetails_pcn_foi_full.eta_decisionreceived as pcn_eta_decisionreceived +,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome +,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby +,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby +,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address +,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address +,pcnfoidetails_pcn_foi_full.corresp_dispute_flag as pcn_corresp_dispute_flag +,pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as pcn_keyworker_corresp_dispute_flag +,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag +,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year +,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref +,pcnfoidetails_pcn_foi_full.nto_printed as pcn_nto_printed +,pcnfoidetails_pcn_foi_full.appeal_accepted as pcn_appeal_accepted +,pcnfoidetails_pcn_foi_full.arrived_in_pound as pcn_arrived_in_pound +,pcnfoidetails_pcn_foi_full.cancellation_reversed as pcn_cancellation_reversed +,pcnfoidetails_pcn_foi_full.cc_printed as pcn_cc_printed +,pcnfoidetails_pcn_foi_full.drr as pcn_drr +,pcnfoidetails_pcn_foi_full.en_printed as pcn_en_printed +,pcnfoidetails_pcn_foi_full.hold_released as pcn_hold_released +,pcnfoidetails_pcn_foi_full.dvla_response as pcn_dvla_response +,pcnfoidetails_pcn_foi_full.dvla_request as pcn_dvla_request +,pcnfoidetails_pcn_foi_full.full_rate_uplift as pcn_full_rate_uplift +,pcnfoidetails_pcn_foi_full.hold_until as pcn_hold_until +,pcnfoidetails_pcn_foi_full.lifted_at as pcn_lifted_at +,pcnfoidetails_pcn_foi_full.lifted_by as pcn_lifted_by +,pcnfoidetails_pcn_foi_full.loaded as pcn_loaded +,pcnfoidetails_pcn_foi_full.nor_sent as pcn_nor_sent +,pcnfoidetails_pcn_foi_full.notice_held as pcn_notice_held +,pcnfoidetails_pcn_foi_full.ofr_printed as pcn_ofr_printed +,pcnfoidetails_pcn_foi_full.pcn_printed as pcn_pcn_printed +,pcnfoidetails_pcn_foi_full.reissue_nto_requested as pcn_reissue_nto_requested +,pcnfoidetails_pcn_foi_full.reissue_pcn as pcn_reissue_pcn +,pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as pcn_set_back_to_pre_cc_stage +,pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as pcn_vehicle_released_for_auction +,pcnfoidetails_pcn_foi_full.warrant_issued as pcn_warrant_issued +,pcnfoidetails_pcn_foi_full.warrant_redistributed as pcn_warrant_redistributed +,pcnfoidetails_pcn_foi_full.warrant_request_granted as pcn_warrant_request_granted +,pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as pcn_ad_hoc_vq4_request +,pcnfoidetails_pcn_foi_full.paper_vq5_received as pcn_paper_vq5_received +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as pcn_pcn_extracted_for_buslane +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as pcn_pcn_extracted_for_pre_debt +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as pcn_pcn_extracted_for_collection +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as pcn_pcn_extracted_for_drr +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as pcn_pcn_extracted_for_cc +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as pcn_pcn_extracted_for_nto +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as pcn_pcn_extracted_for_print +,pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as pcn_warning_notice_extracted_for_print +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as pcn_pcn_extracted_for_ofr +,pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as pcn_pcn_extracted_for_warrant_request +,pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as pcn_pre_debt_new_debtor_details +,pcnfoidetails_pcn_foi_full.importdattime as pcn_importdattime +,pcnfoidetails_pcn_foi_full.importdatetime as pcn_importdatetime +,pcnfoidetails_pcn_foi_full.import_year as pcn_import_year +,pcnfoidetails_pcn_foi_full.import_month as pcn_import_month +,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day +,pcnfoidetails_pcn_foi_full.import_date as pcn_import_date +,team.* + + +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date +left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) + +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - /*Days to assign*/ - CAST(l.days_since_date_received_whenassigned AS VARCHAR) AS days_to_assign, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - THEN - CASE - WHEN l.days_since_date_received_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received_whenassigned > 56 THEN '56 plus days' - --WHEN l.days_since_date_received_whenassigned IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS days_to_assign_group, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - AND l.days_since_date_received_whenassigned <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS Days_to_assign_kpiTotFiftySixLess, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - AND l.days_since_date_received_whenassigned <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS Days_to_assign_kpiTotFourteenLess, - - /*assigned in progress days*/ - CAST(l.days_since_whenassigned AS VARCHAR) AS assigned_in_progress_days, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned > 56 THEN '56 plus days' - --WHEN l.days_since_whenassigned IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_days_group, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS assigned_in_progress_days_kpiTotFiftySixLess, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS assigned_in_progress_days_kpiTotFourteenLess, - - /*assigned response days*/ - CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) AS assignedResponseDays, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - THEN - CASE - WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' - --WHEN l.days_since_whenassigned_response_generated_at IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assignedResponseDays_group, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - AND l.days_since_whenassigned_response_generated_at <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS assignedResponseDays_kpiTotFiftySixLess, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - AND l.days_since_whenassigned_response_generated_at <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS assignedResponseDays_kpiTotFourteenLess, - - /*Response days*/ - CAST(l.days_since_date_received_response_generated_at AS VARCHAR) AS ResponseDays, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - THEN - CASE - WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' - --WHEN l.days_since_date_received_response_generated_at IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS ResponseDays_group, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS ResponseDays_kpiTotFiftySixLess, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS ResponseDays_kpiTotFourteenLess, - - l.Response_generated_at, --AS VARCHAR - l.Date_Received, --AS VARCHAR - - CAST(SUBSTR(TRY_CAST(l.date_received AS VARCHAR(10)), 1, 7) || '-01' AS VARCHAR) AS MonthYear, - - l.Type AS "type", --AS VARCHAR - l.Serviceable, --AS VARCHAR - l.Service_category, --AS VARCHAR - l.Response_written_by, --AS VARCHAR - l.Letter_template, --AS VARCHAR - l.Action_taken, --AS VARCHAR - l.Related_to_PCN, --AS VARCHAR - l.Cancellation_group, --AS VARCHAR - l.Cancellation_reason, --AS VARCHAR - l.whenassigned, --AS VARCHAR - l.ticketserialnumber, --AS VARCHAR - l.noderef, --AS VARCHAR - TRY_CAST(l.record_created AS TIMESTAMP(3)) AS record_created, -- cast from VARCHAR - l.import_timestamp, --AS VARCHAR - - /*** pcn columns taken from "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ***/ - -- Data sourced from the refined zone needs to be handled carefully... - -- Do not assume the following output column datatypes will be the same as "pcnfoidetails_pcn_foi_full". - -- Being left-joined to "pcnfoidetails_pcn_foi_full" will cause NULLs to appear in these columns. - -- And similar transforms elsewhere require different translations to the ones below... - /* These are the current automated translations developed for this transform:- - TRY_CAST(p.[source_column] AS DATE) AS [target_column], --[position]-- try date to date - COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS DATE), '%Y-%m-%d'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try date to string - TRY_CAST(p.[source_column] AS INTEGER) AS [target_column], --[position]-- try int to int - TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try timestamp to string - TRY_CAST(p.[source_column] AS TIMESTAMP) AS [target_column], --[position]-- try timestamp to timestamp - */ - -- These translations try to deal with problems without resorting to throwing errors, tolerating a reasonable degree of schema evolution in the "pcnfoidetails_pcn_foi_full" source. - TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --45-- try string to string (default) - TRY_CAST(p.pcnissuedate AS DATE) AS pcn_pcnissuedate, --46-- try date to date - TRY_CAST(p.pcnissuedatetime AS TIMESTAMP) AS pcn_pcnissuedatetime, --47-- try timestamp to timestamp - TRY_CAST(p.pcn_canx_date AS DATE) AS pcn_pcn_canx_date, --48-- try date to date - TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --49-- try string to string (default) - TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --50-- try string to string (default) - TRY_CAST(p.pcn_casecloseddate AS DATE) AS pcn_pcn_casecloseddate, --51-- try date to date - TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --52-- try string to string (default) - TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --53-- try string to string (default) - TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --54-- try string to string (default) - TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --55-- try string to string (default) - TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --56-- try string to string (default) - TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --57-- try string to string (default) - TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --58-- try string to string (default) - TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --59-- try string to string (default) - TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --60-- try string to string (default) - TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --61-- try string to string (default) - TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --62-- try string to string (default) - TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --63-- try string to string (default) - TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --64-- try string to string (default) - TRY_CAST(p.current_30_day_flag AS INTEGER) AS pcn_current_30_day_flag, --65-- try int to int - TRY_CAST(p.isvda AS INTEGER) AS pcn_isvda, --66-- try int to int - TRY_CAST(p.isvoid AS INTEGER) AS pcn_isvoid, --67-- try int to int - TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --68-- try string to string (default) - TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --69-- try string to string (default) - TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --70-- try string to string (default) - TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --71-- try string to string (default) - TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --72-- try string to string (default) - TRY_CAST(p.warningflag AS INTEGER) AS pcn_warningflag, --73-- try int to int - TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --74-- try string to string (default) - TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --75-- try string to string (default) - TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --76-- try string to string (default) - TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --77-- try string to string (default) - TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --78-- try string to string (default) - TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --79-- try string to string (default) - TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --80-- try string to string (default) - TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --81-- try string to string (default) - TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --82-- try string to string (default) - TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --83-- try string to string (default) - TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --84-- try string to string (default) - TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --85-- try string to string (default) - TRY_CAST(p.bailiff_processedon AS TIMESTAMP) AS pcn_bailiff_processedon, --86-- try timestamp to timestamp - TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --87-- try string to string (default) - TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --88-- try string to string (default) - TRY_CAST(p.warrantissuedate AS TIMESTAMP) AS pcn_warrantissuedate, --89-- try timestamp to timestamp - TRY_CAST(p.allocation AS INTEGER) AS pcn_allocation, --90-- try int to int - TRY_CAST(p.eta_datenotified AS TIMESTAMP) AS pcn_eta_datenotified, --91-- try timestamp to timestamp - TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP) AS pcn_eta_packsubmittedon, --92-- try timestamp to timestamp - TRY_CAST(p.eta_evidencedate AS TIMESTAMP) AS pcn_eta_evidencedate, --93-- try timestamp to timestamp - TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP) AS pcn_eta_adjudicationdate, --94-- try timestamp to timestamp - TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --95-- try string to string (default) - TRY_CAST(p.eta_decisionreceived AS TIMESTAMP) AS pcn_eta_decisionreceived, --96-- try timestamp to timestamp - TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --97-- try string to string (default) - TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --98-- try string to string (default) - TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --99-- try string to string (default) - TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --100-- try string to string (default) - TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --101-- try string to string (default) - TRY_CAST(p.corresp_dispute_flag AS INTEGER) AS pcn_corresp_dispute_flag, --102-- try int to int - TRY_CAST(p.keyworker_corresp_dispute_flag AS INTEGER) AS pcn_keyworker_corresp_dispute_flag, --103-- try int to int - TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --104-- try string to string (default) - TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --105-- try string to string (default) - TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --106-- try string to string (default) - TRY_CAST(p.nto_printed AS TIMESTAMP) AS pcn_nto_printed, --107-- try timestamp to timestamp - TRY_CAST(p.appeal_accepted AS TIMESTAMP) AS pcn_appeal_accepted, --108-- try timestamp to timestamp - TRY_CAST(p.arrived_in_pound AS TIMESTAMP) AS pcn_arrived_in_pound, --109-- try timestamp to timestamp - TRY_CAST(p.cancellation_reversed AS TIMESTAMP) AS pcn_cancellation_reversed, --110-- try timestamp to timestamp - TRY_CAST(p.cc_printed AS TIMESTAMP) AS pcn_cc_printed, --111-- try timestamp to timestamp - TRY_CAST(p.drr AS TIMESTAMP) AS pcn_drr, --112-- try timestamp to timestamp - TRY_CAST(p.en_printed AS TIMESTAMP) AS pcn_en_printed, --113-- try timestamp to timestamp - TRY_CAST(p.hold_released AS TIMESTAMP) AS pcn_hold_released, --114-- try timestamp to timestamp - TRY_CAST(p.dvla_response AS TIMESTAMP) AS pcn_dvla_response, --115-- try timestamp to timestamp - TRY_CAST(p.dvla_request AS TIMESTAMP) AS pcn_dvla_request, --116-- try timestamp to timestamp - TRY_CAST(p.full_rate_uplift AS TIMESTAMP) AS pcn_full_rate_uplift, --117-- try timestamp to timestamp - TRY_CAST(p.hold_until AS TIMESTAMP) AS pcn_hold_until, --118-- try timestamp to timestamp - TRY_CAST(p.lifted_at AS TIMESTAMP) AS pcn_lifted_at, --119-- try timestamp to timestamp - TRY_CAST(p.lifted_by AS TIMESTAMP) AS pcn_lifted_by, --120-- try timestamp to timestamp - TRY_CAST(p.loaded AS TIMESTAMP) AS pcn_loaded, --121-- try timestamp to timestamp - TRY_CAST(p.nor_sent AS TIMESTAMP) AS pcn_nor_sent, --122-- try timestamp to timestamp - TRY_CAST(p.notice_held AS TIMESTAMP) AS pcn_notice_held, --123-- try timestamp to timestamp - TRY_CAST(p.ofr_printed AS TIMESTAMP) AS pcn_ofr_printed, --124-- try timestamp to timestamp - TRY_CAST(p.pcn_printed AS TIMESTAMP) AS pcn_pcn_printed, --125-- try timestamp to timestamp - TRY_CAST(p.reissue_nto_requested AS TIMESTAMP) AS pcn_reissue_nto_requested, --126-- try timestamp to timestamp - TRY_CAST(p.reissue_pcn AS TIMESTAMP) AS pcn_reissue_pcn, --127-- try timestamp to timestamp - TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP) AS pcn_set_back_to_pre_cc_stage, --128-- try timestamp to timestamp - TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP) AS pcn_vehicle_released_for_auction, --129-- try timestamp to timestamp - TRY_CAST(p.warrant_issued AS TIMESTAMP) AS pcn_warrant_issued, --130-- try timestamp to timestamp - TRY_CAST(p.warrant_redistributed AS TIMESTAMP) AS pcn_warrant_redistributed, --131-- try timestamp to timestamp - TRY_CAST(p.warrant_request_granted AS TIMESTAMP) AS pcn_warrant_request_granted, --132-- try timestamp to timestamp - TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP) AS pcn_ad_hoc_vq4_request, --133-- try timestamp to timestamp - TRY_CAST(p.paper_vq5_received AS TIMESTAMP) AS pcn_paper_vq5_received, --134-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP) AS pcn_pcn_extracted_for_buslane, --135-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP) AS pcn_pcn_extracted_for_pre_debt, --136-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP) AS pcn_pcn_extracted_for_collection, --137-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP) AS pcn_pcn_extracted_for_drr, --138-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP) AS pcn_pcn_extracted_for_cc, --139-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP) AS pcn_pcn_extracted_for_nto, --140-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP) AS pcn_pcn_extracted_for_print, --141-- try timestamp to timestamp - TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP) AS pcn_warning_notice_extracted_for_print, --142-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP) AS pcn_pcn_extracted_for_ofr, --143-- try timestamp to timestamp - TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP) AS pcn_pcn_extracted_for_warrant_request, --144-- try timestamp to timestamp - TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP) AS pcn_pre_debt_new_debtor_details, --145-- try timestamp to timestamp - TRY_CAST(p.importdattime AS TIMESTAMP) AS pcn_importdattime, --146-- try timestamp to timestamp - TRY_CAST(p.importdatetime AS TIMESTAMP) AS pcn_importdatetime, --147-- try timestamp to timestamp - TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --148-- try string to string (default) - TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --149-- try string to string (default) - TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --150-- try string to string (default) - TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --151-- try string to string (default) - - /* teams columns expanded by above subquery */ - team.*, - - /* Partition columns moved to the end to keep schema alligned */ - l.import_year, - l.import_month, - l.import_day, - l.import_date - -FROM liberator l -- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" -LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p -/* - ON p.import_date = l.import_date -- joined on import partition - Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... - ...resulting in empty pcn_ columns in the output. - Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... - ...but for now we'll just use the following simple workaround... -*/ - ON p.import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g - ) - AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid -LEFT JOIN team -- < "parking-raw-zone"."parking_correspondence_performance_teams" - ON UPPER(team.t_full_name) = UPPER(l.Response_written_by) -; """ +ApplyMapping_node2 = sparkSqlQuery( + glueContext, + query=SqlQuery0, + mapping={ + "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1, + "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1682353070282, + "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1682353072411, + }, + transformation_ctx="ApplyMapping_node2", +) -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name +# Script generated for node S3 bucket +S3bucket_node3 = glueContext.getSink( + path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn/", + connection_type="s3", + updateBehavior="UPDATE_IN_DATABASE", + partitionKeys=["import_year", "import_month", "import_day", "import_date"], + compression="snappy", + enableUpdateCatalog=True, + transformation_ctx="S3bucket_node3", +) +S3bucket_node3.setCatalogInfo( + catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", + catalogTableName="parking_correspondence_performance_records_with_pcn", ) +S3bucket_node3.setFormat("glueparquet") +S3bucket_node3.writeFrame(ApplyMapping_node2) +job.commit() diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py index fd7b47a8b..e0f61029b 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime.py @@ -1,1067 +1,592 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job +from awsglue import DynamicFrame +from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS + +def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: + for alias, frame in mapping.items(): + frame.toDF().createOrReplaceTempView(alias) + result = spark.sql(query) + return DynamicFrame.fromDF(result, glueContext, transformation_ctx) + + +args = getResolvedOptions(sys.argv, ["JOB_NAME"]) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) +environment = get_glue_env_var("environment") -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var +# Script generated for node parking_raw_zone - parking_correspondence_performance_teams +parking_raw_zoneparking_correspondence_performance_teams_node1682353072411 = glueContext.create_dynamic_frame.from_catalog( + database="parking-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="parking_correspondence_performance_teams", + transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1682353072411", +) -environment = get_glue_env_var("environment") +# Script generated for node S3 bucket - Raw - liberator_pcn_ic +S3bucketRawliberator_pcn_ic_node1682353070282 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="liberator_pcn_ic", + transformation_ctx="S3bucketRawliberator_pcn_ic_node1682353070282", +) + +# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full +AmazonS3refinedpcnfoidetails_pcn_foi_full_node1 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-refined-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="pcnfoidetails_pcn_foi_full", + transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1", +) -# The target table in liberator refined zone -table_name = "parking_correspondence_performance_records_with_pcn_downtime" +# Script generated for node parking_raw_zone - parking_officer_downtime +parking_raw_zoneparking_officer_downtime_node1697211483070 = glueContext.create_dynamic_frame.from_catalog( + database="parking-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="parking_officer_downtime", + transformation_ctx="parking_raw_zoneparking_officer_downtime_node1697211483070", +) -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ --->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn_downtime" ---< "parking-raw-zone"."parking_correspondence_performance_teams" (data from google spreadsheet) ---< "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" ---<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ---< "parking-raw-zone"."parking_officer_downtime" (data from google form) +# Script generated for node ApplyMapping +SqlQuery0 = """ /* Correspondence Performance records last 13 months with PCN FOI records -16/06/2022 - Created +16/06/2022 - Created 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing 02/10/2023 - modified to union data into one output for Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 -22/10/2024 - translated into PrestoSQL to move away from Glue -01/11/2024 + Refactored SQL to separate the inward column conversions from onward business logic using CTE subqueries and allow improved fault handling. - + Downtime's Google Sheets alternate date formats properly handled +06/11/2024 - updated date formats in downtime data as source google sheet changed to yyyy-mm-dd HH:MM:ss from d/m/y 28/11/2024 - updated downtime records for t_team details to pull from parking_correspondence_performance_teams instead on leaving blank -12/12/2024 + Integrated the previous downtime update into the refactored version - + Commented out alternative last 13 month filter that caused unexpectedly higher row count -05/03/2025 - Replaced conditional '' empty string outputs with NULL defaults to be consistent with previous Glue/SparkSQL implementations. -14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. Fixed faulty arithmetic translation of Glue/SparkSQL to AthenaSQL. -16/04/2025 - Workaround orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready */ -/*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ -WITH -team AS ( - SELECT DISTINCT - start_date AS t_start_date, - end_date AS t_end_date, - team AS t_team, - team_name AS t_team_name, - ROLE AS t_role, - forename AS t_forename, - surname AS t_surname, - full_name AS t_full_name, - qa_doc_created_by AS t_qa_doc_created_by, - qa_doc_full_name AS t_qa_doc_full_name, - post_title AS t_post_title, - notes AS t_notes, - import_date AS t_import_date --* - FROM "parking-raw-zone"."parking_correspondence_performance_teams" - WHERE import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "parking-raw-zone"."parking_correspondence_performance_teams" g - ) -), -liberator_pcn_icdate_received_last13months AS ( - SELECT - *, - CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS current_utc_timestamp, - TRY(CAST(date_received AS TIMESTAMP)) AS date_received_timestamp, -- pre-filtered by WHERE clause - - IF(whenassigned <> '', -- contains probable valid timestamp - TRY(CAST(whenassigned AS TIMESTAMP)), - CAST(NULL AS TIMESTAMP) -- edge case - ) AS whenassigned_timestamp, - - IF(Response_generated_at <> '', -- contains probable valid timestamp - TRY(CAST(Response_generated_at AS TIMESTAMP)), - CAST(NULL AS TIMESTAMP) -- edge case - ) AS Response_generated_at_timestamp, - - CAST(DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' - CAST(CURRENT_TIMESTAMP AS DATE) - ) AS BIGINT) AS days_since_date_received, - - CAST(IF( - whenassigned <> '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' - TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_date_recieved_whenassigned, - - CAST(IF( - whenassigned <> '' AND response_generated_at = '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_whenassigned, - - CAST(IF( - whenassigned <> '' AND response_generated_at <> '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), - TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_whenassigned_response_generated_at, - - CAST(IF( - response_generated_at <> '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' - TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_date_received_response_generated_at--, - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" - WHERE import_Date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g - ) - AND LENGTH(ticketserialnumber) = 10 -- ticket filter - AND date_received <>'' -- is the overriding condition for "13 months from todays date"! - AND CAST(SUBSTR(date_received, 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - DATE_DIFF( - 'month', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ) <= 13 -*/ -), -liberator_with_team AS ( - SELECT - /*Fields for union downtime data*/ - CAST('' AS VARCHAR) AS downtime_timestamp, - CAST('' AS VARCHAR) AS downtime_email_address, - CAST('' AS VARCHAR) AS officer_s_first_name, - CAST('' AS VARCHAR) AS officer_s_last_name, - CAST('' AS VARCHAR) AS liberator_system_username, - CAST('' AS VARCHAR) AS liberator_system_id, - CAST('' AS VARCHAR) AS import_datetime, - CAST('' AS VARCHAR) AS multiple_downtime_days_flag, - CAST('' AS VARCHAR) AS response_secs, - CAST('' AS VARCHAR) AS response_mins, - CAST('' AS VARCHAR) AS response_hours, - CAST('' AS VARCHAR) AS response_days_plus_one, - CAST('' AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, - CAST('' AS VARCHAR) AS downtime_total_non_working_mins, - CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch, - CAST('' AS VARCHAR) AS downtime_total_working_mins, - CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, - CAST('' AS VARCHAR) AS downtime_total_working_mins_net, - CAST('' AS VARCHAR) AS start_date_time, - CAST('' AS VARCHAR) AS startdate, - CAST('' AS VARCHAR) AS start_date_datetime, - CAST('' AS VARCHAR) AS end_date_time, - CAST('' AS VARCHAR) AS enddate, - CAST('' AS VARCHAR) AS end_date_datetime, - - /*Liberator Incoming Correspondence Data*/ - CAST(CASE -- having already asserted date_received <>'' - WHEN l.whenassigned = '' THEN 'Unassigned' - WHEN l.whenassigned <> '' AND l.response_generated_at = '' THEN 'Assigned' - WHEN l.whenassigned <> '' AND l.response_generated_at <> '' THEN 'Responded' - --ELSE NULL - END AS VARCHAR) AS response_status, - - CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS unassigned_time, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS to_assigned_time, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_time, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_response_time, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS response_time, - - /*unassigned days*/ - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - THEN CAST(days_since_date_received AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS unassigned_days, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - THEN - CASE - WHEN l.days_since_date_received <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS unassigned_days_group, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - AND l.days_since_date_received <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - AND l.days_since_date_received <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, - - /*Days to assign*/ - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - THEN CAST(l.days_since_date_recieved_whenassigned AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS Days_to_assign, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - THEN - CASE - WHEN l.days_since_date_recieved_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_date_recieved_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_recieved_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_recieved_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_recieved_whenassigned > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS Days_to_assign_group, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - AND l.days_since_date_recieved_whenassigned <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - AND l.days_since_date_recieved_whenassigned <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, - - /*assigned in progress days*/ - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN CAST(l.days_since_whenassigned AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_days, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_days_group, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, - - /*assigned response days*/ - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS assignedResponseDays, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assignedResponseDays_group, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at <> '' - AND l.days_since_whenassigned_response_generated_at <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - AND l.days_since_whenassigned_response_generated_at <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, - - /*Response days*/ - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - THEN CAST(l.days_since_date_received_response_generated_at AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS ResponseDays, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS ResponseDays_group, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, - - Response_generated_at, - Date_Received, - - SUBSTR(CAST(l.date_received AS VARCHAR (10)), 1, 7) || '-01' AS MonthYear, - - l.Type AS "Type", - l.Serviceable, - l.Service_category, - l.Response_written_by, - l.Letter_template, - l.Action_taken, - l.Related_to_PCN, - l.Cancellation_group, - l.Cancellation_reason, - l.whenassigned, - l.ticketserialnumber, - l.noderef, - CAST(l.record_created AS VARCHAR) AS record_created, - - l.import_timestamp, - - /*pcn data*/ - /*** pcn columns taken from "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ***/ - -- Data sourced from the refined zone needs to be handled carefully... - -- Do not assume the following output column datatypes will be the same as "pcnfoidetails_pcn_foi_full". - -- Being left-joined to "pcnfoidetails_pcn_foi_full" will cause NULLs to appear in these columns. - -- And similar transforms elsewhere require different translations to the ones below... - /* These are the current automated translations developed for this transform:- - TRY_CAST(p.[source_column] AS DATE) AS [target_column], --[position]-- try date to date - TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try int to string - COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS DATE), '%Y-%m-%d'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try date to string - TRY_CAST(p.[source_column] AS INTEGER) AS [target_column], --[position]-- try int to int - TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try timestamp to string - TRY_CAST(p.[source_column] AS TIMESTAMP) AS [target_column], --[position]-- try timestamp to timestamp - */ - -- These translations try to deal with problems without resorting to throwing errors, tolerating a reasonable degree of schema evolution in the "pcnfoidetails_pcn_foi_full" source. - -- For example, date and timestamp string formatting works with either string, date and timestamp source columns and works like this... - /* If source can be cast to a date then it can be correctly date-formatted into a string, otherwise just cast whatever it was to a string. - COALESCE( - DATE_FORMAT(TRY_CAST(p.xxxx AS DATE), '%Y-%m-%d'), - CAST(p.xxxx AS VARCHAR) --...and can be extended to handle other date formats. - ) AS pcn_xxxx, -- from DATE - */ - /* If source can be cast to a timestamp then it can be correctly timestamp-formatted into a string, otherwise just cast whatever it was to a string. - COALESCE( - DATE_FORMAT(TRY_CAST(p.xxxx AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), - CAST(p.xxxx AS VARCHAR) -- Alternative for transforming a time-zoned timestamp... - --...COALESCE(DATE_FORMAT(TRY_CAST(TRY(From_iso8601_timestamp(p.xxxx)) AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.xxxx AS VARCHAR)) - --...and can be extended to handle other timestamp formats. - ) AS pcn_xxxx, -- from TIMESTAMP - */ - TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --69-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedate AS DATE), '%Y-%m-%d'), CAST(p.pcnissuedate AS VARCHAR)) AS pcn_pcnissuedate, --70-- try date to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcnissuedatetime AS VARCHAR)) AS pcn_pcnissuedatetime, --71-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_canx_date AS DATE), '%Y-%m-%d'), CAST(p.pcn_canx_date AS VARCHAR)) AS pcn_pcn_canx_date, --72-- try date to string - TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --73-- try string to string (default) - TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --74-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_casecloseddate AS DATE), '%Y-%m-%d'), CAST(p.pcn_casecloseddate AS VARCHAR)) AS pcn_pcn_casecloseddate, --75-- try date to string - TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --76-- try string to string (default) - TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --77-- try string to string (default) - TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --78-- try string to string (default) - TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --79-- try string to string (default) - TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --80-- try string to string (default) - TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --81-- try string to string (default) - TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --82-- try string to string (default) - TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --83-- try string to string (default) - TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --84-- try string to string (default) - TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --85-- try string to string (default) - TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --86-- try string to string (default) - TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --87-- try string to string (default) - TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --88-- try string to string (default) - TRY_CAST(p.current_30_day_flag AS VARCHAR) AS pcn_current_30_day_flag, --89-- try int to string - TRY_CAST(p.isvda AS VARCHAR) AS pcn_isvda, --90-- try int to string - TRY_CAST(p.isvoid AS VARCHAR) AS pcn_isvoid, --91-- try int to string - TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --92-- try string to string (default) - TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --93-- try string to string (default) - TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --94-- try string to string (default) - TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --95-- try string to string (default) - TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --96-- try string to string (default) - TRY_CAST(p.warningflag AS VARCHAR) AS pcn_warningflag, --97-- try int to string - TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --98-- try string to string (default) - TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --99-- try string to string (default) - TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- try string to string (default) - TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --101-- try string to string (default) - TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- try string to string (default) - TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --103-- try string to string (default) - TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --104-- try string to string (default) - TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --105-- try string to string (default) - TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --106-- try string to string (default) - TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --107-- try string to string (default) - TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --108-- try string to string (default) - TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --109-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.bailiff_processedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.bailiff_processedon AS VARCHAR)) AS pcn_bailiff_processedon, --110-- try timestamp to string - TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- try string to string (default) - TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --112-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.warrantissuedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrantissuedate AS VARCHAR)) AS pcn_warrantissuedate, --113-- try timestamp to string - TRY_CAST(p.allocation AS VARCHAR) AS pcn_allocation, --114-- try int to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_datenotified AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_datenotified AS VARCHAR)) AS pcn_eta_datenotified, --115-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_packsubmittedon AS VARCHAR)) AS pcn_eta_packsubmittedon, --116-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_evidencedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_evidencedate AS VARCHAR)) AS pcn_eta_evidencedate, --117-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_adjudicationdate AS VARCHAR)) AS pcn_eta_adjudicationdate, --118-- try timestamp to string - TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --119-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_decisionreceived AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_decisionreceived AS VARCHAR)) AS pcn_eta_decisionreceived, --120-- try timestamp to string - TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --121-- try string to string (default) - TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- try string to string (default) - TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --123-- try string to string (default) - TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --124-- try string to string (default) - TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --125-- try string to string (default) - TRY_CAST(p.corresp_dispute_flag AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- try int to string - TRY_CAST(p.keyworker_corresp_dispute_flag AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- try int to string - TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --128-- try string to string (default) - TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --129-- try string to string (default) - TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --130-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.nto_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nto_printed AS VARCHAR)) AS pcn_nto_printed, --131-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.appeal_accepted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.appeal_accepted AS VARCHAR)) AS pcn_appeal_accepted, --132-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.arrived_in_pound AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.arrived_in_pound AS VARCHAR)) AS pcn_arrived_in_pound, --133-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.cancellation_reversed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cancellation_reversed AS VARCHAR)) AS pcn_cancellation_reversed, --134-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.cc_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cc_printed AS VARCHAR)) AS pcn_cc_printed, --135-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.drr AS VARCHAR)) AS pcn_drr, --136-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.en_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.en_printed AS VARCHAR)) AS pcn_en_printed, --137-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.hold_released AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_released AS VARCHAR)) AS pcn_hold_released, --138-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_response AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_response AS VARCHAR)) AS pcn_dvla_response, --139-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_request AS VARCHAR)) AS pcn_dvla_request, --140-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.full_rate_uplift AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.full_rate_uplift AS VARCHAR)) AS pcn_full_rate_uplift, --141-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.hold_until AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_until AS VARCHAR)) AS pcn_hold_until, --142-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_at AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_at AS VARCHAR)) AS pcn_lifted_at, --143-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_by AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_by AS VARCHAR)) AS pcn_lifted_by, --144-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.loaded AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.loaded AS VARCHAR)) AS pcn_loaded, --145-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.nor_sent AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nor_sent AS VARCHAR)) AS pcn_nor_sent, --146-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.notice_held AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.notice_held AS VARCHAR)) AS pcn_notice_held, --147-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.ofr_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ofr_printed AS VARCHAR)) AS pcn_ofr_printed, --148-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_printed AS VARCHAR)) AS pcn_pcn_printed, --149-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_nto_requested AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_nto_requested AS VARCHAR)) AS pcn_reissue_nto_requested, --150-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_pcn AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_pcn AS VARCHAR)) AS pcn_reissue_pcn, --151-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.set_back_to_pre_cc_stage AS VARCHAR)) AS pcn_set_back_to_pre_cc_stage, --152-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.vehicle_released_for_auction AS VARCHAR)) AS pcn_vehicle_released_for_auction, --153-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_issued AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_issued AS VARCHAR)) AS pcn_warrant_issued, --154-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_redistributed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_redistributed AS VARCHAR)) AS pcn_warrant_redistributed, --155-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_request_granted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_request_granted AS VARCHAR)) AS pcn_warrant_request_granted, --156-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ad_hoc_vq4_request AS VARCHAR)) AS pcn_ad_hoc_vq4_request, --157-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.paper_vq5_received AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.paper_vq5_received AS VARCHAR)) AS pcn_paper_vq5_received, --158-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_buslane AS VARCHAR)) AS pcn_pcn_extracted_for_buslane, --159-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_pre_debt AS VARCHAR)) AS pcn_pcn_extracted_for_pre_debt, --160-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_collection AS VARCHAR)) AS pcn_pcn_extracted_for_collection, --161-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_drr AS VARCHAR)) AS pcn_pcn_extracted_for_drr, --162-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_cc AS VARCHAR)) AS pcn_pcn_extracted_for_cc, --163-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_nto AS VARCHAR)) AS pcn_pcn_extracted_for_nto, --164-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_print AS VARCHAR)) AS pcn_pcn_extracted_for_print, --165-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warning_notice_extracted_for_print AS VARCHAR)) AS pcn_warning_notice_extracted_for_print, --166-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_ofr AS VARCHAR)) AS pcn_pcn_extracted_for_ofr, --167-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_warrant_request AS VARCHAR)) AS pcn_pcn_extracted_for_warrant_request, --168-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pre_debt_new_debtor_details AS VARCHAR)) AS pcn_pre_debt_new_debtor_details, --169-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.importdattime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdattime AS VARCHAR)) AS pcn_importdattime, --170-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.importdatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdatetime AS VARCHAR)) AS pcn_importdatetime, --171-- try timestamp to string - TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --172-- try string to string (default) - TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --173-- try string to string (default) - TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --174-- try string to string (default) - TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --175-- try string to string (default) - - /* teams */ - t.t_start_date, - t.t_end_date, - t.t_team, - t.t_team_name, - t.t_role, - t.t_forename, - t.t_surname, - t.t_full_name, - t.t_qa_doc_created_by, - t.t_qa_doc_full_name, - t.t_post_title, - t.t_notes, - t.t_import_date, - - /* Partition columns moved to the end to keep schema alligned */ - l.import_year, - l.import_month, - l.import_day, - l.import_date - - FROM liberator_pcn_icdate_received_last13months l - LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p - /* - ON p.import_date = l.import_date -- joined on import partition - Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... - ...resulting in empty pcn_ columns in the output. - Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... - ...but for now we'll just use the following simple workaround... - */ - ON p.import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g - ) - AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid - LEFT JOIN team t - ON UPPER(t.t_full_name) = UPPER(l.Response_written_by) -), - -downtime_start_date_end_date_last13months AS ( - SELECT *, - IF(start_date LIKE '%/%/%', -- contains possible valid timestamp - TRY(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone - IF(start_date LIKE '%-%-%', -- contains other possible valid timestamp - TRY(PARSE_DATETIME(start_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone - CAST(NULL AS TIMESTAMP) -- edge case - ) - ) AS start_timestampZ, - - IF(end_date LIKE '%/%/%', -- contains possible valid timestamp - TRY(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone - IF(end_date LIKE '%-%-%', -- contains other possible valid timestamp - TRY(PARSE_DATETIME(end_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone - CAST(NULL AS TIMESTAMP) -- edge case - ) - ) AS end_timestampZ--, - - FROM "parking-raw-zone"."parking_officer_downtime" - WHERE import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "parking-raw-zone"."parking_officer_downtime" g - ) - AND "timestamp" NOT LIKE '' - AND "timestamp" NOT LIKE '#REF!' - AND CAST(SUBSTR(CAST("timestamp" as VARCHAR(30)), 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - DATE_DIFF( - 'month', - TRY(CAST(SUBSTR("timestamp", 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ) <= 13 -*/ -), - -downtime AS ( - SELECT *, - /* - TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS start_end_unixtime_seconds, - TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) AS end_unixtime_seconds, - - TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) - - TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS downtime_secs, - - -- downtime_secs / 60 AS downtime_mins - -- downtime_secs / 3600 AS downtime_hours - */ - - DATE_DIFF( - 'second', - start_timestampZ, - end_timestampZ - ) AS downtime_secs, - - DATE_DIFF( - 'minute', - start_timestampZ, - end_timestampZ - ) AS downtime_mins, - - DATE_DIFF( - 'hour', - start_timestampZ, - end_timestampZ - ) AS downtime_hours, - - DATE_DIFF( - 'day', - start_timestampZ, - end_timestampZ - ) AS downtime_days, - - TRIM(officer_s_first_name) || ' ' || TRIM(officer_s_last_name) AS Response_written_by --downtime officer full name - - FROM downtime_start_date_end_date_last13months -), - -google_form AS ( - --Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 - SELECT - CAST(d."timestamp" AS VARCHAR) AS downtime_timestamp, - d.email_address AS downtime_email_address, - TRIM(d.officer_s_first_name) AS officer_s_first_name, - TRIM(d.officer_s_last_name) AS officer_s_last_name, - d.liberator_system_username, - d.liberator_system_id, - CAST(d.import_datetime AS VARCHAR) AS import_datetime, - - /* Throughout asserts... - "timestamp" NOT LIKE '' - AND "timestamp" NOT LIKE '#REF!' - */ - CAST(CASE - WHEN d.downtime_mins > 1440 /* downtime mins (response_mins) */ - THEN '1' - ELSE '0' - END AS VARCHAR) AS multiple_downtime_days_flag, - - -- Because the Glue job's response_secs string output was cast from a bigint returned from unix_timestamp... - -- ...there was no need to cast it to a DOUBLE here... - CAST(d.downtime_secs AS VARCHAR) AS response_secs, - -- ...unlike the products below where SparkSQL and AthenaSQL differed in their arithmetic outcomes... - - /* This version was rejected because the output differs wildly from the original Glue output... - -- Here all input value was cast beforehand to DOUBLE. - CAST(CAST(d.downtime_secs AS DOUBLE)/60 AS VARCHAR) AS REJECTED_response_mins, - */ - -- So we are keeping this version because it produces closest to the original Glue output... - -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... - CAST(CAST(d.downtime_mins AS DOUBLE) AS VARCHAR) AS response_mins, - /* But might the following version be the better solution? - -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... - CAST(d.downtime_mins AS VARCHAR) AS ALTERNATIVE_response_mins, - */ - - -- We are keeping this version because it produces closest to the original Glue output... - -- Though beware, several decimal places may be produced in the VARCHAR output... - CAST(CAST(d.downtime_secs AS DOUBLE)/3600 AS VARCHAR) AS response_hours, - /* But might the following version be the better solution? - -- This does not produce the fractional mantissa because it is an integer calculation from start to finish... - CAST(d.downtime_hours AS VARCHAR) AS ALTERNATIVE_response_hours, - */ - - /*Downtime calendar days*/ - -- Warning: A small number outlier cases, producing negative values in original Glue output are also produced here... - CAST(d.downtime_days + 1 AS VARCHAR) AS response_days_plus_one, -- downtime days plus one calendar day - - /*Downtime non working mins*/ - CAST(CASE - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1 ) * 948 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 948 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, --if greater than 1440 mins then (days + 1) * 948 " - - CAST(CASE /* downtime mins (response_mins)*/ - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 1008 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 1008 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_non_working_mins, --if greater than 1440 mins then (days + 1) * 1008 " - - /*Downtime working mins*/ - CAST(CASE /* downtime mins (response_mins)*/ - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 492 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_working_mins_with_lunch, -- 492 mins = 8hrs 12 mins working day including lunch" - - CAST(CASE /* downtime mins (response_mins)*/ - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 432 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_working_mins, -- 432 mins = 7hrs 12mins working hours" - - /* Downtime working mins net (less downtime mins)*/ - - /* This version was rejected because the output differs wildly from the original Glue output... - -- Here all input values were cast beforehand to DOUBLE... - CAST(CASE -- downtime mins (response_mins) - WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 492 -- response_days_plus_one - WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 492 - CAST(d.downtime_secs AS DOUBLE)/60 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS REJECTED_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" - */ - -- So we are keeping this version because it produces closest to the original Glue output... - -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... - CAST(CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 - WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" - /* So might the following version be the better solution? - -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... - CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 - WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" - */ - - /* This version was rejected because the output differs wildly from the original Glue output... - -- Here all input values were cast beforehand to DOUBLE... - CAST(CASE -- downtime mins (response_mins) - WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 432 -- response_days_plus_one - WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 432 - CAST(d.downtime_secs AS DOUBLE)/60 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS REJECTED_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" - */ - -- So we are keeping this version because it produces closest to the original Glue output... - -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... - CAST(CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one - WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" - /* So might the following version be the better solution? - -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... - CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one - WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" - */ - - CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 12, 5) AS VARCHAR) AS start_date_time, - CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 1, 10) AS VARCHAR) AS startdate, - - CAST(SUBSTR( - CAST(d.start_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS VARCHAR) AS start_date_datetime, - - CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 12, 5) AS VARCHAR) AS end_date_time, - CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 1, 10) AS VARCHAR) AS enddate, - - CAST(SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS VARCHAR) AS end_date_datetime, - - CAST('Downtime' AS VARCHAR) AS response_status, - CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, - - /*Liberator Incoming Correspondence Data*/ - CAST('' AS VARCHAR) AS unassigned_time, - CAST('' AS VARCHAR) AS to_assigned_time, - CAST('' AS VARCHAR) AS assigned_in_progress_time, - CAST('' AS VARCHAR) AS assigned_response_time, - - -- Is there a better way to do this? Substring captures only the minutes... - 'INTERVAL ''' - || REGEXP_REPLACE(CAST( - TRY(CAST(SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS TIMESTAMP)) - - TRY(CAST(SUBSTR( - CAST(d.start_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS TIMESTAMP)) - AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - AS response_time, --as downtime_duration --line 30 - - - CAST('' AS VARCHAR) AS unassigned_days, - CAST('' AS VARCHAR) AS unassigned_days_group, - CAST('' AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, - CAST('' AS VARCHAR) AS Days_to_assign, - CAST('' AS VARCHAR) AS Days_to_assign_group, - CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, - CAST('' AS VARCHAR) AS assigned_in_progress_days, - CAST('' AS VARCHAR) AS assigned_in_progress_days_group, - CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, - CAST('' AS VARCHAR) AS assignedResponseDays, - CAST('' AS VARCHAR) AS assignedResponseDays_group, - CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, - - CAST(DATE_DIFF( - 'day', - CAST(SUBSTR( - CAST(d.start_timestampZ AS VARCHAR(30)), - 1, - 10 - ) AS DATE), - CAST(SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 10 - ) AS DATE) - ) AS VARCHAR) AS ResponseDays, --as downtime_duration in days - - CAST('' AS VARCHAR) AS ResponseDays_group, - CAST('' AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, - - d.end_date AS Response_generated_at, - d.start_date AS Date_Received, - - SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 7 - ) || '-01' AS MonthYear, - - 'Downtime' AS "Type", - - d.downtime AS Serviceable, -- downtime --,'Downtime' as Serviceable -- downtime - - CAST('' AS VARCHAR) AS Service_category, - - d.Response_written_by, --downtime officer full name - - CAST('' AS VARCHAR) AS Letter_template, - CAST('' AS VARCHAR) AS Action_taken, - CAST('' AS VARCHAR) AS Related_to_PCN, - CAST('' AS VARCHAR) AS Cancellation_group, - CAST('' AS VARCHAR) AS Cancellation_reason, - CAST('' AS VARCHAR) AS whenassigned, - CAST('' AS VARCHAR) AS ticketserialnumber, - CAST('' AS VARCHAR) AS noderef, - CAST('' AS VARCHAR) AS record_created, - d.import_timestamp, - - /*pcn data*/ - -- The original Glue SparkSQL had all of these pcn columns output as '' empty strings which either... - -- ...conflicted with the output column datatypes configured by Glue - -- ...and/or conflicted with the column translations done earlier in the SQL script. - /* Instead here, the null or empty translations are deliberately set to depend on the target datatype... - CAST(NULL AS DATE) AS [target_column], --[position]-- null date - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS INTEGER) AS [target_column], --[position]-- null int - CAST('' AS VARCHAR) AS [target_column], --[position]-- empty string - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS TIMESTAMP) AS [target_column], --[position]-- null timestamp - */ - CAST('' AS VARCHAR) AS pcn_pcn, --69-- empty string - CAST(NULL AS VARCHAR) AS pcn_pcnissuedate, --70-- null string - CAST(NULL AS VARCHAR) AS pcn_pcnissuedatetime, --71-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_canx_date, --72-- null string - CAST('' AS VARCHAR) AS pcn_cancellationgroup, --73-- empty string - CAST('' AS VARCHAR) AS pcn_cancellationreason, --74-- empty string - CAST(NULL AS VARCHAR) AS pcn_pcn_casecloseddate, --75-- null string - CAST('' AS VARCHAR) AS pcn_street_location, --76-- empty string - CAST('' AS VARCHAR) AS pcn_whereonlocation, --77-- empty string - CAST('' AS VARCHAR) AS pcn_zone, --78-- empty string - CAST('' AS VARCHAR) AS pcn_usrn, --79-- empty string - CAST('' AS VARCHAR) AS pcn_contraventioncode, --80-- empty string - CAST('' AS VARCHAR) AS pcn_contraventionsuffix, --81-- empty string - CAST('' AS VARCHAR) AS pcn_debttype, --82-- empty string - CAST('' AS VARCHAR) AS pcn_vrm, --83-- empty string - CAST('' AS VARCHAR) AS pcn_vehiclemake, --84-- empty string - CAST('' AS VARCHAR) AS pcn_vehiclemodel, --85-- empty string - CAST('' AS VARCHAR) AS pcn_vehiclecolour, --86-- empty string - CAST('' AS VARCHAR) AS pcn_ceo, --87-- empty string - CAST('' AS VARCHAR) AS pcn_ceodevice, --88-- empty string - CAST(NULL AS VARCHAR) AS pcn_current_30_day_flag, --89-- null string - CAST(NULL AS VARCHAR) AS pcn_isvda, --90-- null string - CAST(NULL AS VARCHAR) AS pcn_isvoid, --91-- null string - CAST('' AS VARCHAR) AS pcn_isremoval, --92-- empty string - CAST('' AS VARCHAR) AS pcn_driverseen, --93-- empty string - CAST('' AS VARCHAR) AS pcn_allwindows, --94-- empty string - CAST('' AS VARCHAR) AS pcn_parkedonfootway, --95-- empty string - CAST('' AS VARCHAR) AS pcn_doctor, --96-- empty string - CAST(NULL AS VARCHAR) AS pcn_warningflag, --97-- null string - CAST('' AS VARCHAR) AS pcn_progressionstage, --98-- empty string - CAST('' AS VARCHAR) AS pcn_nextprogressionstage, --99-- empty string - CAST('' AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- empty string - CAST('' AS VARCHAR) AS pcn_holdreason, --101-- empty string - CAST('' AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_received, --103-- empty string - CAST('' AS VARCHAR) AS pcn_lib_write_off_amount, --104-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_void, --105-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_method, --106-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_ref, --107-- empty string - CAST('' AS VARCHAR) AS pcn_baliff_from, --108-- empty string - CAST('' AS VARCHAR) AS pcn_bailiff_to, --109-- empty string - CAST(NULL AS VARCHAR) AS pcn_bailiff_processedon, --110-- null string - CAST('' AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- empty string - CAST('' AS VARCHAR) AS pcn_bailiff, --112-- empty string - CAST(NULL AS VARCHAR) AS pcn_warrantissuedate, --113-- null string - CAST(NULL AS VARCHAR) AS pcn_allocation, --114-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_datenotified, --115-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_packsubmittedon, --116-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_evidencedate, --117-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_adjudicationdate, --118-- null string - CAST('' AS VARCHAR) AS pcn_eta_appealgrounds, --119-- empty string - CAST(NULL AS VARCHAR) AS pcn_eta_decisionreceived, --120-- null string - CAST('' AS VARCHAR) AS pcn_eta_outcome, --121-- empty string - CAST('' AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- empty string - CAST('' AS VARCHAR) AS pcn_cancelledby, --123-- empty string - CAST('' AS VARCHAR) AS pcn_registered_keeper_address, --124-- empty string - CAST('' AS VARCHAR) AS pcn_current_ticket_address, --125-- empty string - CAST(NULL AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- null string - CAST(NULL AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- null string - CAST('' AS VARCHAR) AS pcn_fin_year_flag, --128-- empty string - CAST('' AS VARCHAR) AS pcn_fin_year, --129-- empty string - CAST('' AS VARCHAR) AS pcn_ticket_ref, --130-- empty string - CAST(NULL AS VARCHAR) AS pcn_nto_printed, --131-- null string - CAST(NULL AS VARCHAR) AS pcn_appeal_accepted, --132-- null string - CAST(NULL AS VARCHAR) AS pcn_arrived_in_pound, --133-- null string - CAST(NULL AS VARCHAR) AS pcn_cancellation_reversed, --134-- null string - CAST(NULL AS VARCHAR) AS pcn_cc_printed, --135-- null string - CAST(NULL AS VARCHAR) AS pcn_drr, --136-- null string - CAST(NULL AS VARCHAR) AS pcn_en_printed, --137-- null string - CAST(NULL AS VARCHAR) AS pcn_hold_released, --138-- null string - CAST(NULL AS VARCHAR) AS pcn_dvla_response, --139-- null string - CAST(NULL AS VARCHAR) AS pcn_dvla_request, --140-- null string - CAST(NULL AS VARCHAR) AS pcn_full_rate_uplift, --141-- null string - CAST(NULL AS VARCHAR) AS pcn_hold_until, --142-- null string - CAST(NULL AS VARCHAR) AS pcn_lifted_at, --143-- null string - CAST(NULL AS VARCHAR) AS pcn_lifted_by, --144-- null string - CAST(NULL AS VARCHAR) AS pcn_loaded, --145-- null string - CAST(NULL AS VARCHAR) AS pcn_nor_sent, --146-- null string - CAST(NULL AS VARCHAR) AS pcn_notice_held, --147-- null string - CAST(NULL AS VARCHAR) AS pcn_ofr_printed, --148-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_printed, --149-- null string - CAST(NULL AS VARCHAR) AS pcn_reissue_nto_requested, --150-- null string - CAST(NULL AS VARCHAR) AS pcn_reissue_pcn, --151-- null string - CAST(NULL AS VARCHAR) AS pcn_set_back_to_pre_cc_stage, --152-- null string - CAST(NULL AS VARCHAR) AS pcn_vehicle_released_for_auction, --153-- null string - CAST(NULL AS VARCHAR) AS pcn_warrant_issued, --154-- null string - CAST(NULL AS VARCHAR) AS pcn_warrant_redistributed, --155-- null string - CAST(NULL AS VARCHAR) AS pcn_warrant_request_granted, --156-- null string - CAST(NULL AS VARCHAR) AS pcn_ad_hoc_vq4_request, --157-- null string - CAST(NULL AS VARCHAR) AS pcn_paper_vq5_received, --158-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_buslane, --159-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_pre_debt, --160-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_collection, --161-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_drr, --162-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_cc, --163-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_nto, --164-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_print, --165-- null string - CAST(NULL AS VARCHAR) AS pcn_warning_notice_extracted_for_print, --166-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_ofr, --167-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_warrant_request, --168-- null string - CAST(NULL AS VARCHAR) AS pcn_pre_debt_new_debtor_details, --169-- null string - CAST(NULL AS VARCHAR) AS pcn_importdattime, --170-- null string - CAST(NULL AS VARCHAR) AS pcn_importdatetime, --171-- null string - CAST('' AS VARCHAR) AS pcn_import_year, --172-- empty string - CAST('' AS VARCHAR) AS pcn_import_month, --173-- empty string - CAST('' AS VARCHAR) AS pcn_import_day, --174-- empty string - CAST('' AS VARCHAR) AS pcn_import_date, --175-- empty string - - /*Teams data*/ - t.t_start_date, - t.t_end_date, - t.t_team, - t.t_team_name, - t.t_role, - t.t_forename, - t.t_surname, - t.t_full_name, - t.t_qa_doc_created_by, - t.t_qa_doc_full_name, - t.t_post_title, - t.t_notes, - t.t_import_date, +/*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ +With team as ( +select distinct start_date as t_start_date +,end_date as t_end_date +,team as t_team +,team_name as t_team_name +,role as t_role +,forename as t_forename +,surname as t_surname +,full_name as t_full_name +,qa_doc_created_by as t_qa_doc_created_by +,qa_doc_full_name as t_qa_doc_full_name +,post_title as t_post_title +,notes as t_notes +,import_date as t_import_date--* +from parking_correspondence_performance_teams where import_date = (select max(parking_correspondence_performance_teams.import_date) from parking_correspondence_performance_teams ) +) - /* Partition columns moved to the end to keep schema alligned */ - d.import_year, - d.import_month, - d.import_day, - CAST(d.import_date AS VARCHAR) AS import_date +select + +/*Fields for union downtime data*/ +'' as downtime_timestamp +,'' as downtime_email_address +,'' as officer_s_first_name +,'' as officer_s_last_name +,'' as liberator_system_username +,'' as liberator_system_id +,'' as import_datetime +,'' as multiple_downtime_days_flag +,'' as response_secs +,'' as response_mins +,'' as response_hours +,'' as response_days_plus_one +,'' as downtime_total_non_working_mins_with_lunch +, '' as downtime_total_non_working_mins +,'' as downtime_total_working_mins_with_lunch +,'' as downtime_total_working_mins +,'' as downtime_total_working_mins_with_lunch_net +,'' as downtime_total_working_mins_net +, '' as start_date_time +,'' as startdate +,'' as start_date_datetime +,'' as end_date_time +,'' as enddate +,'' as end_date_datetime + +/*Liberator Incoming Correspondence Data*/ +,case +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' +end as response_status, +cast(current_timestamp as string) as Current_time_stamp, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, + +/*unassigned days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' +end as unassigned_days_group +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFourteenLess, + +/*Days to assign*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as Days_to_assign_group +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFourteenLess, + +/*assigned in progress days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' +end as assigned_in_progress_days_group +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFourteenLess, + +/*assigned response days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' +end as assignedResponseDays_group +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFourteenLess, + +/*Response days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as ResponseDays_group +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff(cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFourteenLess + +,Response_generated_at +,Date_Received +,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear +,liberator_pcn_ic.Type +,Serviceable +,Service_category +,Response_written_by +,Letter_template +,Action_taken +,Related_to_PCN +,Cancellation_group +,Cancellation_reason +,whenassigned +,ticketserialnumber +,noderef +,cast(liberator_pcn_ic.record_created as string) as record_created +,liberator_pcn_ic.import_timestamp +,liberator_pcn_ic.import_year +,liberator_pcn_ic.import_month +,liberator_pcn_ic.import_day +,liberator_pcn_ic.import_date + +/*pcn data*/ +,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn +,cast(pcnfoidetails_pcn_foi_full.pcnissuedate as string) as pcn_pcnissuedate +,cast(pcnfoidetails_pcn_foi_full.pcnissuedatetime as string) as pcn_pcnissuedatetime +,cast(pcnfoidetails_pcn_foi_full.pcn_canx_date as string) as pcn_pcn_canx_date +,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup +,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason +,cast(pcnfoidetails_pcn_foi_full.pcn_casecloseddate as string) as pcn_pcn_casecloseddate +,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location +,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation +,pcnfoidetails_pcn_foi_full.zone as pcn_zone +,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn +,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode +,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix +,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype +,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm +,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake +,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel +,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour +,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo +,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice +,cast(pcnfoidetails_pcn_foi_full.current_30_day_flag as string) as pcn_current_30_day_flag +,cast(pcnfoidetails_pcn_foi_full.isvda as string) as pcn_isvda +,cast(pcnfoidetails_pcn_foi_full.isvoid as string) as pcn_isvoid +,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval +,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen +,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows +,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway +,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor +,cast(pcnfoidetails_pcn_foi_full.warningflag as string) as pcn_warningflag +,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts +,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason +,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount +,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received +,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount +,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void +,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method +,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref +,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from +,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to +,cast(pcnfoidetails_pcn_foi_full.bailiff_processedon as string) as pcn_bailiff_processedon +,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason +,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff +,cast(pcnfoidetails_pcn_foi_full.warrantissuedate as string) as pcn_warrantissuedate +,cast(pcnfoidetails_pcn_foi_full.allocation as string) as pcn_allocation +,cast(pcnfoidetails_pcn_foi_full.eta_datenotified as string) as pcn_eta_datenotified +,cast(pcnfoidetails_pcn_foi_full.eta_packsubmittedon as string) as pcn_eta_packsubmittedon +,cast(pcnfoidetails_pcn_foi_full.eta_evidencedate as string) as pcn_eta_evidencedate +,cast(pcnfoidetails_pcn_foi_full.eta_adjudicationdate as string) as pcn_eta_adjudicationdate +,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds +,cast(pcnfoidetails_pcn_foi_full.eta_decisionreceived as string) as pcn_eta_decisionreceived +,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome +,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby +,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby +,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address +,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address +,cast(pcnfoidetails_pcn_foi_full.corresp_dispute_flag as string) as pcn_corresp_dispute_flag +,cast(pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as string) as pcn_keyworker_corresp_dispute_flag +,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag +,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year +,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref +,cast(pcnfoidetails_pcn_foi_full.nto_printed as string) as pcn_nto_printed +,cast(pcnfoidetails_pcn_foi_full.appeal_accepted as string) as pcn_appeal_accepted +,cast(pcnfoidetails_pcn_foi_full.arrived_in_pound as string) as pcn_arrived_in_pound +,cast(pcnfoidetails_pcn_foi_full.cancellation_reversed as string) as pcn_cancellation_reversed +,cast(pcnfoidetails_pcn_foi_full.cc_printed as string) as pcn_cc_printed +,cast(pcnfoidetails_pcn_foi_full.drr as string) as pcn_drr +,cast(pcnfoidetails_pcn_foi_full.en_printed as string) as pcn_en_printed +,cast(pcnfoidetails_pcn_foi_full.hold_released as string) as pcn_hold_released +,cast(pcnfoidetails_pcn_foi_full.dvla_response as string) as pcn_dvla_response +,cast(pcnfoidetails_pcn_foi_full.dvla_request as string) as pcn_dvla_request +,cast(pcnfoidetails_pcn_foi_full.full_rate_uplift as string) as pcn_full_rate_uplift +,cast(pcnfoidetails_pcn_foi_full.hold_until as string) as pcn_hold_until +,cast(pcnfoidetails_pcn_foi_full.lifted_at as string) as pcn_lifted_at +,cast(pcnfoidetails_pcn_foi_full.lifted_by as string) as pcn_lifted_by +,cast(pcnfoidetails_pcn_foi_full.loaded as string) as pcn_loaded +,cast(pcnfoidetails_pcn_foi_full.nor_sent as string) as pcn_nor_sent +,cast(pcnfoidetails_pcn_foi_full.notice_held as string) as pcn_notice_held +,cast(pcnfoidetails_pcn_foi_full.ofr_printed as string) as pcn_ofr_printed +,cast(pcnfoidetails_pcn_foi_full.pcn_printed as string) as pcn_pcn_printed +,cast(pcnfoidetails_pcn_foi_full.reissue_nto_requested as string) as pcn_reissue_nto_requested +,cast(pcnfoidetails_pcn_foi_full.reissue_pcn as string) as pcn_reissue_pcn +,cast(pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as string) as pcn_set_back_to_pre_cc_stage +,cast(pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as string) as pcn_vehicle_released_for_auction +,cast(pcnfoidetails_pcn_foi_full.warrant_issued as string) as pcn_warrant_issued +,cast(pcnfoidetails_pcn_foi_full.warrant_redistributed as string) as pcn_warrant_redistributed +,cast(pcnfoidetails_pcn_foi_full.warrant_request_granted as string) as pcn_warrant_request_granted +,cast(pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as string) as pcn_ad_hoc_vq4_request +,cast(pcnfoidetails_pcn_foi_full.paper_vq5_received as string) as pcn_paper_vq5_received +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as string) as pcn_pcn_extracted_for_buslane +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as string) as pcn_pcn_extracted_for_pre_debt +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as string) as pcn_pcn_extracted_for_collection +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as string) as pcn_pcn_extracted_for_drr +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as string) as pcn_pcn_extracted_for_cc +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as string) as pcn_pcn_extracted_for_nto +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as string) as pcn_pcn_extracted_for_print +,cast(pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as string) as pcn_warning_notice_extracted_for_print +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as string) as pcn_pcn_extracted_for_ofr +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as string) as pcn_pcn_extracted_for_warrant_request +,cast(pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as string) as pcn_pre_debt_new_debtor_details +,cast(pcnfoidetails_pcn_foi_full.importdattime as string) as pcn_importdattime +,cast(pcnfoidetails_pcn_foi_full.importdatetime as string) as pcn_importdatetime -- +,pcnfoidetails_pcn_foi_full.import_year as pcn_import_year +,pcnfoidetails_pcn_foi_full.import_month as pcn_import_month +,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day +,pcnfoidetails_pcn_foi_full.import_date as pcn_import_date + +/*Teams data*/ +,t_start_date +,t_end_date +,t_team +,t_team_name +,t_role +,t_forename +,t_surname +,t_full_name +,t_qa_doc_created_by +,t_qa_doc_full_name +,t_post_title +,t_notes +,t_import_date + +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date +left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) + +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - FROM downtime d - LEFT JOIN team t - ON UPPER(t.t_full_name) = UPPER(d.Response_written_by) --downtime officer full name -) -SELECT * FROM liberator_with_team UNION -SELECT * FROM google_form ---order by downtime_timestamp desc -; + +--Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 +Select +cast(parking_officer_downtime.timestamp as string) as downtime_timestamp +,email_address as downtime_email_address +,TRIM(officer_s_first_name) as officer_s_first_name +,TRIM(officer_s_last_name) as officer_s_last_name +,liberator_system_username +,liberator_system_id +,cast(import_datetime as string) import_datetime +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60) > 1440 then 1 else 0 end as string) as multiple_downtime_days_flag +,cast(unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')) as string) as response_secs +,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 end as string) as response_mins +,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/3600 end as string) as response_hours + +/*Downtime calendar days*/ +,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1 end as string) as response_days_plus_one -- downtime days plus one calendar day + +/*Downtime non working mins*/ +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 948 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 948 end as string) as downtime_total_non_working_mins_with_lunch --if greater than 1440 mins then (days + 1) * 948 " +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 1008 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 1008 end as string) as downtime_total_non_working_mins --if greater than 1440 mins then (days + 1) * 1008 " + +/*Downtime working mins*/ +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 end as string) as downtime_total_working_mins_with_lunch -- 492 mins = 8hrs 12 mins working day including lunch" +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 end as string) as downtime_total_working_mins --432 mins = 7hrs 12mins working hours" + +/*Downtime working mins net (less downtime mins)*/ +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 - /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_with_lunch_net -- 492 mins = 8hrs 12 mins working day including lunch less downtime" +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 - (/* downtime mins (response_mins)*/ (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_net --432 mins = 7hrs 12mins working hours" + +,cast(substr(cast(start_date as string), 12, 5) as string) as start_date_time +,cast(substr(cast(start_date as string), 1, 10) as string) as startdate +,cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as start_date_datetime +,cast(substr(cast(end_date as string), 12, 5) as string) as end_date_time +,cast(substr(cast(end_date as string), 1, 10) as string) as enddate +,cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as end_date_datetime +,'Downtime' as response_status +,cast(current_timestamp as string) as Current_time_stamp + +/*Liberator Incoming Correspondence Data*/ +,'' as unassigned_time +,'' as to_assigned_time +,'' as assigned_in_progress_time +,'' as assigned_response_time +,(cast (cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) - cast ( Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) as string) ) as response_time --as downtime_duration --line 30 +,'' as unassigned_days +,'' as unassigned_days_group +,'' as unassigned_days_kpiTotFiftySixLess +,'' as unassigned_days_kpiTotFourteenLess +,'' as Days_to_assign +,'' as Days_to_assign_group +,'' as Days_to_assign_kpiTotFiftySixLess +,'' as Days_to_assign_kpiTotFourteenLess +,'' as assigned_in_progress_days +,'' as assigned_in_progress_days_group +,'' as assigned_in_progress_days_kpiTotFiftySixLess +,'' as assigned_in_progress_days_kpiTotFourteenLess +,'' as assignedResponseDays +,'' as assignedResponseDays_group +,'' as assignedResponseDays_kpiTotFiftySixLess +,'' as assignedResponseDays_kpiTotFourteenLess +,cast( Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) ) end as string) as ResponseDays --as downtime_duration in days +,'' as ResponseDays_group +,'' as ResponseDays_kpiTotFiftySixLess +,'' as ResponseDays_kpiTotFourteenLess +, end_date as Response_generated_at +, start_date as Date_Received +,concat(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,7), '-01') as MonthYear +,'Downtime' as Type +,parking_officer_downtime.Downtime as Serviceable -- downtime +,'' as Service_category +,concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name)) as Response_written_by --downtime officer full name +,'' as Letter_template +,'' as Action_taken +,'' as Related_to_PCN +,'' as Cancellation_group +,'' as Cancellation_reason +,'' as whenassigned +,'' as ticketserialnumber +,'' as noderef +,'' as record_created +,import_timestamp +,import_year +,import_month +,import_day +,cast(import_date as string) as import_date + +/*pcn data*/ +,'' as pcn_pcn +,'' as pcn_pcnissuedate +,'' as pcn_pcnissuedatetime --line 74 +,'' as pcn_pcn_canx_date +,'' as pcn_cancellationgroup +,'' as pcn_cancellationreason +,'' as pcn_pcn_casecloseddate +,'' as pcn_street_location +,'' as pcn_whereonlocation +,'' as pcn_zone +,'' as pcn_usrn +,'' as pcn_contraventioncode +,'' as pcn_contraventionsuffix +,'' as pcn_debttype +,'' as pcn_vrm +,'' as pcn_vehiclemake +,'' as pcn_vehiclemodel +,'' as pcn_vehiclecolour +,'' as pcn_ceo +,'' as pcn_ceodevice +,'' as pcn_current_30_day_flag +,'' as pcn_isvda +,'' as pcn_isvoid +,'' as pcn_isremoval +,'' as pcn_driverseen +,'' as pcn_allwindows +,'' as pcn_parkedonfootway +,'' as pcn_doctor +,'' as pcn_warningflag +,'' as pcn_progressionstage +,'' as pcn_nextprogressionstage --100 +,'' as pcn_nextprogressionstagestarts +,'' as pcn_holdreason +,'' as pcn_lib_initial_debt_amount +,'' as pcn_lib_payment_received +,'' as pcn_lib_write_off_amount +,'' as pcn_lib_payment_void +,'' as pcn_lib_payment_method +,'' as pcn_lib_payment_ref +,'' as pcn_baliff_from +,'' as pcn_bailiff_to +,'' as pcn_bailiff_processedon +,'' as pcn_bailiff_redistributionreason +,'' as pcn_bailiff +,'' as pcn_warrantissuedate +,'' as pcn_allocation +,'' as pcn_eta_datenotified +,'' as pcn_eta_packsubmittedon +,'' as pcn_eta_evidencedate +,'' as pcn_eta_adjudicationdate +,'' as pcn_eta_appealgrounds +,'' as pcn_eta_decisionreceived +,'' as pcn_eta_outcome +,'' as pcn_eta_packsubmittedby +,'' as pcn_cancelledby +,'' as pcn_registered_keeper_address +,'' as pcn_current_ticket_address +,'' as pcn_corresp_dispute_flag +,'' as pcn_keyworker_corresp_dispute_flag +,'' as pcn_fin_year_flag +,'' as pcn_fin_year +,'' as pcn_ticket_ref +,'' as pcn_nto_printed +,'' as pcn_appeal_accepted +,'' as pcn_arrived_in_pound +,'' as pcn_cancellation_reversed +,'' as pcn_cc_printed +,'' as pcn_drr +,'' as pcn_en_printed +,'' as pcn_hold_released +,'' as pcn_dvla_response +,'' as pcn_dvla_request +,'' as pcn_full_rate_uplift +,'' as pcn_hold_until +,'' as pcn_lifted_at +,'' as pcn_lifted_by +,'' as pcn_loaded +,'' as pcn_nor_sent +,'' as pcn_notice_held +,'' as pcn_ofr_printed +,'' as pcn_pcn_printed +,'' as pcn_reissue_nto_requested +,'' as pcn_reissue_pcn +,'' as pcn_set_back_to_pre_cc_stage +,'' as pcn_vehicle_released_for_auction +,'' as pcn_warrant_issued +,'' as pcn_warrant_redistributed +,'' as pcn_warrant_request_granted +,'' as pcn_ad_hoc_vq4_request +,'' as pcn_paper_vq5_received +,'' as pcn_pcn_extracted_for_buslane +,'' as pcn_pcn_extracted_for_pre_debt +,'' as pcn_pcn_extracted_for_collection +,'' as pcn_pcn_extracted_for_drr +,'' as pcn_pcn_extracted_for_cc +,'' as pcn_pcn_extracted_for_nto +,'' as pcn_pcn_extracted_for_print +,'' as pcn_warning_notice_extracted_for_print +,'' as pcn_pcn_extracted_for_ofr +,'' as pcn_pcn_extracted_for_warrant_request +,'' as pcn_pre_debt_new_debtor_details +,'' as pcn_importdattime +,'' as pcn_importdatetime +,'' as pcn_import_year +,'' as pcn_import_month +,'' as pcn_import_day +,'' as pcn_import_date + +/*Teams data*/ +,t_start_date +,t_end_date +,t_team +,t_team_name +,t_role +,t_forename +,t_surname +,t_full_name +,t_qa_doc_created_by +,t_qa_doc_full_name +,t_post_title +,t_notes +,t_import_date + +from parking_officer_downtime +left join team on upper(team.t_full_name) = upper(concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name))) --as Response_written_by --downtime officer full name +where import_date = (select max(import_date) from parking_officer_downtime) and timestamp not like '' and timestamp not like '#REF!' +AND cast(Substr( cast(parking_officer_downtime.timestamp as string) ,1,10) as date) > current_date - interval '13' month --Last 13 months from todays date +--order by timestamp desc """ +ApplyMapping_node2 = sparkSqlQuery( + glueContext, + query=SqlQuery0, + mapping={ + "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1, + "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1682353070282, + "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1682353072411, + "parking_officer_downtime": parking_raw_zoneparking_officer_downtime_node1697211483070, + }, + transformation_ctx="ApplyMapping_node2", +) -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name +# Script generated for node Liberator Refined - parking_correspondence_performance_records_with_pcn +LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3 = glueContext.getSink( + path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn_downtime/", + connection_type="s3", + updateBehavior="UPDATE_IN_DATABASE", + partitionKeys=["import_year", "import_month", "import_day", "import_date"], + compression="snappy", + enableUpdateCatalog=True, + transformation_ctx="LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3", +) +LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3.setCatalogInfo( + catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", + catalogTableName="parking_correspondence_performance_records_with_pcn_downtime", +) +LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3.setFormat( + "glueparquet" +) +LiberatorRefinedparking_correspondence_performance_records_with_pcn_node3.writeFrame( + ApplyMapping_node2 ) +job.commit() diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py index aba4ecc73..bdfeb6aa3 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_downtime_gds.py @@ -1,1167 +1,683 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job +from awsglue import DynamicFrame +from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS + +def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: + for alias, frame in mapping.items(): + frame.toDF().createOrReplaceTempView(alias) + result = spark.sql(query) + return DynamicFrame.fromDF(result, glueContext, transformation_ctx) + + +args = getResolvedOptions(sys.argv, ["JOB_NAME"]) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) +environment = get_glue_env_var("environment") -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var +# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full +AmazonS3refinedpcnfoidetails_pcn_foi_full_node1697453826330 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-refined-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="pcnfoidetails_pcn_foi_full", + transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1697453826330", +) -environment = get_glue_env_var("environment") +# Script generated for node S3 bucket - Raw - liberator_pcn_ic +S3bucketRawliberator_pcn_ic_node1 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="liberator_pcn_ic", + transformation_ctx="S3bucketRawliberator_pcn_ic_node1", +) -# The target table in liberator refined zone -table_name = "parking_correspondence_performance_records_with_pcn_downtime_gds" +# Script generated for node parking_raw_zone - parking_officer_downtime +parking_raw_zoneparking_officer_downtime_node1697453968832 = glueContext.create_dynamic_frame.from_catalog( + database="parking-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="parking_officer_downtime", + transformation_ctx="parking_raw_zoneparking_officer_downtime_node1697453968832", +) -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ --->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn_downtime_gds" ---< "parking-raw-zone"."parking_correspondence_performance_teams" (data from google spreadsheet) ---< "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" ---<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ---< "parking-raw-zone"."parking_officer_downtime" (data from google form) +# Script generated for node parking_raw_zone - parking_correspondence_performance_teams +parking_raw_zoneparking_correspondence_performance_teams_node1697453966201 = glueContext.create_dynamic_frame.from_catalog( + database="parking-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="parking_correspondence_performance_teams", + transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1697453966201", +) + +# Script generated for node SQL Query +SqlQuery0 = """ /* Correspondence Performance records last 13 months with PCN FOI records -16/06/2022 - Created +16/06/2022 - Created 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing 02/10/2023 - modified to union data into one output for Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 08/11/2023 - modified to add monthly officer totals from original gds job -22/10/2024 - translated into PrestoSQL to move away from Glue -05/11/2024 + Refactored SQL to separate the inward column conversions from onward business logic using CTE subqueries and allow improved fault handling. - + Downtime's Google Sheets alternate date formats properly handled +06/11/2024 - updated date formats in downtime data as source google sheet changed to yyyy-mm-dd HH:MM:ss from d/m/y 28/11/2024 - updated downtime records for t_team details to pull from parking_correspondence_performance_teams instead on leaving blank -12/12/2024 + Integrated the previous downtime update into the refactored version - + Commented out alternative last 13 month filter that caused unexpectedly higher row count -05/03/2025 - Replaced conditional '' empty string outputs with NULL defaults to be consistent with previous Glue/SparkSQL implementations. -05/03/2025 - Removed redundant left joins from subquery -14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. Fixed faulty arithmetic translation of Glue/SparkSQL to AthenaSQL. -16/04/2025 - Workaround orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready -*/ -/*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ -WITH -responses AS ( - SELECT - /* officer_total_rep_dates*/ - l.Response_written_by AS tot_rep_Response_written_by, - SUBSTR(l.response_generated_at, 1, 10) AS tot_rep_response_date, - SUBSTR(l.response_generated_at, 1, 7) || '-01' AS tot_rep_monthYear_response, - -- Empty "response_generated_at" outputs '-01' which would get counted! - -- Makes no difference: SUBSTR(CAST(liberator_pcn_ic.response_generated_at AS VARCHAR(10)), 1, 7) || '-01' AS tot_rep_monthYear_response, - l.Response_written_by || SUBSTR(l.response_generated_at, 1, 10) AS tot_rep_unique_link, - - /* total_response_month*/ - l.Response_written_by AS mth_tot_rep_Response_written_by, - SUBSTR(l.response_generated_at, 1, 10) AS mth_tot_rep_response_date, - SUBSTR(l.response_generated_at, 1, 7) || '-01' AS mth_tot_rep_monthYear_response, - l.Response_written_by || SUBSTR(l.response_generated_at, 1, 7) || '-01' AS mth_tot_rep_unique_link - -- Q. How do empty "response_generated_at" rows effect subsequent business logic? - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" l - -- The unreferenced p. caused SQL linter error ST11 - -- Then discoverd the row count was not expanded by the left join to p because p.pcn is unique within import_date -/* -- Therefore removed this redundant left join... - LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p - ON l.ticketserialnumber = p.pcn - AND l.import_date = p.import_date */ - WHERE l.import_Date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g - ) - AND l.response_generated_at <> '' -- Q. Should I remove empty "response_generated_at" to prevent side-effects? - AND LENGTH(l.ticketserialnumber) = 10 - AND CAST(SUBSTR(l.date_received, 1, 10) AS DATE) > CURRENT_DATE - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - DATE_DIFF( - 'month', - TRY(CAST(SUBSTR(l.date_received, 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ) <= 13 -*/ -), - -officer_total_rep_dates AS ( - SELECT - tot_rep_Response_written_by, - tot_rep_response_date, - tot_rep_monthYear_response, - tot_rep_unique_link, - - COUNT(DISTINCT tot_rep_response_date) AS tot_rep_total_dates, - -- COUNT(DISTINCT SUBSTR(liberator_pcn_ic.response_generated_at, 1, 10)) AS tot_rep_total_dates, - COUNT(*) AS tot_rep_records - FROM - responses - GROUP BY - tot_rep_Response_written_by, - tot_rep_response_date, - tot_rep_monthYear_response, - tot_rep_unique_link -), - -total_response_month AS ( - SELECT - mth_tot_rep_Response_written_by, - mth_tot_rep_monthYear_response, - mth_tot_rep_unique_link, - - COUNT(DISTINCT mth_tot_rep_response_date) AS mth_tot_rep_total_dates, - -- COUNT(DISTINCT SUBSTR(liberator_pcn_ic.response_generated_at, 1, 10)) AS mth_tot_rep_total_dates, - COUNT(*) AS mth_tot_rep_records - FROM - responses - GROUP BY - mth_tot_rep_Response_written_by, - mth_tot_rep_monthYear_response, - mth_tot_rep_unique_link -), /*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ -team AS ( - SELECT DISTINCT - start_date AS t_start_date, - end_date AS t_end_date, - team AS t_team, - team_name AS t_team_name, - ROLE AS t_role, - forename AS t_forename, - surname AS t_surname, - full_name AS t_full_name, - qa_doc_created_by AS t_qa_doc_created_by, - qa_doc_full_name AS t_qa_doc_full_name, - post_title AS t_post_title, - notes AS t_notes, - import_date AS t_import_date --* - FROM "parking-raw-zone"."parking_correspondence_performance_teams" - WHERE import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "parking-raw-zone"."parking_correspondence_performance_teams" g - ) -), - -liberator_pcn_icdate_received_last13months AS ( - SELECT - *, - CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS current_utc_timestamp, - TRY(CAST(date_received AS TIMESTAMP)) AS date_received_timestamp, -- pre-filtered by WHERE clause - - IF(whenassigned <> '', -- contains probable valid timestamp - TRY(CAST(whenassigned AS TIMESTAMP)), - CAST(NULL AS TIMESTAMP) -- edge case - ) AS whenassigned_timestamp, - - IF(Response_generated_at <> '', -- contains probable valid timestamp - TRY(CAST(Response_generated_at AS TIMESTAMP)), - CAST(NULL AS TIMESTAMP) -- edge case - ) AS Response_generated_at_timestamp, - - CAST(DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' - CAST(CURRENT_TIMESTAMP AS DATE) - ) AS BIGINT) AS days_since_date_received, - - CAST(IF( - whenassigned <> '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' - TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_date_recieved_whenassigned, - - CAST(IF( - whenassigned <> '' AND response_generated_at = '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_whenassigned, - - CAST(IF( - whenassigned <> '' AND response_generated_at <> '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(whenassigned, 1, 10) AS DATE)), - TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_whenassigned_response_generated_at, - - CAST(IF( - response_generated_at <> '', - DATE_DIFF( - 'day', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), -- under condition of date_received <>'' - TRY(CAST(SUBSTR(response_generated_at, 1, 10) AS DATE)) - ), - NULL -- edge case - ) AS BIGINT) AS days_since_date_received_response_generated_at, - - /*links to officer monthly totals*/ - CAST(Response_written_by || SUBSTR(response_generated_at, 1, 10) AS VARCHAR) AS link_officer_total_rep_dates, - CAST(Response_written_by || IF(response_generated_at <> '', SUBSTR(response_generated_at, 1, 7) || '-01', '') AS VARCHAR) AS link_month_total--, - - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" - WHERE import_Date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g - ) - AND LENGTH(ticketserialnumber) = 10 -- ticket filter - AND date_received <>'' -- is the overriding condition for "13 months from todays date"! - AND CAST(SUBSTR(date_received, 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - DATE_DIFF( - 'month', - TRY(CAST(SUBSTR(date_received, 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ) <= 13 -*/ -), - -liberator_with_team_gds AS ( - SELECT - /*Fields for union downtime data*/ - CAST('' AS VARCHAR) AS downtime_timestamp, - CAST('' AS VARCHAR) AS downtime_email_address, - CAST('' AS VARCHAR) AS officer_s_first_name, - CAST('' AS VARCHAR) AS officer_s_last_name, - CAST('' AS VARCHAR) AS liberator_system_username, - CAST('' AS VARCHAR) AS liberator_system_id, - CAST('' AS VARCHAR) AS import_datetime, - CAST('' AS VARCHAR) AS multiple_downtime_days_flag, - CAST('' AS VARCHAR) AS response_secs, - CAST('' AS VARCHAR) AS response_mins, - CAST('' AS VARCHAR) AS response_hours, - CAST('' AS VARCHAR) AS response_days_plus_one, - CAST('' AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, - CAST('' AS VARCHAR) AS downtime_total_non_working_mins, - CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch, - CAST('' AS VARCHAR) AS downtime_total_working_mins, - CAST('' AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, - CAST('' AS VARCHAR) AS downtime_total_working_mins_net, - CAST('' AS VARCHAR) AS start_date_time, - CAST('' AS VARCHAR) AS startdate, - CAST('' AS VARCHAR) AS start_date_datetime, - CAST('' AS VARCHAR) AS end_date_time, - CAST('' AS VARCHAR) AS enddate, - CAST('' AS VARCHAR) AS end_date_datetime, - - /*Liberator Incoming Correspondence Data*/ - CAST(CASE -- having already asserted date_received <>'' - WHEN l.whenassigned = '' THEN 'Unassigned' - WHEN l.whenassigned <> '' AND l.response_generated_at = '' THEN 'Assigned' - WHEN l.whenassigned <> '' AND l.response_generated_at <> '' THEN 'Responded' - --ELSE NULL - END AS VARCHAR) AS response_status, - - CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS unassigned_time, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS to_assigned_time, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_time, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_response_time, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS response_time, - - /*unassigned days*/ - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - THEN CAST(days_since_date_received AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS unassigned_days, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - THEN - CASE - WHEN l.days_since_date_received <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS unassigned_days_group, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - AND l.days_since_date_received <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned = '' - AND l.days_since_date_received <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, - - /*Days to assign*/ - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - THEN CAST(l.days_since_date_recieved_whenassigned AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS Days_to_assign, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - THEN - CASE - WHEN l.days_since_date_recieved_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_date_recieved_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_recieved_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_recieved_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_recieved_whenassigned > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS Days_to_assign_group, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - AND l.days_since_date_recieved_whenassigned <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, - - CAST(CASE -- asserted date_received <>'' - WHEN l.whenassigned <> '' - AND l.days_since_date_recieved_whenassigned <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, - - /*assigned in progress days*/ - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN CAST(l.days_since_whenassigned AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_days, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_days_group, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, - - /*assigned response days*/ - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS assignedResponseDays, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assignedResponseDays_group, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at <> '' - AND l.days_since_whenassigned_response_generated_at <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, - - CAST(CASE - WHEN l.whenassigned <> '' - AND l.response_generated_at = '' - AND l.days_since_whenassigned_response_generated_at <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, - - /*Response days*/ - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - THEN CAST(l.days_since_date_received_response_generated_at AS VARCHAR) - --ELSE NULL - END AS VARCHAR) AS ResponseDays, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS ResponseDays_group, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 56 - THEN '1' - ELSE '0' - END AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, - - CAST(CASE -- asserted date_received <>'' - WHEN l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 14 - THEN '1' - ELSE '0' - END AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, - - Response_generated_at, - Date_Received, +With officer_total_rep_dates as( +select +liberator_pcn_ic.Response_written_by as tot_rep_Response_written_by +,substr(liberator_pcn_ic.response_generated_at, 1, 10) as tot_rep_response_date +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as tot_rep_unique_link - SUBSTR(CAST(l.date_received AS VARCHAR(10)), 1, 7) || '-01' AS MonthYear, -- asserted date_received <>'' +,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as tot_rep_total_dates +,count (*) as tot_rep_records - l.Type AS "Type", - l.Serviceable, - l.Service_category, - l.Response_written_by, - l.Letter_template, - l.Action_taken, - l.Related_to_PCN, - l.Cancellation_group, - l.Cancellation_reason, - l.whenassigned, - l.ticketserialnumber, - l.noderef, - CAST(l.record_created AS VARCHAR) AS record_created, - l.import_timestamp, +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date - /*pcn data*/ - -- The original Glue SparkSQL had all of these pcn columns output as '' empty strings which either... - -- ...conflicted with the output column datatypes configured by Glue - -- ...and/or conflicted with the column translations done earlier in the SQL script. - /* Instead here, the null or empty translations are deliberately set to depend on the target datatype... - CAST(NULL AS DATE) AS [target_column], --[position]-- null date - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS INTEGER) AS [target_column], --[position]-- null int - CAST('' AS VARCHAR) AS [target_column], --[position]-- empty string - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS TIMESTAMP) AS [target_column], --[position]-- null timestamp - */ - TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --69-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedate AS DATE), '%Y-%m-%d'), CAST(p.pcnissuedate AS VARCHAR)) AS pcn_pcnissuedate, --70-- try date to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcnissuedatetime AS VARCHAR)) AS pcn_pcnissuedatetime, --71-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_canx_date AS DATE), '%Y-%m-%d'), CAST(p.pcn_canx_date AS VARCHAR)) AS pcn_pcn_canx_date, --72-- try date to string - TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --73-- try string to string (default) - TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --74-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_casecloseddate AS DATE), '%Y-%m-%d'), CAST(p.pcn_casecloseddate AS VARCHAR)) AS pcn_pcn_casecloseddate, --75-- try date to string - TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --76-- try string to string (default) - TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --77-- try string to string (default) - TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --78-- try string to string (default) - TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --79-- try string to string (default) - TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --80-- try string to string (default) - TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --81-- try string to string (default) - TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --82-- try string to string (default) - TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --83-- try string to string (default) - TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --84-- try string to string (default) - TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --85-- try string to string (default) - TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --86-- try string to string (default) - TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --87-- try string to string (default) - TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --88-- try string to string (default) - TRY_CAST(p.current_30_day_flag AS VARCHAR) AS pcn_current_30_day_flag, --89-- try int to string - TRY_CAST(p.isvda AS VARCHAR) AS pcn_isvda, --90-- try int to string - TRY_CAST(p.isvoid AS VARCHAR) AS pcn_isvoid, --91-- try int to string - TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --92-- try string to string (default) - TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --93-- try string to string (default) - TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --94-- try string to string (default) - TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --95-- try string to string (default) - TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --96-- try string to string (default) - TRY_CAST(p.warningflag AS VARCHAR) AS pcn_warningflag, --97-- try int to string - TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --98-- try string to string (default) - TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --99-- try string to string (default) - TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- try string to string (default) - TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --101-- try string to string (default) - TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- try string to string (default) - TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --103-- try string to string (default) - TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --104-- try string to string (default) - TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --105-- try string to string (default) - TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --106-- try string to string (default) - TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --107-- try string to string (default) - TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --108-- try string to string (default) - TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --109-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.bailiff_processedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.bailiff_processedon AS VARCHAR)) AS pcn_bailiff_processedon, --110-- try timestamp to string - TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- try string to string (default) - TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --112-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.warrantissuedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrantissuedate AS VARCHAR)) AS pcn_warrantissuedate, --113-- try timestamp to string - TRY_CAST(p.allocation AS VARCHAR) AS pcn_allocation, --114-- try int to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_datenotified AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_datenotified AS VARCHAR)) AS pcn_eta_datenotified, --115-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_packsubmittedon AS VARCHAR)) AS pcn_eta_packsubmittedon, --116-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_evidencedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_evidencedate AS VARCHAR)) AS pcn_eta_evidencedate, --117-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_adjudicationdate AS VARCHAR)) AS pcn_eta_adjudicationdate, --118-- try timestamp to string - TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --119-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_decisionreceived AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_decisionreceived AS VARCHAR)) AS pcn_eta_decisionreceived, --120-- try timestamp to string - TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --121-- try string to string (default) - TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- try string to string (default) - TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --123-- try string to string (default) - TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --124-- try string to string (default) - TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --125-- try string to string (default) - TRY_CAST(p.corresp_dispute_flag AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- try int to string - TRY_CAST(p.keyworker_corresp_dispute_flag AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- try int to string - TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --128-- try string to string (default) - TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --129-- try string to string (default) - TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --130-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.nto_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nto_printed AS VARCHAR)) AS pcn_nto_printed, --131-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.appeal_accepted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.appeal_accepted AS VARCHAR)) AS pcn_appeal_accepted, --132-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.arrived_in_pound AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.arrived_in_pound AS VARCHAR)) AS pcn_arrived_in_pound, --133-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.cancellation_reversed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cancellation_reversed AS VARCHAR)) AS pcn_cancellation_reversed, --134-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.cc_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cc_printed AS VARCHAR)) AS pcn_cc_printed, --135-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.drr AS VARCHAR)) AS pcn_drr, --136-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.en_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.en_printed AS VARCHAR)) AS pcn_en_printed, --137-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.hold_released AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_released AS VARCHAR)) AS pcn_hold_released, --138-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_response AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_response AS VARCHAR)) AS pcn_dvla_response, --139-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_request AS VARCHAR)) AS pcn_dvla_request, --140-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.full_rate_uplift AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.full_rate_uplift AS VARCHAR)) AS pcn_full_rate_uplift, --141-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.hold_until AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_until AS VARCHAR)) AS pcn_hold_until, --142-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_at AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_at AS VARCHAR)) AS pcn_lifted_at, --143-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_by AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_by AS VARCHAR)) AS pcn_lifted_by, --144-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.loaded AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.loaded AS VARCHAR)) AS pcn_loaded, --145-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.nor_sent AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nor_sent AS VARCHAR)) AS pcn_nor_sent, --146-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.notice_held AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.notice_held AS VARCHAR)) AS pcn_notice_held, --147-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.ofr_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ofr_printed AS VARCHAR)) AS pcn_ofr_printed, --148-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_printed AS VARCHAR)) AS pcn_pcn_printed, --149-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_nto_requested AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_nto_requested AS VARCHAR)) AS pcn_reissue_nto_requested, --150-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_pcn AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_pcn AS VARCHAR)) AS pcn_reissue_pcn, --151-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.set_back_to_pre_cc_stage AS VARCHAR)) AS pcn_set_back_to_pre_cc_stage, --152-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.vehicle_released_for_auction AS VARCHAR)) AS pcn_vehicle_released_for_auction, --153-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_issued AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_issued AS VARCHAR)) AS pcn_warrant_issued, --154-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_redistributed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_redistributed AS VARCHAR)) AS pcn_warrant_redistributed, --155-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_request_granted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_request_granted AS VARCHAR)) AS pcn_warrant_request_granted, --156-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ad_hoc_vq4_request AS VARCHAR)) AS pcn_ad_hoc_vq4_request, --157-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.paper_vq5_received AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.paper_vq5_received AS VARCHAR)) AS pcn_paper_vq5_received, --158-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_buslane AS VARCHAR)) AS pcn_pcn_extracted_for_buslane, --159-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_pre_debt AS VARCHAR)) AS pcn_pcn_extracted_for_pre_debt, --160-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_collection AS VARCHAR)) AS pcn_pcn_extracted_for_collection, --161-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_drr AS VARCHAR)) AS pcn_pcn_extracted_for_drr, --162-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_cc AS VARCHAR)) AS pcn_pcn_extracted_for_cc, --163-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_nto AS VARCHAR)) AS pcn_pcn_extracted_for_nto, --164-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_print AS VARCHAR)) AS pcn_pcn_extracted_for_print, --165-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warning_notice_extracted_for_print AS VARCHAR)) AS pcn_warning_notice_extracted_for_print, --166-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_ofr AS VARCHAR)) AS pcn_pcn_extracted_for_ofr, --167-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_warrant_request AS VARCHAR)) AS pcn_pcn_extracted_for_warrant_request, --168-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pre_debt_new_debtor_details AS VARCHAR)) AS pcn_pre_debt_new_debtor_details, --169-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.importdattime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdattime AS VARCHAR)) AS pcn_importdattime, --170-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.importdatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdatetime AS VARCHAR)) AS pcn_importdatetime, --171-- try timestamp to string - TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --172-- try string to string (default) - TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --173-- try string to string (default) - TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --174-- try string to string (default) - TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --175-- try string to string (default) +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - /*links to officer monthly totals*/ - l.link_officer_total_rep_dates, - l.link_month_total, +group by +liberator_pcn_ic.Response_written_by --as tot_rep_Response_written_by +,substr(liberator_pcn_ic.response_generated_at, 1, 10) --as tot_rep_response_date +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) --as tot_rep_unique_link - /* officer monthly totals*/ - CAST(m.mth_tot_rep_monthYear_response AS VARCHAR) AS mth_tot_rep_monthYear_response, - CAST(m.mth_tot_rep_total_dates AS VARCHAR) AS mth_tot_rep_total_dates, - CAST(m.mth_tot_rep_records AS VARCHAR) AS mth_tot_rep_records, - CAST(o.tot_rep_response_date AS VARCHAR) AS tot_rep_response_date, - CAST(o.tot_rep_total_dates AS VARCHAR) AS tot_rep_total_date, - CAST(o.tot_rep_records AS VARCHAR) AS tot_rep_records, - - /* teams */ - t.t_start_date, - t.t_end_date, - t.t_team, - t.t_team_name, - t.t_role, - t.t_forename, - t.t_surname, - t.t_full_name, - t.t_qa_doc_created_by, - t.t_qa_doc_full_name, - t.t_post_title, - t.t_notes, - t.t_import_date, - - /* Partition columns moved to the end to keep schema alligned */ - l.import_year, - l.import_month, - l.import_day, - l.import_date - - FROM liberator_pcn_icdate_received_last13months l - - LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p - /* - ON p.import_date = l.import_date -- joined on import partition - Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... - ...resulting in empty pcn_ columns in the output. - Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... - ...but for now we'll just use the following simple workaround... - */ - ON p.import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g - ) - AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid - - LEFT JOIN team t - ON UPPER(t.t_full_name) = UPPER(l.Response_written_by) - - LEFT JOIN officer_total_rep_dates o - ON l.link_officer_total_rep_dates = o.tot_rep_unique_link - - LEFT JOIN total_response_month m - ON l.link_month_total = m.mth_tot_rep_unique_link -), - - -downtime_start_date_end_date_last13months AS ( - SELECT *, - - IF(start_date LIKE '%/%/%', -- contains possible valid timestamp - TRY(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone - IF(start_date LIKE '%-%-%', -- contains other possible valid timestamp - TRY(PARSE_DATETIME(start_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone - CAST(NULL AS TIMESTAMP) -- edge case - ) - ) AS start_timestampZ, - - IF(end_date LIKE '%/%/%', -- contains possible valid timestamp - TRY(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')), --> TIMESTAMP with time zone - IF(end_date LIKE '%-%-%', -- contains other possible valid timestamp - TRY(PARSE_DATETIME(end_date, 'yyyy-MM-dd HH:mm:ss')), --> TIMESTAMP with time zone - CAST(NULL AS TIMESTAMP) -- edge case - ) - ) AS end_timestampZ--, - - FROM "parking-raw-zone"."parking_officer_downtime" - WHERE import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "parking-raw-zone"."parking_officer_downtime" g - ) - AND "timestamp" NOT LIKE '' - AND "timestamp" NOT LIKE '#REF!' - AND CAST(SUBSTR(CAST("timestamp" as VARCHAR(30)), 1, 10) AS DATE) > CAST(CURRENT_TIMESTAMP AS DATE) - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - DATE_DIFF( - 'month', - TRY(CAST(SUBSTR("timestamp", 1, 10) AS DATE)), - CAST(CURRENT_TIMESTAMP AS DATE) - ) <= 13 -*/ -), - -downtime AS ( - SELECT *, - /* - TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS start_end_unixtime_seconds, - TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) AS end_unixtime_seconds, - - TO_UNIXTIME(PARSE_DATETIME(end_date, 'dd/MM/yyyy HH:mm:ss')) - - TO_UNIXTIME(PARSE_DATETIME(start_date, 'dd/MM/yyyy HH:mm:ss')) AS downtime_secs, - - -- downtime_secs / 60 AS downtime_mins - -- downtime_secs / 3600 AS downtime_hours - */ - - DATE_DIFF( - 'second', - start_timestampZ, - end_timestampZ - ) AS downtime_secs, - - DATE_DIFF( - 'minute', - start_timestampZ, - end_timestampZ - ) AS downtime_mins, - - DATE_DIFF( - 'hour', - start_timestampZ, - end_timestampZ - ) AS downtime_hours, - - DATE_DIFF( - 'day', - start_timestampZ, - end_timestampZ - ) AS downtime_days, - - TRIM(officer_s_first_name) || ' ' || TRIM(officer_s_last_name) AS Response_written_by --downtime officer full name - - FROM downtime_start_date_end_date_last13months -), - -google_form AS ( - --Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 - SELECT - CAST(d."timestamp" AS VARCHAR) AS downtime_timestamp, - d.email_address AS downtime_email_address, - TRIM(d.officer_s_first_name) AS officer_s_first_name, - TRIM(d.officer_s_last_name) AS officer_s_last_name, - d.liberator_system_username, - d.liberator_system_id, - CAST(d.import_datetime AS VARCHAR) AS import_datetime, - - /* Throughout asserts... - "timestamp" NOT LIKE '' - AND "timestamp" NOT LIKE '#REF!' - */ - CAST(CASE - WHEN d.downtime_mins > 1440 /* downtime mins (response_mins) */ - THEN '1' - ELSE '0' - END AS VARCHAR) AS multiple_downtime_days_flag, - - -- Because the Glue job's response_secs string output was cast from a bigint returned from unix_timestamp... - -- ...there was no need to cast it to a DOUBLE here... - CAST(d.downtime_secs AS VARCHAR) AS response_secs, - -- ...unlike the products below where SparkSQL and AthenaSQL differed in their arithmetic outcomes... - - /* This version was rejected because the output differs wildly from the original Glue output... - -- Here all input value was cast beforehand to DOUBLE. - CAST(CAST(d.downtime_secs AS DOUBLE)/60 AS VARCHAR) AS REJECTED_response_mins, - */ - -- So we are keeping this version because it produces closest to the original Glue output... - -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... - CAST(CAST(d.downtime_mins AS DOUBLE) AS VARCHAR) AS response_mins, - /* But might the following version be the better solution? - -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... - CAST(d.downtime_mins AS VARCHAR) AS ALTERNATIVE_response_mins, - */ - - -- We are keeping this version because it produces closest to the original Glue output... - -- Though beware, several decimal places may be produced in the VARCHAR output... - CAST(CAST(d.downtime_secs AS DOUBLE)/3600 AS VARCHAR) AS response_hours, - /* But might the following version be the better solution? - -- This does not produce the fractional mantissa because it is an integer calculation from start to finish... - CAST(d.downtime_hours AS VARCHAR) AS ALTERNATIVE_response_hours, - */ - - /*Downtime calendar days*/ - -- Warning: A small number outlier cases, producing negative values in the original Glue output are also produced here... - CAST(d.downtime_days + 1 AS VARCHAR) AS response_days_plus_one, -- downtime days plus one calendar day - - /*Downtime non working mins*/ - CAST(CASE - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1 ) * 948 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 948 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_non_working_mins_with_lunch, --if greater than 1440 mins then (days + 1) * 948 " - - CAST(CASE /* downtime mins (response_mins)*/ - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 1008 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 1008 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_non_working_mins, --if greater than 1440 mins then (days + 1) * 1008 " - - /*Downtime working mins*/ - CAST(CASE /* downtime mins (response_mins)*/ - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 492 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_working_mins_with_lunch, -- 492 mins = 8hrs 12 mins working day including lunch" - - CAST(CASE /* downtime mins (response_mins)*/ - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 /*response_days_plus_one*/ - WHEN d.downtime_mins <= 1440 THEN 432 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS downtime_total_working_mins, -- 432 mins = 7hrs 12mins working hours" - - /* Downtime working mins net (less downtime mins)*/ - - /* This version was rejected because the output differs wildly from the original Glue output... - -- Here all input values were cast beforehand to DOUBLE... - CAST(CASE -- downtime mins (response_mins) - WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 492 -- response_days_plus_one - WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 492 - CAST(d.downtime_secs AS DOUBLE)/60 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS REJECTED_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" - */ - -- So we are keeping this version because it produces closest to the original Glue output... - -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... - CAST(CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 - WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" - /* So might the following version be the better solution? - -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... - CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 492 - WHEN d.downtime_mins <= 1440 THEN 492 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_with_lunch_net, -- 492 mins = 8hrs 12 mins "working day including lunch less downtime" - */ - - /* This version was rejected because the output differs wildly from the original Glue output... - -- Here all input values were cast beforehand to DOUBLE... - CAST(CASE -- downtime mins (response_mins) - WHEN CAST(d.downtime_secs AS DOUBLE)/60 > 1440 THEN (CAST(d.downtime_secs AS DOUBLE)/86400 + 1) * 432 -- response_days_plus_one - WHEN CAST(d.downtime_secs AS DOUBLE)/60 <= 1440 THEN 432 - CAST(d.downtime_secs AS DOUBLE)/60 - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS REJECTED_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" - */ - -- So we are keeping this version because it produces closest to the original Glue output... - -- Though beware, the final cast to DOUBLE causes a misleading .0 to be formatted in the VARCHAR output... - CAST(CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one - WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS DOUBLE) AS VARCHAR) AS downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" - /* So might the following version be the better solution? - -- This does not add the misleading .0 output in the previous version because it is an integer calculation from start to finish... - CAST(CASE -- downtime mins (response_mins) - WHEN d.downtime_mins > 1440 THEN (d.downtime_days + 1) * 432 -- response_days_plus_one - WHEN d.downtime_mins <= 1440 THEN 432 - d.downtime_mins - --ELSE NULL --edge case NULL VARCHAR will be materialized in denormalized target table - END AS VARCHAR) AS ALTERNATIVE_downtime_total_working_mins_net, -- 432 mins = 7hrs 12mins "working hours" - */ - - CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 12, 5) AS VARCHAR) AS start_date_time, - CAST(SUBSTR(CAST(d.start_date AS VARCHAR), 1, 10) AS VARCHAR) AS startdate, - - CAST(SUBSTR( - CAST(d.start_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS VARCHAR) AS start_date_datetime, - - CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 12, 5) AS VARCHAR) AS end_date_time, - CAST(SUBSTR(CAST(d.end_date AS VARCHAR), 1, 10) AS VARCHAR) AS enddate, - - CAST(SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS VARCHAR) AS end_date_datetime, - - CAST('Downtime' AS VARCHAR) AS response_status, - CAST(CAST(CURRENT_TIMESTAMP AS TIMESTAMP) AS VARCHAR) AS Current_time_stamp, - - /*Liberator Incoming Correspondence Data*/ - CAST('' AS VARCHAR) AS unassigned_time, - CAST('' AS VARCHAR) AS to_assigned_time, - CAST('' AS VARCHAR) AS assigned_in_progress_time, - CAST('' AS VARCHAR) AS assigned_response_time, - - -- Is there a better way to do this? Substring captures only the minutes... - 'INTERVAL ''' - || REGEXP_REPLACE(CAST( - TRY(CAST(SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS TIMESTAMP)) - - TRY(CAST(SUBSTR( - CAST(d.start_timestampZ AS VARCHAR(30)), - 1, - 16 - ) AS TIMESTAMP)) - AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - AS response_time, --as downtime_duration --line 30 - - CAST('' AS VARCHAR) AS unassigned_days, - CAST('' AS VARCHAR) AS unassigned_days_group, - CAST('' AS VARCHAR) AS unassigned_days_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS unassigned_days_kpiTotFourteenLess, - CAST('' AS VARCHAR) AS Days_to_assign, - CAST('' AS VARCHAR) AS Days_to_assign_group, - CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS Days_to_assign_kpiTotFourteenLess, - CAST('' AS VARCHAR) AS assigned_in_progress_days, - CAST('' AS VARCHAR) AS assigned_in_progress_days_group, - CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS assigned_in_progress_days_kpiTotFourteenLess, - CAST('' AS VARCHAR) AS assignedResponseDays, - CAST('' AS VARCHAR) AS assignedResponseDays_group, - CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS assignedResponseDays_kpiTotFourteenLess, - - CAST(DATE_DIFF( - 'day', - CAST(SUBSTR( - CAST(d.start_timestampZ AS VARCHAR(30)), - 1, - 10 - ) AS DATE), - CAST(SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 10 - ) AS DATE) - ) AS VARCHAR) AS ResponseDays, --as downtime_duration in days - - CAST('' AS VARCHAR) AS ResponseDays_group, - CAST('' AS VARCHAR) AS ResponseDays_kpiTotFiftySixLess, - CAST('' AS VARCHAR) AS ResponseDays_kpiTotFourteenLess, - - d.end_date AS Response_generated_at, - d.start_date AS Date_Received, +) - SUBSTR( - CAST(d.end_timestampZ AS VARCHAR(30)), - 1, - 7 - ) || '-01' AS MonthYear, +, total_response_month as ( +select +liberator_pcn_ic.Response_written_by as mth_tot_rep_Response_written_by +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as mth_tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as mth_tot_rep_unique_link - 'Downtime' AS "Type", +,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as mth_tot_rep_total_dates +,count (*) as mth_tot_rep_records - d.downtime AS Serviceable, -- downtime --,'Downtime' as Serviceable -- downtime - CAST('' AS VARCHAR) AS Service_category, +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date - d.Response_written_by, --downtime officer full name +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - CAST('' AS VARCHAR) AS Letter_template, - CAST('' AS VARCHAR) AS Action_taken, - CAST('' AS VARCHAR) AS Related_to_PCN, - CAST('' AS VARCHAR) AS Cancellation_group, - CAST('' AS VARCHAR) AS Cancellation_reason, - CAST('' AS VARCHAR) AS whenassigned, - CAST('' AS VARCHAR) AS ticketserialnumber, - CAST('' AS VARCHAR) AS noderef, - CAST('' AS VARCHAR) AS record_created, - d.import_timestamp, - /*pcn data*/ - -- The original Glue SparkSQL had all of these pcn columns output as '' empty strings which either... - -- ...conflicted with the output column datatypes configured by Glue - -- ...and/or conflicted with the column translations done earlier in the SQL script. - /* Instead here, the null or empty translations are deliberately dependent upon the target datatype... - CAST(NULL AS DATE) AS [target_column], --[position]-- null date - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS INTEGER) AS [target_column], --[position]-- null int - CAST('' AS VARCHAR) AS [target_column], --[position]-- empty string - CAST(NULL AS VARCHAR) AS [target_column], --[position]-- null string - CAST(NULL AS TIMESTAMP) AS [target_column], --[position]-- null timestamp - */ - CAST('' AS VARCHAR) AS pcn_pcn, --69-- empty string - CAST(NULL AS VARCHAR) AS pcn_pcnissuedate, --70-- null string - CAST(NULL AS VARCHAR) AS pcn_pcnissuedatetime, --71-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_canx_date, --72-- null string - CAST('' AS VARCHAR) AS pcn_cancellationgroup, --73-- empty string - CAST('' AS VARCHAR) AS pcn_cancellationreason, --74-- empty string - CAST(NULL AS VARCHAR) AS pcn_pcn_casecloseddate, --75-- null string - CAST('' AS VARCHAR) AS pcn_street_location, --76-- empty string - CAST('' AS VARCHAR) AS pcn_whereonlocation, --77-- empty string - CAST('' AS VARCHAR) AS pcn_zone, --78-- empty string - CAST('' AS VARCHAR) AS pcn_usrn, --79-- empty string - CAST('' AS VARCHAR) AS pcn_contraventioncode, --80-- empty string - CAST('' AS VARCHAR) AS pcn_contraventionsuffix, --81-- empty string - CAST('' AS VARCHAR) AS pcn_debttype, --82-- empty string - CAST('' AS VARCHAR) AS pcn_vrm, --83-- empty string - CAST('' AS VARCHAR) AS pcn_vehiclemake, --84-- empty string - CAST('' AS VARCHAR) AS pcn_vehiclemodel, --85-- empty string - CAST('' AS VARCHAR) AS pcn_vehiclecolour, --86-- empty string - CAST('' AS VARCHAR) AS pcn_ceo, --87-- empty string - CAST('' AS VARCHAR) AS pcn_ceodevice, --88-- empty string - CAST(NULL AS VARCHAR) AS pcn_current_30_day_flag, --89-- null string - CAST(NULL AS VARCHAR) AS pcn_isvda, --90-- null string - CAST(NULL AS VARCHAR) AS pcn_isvoid, --91-- null string - CAST('' AS VARCHAR) AS pcn_isremoval, --92-- empty string - CAST('' AS VARCHAR) AS pcn_driverseen, --93-- empty string - CAST('' AS VARCHAR) AS pcn_allwindows, --94-- empty string - CAST('' AS VARCHAR) AS pcn_parkedonfootway, --95-- empty string - CAST('' AS VARCHAR) AS pcn_doctor, --96-- empty string - CAST(NULL AS VARCHAR) AS pcn_warningflag, --97-- null string - CAST('' AS VARCHAR) AS pcn_progressionstage, --98-- empty string - CAST('' AS VARCHAR) AS pcn_nextprogressionstage, --99-- empty string - CAST('' AS VARCHAR) AS pcn_nextprogressionstagestarts, --100-- empty string - CAST('' AS VARCHAR) AS pcn_holdreason, --101-- empty string - CAST('' AS VARCHAR) AS pcn_lib_initial_debt_amount, --102-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_received, --103-- empty string - CAST('' AS VARCHAR) AS pcn_lib_write_off_amount, --104-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_void, --105-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_method, --106-- empty string - CAST('' AS VARCHAR) AS pcn_lib_payment_ref, --107-- empty string - CAST('' AS VARCHAR) AS pcn_baliff_from, --108-- empty string - CAST('' AS VARCHAR) AS pcn_bailiff_to, --109-- empty string - CAST(NULL AS VARCHAR) AS pcn_bailiff_processedon, --110-- null string - CAST('' AS VARCHAR) AS pcn_bailiff_redistributionreason, --111-- empty string - CAST('' AS VARCHAR) AS pcn_bailiff, --112-- empty string - CAST(NULL AS VARCHAR) AS pcn_warrantissuedate, --113-- null string - CAST(NULL AS VARCHAR) AS pcn_allocation, --114-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_datenotified, --115-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_packsubmittedon, --116-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_evidencedate, --117-- null string - CAST(NULL AS VARCHAR) AS pcn_eta_adjudicationdate, --118-- null string - CAST('' AS VARCHAR) AS pcn_eta_appealgrounds, --119-- empty string - CAST(NULL AS VARCHAR) AS pcn_eta_decisionreceived, --120-- null string - CAST('' AS VARCHAR) AS pcn_eta_outcome, --121-- empty string - CAST('' AS VARCHAR) AS pcn_eta_packsubmittedby, --122-- empty string - CAST('' AS VARCHAR) AS pcn_cancelledby, --123-- empty string - CAST('' AS VARCHAR) AS pcn_registered_keeper_address, --124-- empty string - CAST('' AS VARCHAR) AS pcn_current_ticket_address, --125-- empty string - CAST(NULL AS VARCHAR) AS pcn_corresp_dispute_flag, --126-- null string - CAST(NULL AS VARCHAR) AS pcn_keyworker_corresp_dispute_flag, --127-- null string - CAST('' AS VARCHAR) AS pcn_fin_year_flag, --128-- empty string - CAST('' AS VARCHAR) AS pcn_fin_year, --129-- empty string - CAST('' AS VARCHAR) AS pcn_ticket_ref, --130-- empty string - CAST(NULL AS VARCHAR) AS pcn_nto_printed, --131-- null string - CAST(NULL AS VARCHAR) AS pcn_appeal_accepted, --132-- null string - CAST(NULL AS VARCHAR) AS pcn_arrived_in_pound, --133-- null string - CAST(NULL AS VARCHAR) AS pcn_cancellation_reversed, --134-- null string - CAST(NULL AS VARCHAR) AS pcn_cc_printed, --135-- null string - CAST(NULL AS VARCHAR) AS pcn_drr, --136-- null string - CAST(NULL AS VARCHAR) AS pcn_en_printed, --137-- null string - CAST(NULL AS VARCHAR) AS pcn_hold_released, --138-- null string - CAST(NULL AS VARCHAR) AS pcn_dvla_response, --139-- null string - CAST(NULL AS VARCHAR) AS pcn_dvla_request, --140-- null string - CAST(NULL AS VARCHAR) AS pcn_full_rate_uplift, --141-- null string - CAST(NULL AS VARCHAR) AS pcn_hold_until, --142-- null string - CAST(NULL AS VARCHAR) AS pcn_lifted_at, --143-- null string - CAST(NULL AS VARCHAR) AS pcn_lifted_by, --144-- null string - CAST(NULL AS VARCHAR) AS pcn_loaded, --145-- null string - CAST(NULL AS VARCHAR) AS pcn_nor_sent, --146-- null string - CAST(NULL AS VARCHAR) AS pcn_notice_held, --147-- null string - CAST(NULL AS VARCHAR) AS pcn_ofr_printed, --148-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_printed, --149-- null string - CAST(NULL AS VARCHAR) AS pcn_reissue_nto_requested, --150-- null string - CAST(NULL AS VARCHAR) AS pcn_reissue_pcn, --151-- null string - CAST(NULL AS VARCHAR) AS pcn_set_back_to_pre_cc_stage, --152-- null string - CAST(NULL AS VARCHAR) AS pcn_vehicle_released_for_auction, --153-- null string - CAST(NULL AS VARCHAR) AS pcn_warrant_issued, --154-- null string - CAST(NULL AS VARCHAR) AS pcn_warrant_redistributed, --155-- null string - CAST(NULL AS VARCHAR) AS pcn_warrant_request_granted, --156-- null string - CAST(NULL AS VARCHAR) AS pcn_ad_hoc_vq4_request, --157-- null string - CAST(NULL AS VARCHAR) AS pcn_paper_vq5_received, --158-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_buslane, --159-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_pre_debt, --160-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_collection, --161-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_drr, --162-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_cc, --163-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_nto, --164-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_print, --165-- null string - CAST(NULL AS VARCHAR) AS pcn_warning_notice_extracted_for_print, --166-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_ofr, --167-- null string - CAST(NULL AS VARCHAR) AS pcn_pcn_extracted_for_warrant_request, --168-- null string - CAST(NULL AS VARCHAR) AS pcn_pre_debt_new_debtor_details, --169-- null string - CAST(NULL AS VARCHAR) AS pcn_importdattime, --170-- null string - CAST(NULL AS VARCHAR) AS pcn_importdatetime, --171-- null string - CAST('' AS VARCHAR) AS pcn_import_year, --172-- empty string - CAST('' AS VARCHAR) AS pcn_import_month, --173-- empty string - CAST('' AS VARCHAR) AS pcn_import_day, --174-- empty string - CAST('' AS VARCHAR) AS pcn_import_date, --175-- empty string +group by +liberator_pcn_ic.Response_written_by --as mth_tot_rep_Response_written_by +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as mth_tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) --as mth_tot_rep_unique_link +order by +concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') desc --as mth_tot_rep_monthYear_response - /*officer monthly totals*/ - CAST('' AS VARCHAR) AS link_officer_total_rep_dates, - CAST('' AS VARCHAR) AS link_month_total, - CAST('' AS VARCHAR) AS mth_tot_rep_monthYear_response, - CAST('' AS VARCHAR) AS mth_tot_rep_total_dates, - CAST('' AS VARCHAR) AS mth_tot_rep_records, - CAST('' AS VARCHAR) AS tot_rep_response_date, - CAST('' AS VARCHAR) AS tot_rep_total_date, - CAST('' AS VARCHAR) AS tot_rep_records, +) - /*Teams data*/ - t.t_start_date, - t.t_end_date, - t.t_team, - t.t_team_name, - t.t_role, - t.t_forename, - t.t_surname, - t.t_full_name, - t.t_qa_doc_created_by, - t.t_qa_doc_full_name, - t.t_post_title, - t.t_notes, - t.t_import_date, +/*Teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing*/ +,team as ( +select distinct start_date as t_start_date +,end_date as t_end_date +,team as t_team +,team_name as t_team_name +,role as t_role +,forename as t_forename +,surname as t_surname +,full_name as t_full_name +,qa_doc_created_by as t_qa_doc_created_by +,qa_doc_full_name as t_qa_doc_full_name +,post_title as t_post_title +,notes as t_notes +,import_date as t_import_date--* +from parking_correspondence_performance_teams where import_date = (select max(parking_correspondence_performance_teams.import_date) from parking_correspondence_performance_teams ) +) - /* Partition columns moved to the end to keep schema alligned */ - d.import_year, - d.import_month, - d.import_day, - CAST(d.import_date AS VARCHAR) AS import_date +select + +/*Fields for union downtime data*/ +'' as downtime_timestamp +,'' as downtime_email_address +,'' as officer_s_first_name +,'' as officer_s_last_name +,'' as liberator_system_username +,'' as liberator_system_id +,'' as import_datetime +,'' as multiple_downtime_days_flag +,'' as response_secs +,'' as response_mins +,'' as response_hours +,'' as response_days_plus_one +,'' as downtime_total_non_working_mins_with_lunch +, '' as downtime_total_non_working_mins +,'' as downtime_total_working_mins_with_lunch +,'' as downtime_total_working_mins +,'' as downtime_total_working_mins_with_lunch_net +,'' as downtime_total_working_mins_net +, '' as start_date_time +,'' as startdate +,'' as start_date_datetime +,'' as end_date_time +,'' as enddate +,'' as end_date_datetime + +/*Liberator Incoming Correspondence Data*/ +,case +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' +end as response_status, +cast(current_timestamp as string) as Current_time_stamp, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, + +/*unassigned days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' +end as unassigned_days_group +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as unassigned_days_kpiTotFourteenLess, + +/*Days to assign*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as Days_to_assign_group +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as Days_to_assign_kpiTotFourteenLess, + +/*assigned in progress days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' +end as assigned_in_progress_days_group +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as string) as assigned_in_progress_days_kpiTotFourteenLess, + +/*assigned response days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' +end as assignedResponseDays_group +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as assignedResponseDays_kpiTotFourteenLess, + +/*Response days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as ResponseDays_group +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff(cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFiftySixLess +,cast(Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as string) as ResponseDays_kpiTotFourteenLess + +,Response_generated_at +,Date_Received +,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear +,liberator_pcn_ic.Type +,Serviceable +,Service_category +,Response_written_by +,Letter_template +,Action_taken +,Related_to_PCN +,Cancellation_group +,Cancellation_reason +,whenassigned +,ticketserialnumber +,noderef +,cast(liberator_pcn_ic.record_created as string) as record_created +,liberator_pcn_ic.import_timestamp +,liberator_pcn_ic.import_year +,liberator_pcn_ic.import_month +,liberator_pcn_ic.import_day +,liberator_pcn_ic.import_date + +/*pcn data*/ +,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn +,cast(pcnfoidetails_pcn_foi_full.pcnissuedate as string) as pcn_pcnissuedate +,cast(pcnfoidetails_pcn_foi_full.pcnissuedatetime as string) as pcn_pcnissuedatetime +,cast(pcnfoidetails_pcn_foi_full.pcn_canx_date as string) as pcn_pcn_canx_date +,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup +,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason +,cast(pcnfoidetails_pcn_foi_full.pcn_casecloseddate as string) as pcn_pcn_casecloseddate +,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location +,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation +,pcnfoidetails_pcn_foi_full.zone as pcn_zone +,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn +,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode +,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix +,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype +,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm +,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake +,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel +,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour +,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo +,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice +,cast(pcnfoidetails_pcn_foi_full.current_30_day_flag as string) as pcn_current_30_day_flag +,cast(pcnfoidetails_pcn_foi_full.isvda as string) as pcn_isvda +,cast(pcnfoidetails_pcn_foi_full.isvoid as string) as pcn_isvoid +,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval +,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen +,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows +,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway +,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor +,cast(pcnfoidetails_pcn_foi_full.warningflag as string) as pcn_warningflag +,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts +,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason +,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount +,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received +,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount +,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void +,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method +,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref +,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from +,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to +,cast(pcnfoidetails_pcn_foi_full.bailiff_processedon as string) as pcn_bailiff_processedon +,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason +,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff +,cast(pcnfoidetails_pcn_foi_full.warrantissuedate as string) as pcn_warrantissuedate +,cast(pcnfoidetails_pcn_foi_full.allocation as string) as pcn_allocation +,cast(pcnfoidetails_pcn_foi_full.eta_datenotified as string) as pcn_eta_datenotified +,cast(pcnfoidetails_pcn_foi_full.eta_packsubmittedon as string) as pcn_eta_packsubmittedon +,cast(pcnfoidetails_pcn_foi_full.eta_evidencedate as string) as pcn_eta_evidencedate +,cast(pcnfoidetails_pcn_foi_full.eta_adjudicationdate as string) as pcn_eta_adjudicationdate +,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds +,cast(pcnfoidetails_pcn_foi_full.eta_decisionreceived as string) as pcn_eta_decisionreceived +,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome +,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby +,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby +,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address +,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address +,cast(pcnfoidetails_pcn_foi_full.corresp_dispute_flag as string) as pcn_corresp_dispute_flag +,cast(pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as string) as pcn_keyworker_corresp_dispute_flag +,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag +,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year +,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref +,cast(pcnfoidetails_pcn_foi_full.nto_printed as string) as pcn_nto_printed +,cast(pcnfoidetails_pcn_foi_full.appeal_accepted as string) as pcn_appeal_accepted +,cast(pcnfoidetails_pcn_foi_full.arrived_in_pound as string) as pcn_arrived_in_pound +,cast(pcnfoidetails_pcn_foi_full.cancellation_reversed as string) as pcn_cancellation_reversed +,cast(pcnfoidetails_pcn_foi_full.cc_printed as string) as pcn_cc_printed +,cast(pcnfoidetails_pcn_foi_full.drr as string) as pcn_drr +,cast(pcnfoidetails_pcn_foi_full.en_printed as string) as pcn_en_printed +,cast(pcnfoidetails_pcn_foi_full.hold_released as string) as pcn_hold_released +,cast(pcnfoidetails_pcn_foi_full.dvla_response as string) as pcn_dvla_response +,cast(pcnfoidetails_pcn_foi_full.dvla_request as string) as pcn_dvla_request +,cast(pcnfoidetails_pcn_foi_full.full_rate_uplift as string) as pcn_full_rate_uplift +,cast(pcnfoidetails_pcn_foi_full.hold_until as string) as pcn_hold_until +,cast(pcnfoidetails_pcn_foi_full.lifted_at as string) as pcn_lifted_at +,cast(pcnfoidetails_pcn_foi_full.lifted_by as string) as pcn_lifted_by +,cast(pcnfoidetails_pcn_foi_full.loaded as string) as pcn_loaded +,cast(pcnfoidetails_pcn_foi_full.nor_sent as string) as pcn_nor_sent +,cast(pcnfoidetails_pcn_foi_full.notice_held as string) as pcn_notice_held +,cast(pcnfoidetails_pcn_foi_full.ofr_printed as string) as pcn_ofr_printed +,cast(pcnfoidetails_pcn_foi_full.pcn_printed as string) as pcn_pcn_printed +,cast(pcnfoidetails_pcn_foi_full.reissue_nto_requested as string) as pcn_reissue_nto_requested +,cast(pcnfoidetails_pcn_foi_full.reissue_pcn as string) as pcn_reissue_pcn +,cast(pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as string) as pcn_set_back_to_pre_cc_stage +,cast(pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as string) as pcn_vehicle_released_for_auction +,cast(pcnfoidetails_pcn_foi_full.warrant_issued as string) as pcn_warrant_issued +,cast(pcnfoidetails_pcn_foi_full.warrant_redistributed as string) as pcn_warrant_redistributed +,cast(pcnfoidetails_pcn_foi_full.warrant_request_granted as string) as pcn_warrant_request_granted +,cast(pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as string) as pcn_ad_hoc_vq4_request +,cast(pcnfoidetails_pcn_foi_full.paper_vq5_received as string) as pcn_paper_vq5_received +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as string) as pcn_pcn_extracted_for_buslane +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as string) as pcn_pcn_extracted_for_pre_debt +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as string) as pcn_pcn_extracted_for_collection +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as string) as pcn_pcn_extracted_for_drr +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as string) as pcn_pcn_extracted_for_cc +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as string) as pcn_pcn_extracted_for_nto +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as string) as pcn_pcn_extracted_for_print +,cast(pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as string) as pcn_warning_notice_extracted_for_print +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as string) as pcn_pcn_extracted_for_ofr +,cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as string) as pcn_pcn_extracted_for_warrant_request +,cast(pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as string) as pcn_pre_debt_new_debtor_details +,cast(pcnfoidetails_pcn_foi_full.importdattime as string) as pcn_importdattime +,cast(pcnfoidetails_pcn_foi_full.importdatetime as string) as pcn_importdatetime -- +,pcnfoidetails_pcn_foi_full.import_year as pcn_import_year +,pcnfoidetails_pcn_foi_full.import_month as pcn_import_month +,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day +,pcnfoidetails_pcn_foi_full.import_date as pcn_import_date + + +/*officer monthly totals*/ +,cast(concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as string) as link_officer_total_rep_dates +,cast(concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as string) as link_month_total + +,cast(total_response_month.mth_tot_rep_monthYear_response as string) as mth_tot_rep_monthYear_response +,cast(total_response_month.mth_tot_rep_total_dates as string) as mth_tot_rep_total_dates +,cast(total_response_month.mth_tot_rep_records as string) as mth_tot_rep_records + + +,cast(officer_total_rep_dates.tot_rep_response_date as string) as tot_rep_response_date +,cast(officer_total_rep_dates.tot_rep_total_dates as string) as tot_rep_total_date +,cast(officer_total_rep_dates.tot_rep_records as string) as tot_rep_records + + +/*Teams data*/ +,t_start_date +,t_end_date +,t_team +,t_team_name +,t_role +,t_forename +,t_surname +,t_full_name +,t_qa_doc_created_by +,t_qa_doc_full_name +,t_post_title +,t_notes +,t_import_date + +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date + +left join officer_total_rep_dates on concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) = officer_total_rep_dates.tot_rep_unique_link + +left join total_response_month on concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) = total_response_month.mth_tot_rep_unique_link + + +left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) + +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date - FROM downtime d - LEFT JOIN team t - ON UPPER(t.t_full_name) = UPPER(d.Response_written_by) --downtime officer full name -) -SELECT * FROM liberator_with_team_gds UNION -SELECT * FROM google_form ---order by downtime_timestamp desc -; + +--Downtime data gathered from Google form https://forms.gle/bB53jAayiZ2Ykwjk6 +Select +cast(parking_officer_downtime.timestamp as string) as downtime_timestamp +,email_address as downtime_email_address +,TRIM(officer_s_first_name) as officer_s_first_name +,TRIM(officer_s_last_name) as officer_s_last_name +,liberator_system_username +,liberator_system_id +,cast(import_datetime as string) import_datetime +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60) > 1440 then 1 else 0 end as string) as multiple_downtime_days_flag +,cast(unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')) as string) as response_secs +,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 end as string) as response_mins +,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/3600 end as string) as response_hours + +/*Downtime calendar days*/ +,cast(Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1 end as string) as response_days_plus_one -- downtime days plus one calendar day + +/*Downtime non working mins*/ +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 948 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 948 end as string) as downtime_total_non_working_mins_with_lunch --if greater than 1440 mins then (days + 1) * 948 " +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 1008 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 1008 end as string) as downtime_total_non_working_mins --if greater than 1440 mins then (days + 1) * 1008 " + +/*Downtime working mins*/ +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 end as string) as downtime_total_working_mins_with_lunch -- 492 mins = 8hrs 12 mins working day including lunch" +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 end as string) as downtime_total_working_mins --432 mins = 7hrs 12mins working hours" + +/*Downtime working mins net (less downtime mins)*/ +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 492 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 492 - /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_with_lunch_net -- 492 mins = 8hrs 12 mins working day including lunch less downtime" +,cast(case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) > 1440 then /*response_days_plus_one*/(datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) )+1) * 432 + When parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' and /* downtime mins (response_mins)*/ ((unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) < 1440 then 432 - (/* downtime mins (response_mins)*/ (unix_timestamp(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss')) - unix_timestamp(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss')))/60 ) end as string) as downtime_total_working_mins_net --432 mins = 7hrs 12mins working hours" + +,cast(substr(cast(start_date as string), 12, 5) as string) as start_date_time +,cast(substr(cast(start_date as string), 1, 10) as string) as startdate +,cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as start_date_datetime +,cast(substr(cast(end_date as string), 12, 5) as string) as end_date_time +,cast(substr(cast(end_date as string), 1, 10) as string) as enddate +,cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as string) as end_date_datetime +,'Downtime' as response_status +,cast(current_timestamp as string) as Current_time_stamp + +/*Liberator Incoming Correspondence Data*/ +,'' as unassigned_time +,'' as to_assigned_time +,'' as assigned_in_progress_time +,'' as assigned_response_time +,(cast (cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) - cast ( Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,16) as timestamp) as string) ) as response_time --as downtime_duration --line 30 +,'' as unassigned_days +,'' as unassigned_days_group +,'' as unassigned_days_kpiTotFiftySixLess +,'' as unassigned_days_kpiTotFourteenLess +,'' as Days_to_assign +,'' as Days_to_assign_group +,'' as Days_to_assign_kpiTotFiftySixLess +,'' as Days_to_assign_kpiTotFourteenLess +,'' as assigned_in_progress_days +,'' as assigned_in_progress_days_group +,'' as assigned_in_progress_days_kpiTotFiftySixLess +,'' as assigned_in_progress_days_kpiTotFourteenLess +,'' as assignedResponseDays +,'' as assignedResponseDays_group +,'' as assignedResponseDays_kpiTotFiftySixLess +,'' as assignedResponseDays_kpiTotFourteenLess +,cast( Case when parking_officer_downtime.timestamp not like '' and parking_officer_downtime.timestamp not like '#REF!' then datediff( cast(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date), cast(Substr(cast(to_timestamp(start_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,10) as date) ) end as string) as ResponseDays --as downtime_duration in days +,'' as ResponseDays_group +,'' as ResponseDays_kpiTotFiftySixLess +,'' as ResponseDays_kpiTotFourteenLess +, end_date as Response_generated_at +, start_date as Date_Received +,concat(Substr(cast(to_timestamp(end_date,'yyyy-MM-dd HH:mm:ss') as varchar(30)),1,7), '-01') as MonthYear +,'Downtime' as Type +,parking_officer_downtime.Downtime as Serviceable -- downtime +,'' as Service_category +,concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name)) as Response_written_by --downtime officer full name +,'' as Letter_template +,'' as Action_taken +,'' as Related_to_PCN +,'' as Cancellation_group +,'' as Cancellation_reason +,'' as whenassigned +,'' as ticketserialnumber +,'' as noderef +,'' as record_created +,import_timestamp +,import_year +,import_month +,import_day +,cast(import_date as string) as import_date + +/*pcn data*/ +,'' as pcn_pcn +,'' as pcn_pcnissuedate +,'' as pcn_pcnissuedatetime --line 74 +,'' as pcn_pcn_canx_date +,'' as pcn_cancellationgroup +,'' as pcn_cancellationreason +,'' as pcn_pcn_casecloseddate +,'' as pcn_street_location +,'' as pcn_whereonlocation +,'' as pcn_zone +,'' as pcn_usrn +,'' as pcn_contraventioncode +,'' as pcn_contraventionsuffix +,'' as pcn_debttype +,'' as pcn_vrm +,'' as pcn_vehiclemake +,'' as pcn_vehiclemodel +,'' as pcn_vehiclecolour +,'' as pcn_ceo +,'' as pcn_ceodevice +,'' as pcn_current_30_day_flag +,'' as pcn_isvda +,'' as pcn_isvoid +,'' as pcn_isremoval +,'' as pcn_driverseen +,'' as pcn_allwindows +,'' as pcn_parkedonfootway +,'' as pcn_doctor +,'' as pcn_warningflag +,'' as pcn_progressionstage +,'' as pcn_nextprogressionstage --100 +,'' as pcn_nextprogressionstagestarts +,'' as pcn_holdreason +,'' as pcn_lib_initial_debt_amount +,'' as pcn_lib_payment_received +,'' as pcn_lib_write_off_amount +,'' as pcn_lib_payment_void +,'' as pcn_lib_payment_method +,'' as pcn_lib_payment_ref +,'' as pcn_baliff_from +,'' as pcn_bailiff_to +,'' as pcn_bailiff_processedon +,'' as pcn_bailiff_redistributionreason +,'' as pcn_bailiff +,'' as pcn_warrantissuedate +,'' as pcn_allocation +,'' as pcn_eta_datenotified +,'' as pcn_eta_packsubmittedon +,'' as pcn_eta_evidencedate +,'' as pcn_eta_adjudicationdate +,'' as pcn_eta_appealgrounds +,'' as pcn_eta_decisionreceived +,'' as pcn_eta_outcome +,'' as pcn_eta_packsubmittedby +,'' as pcn_cancelledby +,'' as pcn_registered_keeper_address +,'' as pcn_current_ticket_address +,'' as pcn_corresp_dispute_flag +,'' as pcn_keyworker_corresp_dispute_flag +,'' as pcn_fin_year_flag +,'' as pcn_fin_year +,'' as pcn_ticket_ref +,'' as pcn_nto_printed +,'' as pcn_appeal_accepted +,'' as pcn_arrived_in_pound +,'' as pcn_cancellation_reversed +,'' as pcn_cc_printed +,'' as pcn_drr +,'' as pcn_en_printed +,'' as pcn_hold_released +,'' as pcn_dvla_response +,'' as pcn_dvla_request +,'' as pcn_full_rate_uplift +,'' as pcn_hold_until +,'' as pcn_lifted_at +,'' as pcn_lifted_by +,'' as pcn_loaded +,'' as pcn_nor_sent +,'' as pcn_notice_held +,'' as pcn_ofr_printed +,'' as pcn_pcn_printed +,'' as pcn_reissue_nto_requested +,'' as pcn_reissue_pcn +,'' as pcn_set_back_to_pre_cc_stage +,'' as pcn_vehicle_released_for_auction +,'' as pcn_warrant_issued +,'' as pcn_warrant_redistributed +,'' as pcn_warrant_request_granted +,'' as pcn_ad_hoc_vq4_request +,'' as pcn_paper_vq5_received +,'' as pcn_pcn_extracted_for_buslane +,'' as pcn_pcn_extracted_for_pre_debt +,'' as pcn_pcn_extracted_for_collection +,'' as pcn_pcn_extracted_for_drr +,'' as pcn_pcn_extracted_for_cc +,'' as pcn_pcn_extracted_for_nto +,'' as pcn_pcn_extracted_for_print +,'' as pcn_warning_notice_extracted_for_print +,'' as pcn_pcn_extracted_for_ofr +,'' as pcn_pcn_extracted_for_warrant_request +,'' as pcn_pre_debt_new_debtor_details +,'' as pcn_importdattime +,'' as pcn_importdatetime +,'' as pcn_import_year +,'' as pcn_import_month +,'' as pcn_import_day +,'' as pcn_import_date + + +/*officer monthly totals*/ +,'' as link_officer_total_rep_dates +,'' as link_month_total + +,'' as mth_tot_rep_monthYear_response +,'' as mth_tot_rep_total_dates +,'' as mth_tot_rep_records + + +,'' as tot_rep_response_date +,'' as tot_rep_total_date +,'' as tot_rep_records + +/*Teams data*/ +,t_start_date +,t_end_date +,t_team +,t_team_name +,t_role +,t_forename +,t_surname +,t_full_name +,t_qa_doc_created_by +,t_qa_doc_full_name +,t_post_title +,t_notes +,t_import_date + +from parking_officer_downtime +left join team on upper(team.t_full_name) = upper(concat(TRIM(officer_s_first_name),' ',TRIM(officer_s_last_name))) --as Response_written_by --downtime officer full name +where import_date = (select max(import_date) from parking_officer_downtime) and timestamp not like '' and timestamp not like '#REF!' +AND cast(Substr( cast(parking_officer_downtime.timestamp as string) ,1,10) as date) > current_date - interval '13' month --Last 13 months from todays date +--order by timestamp desc """ +SQLQuery_node1697453694651 = sparkSqlQuery( + glueContext, + query=SqlQuery0, + mapping={ + "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1, + "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1697453826330, + "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1697453966201, + "parking_officer_downtime": parking_raw_zoneparking_officer_downtime_node1697453968832, + }, + transformation_ctx="SQLQuery_node1697453694651", +) -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name +# Script generated for node Liberator Refined - parking_correspondence_performance_records_with_pcn_downtime_gds +LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2 = glueContext.getSink( + path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn_downtime_gds/", + connection_type="s3", + updateBehavior="UPDATE_IN_DATABASE", + partitionKeys=["import_year", "import_month", "import_day", "import_date"], + compression="snappy", + enableUpdateCatalog=True, + transformation_ctx="LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2", +) +LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2.setCatalogInfo( + catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", + catalogTableName="parking_correspondence_performance_records_with_pcn_downtime_gds", +) +LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2.setFormat( + "glueparquet" +) +LiberatorRefinedparking_correspondence_performance_records_with_pcn_downtime_gds_node2.writeFrame( + SQLQuery_node1697453694651 ) +job.commit() diff --git a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py index 9c23b41fe..bd861c5e6 100644 --- a/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py +++ b/scripts/jobs/parking/parking_correspondence_performance_records_with_pcn_gds.py @@ -1,601 +1,383 @@ -""" -Only need to change the table name and the query prototyped on the Athena UI -by replacing table_name and query_on_athena -""" - -from scripts.helpers.athena_helpers import create_update_table_with_partition -from scripts.helpers.helpers import get_glue_env_var - +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job +from awsglue import DynamicFrame +from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS + +def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: + for alias, frame in mapping.items(): + frame.toDF().createOrReplaceTempView(alias) + result = spark.sql(query) + return DynamicFrame.fromDF(result, glueContext, transformation_ctx) + + +args = getResolvedOptions(sys.argv, ["JOB_NAME"]) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) environment = get_glue_env_var("environment") -# The target table in liberator refined zone -table_name = "parking_correspondence_performance_records_with_pcn_gds" +# Script generated for node Amazon S3 - refined - pcnfoidetails_pcn_foi_full +AmazonS3refinedpcnfoidetails_pcn_foi_full_node1657284942794 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-refined-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="pcnfoidetails_pcn_foi_full", + transformation_ctx="AmazonS3refinedpcnfoidetails_pcn_foi_full_node1657284942794", +) -# The exact same query prototyped in pre-prod(stg) or prod Athena -query_on_athena = """ -/* -Correspondence Performance records last 13 months with PCN FOI records +# Script generated for node S3 bucket - Raw - liberator_pcn_ic +S3bucketRawliberator_pcn_ic_node1 = glueContext.create_dynamic_frame.from_catalog( + database="dataplatform-"+environment+"-liberator-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="liberator_pcn_ic", + transformation_ctx="S3bucketRawliberator_pcn_ic_node1", +) --->> "dataplatform-prod-liberator-refined-zone"."parking_correspondence_performance_records_with_pcn_gds" ---<< "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" --- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" --- < "parking-raw-zone"."parking_correspondence_performance_teams" +# Script generated for node parking_raw_zone - parking_correspondence_performance_teams +parking_raw_zoneparking_correspondence_performance_teams_node1682094127227 = glueContext.create_dynamic_frame.from_catalog( + database="parking-raw-zone", + push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 14)", + table_name="parking_correspondence_performance_teams", + transformation_ctx="parking_raw_zoneparking_correspondence_performance_teams_node1682094127227", +) -16/06/2022 - Created +# Script generated for node SQL +SqlQuery0 = """ +/* +Correspondence Performance records last 13 months with PCN FOI records +16/06/2022 - Created 30/11/2022 - with totals for Month and response dates for each officer 21/04/2023 - added teams data from google spreadsheet load - https://docs.google.com/spreadsheets/d/1zxZXX1_qU9NW93Ug1JUy7aXsnTz45qIj7Zftmi9trbI/edit?usp=sharing -04/03/2025 - Refactored SQL for migration back to AthenaSQL for dap-airflow -05/03/2025 - Removed redundant left joins from subqueries -14/04/2025 - Strictly applied column formatting consistent with previous Glue outputs. -16/04/2025 - Workaround to orchestration issue caused by latest "pcnfoidetails_pcn_foi_full" not yet ready -*/ -WITH - -liberator AS ( - - SELECT *, - - CAST(CURRENT_TIMESTAMP AS TIMESTAMP(3)) AS current_utc_timestamp, - TRY(CAST(date_received AS TIMESTAMP(3))) AS date_received_timestamp, -- pre-filtered by WHERE clause - - IF(whenassigned <> '', -- contains probable valid timestamp - TRY(CAST(whenassigned AS TIMESTAMP(3))), - CAST(NULL AS TIMESTAMP(3)) -- edge case - ) AS whenassigned_timestamp, - - IF(Response_generated_at <> '', -- contains probable valid timestamp - TRY(CAST(Response_generated_at AS TIMESTAMP(3))), - CAST(NULL AS TIMESTAMP(3)) -- edge case - ) AS Response_generated_at_timestamp, - - CAST(CASE WHEN date_received <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' - CURRENT_DATE - ) - --ELSE NULL - END AS INTEGER) AS days_since_date_received, - - CAST(CASE WHEN date_received <> '' AND whenassigned <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), -- under condition of date_received <>'' - TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE) - ) - --ELSE NULL - END AS INTEGER) AS days_since_date_received_whenassigned, - - CAST(CASE WHEN whenassigned <> '' AND response_generated_at = '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), - CURRENT_DATE - ) - --ELSE NULL -- edge case - END AS INTEGER) AS days_since_whenassigned, - - CAST(CASE WHEN whenassigned <> '' AND response_generated_at <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(whenassigned, 1, 10) AS DATE), - TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) - ) - --ELSE NULL -- edge case - END AS INTEGER) AS days_since_whenassigned_response_generated_at, - - CAST(CASE WHEN date_received <> '' AND response_generated_at <> '' - THEN DATE_DIFF( - 'day', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), - TRY_CAST(SUBSTR(response_generated_at, 1, 10) AS DATE) - ) - --ELSE NULL -- edge case - END AS INTEGER) AS days_since_date_received_response_generated_at, - - /* officer_total_rep_dates */ - Response_written_by AS tot_rep_Response_written_by, - SUBSTR(response_generated_at, 1, 10) AS tot_rep_response_date, - CONCAT(SUBSTR(CAST(response_generated_at AS VARCHAR(10)), 1, 7), '-01') AS tot_rep_monthYear_response, - CONCAT( - Response_written_by, - SUBSTR(response_generated_at, 1, 10) - ) AS tot_rep_unique_link, - - /* total_response_month */ - Response_written_by AS mth_tot_rep_Response_written_by, - SUBSTR(response_generated_at, 1, 10) AS mth_tot_rep_response_date, - CONCAT(SUBSTR(CAST(response_generated_at AS VARCHAR(10)), 1, 7), '-01') AS mth_tot_rep_monthYear_response, - CONCAT( - Response_written_by, - CONCAT(SUBSTR(CAST(response_generated_at AS VARCHAR(10)), 1, 7), '-01') - ) AS mth_tot_rep_unique_link - - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" - WHERE import_Date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" g - ) - AND LENGTH(ticketserialnumber) = 10 -- ticket filter - AND date_received <> '' -- is the overriding condition for "13 months from todays date"! - AND TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE) > CURRENT_DATE - INTERVAL '13' MONTH - -- Last 13 months from todays date -/* -- This alternative method captures slightly more records... - AND DATE_DIFF( - 'month', - TRY_CAST(SUBSTR(date_received, 1, 10) AS DATE), - CURRENT_DATE - ) <= 13 -*/ -), - -officer_total_rep_dates AS ( - SELECT - l.tot_rep_Response_written_by, - l.tot_rep_response_date, - l.tot_rep_monthYear_response, - l.tot_rep_unique_link, - COUNT(DISTINCT l.tot_rep_response_date) AS tot_rep_total_dates, - COUNT(*) AS tot_rep_records - -- The unreferenced p. caused SQL linter error ST11 - -- Then discoverd COUNT(*) was not expanded by the left join to p because p.pcn is unique within import_date - FROM liberator l -/* -- Therefore removed this redundant left join... - LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p - ON l.ticketserialnumber = p.pcn - AND l.import_date = p.import_date -*/ - GROUP BY - l.tot_rep_Response_written_by, - l.tot_rep_response_date, - l.tot_rep_monthYear_response, - l.tot_rep_unique_link -), - -total_response_month AS ( - SELECT - l.mth_tot_rep_Response_written_by, - l.mth_tot_rep_monthYear_response, - l.mth_tot_rep_unique_link, - COUNT(DISTINCT l.mth_tot_rep_response_date) AS mth_tot_rep_total_dates, - COUNT(*) AS mth_tot_rep_records - -- The unreferenced p. caused SQL linter error ST11 - -- Then discoverd COUNT(*) was not expanded by the left join to p because p.pcn is unique within import_date - FROM liberator l -/* -- Therefore removed this redundant left join... - LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p - ON l.ticketserialnumber = p.pcn - AND l.import_date = p.import_date + */ - GROUP BY - l.mth_tot_rep_Response_written_by, - l.mth_tot_rep_monthYear_response, - l.mth_tot_rep_unique_link - --ORDER BY - -- l.mth_rep_monthYear_response DESC -), - -team AS ( - SELECT DISTINCT - "start_date" AS t_start_date, - end_date AS t_end_date, - team AS t_team, - team_name AS t_team_name, - "role" AS t_role, - forename AS t_forename, - surname AS t_surname, - full_name AS t_full_name, - qa_doc_created_by AS t_qa_doc_created_by, - qa_doc_full_name AS t_qa_doc_full_name, - post_title AS t_post_title, - notes AS t_notes, - import_date AS t_import_date - FROM "parking-raw-zone"."parking_correspondence_performance_teams" - WHERE import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "parking-raw-zone"."parking_correspondence_performance_teams" g - ) +With officer_total_rep_dates as( +select +liberator_pcn_ic.Response_written_by as tot_rep_Response_written_by +,substr(liberator_pcn_ic.response_generated_at, 1, 10) as tot_rep_response_date +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as tot_rep_unique_link + +,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as tot_rep_total_dates +,count (*) as tot_rep_records + + +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date + +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + +group by +liberator_pcn_ic.Response_written_by --as tot_rep_Response_written_by +,substr(liberator_pcn_ic.response_generated_at, 1, 10) --as tot_rep_response_date +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) --as tot_rep_unique_link + ) +, total_response_month as ( +select +liberator_pcn_ic.Response_written_by as mth_tot_rep_Response_written_by +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') as mth_tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as mth_tot_rep_unique_link + +,count (distinct substr(liberator_pcn_ic.response_generated_at, 1, 10)) as mth_tot_rep_total_dates +,count (*) as mth_tot_rep_records + + +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date + +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + + +group by +liberator_pcn_ic.Response_written_by --as mth_tot_rep_Response_written_by +,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') --as mth_tot_rep_monthYear_response +,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) --as mth_tot_rep_unique_link +order by +concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') desc --as mth_tot_rep_monthYear_response + +) +, team as ( +select distinct start_date as t_start_date +,end_date as t_end_date +,team as t_team +,team_name as t_team_name +,role as t_role +,forename as t_forename +,surname as t_surname +,full_name as t_full_name +,qa_doc_created_by as t_qa_doc_created_by +,qa_doc_full_name as t_qa_doc_full_name +,post_title as t_post_title +,notes as t_notes +,import_date as t_import_date--* +from parking_correspondence_performance_teams where import_date = (select max(import_date) from parking_correspondence_performance_teams ) +) + + +Select +case +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then 'Unassigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then 'Assigned' +when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then 'Responded' +end as response_status, +cast(current_timestamp as string) as Current_time_stamp, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then current_timestamp - cast( liberator_pcn_ic.date_received as timestamp) end as string) as unassigned_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then cast( liberator_pcn_ic.whenassigned as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as to_assigned_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then current_timestamp - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_in_progress_time, +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.whenassigned as timestamp) end as string) as assigned_response_time, +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then cast( liberator_pcn_ic.Response_generated_at as timestamp) - cast( liberator_pcn_ic.date_received as timestamp) end as string) as response_time, + +/*unassigned days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' then datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) ) end as string) as unassigned_days, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff(current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) > 56 Then '56 plus days' +end as unassigned_days_group +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as unassigned_days_kpiTotFiftySixLess +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as unassigned_days_kpiTotFourteenLess, + +/*Days to assign*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' then datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as Days_to_assign, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as Days_to_assign_group +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as Days_to_assign_kpiTotFiftySixLess +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.whenassigned != '' and (datediff( cast(substr( liberator_pcn_ic.whenassigned, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as Days_to_assign_kpiTotFourteenLess, + +/*assigned in progress days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' then datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) ) end as string) as assigned_in_progress_days, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 6 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 15 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) >= 48 AND (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) > 56 Then '56 plus days' +end as assigned_in_progress_days_group +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 56 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFiftySixLess +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at = '' and (datediff( current_timestamp, cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date) )) <= 14 Then 1 ELSE 0 END as assigned_in_progress_days_kpiTotFourteenLess, + +/*assigned response days*/ +cast(Case when liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date)) end as string) as assignedResponseDays, +Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) > 56 Then '56 plus days' +end as assignedResponseDays_group +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFiftySixLess +,Case When liberator_pcn_ic.whenassigned != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.whenassigned, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as assignedResponseDays_kpiTotFourteenLess, + +/*Response days*/ +cast(Case when liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' then datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date)) end as string) as ResponseDays, +Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 5 Then '5 or Less days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 6 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=14 Then '6 to 14 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 15 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=47 Then '15 to 47 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) >= 48 AND (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <=56 Then '48 to 56 days' +When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) > 56 Then '56 plus days' +end as ResponseDays_group +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 56 Then 1 ELSE 0 END as ResponseDays_kpiTotFiftySixLess +,Case When liberator_pcn_ic.date_received != '' AND liberator_pcn_ic.response_generated_at != '' and (datediff( cast(substr(liberator_pcn_ic.response_generated_at, 1, 10) as date), cast(substr(liberator_pcn_ic.date_received, 1, 10) as date))) <= 14 Then 1 ELSE 0 END as ResponseDays_kpiTotFourteenLess, + +Response_generated_at +,Date_Received +,concat(substr(Cast(liberator_pcn_ic.date_received as varchar(10)),1, 7), '-01') as MonthYear +,liberator_pcn_ic.Type +,Serviceable +,Service_category +,Response_written_by +,Letter_template +,Action_taken +,Related_to_PCN +,Cancellation_group +,Cancellation_reason +,whenassigned +,ticketserialnumber +,noderef +,substr(cast(liberator_pcn_ic.record_created as string), 1, 19) as record_created +,substr(cast(liberator_pcn_ic.import_timestamp as string), 1, 19) as import_timestamp +,substr(cast(liberator_pcn_ic.import_year as string), 1, 04) as import_year +,substr(cast(liberator_pcn_ic.import_month as string), 1, 02) as import_month +,liberator_pcn_ic.import_day +,substr(cast(liberator_pcn_ic.import_date as string), 1, 08) as import_date + +/*pcn data*/ +,pcnfoidetails_pcn_foi_full.pcn as pcn_pcn +,substr(cast(pcnfoidetails_pcn_foi_full.pcnissuedate as string), 1, 10) as pcn_pcnissuedate +,substr(cast(pcnfoidetails_pcn_foi_full.pcnissuedatetime as string), 1, 19) as pcn_pcnissuedatetime +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_canx_date as string), 1, 10) as pcn_pcn_canx_date +,pcnfoidetails_pcn_foi_full.cancellationgroup as pcn_cancellationgroup +,pcnfoidetails_pcn_foi_full.cancellationreason as pcn_cancellationreason +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_casecloseddate as string), 1, 10) as pcn_pcn_casecloseddate +,pcnfoidetails_pcn_foi_full.street_location as pcn_street_location +,pcnfoidetails_pcn_foi_full.whereonlocation as pcn_whereonlocation +,pcnfoidetails_pcn_foi_full.zone as pcn_zone +,pcnfoidetails_pcn_foi_full.usrn as pcn_usrn +,pcnfoidetails_pcn_foi_full.contraventioncode as pcn_contraventioncode +,pcnfoidetails_pcn_foi_full.contraventionsuffix as pcn_contraventionsuffix +,pcnfoidetails_pcn_foi_full.debttype as pcn_debttype +,pcnfoidetails_pcn_foi_full.vrm as pcn_vrm +,pcnfoidetails_pcn_foi_full.vehiclemake as pcn_vehiclemake +,pcnfoidetails_pcn_foi_full.vehiclemodel as pcn_vehiclemodel +,pcnfoidetails_pcn_foi_full.vehiclecolour as pcn_vehiclecolour +,pcnfoidetails_pcn_foi_full.ceo as pcn_ceo +,pcnfoidetails_pcn_foi_full.ceodevice as pcn_ceodevice +,pcnfoidetails_pcn_foi_full.current_30_day_flag as pcn_current_30_day_flag +,pcnfoidetails_pcn_foi_full.isvda as pcn_isvda +,pcnfoidetails_pcn_foi_full.isvoid as pcn_isvoid +,pcnfoidetails_pcn_foi_full.isremoval as pcn_isremoval +,pcnfoidetails_pcn_foi_full.driverseen as pcn_driverseen +,pcnfoidetails_pcn_foi_full.allwindows as pcn_allwindows +,pcnfoidetails_pcn_foi_full.parkedonfootway as pcn_parkedonfootway +,pcnfoidetails_pcn_foi_full.doctor as pcn_doctor +,pcnfoidetails_pcn_foi_full.warningflag as pcn_warningflag +,pcnfoidetails_pcn_foi_full.progressionstage as pcn_progressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstage as pcn_nextprogressionstage +,pcnfoidetails_pcn_foi_full.nextprogressionstagestarts as pcn_nextprogressionstagestarts +,pcnfoidetails_pcn_foi_full.holdreason as pcn_holdreason +,pcnfoidetails_pcn_foi_full.lib_initial_debt_amount as pcn_lib_initial_debt_amount +,pcnfoidetails_pcn_foi_full.lib_payment_received as pcn_lib_payment_received +,pcnfoidetails_pcn_foi_full.lib_write_off_amount as pcn_lib_write_off_amount +,pcnfoidetails_pcn_foi_full.lib_payment_void as pcn_lib_payment_void +,pcnfoidetails_pcn_foi_full.lib_payment_method as pcn_lib_payment_method +,pcnfoidetails_pcn_foi_full.lib_payment_ref as pcn_lib_payment_ref +,pcnfoidetails_pcn_foi_full.baliff_from as pcn_baliff_from +,pcnfoidetails_pcn_foi_full.bailiff_to as pcn_bailiff_to +,substr(cast(pcnfoidetails_pcn_foi_full.bailiff_processedon as string), 1, 19) as pcn_bailiff_processedon +,pcnfoidetails_pcn_foi_full.bailiff_redistributionreason as pcn_bailiff_redistributionreason +,pcnfoidetails_pcn_foi_full.bailiff as pcn_bailiff +,substr(cast(pcnfoidetails_pcn_foi_full.warrantissuedate as string), 1, 19) as pcn_warrantissuedate +,pcnfoidetails_pcn_foi_full.allocation as pcn_allocation +,substr(cast(pcnfoidetails_pcn_foi_full.eta_datenotified as string), 1, 19) as pcn_eta_datenotified +,substr(cast(pcnfoidetails_pcn_foi_full.eta_packsubmittedon as string), 1, 19) as pcn_eta_packsubmittedon +,substr(cast(pcnfoidetails_pcn_foi_full.eta_evidencedate as string), 1, 19) as pcn_eta_evidencedate +,substr(cast(pcnfoidetails_pcn_foi_full.eta_adjudicationdate as string), 1, 19) as pcn_eta_adjudicationdate +,pcnfoidetails_pcn_foi_full.eta_appealgrounds as pcn_eta_appealgrounds +,pcnfoidetails_pcn_foi_full.eta_decisionreceived as pcn_eta_decisionreceived +,pcnfoidetails_pcn_foi_full.eta_outcome as pcn_eta_outcome +,pcnfoidetails_pcn_foi_full.eta_packsubmittedby as pcn_eta_packsubmittedby +,pcnfoidetails_pcn_foi_full.cancelledby as pcn_cancelledby +,pcnfoidetails_pcn_foi_full.registered_keeper_address as pcn_registered_keeper_address +,pcnfoidetails_pcn_foi_full.current_ticket_address as pcn_current_ticket_address +,pcnfoidetails_pcn_foi_full.corresp_dispute_flag as pcn_corresp_dispute_flag +,pcnfoidetails_pcn_foi_full.keyworker_corresp_dispute_flag as pcn_keyworker_corresp_dispute_flag +,pcnfoidetails_pcn_foi_full.fin_year_flag as pcn_fin_year_flag +,pcnfoidetails_pcn_foi_full.fin_year as pcn_fin_year +,pcnfoidetails_pcn_foi_full.ticket_ref as pcn_ticket_ref +,substr(cast(pcnfoidetails_pcn_foi_full.nto_printed as string), 1, 19) as pcn_nto_printed +,substr(cast(pcnfoidetails_pcn_foi_full.appeal_accepted as string), 1, 19) as pcn_appeal_accepted +,substr(cast(pcnfoidetails_pcn_foi_full.arrived_in_pound as string), 1, 19) as pcn_arrived_in_pound +,substr(cast(pcnfoidetails_pcn_foi_full.cancellation_reversed as string), 1, 19) as pcn_cancellation_reversed +,substr(cast(pcnfoidetails_pcn_foi_full.cc_printed as string), 1, 19) as pcn_cc_printed +,substr(cast(pcnfoidetails_pcn_foi_full.drr as string), 1, 19) as pcn_drr +,substr(cast(pcnfoidetails_pcn_foi_full.en_printed as string), 1, 19) as pcn_en_printed +,substr(cast(pcnfoidetails_pcn_foi_full.hold_released as string), 1, 19) as pcn_hold_released +,substr(cast(pcnfoidetails_pcn_foi_full.dvla_response as string), 1, 19) as pcn_dvla_response +,substr(cast(pcnfoidetails_pcn_foi_full.dvla_request as string), 1, 19) as pcn_dvla_request +,substr(cast(pcnfoidetails_pcn_foi_full.full_rate_uplift as string), 1, 19) as pcn_full_rate_uplift +,substr(cast(pcnfoidetails_pcn_foi_full.hold_until as string), 1, 19) as pcn_hold_until +,substr(cast(pcnfoidetails_pcn_foi_full.lifted_at as string), 1, 19) as pcn_lifted_at +,substr(cast(pcnfoidetails_pcn_foi_full.lifted_by as string), 1, 19) as pcn_lifted_by +,substr(cast(pcnfoidetails_pcn_foi_full.loaded as string), 1, 19) as pcn_loaded +,substr(cast(pcnfoidetails_pcn_foi_full.nor_sent as string), 1, 19) as pcn_nor_sent +,substr(cast(pcnfoidetails_pcn_foi_full.notice_held as string), 1, 19) as pcn_notice_held +,substr(cast(pcnfoidetails_pcn_foi_full.ofr_printed as string), 1, 19) as pcn_ofr_printed +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_printed as string), 1, 19) as pcn_pcn_printed +,substr(cast(pcnfoidetails_pcn_foi_full.reissue_nto_requested as string), 1, 19) as pcn_reissue_nto_requested +,substr(cast(pcnfoidetails_pcn_foi_full.reissue_pcn as string), 1, 19) as pcn_reissue_pcn +,substr(cast(pcnfoidetails_pcn_foi_full.set_back_to_pre_cc_stage as string), 1, 19) as pcn_set_back_to_pre_cc_stage +,substr(cast(pcnfoidetails_pcn_foi_full.vehicle_released_for_auction as string), 1, 19) as pcn_vehicle_released_for_auction +,substr(cast(pcnfoidetails_pcn_foi_full.warrant_issued as string), 1, 19) as pcn_warrant_issued +,substr(cast(pcnfoidetails_pcn_foi_full.warrant_redistributed as string), 1, 19) as pcn_warrant_redistributed +,substr(cast(pcnfoidetails_pcn_foi_full.warrant_request_granted as string), 1, 19) as pcn_warrant_request_granted +,substr(cast(pcnfoidetails_pcn_foi_full.ad_hoc_vq4_request as string), 1, 19) as pcn_ad_hoc_vq4_request +,substr(cast(pcnfoidetails_pcn_foi_full.paper_vq5_received as string), 1, 19) as pcn_paper_vq5_received +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_buslane as string), 1, 19) as pcn_pcn_extracted_for_buslane +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_pre_debt as string), 1, 19) as pcn_pcn_extracted_for_pre_debt +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_collection as string), 1, 19) as pcn_pcn_extracted_for_collection +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_drr as string), 1, 19) as pcn_pcn_extracted_for_drr +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_cc as string), 1, 19) as pcn_pcn_extracted_for_cc +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_nto as string), 1, 19) as pcn_pcn_extracted_for_nto +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_print as string), 1, 19) as pcn_pcn_extracted_for_print +,substr(cast(pcnfoidetails_pcn_foi_full.warning_notice_extracted_for_print as string), 1, 19) as pcn_warning_notice_extracted_for_print +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_ofr as string), 1, 19) as pcn_pcn_extracted_for_ofr +,substr(cast(pcnfoidetails_pcn_foi_full.pcn_extracted_for_warrant_request as string), 1, 19) as pcn_pcn_extracted_for_warrant_request +,substr(cast(pcnfoidetails_pcn_foi_full.pre_debt_new_debtor_details as string), 1, 19) as pcn_pre_debt_new_debtor_details +,substr(cast(pcnfoidetails_pcn_foi_full.importdattime as string), 1, 19) as pcn_importdattime +,substr(cast(pcnfoidetails_pcn_foi_full.importdatetime as string), 1, 19) as pcn_importdatetime +,substr(cast(pcnfoidetails_pcn_foi_full.import_year as string), 1, 04) as pcn_import_year +,substr(cast(pcnfoidetails_pcn_foi_full.import_month as string), 1, 02) as pcn_import_month +,pcnfoidetails_pcn_foi_full.import_day as pcn_import_day +,substr(cast(pcnfoidetails_pcn_foi_full.import_date as string), 1, 08) as pcn_import_date + + +,concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) as link_officer_total_rep_dates +,concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) as link_month_total + +,total_response_month.mth_tot_rep_monthYear_response +,total_response_month.mth_tot_rep_total_dates +,total_response_month.mth_tot_rep_records + + +,officer_total_rep_dates.tot_rep_response_date +,officer_total_rep_dates.tot_rep_total_dates +,officer_total_rep_dates.tot_rep_records +,team.* + +from liberator_pcn_ic +left join pcnfoidetails_pcn_foi_full on liberator_pcn_ic.ticketserialnumber = pcnfoidetails_pcn_foi_full.pcn and pcnfoidetails_pcn_foi_full.import_date = liberator_pcn_ic.import_date + +left join officer_total_rep_dates on concat(liberator_pcn_ic.Response_written_by,substr(liberator_pcn_ic.response_generated_at, 1, 10) ) = officer_total_rep_dates.tot_rep_unique_link + +left join total_response_month on concat(liberator_pcn_ic.Response_written_by,concat(substr(Cast(liberator_pcn_ic.response_generated_at as varchar(10)),1, 7), '-01') ) = total_response_month.mth_tot_rep_unique_link + +left join team on upper(team.t_full_name) = upper(liberator_pcn_ic.Response_written_by) + +where liberator_pcn_ic.import_Date = (Select MAX(liberator_pcn_ic.import_date) from liberator_pcn_ic) +AND length(liberator_pcn_ic.ticketserialnumber) = 10 +AND cast(substr(liberator_pcn_ic.date_received, 1, 10) as date) > current_date - interval '13' month --Last 13 months from todays date + -SELECT - CAST(CASE - WHEN l.date_received <> '' AND l.whenassigned = '' - THEN 'Unassigned' - WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at = '' - THEN 'Assigned' - WHEN l.date_received <> '' AND l.whenassigned <> '' AND l.response_generated_at <> '' - THEN 'Responded' - --ELSE NULL - END AS VARCHAR) AS response_status, - - CAST(l.current_utc_timestamp AS VARCHAR) AS current_time_stamp, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS unassigned_time, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.whenassigned_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS to_assigned_time, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.current_utc_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_time, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.whenassigned_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS assigned_response_time, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - THEN 'INTERVAL ''' - || REGEXP_REPLACE(CAST(l.Response_generated_at_timestamp - l.date_received_timestamp AS VARCHAR), '(\.\d+)\s*$', '') --removes mantissa - || ''' DAY TO SECOND' - --ELSE NULL - END AS VARCHAR) AS response_time, - - /*unassigned days*/ - CAST(CASE WHEN l.whenassigned = '' - THEN l.days_since_date_received - --ELSE NULL - END AS VARCHAR) AS unassigned_days, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - THEN - CASE - WHEN l.days_since_date_received <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received > 56 THEN '56 plus days' - --WHEN l.days_since_date_received IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS unassigned_days_group, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - AND l.days_since_date_received <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS unassigned_days_kpiTotFiftySixLess, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned = '' - AND l.days_since_date_received <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS unassigned_days_kpiTotFourteenLess, - - /*Days to assign*/ - CAST(l.days_since_date_received_whenassigned AS VARCHAR) AS days_to_assign, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - THEN - CASE - WHEN l.days_since_date_received_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received_whenassigned > 56 THEN '56 plus days' - --WHEN l.days_since_date_received_whenassigned IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS days_to_assign_group, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - AND l.days_since_date_received_whenassigned <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS Days_to_assign_kpiTotFiftySixLess, - - CAST(CASE WHEN l.date_received <> '' AND l.whenassigned <> '' - AND l.days_since_date_received_whenassigned <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS Days_to_assign_kpiTotFourteenLess, - - /*assigned in progress days*/ - CAST(l.days_since_whenassigned AS VARCHAR) AS assigned_in_progress_days, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - THEN - CASE - WHEN l.days_since_whenassigned <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned > 56 THEN '56 plus days' - --WHEN l.days_since_whenassigned IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assigned_in_progress_days_group, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS assigned_in_progress_days_kpiTotFiftySixLess, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at = '' - AND l.days_since_whenassigned <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS assigned_in_progress_days_kpiTotFourteenLess, - - /*assigned response days*/ - CAST(l.days_since_whenassigned_response_generated_at AS VARCHAR) AS assignedResponseDays, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - THEN - CASE - WHEN l.days_since_whenassigned_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_whenassigned_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_whenassigned_response_generated_at > 56 THEN '56 plus days' - --WHEN l.days_since_whenassigned_response_generated_at IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS assignedResponseDays_group, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - AND l.days_since_whenassigned_response_generated_at <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS assignedResponseDays_kpiTotFiftySixLess, - - CAST(CASE WHEN l.whenassigned <> '' AND l.response_generated_at <> '' - AND l.days_since_whenassigned_response_generated_at <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS assignedResponseDays_kpiTotFourteenLess, - - /*Response days*/ - CAST(l.days_since_date_received_response_generated_at AS VARCHAR) AS ResponseDays, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - THEN - CASE - WHEN l.days_since_date_received_response_generated_at <= 5 THEN '5 or Less days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 6 AND 14 THEN '6 to 14 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 15 AND 47 THEN '15 to 47 days' - WHEN l.days_since_date_received_response_generated_at BETWEEN 48 AND 56 THEN '48 to 56 days' - WHEN l.days_since_date_received_response_generated_at > 56 THEN '56 plus days' - --WHEN l.days_since_date_received_response_generated_at IS NULL THEN NULL -- edge case examined - --ELSE NULL - END - --ELSE NULL - END AS VARCHAR) AS ResponseDays_group, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 56 - THEN 1 - ELSE 0 - END AS INTEGER) AS ResponseDays_kpiTotFiftySixLess, - - CAST(CASE WHEN l.date_received <> '' AND l.response_generated_at <> '' - AND l.days_since_date_received_response_generated_at <= 14 - THEN 1 - ELSE 0 - END AS INTEGER) AS ResponseDays_kpiTotFourteenLess, - - l.Response_generated_at, --AS VARCHAR - l.Date_Received, --AS VARCHAR - - CAST(SUBSTR(TRY_CAST(l.date_received AS VARCHAR(10)), 1, 7) || '-01' AS VARCHAR) AS MonthYear, - - l.Type AS "type", --AS VARCHAR - l.Serviceable, --AS VARCHAR - l.Service_category, --AS VARCHAR - l.Response_written_by, --AS VARCHAR - l.Letter_template, --AS VARCHAR - l.Action_taken, --AS VARCHAR - l.Related_to_PCN, --AS VARCHAR - l.Cancellation_group, --AS VARCHAR - l.Cancellation_reason, --AS VARCHAR - l.whenassigned, --AS VARCHAR - l.ticketserialnumber, --AS VARCHAR - l.noderef, --AS VARCHAR - - CAST(l.record_created AS VARCHAR) AS record_created, -- cast from VARCHAR see **NOTE** below - l.import_timestamp, --AS VARCHAR - - /*** pcn columns taken from "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" ***/ - -- Data sourced from the refined zone needs to be handled carefully... - -- Do not assume the following output column datatypes will be the same as "pcnfoidetails_pcn_foi_full". - -- Being left-joined to "pcnfoidetails_pcn_foi_full" will cause NULLs to appear in these columns. - -- And similar transforms elsewhere require different translations to the ones below... - /* These are the current automated translations developed for this transform:- - COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS DATE), '%Y-%m-%d'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try date to string - TRY_CAST(p.[source_column] AS INTEGER) AS [target_column], --[position]-- try int to int - TRY_CAST(p.[source_column] AS VARCHAR) AS [target_column], --[position]-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.[source_column] AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.[source_column] AS VARCHAR)) AS [target_column], --[position]-- try timestamp to string - TRY_CAST(p.[source_column] AS TIMESTAMP) AS [target_column], --[position]-- try timestamp to timestamp - */ - -- These translations try to deal with problems without resorting to throwing errors, tolerating a reasonable degree of schema evolution in the "pcnfoidetails_pcn_foi_full" source. - -- For example, date and timestamp string formatting works with either string, date and timestamp source columns and works like this... - /* If source can be cast to a date then it can be correctly date-formatted into a string, otherwise just cast whatever it was to a string. - COALESCE( - DATE_FORMAT(TRY_CAST(p.xxxx AS DATE), '%Y-%m-%d'), - CAST(p.xxxx AS VARCHAR) --...and can be extended to handle other date formats. - ) AS pcn_xxxx, -- from DATE - */ - /* If source can be cast to a timestamp then it can be correctly timestamp-formatted into a string, otherwise just cast whatever it was to a string. - COALESCE( - DATE_FORMAT(TRY_CAST(p.xxxx AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), - CAST(p.xxxx AS VARCHAR) -- Alternative for transforming a time-zoned timestamp... - --...COALESCE(DATE_FORMAT(TRY_CAST(TRY(From_iso8601_timestamp(p.xxxx)) AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.xxxx AS VARCHAR)) - --...and can be extended to handle other timestamp formats. - ) AS pcn_xxxx, -- from TIMESTAMP - */ - TRY_CAST(p.pcn AS VARCHAR) AS pcn_pcn, --45-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedate AS DATE), '%Y-%m-%d'), CAST(p.pcnissuedate AS VARCHAR)) AS pcn_pcnissuedate, --46-- try date to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcnissuedatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcnissuedatetime AS VARCHAR)) AS pcn_pcnissuedatetime, --47-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_canx_date AS DATE), '%Y-%m-%d'), CAST(p.pcn_canx_date AS VARCHAR)) AS pcn_pcn_canx_date, --48-- try date to string - TRY_CAST(p.cancellationgroup AS VARCHAR) AS pcn_cancellationgroup, --49-- try string to string (default) - TRY_CAST(p.cancellationreason AS VARCHAR) AS pcn_cancellationreason, --50-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_casecloseddate AS DATE), '%Y-%m-%d'), CAST(p.pcn_casecloseddate AS VARCHAR)) AS pcn_pcn_casecloseddate, --51-- try date to string - TRY_CAST(p.street_location AS VARCHAR) AS pcn_street_location, --52-- try string to string (default) - TRY_CAST(p.whereonlocation AS VARCHAR) AS pcn_whereonlocation, --53-- try string to string (default) - TRY_CAST(p.zone AS VARCHAR) AS pcn_zone, --54-- try string to string (default) - TRY_CAST(p.usrn AS VARCHAR) AS pcn_usrn, --55-- try string to string (default) - TRY_CAST(p.contraventioncode AS VARCHAR) AS pcn_contraventioncode, --56-- try string to string (default) - TRY_CAST(p.contraventionsuffix AS VARCHAR) AS pcn_contraventionsuffix, --57-- try string to string (default) - TRY_CAST(p.debttype AS VARCHAR) AS pcn_debttype, --58-- try string to string (default) - TRY_CAST(p.vrm AS VARCHAR) AS pcn_vrm, --59-- try string to string (default) - TRY_CAST(p.vehiclemake AS VARCHAR) AS pcn_vehiclemake, --60-- try string to string (default) - TRY_CAST(p.vehiclemodel AS VARCHAR) AS pcn_vehiclemodel, --61-- try string to string (default) - TRY_CAST(p.vehiclecolour AS VARCHAR) AS pcn_vehiclecolour, --62-- try string to string (default) - TRY_CAST(p.ceo AS VARCHAR) AS pcn_ceo, --63-- try string to string (default) - TRY_CAST(p.ceodevice AS VARCHAR) AS pcn_ceodevice, --64-- try string to string (default) - TRY_CAST(p.current_30_day_flag AS INTEGER) AS pcn_current_30_day_flag, --65-- try int to int - TRY_CAST(p.isvda AS INTEGER) AS pcn_isvda, --66-- try int to int - TRY_CAST(p.isvoid AS INTEGER) AS pcn_isvoid, --67-- try int to int - TRY_CAST(p.isremoval AS VARCHAR) AS pcn_isremoval, --68-- try string to string (default) - TRY_CAST(p.driverseen AS VARCHAR) AS pcn_driverseen, --69-- try string to string (default) - TRY_CAST(p.allwindows AS VARCHAR) AS pcn_allwindows, --70-- try string to string (default) - TRY_CAST(p.parkedonfootway AS VARCHAR) AS pcn_parkedonfootway, --71-- try string to string (default) - TRY_CAST(p.doctor AS VARCHAR) AS pcn_doctor, --72-- try string to string (default) - TRY_CAST(p.warningflag AS INTEGER) AS pcn_warningflag, --73-- try int to int - TRY_CAST(p.progressionstage AS VARCHAR) AS pcn_progressionstage, --74-- try string to string (default) - TRY_CAST(p.nextprogressionstage AS VARCHAR) AS pcn_nextprogressionstage, --75-- try string to string (default) - TRY_CAST(p.nextprogressionstagestarts AS VARCHAR) AS pcn_nextprogressionstagestarts, --76-- try string to string (default) - TRY_CAST(p.holdreason AS VARCHAR) AS pcn_holdreason, --77-- try string to string (default) - TRY_CAST(p.lib_initial_debt_amount AS VARCHAR) AS pcn_lib_initial_debt_amount, --78-- try string to string (default) - TRY_CAST(p.lib_payment_received AS VARCHAR) AS pcn_lib_payment_received, --79-- try string to string (default) - TRY_CAST(p.lib_write_off_amount AS VARCHAR) AS pcn_lib_write_off_amount, --80-- try string to string (default) - TRY_CAST(p.lib_payment_void AS VARCHAR) AS pcn_lib_payment_void, --81-- try string to string (default) - TRY_CAST(p.lib_payment_method AS VARCHAR) AS pcn_lib_payment_method, --82-- try string to string (default) - TRY_CAST(p.lib_payment_ref AS VARCHAR) AS pcn_lib_payment_ref, --83-- try string to string (default) - TRY_CAST(p.baliff_from AS VARCHAR) AS pcn_baliff_from, --84-- try string to string (default) - TRY_CAST(p.bailiff_to AS VARCHAR) AS pcn_bailiff_to, --85-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.bailiff_processedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.bailiff_processedon AS VARCHAR)) AS pcn_bailiff_processedon, --86-- try timestamp to string - TRY_CAST(p.bailiff_redistributionreason AS VARCHAR) AS pcn_bailiff_redistributionreason, --87-- try string to string (default) - TRY_CAST(p.bailiff AS VARCHAR) AS pcn_bailiff, --88-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.warrantissuedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrantissuedate AS VARCHAR)) AS pcn_warrantissuedate, --89-- try timestamp to string - TRY_CAST(p.allocation AS INTEGER) AS pcn_allocation, --90-- try int to int - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_datenotified AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_datenotified AS VARCHAR)) AS pcn_eta_datenotified, --91-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_packsubmittedon AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_packsubmittedon AS VARCHAR)) AS pcn_eta_packsubmittedon, --92-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_evidencedate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_evidencedate AS VARCHAR)) AS pcn_eta_evidencedate, --93-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.eta_adjudicationdate AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.eta_adjudicationdate AS VARCHAR)) AS pcn_eta_adjudicationdate, --94-- try timestamp to string - TRY_CAST(p.eta_appealgrounds AS VARCHAR) AS pcn_eta_appealgrounds, --95-- try string to string (default) - TRY_CAST(p.eta_decisionreceived AS TIMESTAMP) AS pcn_eta_decisionreceived, --96-- try timestamp to timestamp - TRY_CAST(p.eta_outcome AS VARCHAR) AS pcn_eta_outcome, --97-- try string to string (default) - TRY_CAST(p.eta_packsubmittedby AS VARCHAR) AS pcn_eta_packsubmittedby, --98-- try string to string (default) - TRY_CAST(p.cancelledby AS VARCHAR) AS pcn_cancelledby, --99-- try string to string (default) - TRY_CAST(p.registered_keeper_address AS VARCHAR) AS pcn_registered_keeper_address, --100-- try string to string (default) - TRY_CAST(p.current_ticket_address AS VARCHAR) AS pcn_current_ticket_address, --101-- try string to string (default) - TRY_CAST(p.corresp_dispute_flag AS INTEGER) AS pcn_corresp_dispute_flag, --102-- try int to int - TRY_CAST(p.keyworker_corresp_dispute_flag AS INTEGER) AS pcn_keyworker_corresp_dispute_flag, --103-- try int to int - TRY_CAST(p.fin_year_flag AS VARCHAR) AS pcn_fin_year_flag, --104-- try string to string (default) - TRY_CAST(p.fin_year AS VARCHAR) AS pcn_fin_year, --105-- try string to string (default) - TRY_CAST(p.ticket_ref AS VARCHAR) AS pcn_ticket_ref, --106-- try string to string (default) - COALESCE(DATE_FORMAT(TRY_CAST(p.nto_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nto_printed AS VARCHAR)) AS pcn_nto_printed, --107-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.appeal_accepted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.appeal_accepted AS VARCHAR)) AS pcn_appeal_accepted, --108-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.arrived_in_pound AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.arrived_in_pound AS VARCHAR)) AS pcn_arrived_in_pound, --109-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.cancellation_reversed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cancellation_reversed AS VARCHAR)) AS pcn_cancellation_reversed, --110-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.cc_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.cc_printed AS VARCHAR)) AS pcn_cc_printed, --111-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.drr AS VARCHAR)) AS pcn_drr, --112-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.en_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.en_printed AS VARCHAR)) AS pcn_en_printed, --113-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.hold_released AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_released AS VARCHAR)) AS pcn_hold_released, --114-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_response AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_response AS VARCHAR)) AS pcn_dvla_response, --115-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.dvla_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.dvla_request AS VARCHAR)) AS pcn_dvla_request, --116-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.full_rate_uplift AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.full_rate_uplift AS VARCHAR)) AS pcn_full_rate_uplift, --117-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.hold_until AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.hold_until AS VARCHAR)) AS pcn_hold_until, --118-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_at AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_at AS VARCHAR)) AS pcn_lifted_at, --119-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.lifted_by AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.lifted_by AS VARCHAR)) AS pcn_lifted_by, --120-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.loaded AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.loaded AS VARCHAR)) AS pcn_loaded, --121-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.nor_sent AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.nor_sent AS VARCHAR)) AS pcn_nor_sent, --122-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.notice_held AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.notice_held AS VARCHAR)) AS pcn_notice_held, --123-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.ofr_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ofr_printed AS VARCHAR)) AS pcn_ofr_printed, --124-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_printed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_printed AS VARCHAR)) AS pcn_pcn_printed, --125-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_nto_requested AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_nto_requested AS VARCHAR)) AS pcn_reissue_nto_requested, --126-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.reissue_pcn AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.reissue_pcn AS VARCHAR)) AS pcn_reissue_pcn, --127-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.set_back_to_pre_cc_stage AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.set_back_to_pre_cc_stage AS VARCHAR)) AS pcn_set_back_to_pre_cc_stage, --128-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.vehicle_released_for_auction AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.vehicle_released_for_auction AS VARCHAR)) AS pcn_vehicle_released_for_auction, --129-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_issued AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_issued AS VARCHAR)) AS pcn_warrant_issued, --130-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_redistributed AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_redistributed AS VARCHAR)) AS pcn_warrant_redistributed, --131-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warrant_request_granted AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warrant_request_granted AS VARCHAR)) AS pcn_warrant_request_granted, --132-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.ad_hoc_vq4_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.ad_hoc_vq4_request AS VARCHAR)) AS pcn_ad_hoc_vq4_request, --133-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.paper_vq5_received AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.paper_vq5_received AS VARCHAR)) AS pcn_paper_vq5_received, --134-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_buslane AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_buslane AS VARCHAR)) AS pcn_pcn_extracted_for_buslane, --135-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_pre_debt AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_pre_debt AS VARCHAR)) AS pcn_pcn_extracted_for_pre_debt, --136-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_collection AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_collection AS VARCHAR)) AS pcn_pcn_extracted_for_collection, --137-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_drr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_drr AS VARCHAR)) AS pcn_pcn_extracted_for_drr, --138-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_cc AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_cc AS VARCHAR)) AS pcn_pcn_extracted_for_cc, --139-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_nto AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_nto AS VARCHAR)) AS pcn_pcn_extracted_for_nto, --140-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_print AS VARCHAR)) AS pcn_pcn_extracted_for_print, --141-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.warning_notice_extracted_for_print AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.warning_notice_extracted_for_print AS VARCHAR)) AS pcn_warning_notice_extracted_for_print, --142-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_ofr AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_ofr AS VARCHAR)) AS pcn_pcn_extracted_for_ofr, --143-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pcn_extracted_for_warrant_request AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pcn_extracted_for_warrant_request AS VARCHAR)) AS pcn_pcn_extracted_for_warrant_request, --144-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.pre_debt_new_debtor_details AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.pre_debt_new_debtor_details AS VARCHAR)) AS pcn_pre_debt_new_debtor_details, --145-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.importdattime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdattime AS VARCHAR)) AS pcn_importdattime, --146-- try timestamp to string - COALESCE(DATE_FORMAT(TRY_CAST(p.importdatetime AS TIMESTAMP), '%Y-%m-%d %H:%i:%s'), CAST(p.importdatetime AS VARCHAR)) AS pcn_importdatetime, --147-- try timestamp to string - TRY_CAST(p.import_year AS VARCHAR) AS pcn_import_year, --148-- try string to string (default) - TRY_CAST(p.import_month AS VARCHAR) AS pcn_import_month, --149-- try string to string (default) - TRY_CAST(p.import_day AS VARCHAR) AS pcn_import_day, --150-- try string to string (default) - TRY_CAST(p.import_date AS VARCHAR) AS pcn_import_date, --151-- try string to string (default) - - /* Links to aggregations */ - CAST(l.tot_rep_unique_link AS VARCHAR) AS link_officer_total_rep_dates, - CAST(l.mth_tot_rep_unique_link AS VARCHAR) AS link_month_total, - - /* Monthly aggregations */ - CAST(m.mth_tot_rep_monthYear_response AS VARCHAR) AS mth_tot_rep_monthYear_response, - TRY_CAST(m.mth_tot_rep_total_dates AS BIGINT) AS mth_tot_rep_total_dates, - TRY_CAST(m.mth_tot_rep_records AS BIGINT) AS mth_tot_rep_records, - - /* Daily Aggregations */ - CAST(d.tot_rep_response_date AS VARCHAR) AS tot_rep_response_date, - TRY_CAST(d.tot_rep_total_dates AS BIGINT) AS tot_rep_total_dates, - TRY_CAST(d.tot_rep_records AS BIGINT) AS tot_rep_records, - - /* teams columns expanded by above subquery */ - team.*, - - /* Partition columns moved to the end to keep schema alligned */ - l.import_year, - l.import_month, - l.import_day, - l.import_date - -FROM liberator l -- < "dataplatform-prod-liberator-raw-zone"."liberator_pcn_ic" -LEFT JOIN "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" p -/* - ON p.import_date = l.import_date -- joined on import partition - Fails when "pcnfoidetails_pcn_foi_full" hasn't produced data for l.import_date yet... - ...resulting in empty pcn_ columns in the output. - Airflow's orchestration should avoid this after PCNFOIDetails_PCN_FOI_FULL.sql has been migrated... - ...but for now we'll just use the following simple workaround... -*/ - ON p.import_date IN ( - SELECT MAX(g.import_date) AS import_date - FROM "dataplatform-prod-liberator-refined-zone"."pcnfoidetails_pcn_foi_full" g - ) - AND l.ticketserialnumber = p.pcn -- ticketserialnumber is valid -LEFT JOIN officer_total_rep_dates d -- < Daily Aggregations of (l LEFT JOIN p) - ON l.tot_rep_unique_link = d.tot_rep_unique_link -LEFT JOIN total_response_month m -- < Monthly Aggregations of (l LEFT JOIN p) - ON l.mth_tot_rep_unique_link = m.mth_tot_rep_unique_link - LEFT JOIN team -- < "parking-raw-zone"."parking_correspondence_performance_teams" - ON UPPER(team.t_full_name) = UPPER(l.Response_written_by) -- Can this name match be improved? -; """ +SQL_node1658765472050 = sparkSqlQuery( + glueContext, + query=SqlQuery0, + mapping={ + "pcnfoidetails_pcn_foi_full": AmazonS3refinedpcnfoidetails_pcn_foi_full_node1657284942794, + "liberator_pcn_ic": S3bucketRawliberator_pcn_ic_node1, + "parking_correspondence_performance_teams": parking_raw_zoneparking_correspondence_performance_teams_node1682094127227, + }, + transformation_ctx="SQL_node1658765472050", +) -create_update_table_with_partition( - environment=environment, query_on_athena=query_on_athena, table_name=table_name +# Script generated for node Amazon S3 +AmazonS3_node1658765590649 = glueContext.getSink( + path="s3://dataplatform-"+environment+"-refined-zone/parking/liberator/parking_correspondence_performance_records_with_pcn_gds/", + connection_type="s3", + updateBehavior="UPDATE_IN_DATABASE", + partitionKeys=["import_year", "import_month", "import_day", "import_date"], + compression="snappy", + enableUpdateCatalog=True, + transformation_ctx="AmazonS3_node1658765590649", ) +AmazonS3_node1658765590649.setCatalogInfo( + catalogDatabase="dataplatform-"+environment+"-liberator-refined-zone", + catalogTableName="parking_correspondence_performance_records_with_pcn_gds", +) +AmazonS3_node1658765590649.setFormat("glueparquet") +AmazonS3_node1658765590649.writeFrame(SQL_node1658765472050) +job.commit() +