From 2b681d5edf761a4386d82aee7fa2ae96851a4eba Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Thu, 14 Aug 2025 16:32:28 +0100 Subject: [PATCH 1/5] Add template name --- ...resource_request_item_plan_status_table.tf | 16 +++++++++++++++ .../ingestion/request_item_plan_status.sql | 20 +++++++++++++++++++ .../sql/tables/request_item_plan_status.sql | 1 + 3 files changed, 37 insertions(+) diff --git a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf index 594d05e2..0d392693 100644 --- a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf +++ b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf @@ -60,3 +60,19 @@ resource "null_resource" "request_item_plan_status_recipientcontactid_column" { depends_on = [null_resource.request_item_plan_status_ordernumber_column] } + +resource "null_resource" "request_item_plan_status_templatename_column" { + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = <), + (k, v) -> UPPER(k) = UPPER(supplier) + ) + ), + ARRAY[ROW(NULL, NULL)] + ) + END + ) AS t(templatekv) WHERE (sk LIKE 'REQUEST_ITEM_PLAN#%') AND ( -- Moving 1-week ingestion window @@ -63,6 +80,7 @@ WHEN MATCHED AND (source.timestamp > target.timestamp) THEN UPDATE SET channeltype = source.channeltype, ordernumber = source.ordernumber, recipientcontactid = source.recipientcontactid, + templatename = source.templatename, timestamp = source.timestamp WHEN NOT MATCHED THEN INSERT ( clientid, @@ -85,6 +103,7 @@ WHEN NOT MATCHED THEN INSERT ( channeltype, ordernumber, recipientcontactid, + templatename, timestamp ) VALUES ( @@ -108,5 +127,6 @@ VALUES ( source.channeltype, source.ordernumber, source.recipientcontactid, + source.templatename, source.timestamp ) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql index b8167ea3..1daf0d2d 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql @@ -19,6 +19,7 @@ CREATE TABLE IF NOT EXISTS ${table_name} ( channeltype string, ordernumber int, recipientcontactid string, + templatename string, timestamp bigint ) PARTITIONED BY (bucket(32, clientid), month(createdtime), month(completedtime)) From f9214950f9a4ddfb6bea48e84131d88211f0ff68 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Fri, 15 Aug 2025 09:43:40 +0100 Subject: [PATCH 2/5] Add templatename to reports --- .../reporting/scripts/sql/reports/completed_batch_report.sql | 3 ++- .../reporting/scripts/sql/reports/completed_comms_report.sql | 3 ++- .../components/reporting/scripts/sql/views/completed_comms.sql | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql index 1dcd8cfa..95806996 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql @@ -15,6 +15,7 @@ SELECT requestitemplanstatus, communicationtype, channeltype, - requestitemplanfailedreason + requestitemplanfailedreason, + templatename FROM completed_comms WHERE clientid = ? AND requestid = ? diff --git a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql index 112aebac..a507b96b 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql @@ -15,7 +15,8 @@ SELECT requestitemplanstatus, communicationtype, channeltype, - requestitemplanfailedreason + requestitemplanfailedreason, + templatename FROM completed_comms WHERE clientid = ? AND diff --git a/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql b/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql index 47fc3b96..7ed02224 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql @@ -17,7 +17,8 @@ SELECT rip.status as requestitemplanstatus, rip.communicationtype as communicationtype, rip.channeltype as channeltype, - rip.failedreason as requestitemplanfailedreason + rip.failedreason as requestitemplanfailedreason, + rip.templatename as templatename FROM request_item_status ri LEFT OUTER JOIN request_item_plan_status rip ON ri.requestitemid = rip.requestitemid AND From 42d371826cd78ed12954dbaedddc603f57ac962c Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Fri, 15 Aug 2025 10:10:37 +0100 Subject: [PATCH 3/5] Change column name to templateid --- .../null_resource_request_item_plan_status_table.tf | 4 ++-- .../scripts/sql/ingestion/request_item_plan_status.sql | 8 ++++---- .../scripts/sql/reports/completed_batch_report.sql | 2 +- .../scripts/sql/reports/completed_comms_report.sql | 2 +- .../scripts/sql/tables/request_item_plan_status.sql | 2 +- .../reporting/scripts/sql/views/completed_comms.sql | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf index 0d392693..994bbd77 100644 --- a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf +++ b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf @@ -61,7 +61,7 @@ resource "null_resource" "request_item_plan_status_recipientcontactid_column" { depends_on = [null_resource.request_item_plan_status_ordernumber_column] } -resource "null_resource" "request_item_plan_status_templatename_column" { +resource "null_resource" "request_item_plan_status_templateid_column" { triggers = { always_run = timestamp() } @@ -70,7 +70,7 @@ resource "null_resource" "request_item_plan_status_templatename_column" { ${path.module}/scripts/add_column.sh \ ${aws_athena_workgroup.setup.name} \ ${aws_glue_catalog_database.reporting.name} \ - request_item_plan_status templatename string + request_item_plan_status templateid string EOT } diff --git a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql index d1bc5267..0661db3f 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql @@ -30,7 +30,7 @@ USING ( channeltype, ordernumber, recipientcontactid, - templatekv[2] AS templatename, + templatekv[2] AS templateid, CAST("$classification".timestamp AS BIGINT) AS timestamp FROM ${source_table} --CROSS JOIN needed to unpack template name from struct @@ -80,7 +80,7 @@ WHEN MATCHED AND (source.timestamp > target.timestamp) THEN UPDATE SET channeltype = source.channeltype, ordernumber = source.ordernumber, recipientcontactid = source.recipientcontactid, - templatename = source.templatename, + templateid = source.templateid, timestamp = source.timestamp WHEN NOT MATCHED THEN INSERT ( clientid, @@ -103,7 +103,7 @@ WHEN NOT MATCHED THEN INSERT ( channeltype, ordernumber, recipientcontactid, - templatename, + templateid, timestamp ) VALUES ( @@ -127,6 +127,6 @@ VALUES ( source.channeltype, source.ordernumber, source.recipientcontactid, - source.templatename, + source.templateid, source.timestamp ) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql index 95806996..a6420042 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_batch_report.sql @@ -16,6 +16,6 @@ SELECT communicationtype, channeltype, requestitemplanfailedreason, - templatename + templateid FROM completed_comms WHERE clientid = ? AND requestid = ? diff --git a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql index a507b96b..0cb6798b 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/reports/completed_comms_report.sql @@ -16,7 +16,7 @@ SELECT communicationtype, channeltype, requestitemplanfailedreason, - templatename + templateid FROM completed_comms WHERE clientid = ? AND diff --git a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql index 1daf0d2d..751aa1c0 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS ${table_name} ( channeltype string, ordernumber int, recipientcontactid string, - templatename string, + templateid string, timestamp bigint ) PARTITIONED BY (bucket(32, clientid), month(createdtime), month(completedtime)) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql b/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql index 7ed02224..ae1e2178 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/views/completed_comms.sql @@ -18,7 +18,7 @@ SELECT rip.communicationtype as communicationtype, rip.channeltype as channeltype, rip.failedreason as requestitemplanfailedreason, - rip.templatename as templatename + rip.templateid as templateid FROM request_item_status ri LEFT OUTER JOIN request_item_plan_status rip ON ri.requestitemid = rip.requestitemid AND From a0413e7f16a9df8642f4d0cca3078dbc723d44e0 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 18 Aug 2025 12:54:18 +0100 Subject: [PATCH 4/5] SQL simplification suggestion from Chris --- .../sql/ingestion/request_item_plan_status.sql | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql index 0661db3f..5cc6d907 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql @@ -30,25 +30,9 @@ USING ( channeltype, ordernumber, recipientcontactid, - templatekv[2] AS templateid, + CAST(CAST(th.templates.suppliers AS json) AS map)[LOWER(supplier)] AS templateid, CAST("$classification".timestamp AS BIGINT) AS timestamp FROM ${source_table} - --CROSS JOIN needed to unpack template name from struct - CROSS JOIN UNNEST( - CASE - WHEN supplier IS NULL THEN ARRAY[ROW(NULL, NULL)] - ELSE - COALESCE( - MAP_ENTRIES( - MAP_FILTER( - CAST(CAST(templates.suppliers AS json) AS map), - (k, v) -> UPPER(k) = UPPER(supplier) - ) - ), - ARRAY[ROW(NULL, NULL)] - ) - END - ) AS t(templatekv) WHERE (sk LIKE 'REQUEST_ITEM_PLAN#%') AND ( -- Moving 1-week ingestion window From 6bc16cdcf7f2e10de7a13f7ccae406f59fd0b0d4 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 18 Aug 2025 13:04:59 +0100 Subject: [PATCH 5/5] Bug fix --- .../scripts/sql/ingestion/request_item_plan_status.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql index 5cc6d907..70ceec45 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql @@ -30,7 +30,7 @@ USING ( channeltype, ordernumber, recipientcontactid, - CAST(CAST(th.templates.suppliers AS json) AS map)[LOWER(supplier)] AS templateid, + CAST(CAST(templates.suppliers AS json) AS map)[LOWER(supplier)] AS templateid, CAST("$classification".timestamp AS BIGINT) AS timestamp FROM ${source_table} WHERE (sk LIKE 'REQUEST_ITEM_PLAN#%') AND