NRL-1386 Add infra and config for prod dashboards

terraform/account-wide-infrastructure/dev/athena.tf

@@ -1,5 +1,7 @@
 module "dev-athena" {
+  count              = var.enable_reporting ? 1 : 0
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--dev"
   target_bucket_name = module.dev-glue.target_bucket_name
+  glue_database      = module.dev-glue.glue_database
 }

terraform/account-wide-infrastructure/dev/ec2.tf

@@ -1,4 +1,5 @@
 module "vpc" {
+  count                          = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source                         = "../modules/vpc"
   vpc_cidr_block                 = var.vpc_cidr_block
   enable_dns_hostnames           = var.enable_dns_hostnames
@@ -8,16 +9,17 @@ module "vpc" {
   name_prefix                    = "nhsd-nrlf--dev"
 }
 
-module "powerbi_gw_instance_v2" {
-  source             = "../modules/ec2"
-  use_custom_ami     = true
-  instance_type      = var.instance_type
-  name_prefix        = "nhsd-nrlf--dev-powerbi-gw-v2"
+module "powerbi_gw_instance" {
+  count              = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
+  source             = "../modules/powerbi-gw-ec2"
+  use_custom_ami     = var.use_powerbi_gw_custom_ami
+  instance_type      = var.powerbi_gw_instance_type
+  name_prefix        = "nhsd-nrlf--dev-powerbi-gw"
   target_bucket_arn  = module.dev-glue.target_bucket_arn
   glue_kms_key_arn   = module.dev-glue.aws_kms_key_arn
-  athena_kms_key_arn = module.dev-athena.kms_key_arn
-  athena_bucket_arn  = module.dev-athena.bucket_arn
+  athena_kms_key_arn = module.dev-athena[0].kms_key_arn
+  athena_bucket_arn  = module.dev-athena[0].bucket_arn
 
-  subnet_id       = module.vpc.private_subnet_id
-  security_groups = [module.vpc.powerbi_gw_security_group_id]
+  subnet_id       = module.vpc[0].private_subnet_id
+  security_groups = [module.vpc[0].powerbi_gw_security_group_id]
 }

terraform/account-wide-infrastructure/dev/glue.tf

@@ -1,4 +1,5 @@
 module "dev-glue" {
+  is_enabled     = var.enable_reporting
   source         = "../modules/glue"
   name_prefix    = "nhsd-nrlf--dev"
   python_version = 3

terraform/account-wide-infrastructure/dev/vars.tf

@@ -14,6 +14,12 @@ variable "devsandbox_api_domain_name" {
   default     = "dev-sandbox.api.record-locator.dev.national.nhs.uk"
 }
 
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable account-wide reporting services in the dev account"
+  default     = false
+}
+
 variable "aws_azs" {
   type        = string
   description = "AWS Availability Zones"
@@ -44,14 +50,20 @@ variable "vpc_private_subnets_cidr_block" {
   default     = "10.0.1.0/24"
 }
 
-variable "instance_type" {
+variable "enable_powerbi_auto_push" {
+  type        = bool
+  description = "Enable automatic pushing of info into PowerBI"
+  default     = true
+}
+
+variable "powerbi_gw_instance_type" {
   type        = string
-  description = "Type for EC2 Instance"
+  description = "Type for PowerBI GW EC2 Instance"
   default     = "t2.micro"
 }
 
-variable "use_custom_ami" {
+variable "use_powerbi_gw_custom_ami" {
   type        = bool
-  description = "Use custom image"
-  default     = false
+  description = "Use custom image for PowerBI GW instance"
+  default     = true
 }

terraform/account-wide-infrastructure/modules/athena/athena.tf

@@ -16,3 +16,17 @@ resource "aws_athena_workgroup" "athena" {
   }
 
 }
+
+resource "aws_athena_named_query" "rep_consumer" {
+  name      = "rep_consumer"
+  workgroup = aws_athena_workgroup.athena.id
+  database  = var.glue_database
+  query     = file("${path.module}/sql/rep_consumer.sql")
+}
+
+resource "aws_athena_named_query" "rep_producer" {
+  name      = "rep_producer"
+  workgroup = aws_athena_workgroup.athena.id
+  database  = var.glue_database
+  query     = file("${path.module}/sql/rep_producer.sql")
+}

terraform/account-wide-infrastructure/modules/athena/sql/rep_consumer.sql

@@ -0,0 +1,123 @@
+CREATE OR REPLACE VIEW "rep_consumer" AS
+WITH
+  cc AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , '' event_custodian
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_countdocumentreference
+)
+, cr AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , event_custodian
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_readdocumentreference
+)
+, cs AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , event_custodian
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_searchdocumentreference
+)
+, csp AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , event_custodian
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_searchpostdocumentreference
+)
+, base AS (
+   SELECT *
+   FROM
+     cc
+UNION    SELECT *
+   FROM
+     cr
+UNION    SELECT *
+   FROM
+     cs
+UNION    SELECT *
+   FROM
+     csp
+)
+, ods_codes AS (
+   SELECT DISTINCT
+     user_ods
+   , event_xray_trace_id
+   FROM
+     base
+   WHERE (user_ods IS NOT NULL)
+)
+SELECT
+  time
+, event_timestamp
+, date
+, host
+, event_log_reference
+, event_level
+, event_location
+, event_message
+, event_service
+, event_function_request_id
+, b.event_correlation_id
+, b.event_xray_trace_id
+, event_pointer_types
+, COALESCE(COALESCE(event_custodian, LAG(event_custodian) IGNORE NULLS OVER (PARTITION BY b.event_xray_trace_id ORDER BY event_timestamp ASC)), COALESCE(event_custodian, LEAD(event_custodian) IGNORE NULLS OVER (PARTITION BY b.event_xray_trace_id ORDER BY event_timestamp ASC))) event_custodian
+, oc.user_ods
+FROM
+  (base b
+LEFT JOIN ods_codes oc ON (b.event_xray_trace_id = oc.event_xray_trace_id))