Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions terraform/etl/48-lambda-gov-notify-ingestion.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
locals {
govnotify_tables = ["notifications", "received_text_messages"]
govnotify_tables = ["notifications", "received_text_messages"]
create_govnotify_resource_count = local.is_live_environment ? 1 : 0
}

Expand Down Expand Up @@ -45,7 +45,7 @@ data "aws_iam_policy_document" "gov_notify_lambda_logs" {
"logs:CreateLogStream",
"logs:PutLogEvents",
]
effect = "Allow"
effect = "Allow"
resources = ["*"]
}
}
Expand All @@ -66,7 +66,7 @@ data "aws_iam_policy_document" "lambda_assume_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
type = "Service"
identifiers = ["lambda.amazonaws.com"]
}
}
Expand Down Expand Up @@ -135,8 +135,8 @@ resource "aws_iam_role_policy_attachment" "gov_notify_lambda_secret_access" {
# Define a IAM Policy Document for Glue StartCrawler Permissions:
data "aws_iam_policy_document" "gov_notify_glue_crawler" {
statement {
actions = ["glue:StartCrawler"]
effect = "Allow"
actions = ["glue:StartCrawler"]
effect = "Allow"
resources = ["*"]
}
}
Expand Down Expand Up @@ -190,7 +190,7 @@ resource "aws_cloudwatch_event_rule" "govnotify_housing_repairs_trigger_event" {
name = "${local.short_identifier_prefix}govnotify_housing_repairs_trigger_event"
description = "Trigger event for GovNotify Housing API ingestion"
schedule_expression = "cron(0 0 * * ? *)"
is_enabled = false
state = "DISABLED"
tags = module.tags.values
}

Expand All @@ -205,11 +205,11 @@ resource "aws_lambda_permission" "allow_cloudwatch_to_call_govnotify" {

# Create a CloudWatch Event Target to trigger the GovNotify Housing Repairs Lambda function.
resource "aws_cloudwatch_event_target" "govnotify_housing_repairs_trigger_event_target" {
count = local.create_govnotify_resource_count
rule = aws_cloudwatch_event_rule.govnotify_housing_repairs_trigger_event[0].name
target_id = "govnotify-housing-repairs-trigger-event-target"
arn = module.gov-notify-ingestion-housing-repairs[0].lambda_function_arn
input = <<EOF
count = local.create_govnotify_resource_count
rule = aws_cloudwatch_event_rule.govnotify_housing_repairs_trigger_event[0].name
target_id = "govnotify-housing-repairs-trigger-event-target"
arn = module.gov-notify-ingestion-housing-repairs[0].lambda_function_arn
input = <<EOF
{
"table_names": ${jsonencode(local.govnotify_tables)}
}
Expand All @@ -218,7 +218,7 @@ resource "aws_cloudwatch_event_target" "govnotify_housing_repairs_trigger_event_
}

resource "aws_glue_crawler" "govnotify_housing_repairs_landing_zone" {
for_each = {for idx, source in local.govnotify_tables : idx => source}
for_each = { for idx, source in local.govnotify_tables : idx => source }

database_name = "${local.identifier_prefix}-landing-zone-database"
name = "${local.short_identifier_prefix}GovNotify Housing Repairs Landing Zone ${each.value}"
Expand All @@ -239,7 +239,7 @@ resource "aws_glue_crawler" "govnotify_housing_repairs_landing_zone" {
}

resource "aws_glue_crawler" "govnotify_housing_repairs_raw_zone" {
for_each = {for idx, source in local.govnotify_tables : idx => source}
for_each = { for idx, source in local.govnotify_tables : idx => source }

database_name = module.department_housing_data_source.raw_zone_catalog_database_name
name = "${local.short_identifier_prefix}GovNotify Housing Repairs Raw Zone ${each.value}"
Expand All @@ -257,4 +257,4 @@ resource "aws_glue_crawler" "govnotify_housing_repairs_raw_zone" {
TableGroupingPolicy = "CombineCompatibleSchemas"
}
})
}
}
27 changes: 13 additions & 14 deletions terraform/etl/49-lambda-gov-notify-ingestion-customer-services.tf
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ data "aws_iam_policy_document" "lambda_assume_role_customer_services" {

data "aws_iam_policy_document" "customer_services_gov_notify_lambda_execution" {
statement {
effect = "Allow"
effect = "Allow"
actions = [
"lambda:InvokeFunction"
]
Expand Down Expand Up @@ -113,7 +113,7 @@ data "aws_iam_policy_document" "gov_notify_customer_services_lambda_secret_acces
actions = [
"secretsmanager:GetSecretValue",
]
effect = "Allow"
effect = "Allow"
resources = [
"arn:aws:secretsmanager:eu-west-2:${data.aws_caller_identity.data_platform.account_id}:secret:customer-services/gov-notify*"
]
Expand Down Expand Up @@ -168,7 +168,7 @@ module "gov-notify-ingestion-customer-services" {
lambda_source_dir = "../../lambdas/govnotify_api_ingestion_customer_services"
lambda_output_path = "../../lambdas/govnotify_api_ingestion_customer_services.zip"
runtime = "python3.9"
environment_variables = {
environment_variables = {

API_SECRET_NAME = "customer-services/gov-notify_live_api_key"
TARGET_S3_BUCKET_LANDING = module.landing_zone_data_source.bucket_id
Expand All @@ -190,7 +190,7 @@ resource "aws_cloudwatch_event_rule" "govnotify_customer_services_trigger_event"
name = "${local.short_identifier_prefix}govnotify_customer_services_trigger_event"
description = "Trigger event for Customer Services GovNotify API ingestion"
schedule_expression = "cron(0 0 * * ? *)"
is_enabled = local.is_production_environment ? true : false
state = local.is_production_environment ? "ENABLED" : "DISABLED"
tags = module.tags.values
}

Expand All @@ -205,11 +205,11 @@ resource "aws_lambda_permission" "allow_cloudwatch_to_call_govnotify_customer_se

# Create a CloudWatch Event Target to trigger the GovNotify Customer Services Lambda function.
resource "aws_cloudwatch_event_target" "govnotify_customer_services_trigger_event_target" {
count = local.create_govnotify_customer_services_resource_count
rule = aws_cloudwatch_event_rule.govnotify_customer_services_trigger_event[0].name
target_id = "govnotify-customer-services-trigger-event-target"
arn = module.gov-notify-ingestion-customer-services[0].lambda_function_arn
input = <<EOF
count = local.create_govnotify_customer_services_resource_count
rule = aws_cloudwatch_event_rule.govnotify_customer_services_trigger_event[0].name
target_id = "govnotify-customer-services-trigger-event-target"
arn = module.gov-notify-ingestion-customer-services[0].lambda_function_arn
input = <<EOF
{
"table_names": ${jsonencode(local.govnotify_tables)}
}
Expand All @@ -221,7 +221,7 @@ resource "aws_cloudwatch_event_target" "govnotify_customer_services_trigger_even
}

resource "aws_glue_crawler" "govnotify_customer_services_landing_zone" {
for_each = {for idx, source in local.govnotify_tables_customer_services : idx => source}
for_each = { for idx, source in local.govnotify_tables_customer_services : idx => source }

database_name = "${local.identifier_prefix}-landing-zone-database"
name = "${local.short_identifier_prefix}GovNotify Customer Services Landing Zone ${each.value}"
Expand All @@ -233,15 +233,15 @@ resource "aws_glue_crawler" "govnotify_customer_services_landing_zone" {
path = "s3://${module.landing_zone_data_source.bucket_id}/customer-services/govnotify/${each.value}/"
}
configuration = jsonencode({
Version = 1.0
Version = 1.0
Grouping = {
TableLevelConfiguration = 5
}
})
}

resource "aws_glue_crawler" "govnotify_customer_services_raw_zone" {
for_each = {for idx, source in local.govnotify_tables_customer_services : idx => source}
for_each = { for idx, source in local.govnotify_tables_customer_services : idx => source }

database_name = module.department_customer_services_data_source.raw_zone_catalog_database_name
name = "${local.short_identifier_prefix}GovNotify Customer Services Raw Zone ${each.value}"
Expand All @@ -253,10 +253,9 @@ resource "aws_glue_crawler" "govnotify_customer_services_raw_zone" {
path = "s3://${module.raw_zone_data_source.bucket_id}/customer-services/govnotify/${each.value}/"
}
configuration = jsonencode({
Version = 1.0
Version = 1.0
Grouping = {
TableLevelConfiguration = 4
}
})
}

Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ data "aws_iam_policy_document" "gov_notify_housing_communal_repairs_lambda_secre
actions = [
"secretsmanager:GetSecretValue",
]
effect = "Allow"
effect = "Allow"
resources = [
"arn:aws:secretsmanager:eu-west-2:${data.aws_caller_identity.data_platform.account_id}:secret:housing-lbh-communal-repairs/gov-notify*"
]
Expand Down Expand Up @@ -41,7 +41,7 @@ module "gov-notify-ingestion-housing-communal-repairs" {
lambda_output_path = "../../lambdas/govnotify_api_ingestion_housing_lbh_communal_repairs.zip"
runtime = "python3.9"
lambda_memory_size = 512
environment_variables = {
environment_variables = {

API_SECRET_NAME = "housing-lbh-communal-repairs/gov-notify_live_api_key"
TARGET_S3_BUCKET_LANDING = module.landing_zone_data_source.bucket_id
Expand All @@ -63,7 +63,7 @@ resource "aws_cloudwatch_event_rule" "govnotify_housing_lbh_communal_repairs_tri
description = "Trigger event for Housing LBH Communal Repairs GovNotify API ingestion"
tags = module.tags.values
schedule_expression = "cron(0 0 * * ? *)"
is_enabled = local.is_production_environment ? true : false
state = local.is_production_environment ? "ENABLED" : "DISABLED"
}

resource "aws_lambda_permission" "allow_cloudwatch_to_call_govnotify_housing_lbh_communal_repairs" {
Expand All @@ -75,11 +75,11 @@ resource "aws_lambda_permission" "allow_cloudwatch_to_call_govnotify_housing_lbh
}

resource "aws_cloudwatch_event_target" "govnotify_housing_lbh_communal_repairs_trigger_event_target" {
count = local.create_govnotify_resource_count
rule = aws_cloudwatch_event_rule.govnotify_housing_lbh_communal_repairs_trigger_event[0].name
target_id = "govnotify-housing-communal-repairs-event-target"
arn = module.gov-notify-ingestion-housing-communal-repairs[0].lambda_function_arn
input = <<EOF
count = local.create_govnotify_resource_count
rule = aws_cloudwatch_event_rule.govnotify_housing_lbh_communal_repairs_trigger_event[0].name
target_id = "govnotify-housing-communal-repairs-event-target"
arn = module.gov-notify-ingestion-housing-communal-repairs[0].lambda_function_arn
input = <<EOF
{
"table_names": ${jsonencode(local.govnotify_tables_housing_communal_repairs)}
}
Expand All @@ -91,7 +91,7 @@ resource "aws_cloudwatch_event_target" "govnotify_housing_lbh_communal_repairs_t
}

resource "aws_glue_crawler" "govnotify_housing_lbh_communal_repairs_landing_zone" {
for_each = {for idx, source in local.govnotify_tables_housing_communal_repairs : idx => source}
for_each = { for idx, source in local.govnotify_tables_housing_communal_repairs : idx => source }

database_name = "${local.identifier_prefix}-landing-zone-database"
name = "${local.short_identifier_prefix}GovNotify Housing LBH Communal Repairs Landing Zone ${each.value}"
Expand All @@ -103,15 +103,15 @@ resource "aws_glue_crawler" "govnotify_housing_lbh_communal_repairs_landing_zone
path = "s3://${module.landing_zone_data_source.bucket_id}/housing/govnotify/lbh_communal_repairs/${each.value}/"
}
configuration = jsonencode({
Version = 1.0
Version = 1.0
Grouping = {
TableLevelConfiguration = 5
}
})
}

resource "aws_glue_crawler" "govnotify_housing_lbh_communal_repairs_raw_zone" {
for_each = {for idx, source in local.govnotify_tables_housing_communal_repairs : idx => source}
for_each = { for idx, source in local.govnotify_tables_housing_communal_repairs : idx => source }

database_name = module.department_housing_data_source.raw_zone_catalog_database_name
name = "${local.short_identifier_prefix}GovNotify Housing LBH Communal Repairs Raw Zone ${each.value}"
Expand All @@ -123,10 +123,9 @@ resource "aws_glue_crawler" "govnotify_housing_lbh_communal_repairs_raw_zone" {
path = "s3://${module.raw_zone_data_source.bucket_id}/housing/govnotify/lbh_communal_repairs/${each.value}/"
}
configuration = jsonencode({
Version = 1.0
Version = 1.0
Grouping = {
TableLevelConfiguration = 5
}
})
}

2 changes: 1 addition & 1 deletion terraform/etl/50-aws-lambda-export-dynamodb-pitr.tf
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ resource "aws_cloudwatch_event_rule" "mtfh_export_trigger_event" {
name = "${local.short_identifier_prefix}mtfh-export-trigger-event"
description = "Trigger event for MTFH export"
schedule_expression = "cron(0 0 * * ? *)"
is_enabled = local.is_production_environment ? true : false
state = local.is_production_environment ? "ENABLED" : "DISABLED"
tags = module.tags.values
}

Expand Down
2 changes: 1 addition & 1 deletion terraform/modules/api-ingestion-lambda/10-lambda.tf
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ resource "aws_cloudwatch_event_rule" "run_lambda" {
name_prefix = "${var.lambda_name}-lambda-"
description = "Fires every day at "
schedule_expression = var.lambda_execution_cron_schedule
is_enabled = var.is_production_environment || !var.is_live_environment
state = (var.is_production_environment || !var.is_live_environment) ? "ENABLED" : "DISABLED"
}

resource "aws_cloudwatch_event_target" "run_lambda" {
Expand Down
2 changes: 1 addition & 1 deletion terraform/modules/copy-from-s3-to-s3/10-lambda.tf
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ resource "aws_cloudwatch_event_rule" "run_s3_copier_lambda_on_glue_job_success"
}
EOF

is_enabled = var.is_live_environment
state = var.is_live_environment ? "ENABLED" : "DISABLED"
}

resource "aws_cloudwatch_event_target" "run_s3_copier_lambda" {
Expand Down
22 changes: 11 additions & 11 deletions terraform/modules/g-drive-to-s3/10-lambda.tf
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ resource "aws_iam_role_policy_attachment" "g_drive_to_s3_copier_lambda" {

data "archive_file" "lambda" {
type = "zip"
source_dir = "../../lambdas/g_drive_to_s3"
source_dir = "../../lambdas/g_drive_to_s3"
output_path = "../../lambdas/g_drive_to_s3.zip"
depends_on = [null_resource.run_install_requirements]
}
Expand All @@ -110,11 +110,11 @@ resource "null_resource" "run_install_requirements" {
dir_sha1 = sha1(join("", [for f in fileset(path.module, "../../../lambdas/g_drive_to_s3/*") : filesha1("${path.module}/${f}")]))
}

# provisioner "local-exec" {
# interpreter = ["bash", "-c"]
# command = "make install-requirements"
# working_dir = "${path.module}/../../../lambdas/g_drive_to_s3/"
# }
# provisioner "local-exec" {
# interpreter = ["bash", "-c"]
# command = "make install-requirements"
# working_dir = "${path.module}/../../../lambdas/g_drive_to_s3/"
# }
}

resource "aws_s3_object" "g_drive_to_s3_copier_lambda" {
Expand All @@ -138,12 +138,12 @@ resource "aws_lambda_function" "g_drive_to_s3_copier_lambda" {
s3_bucket = var.lambda_artefact_storage_bucket
s3_key = aws_s3_object.g_drive_to_s3_copier_lambda.key
source_code_hash = data.archive_file.lambda.output_base64sha256
layers = [
layers = [
"arn:aws:lambda:eu-west-2:${data.aws_caller_identity.current.account_id}:layer:google-apis-layer:1",
"arn:aws:lambda:eu-west-2:${data.aws_caller_identity.current.account_id}:layer:urllib3-1-26-18-layer:1"
]
timeout = local.lambda_timeout
memory_size = local.lambda_memory_size
]
timeout = local.lambda_timeout
memory_size = local.lambda_memory_size

environment {
variables = {
Expand Down Expand Up @@ -175,7 +175,7 @@ resource "aws_cloudwatch_event_rule" "ingestion_schedule" {
name_prefix = "g-drive-to-s3-copier-schedule"
description = "Ingestion Schedule"
schedule_expression = var.ingestion_schedule
is_enabled = var.ingestion_schedule_enabled ? true : false
state = var.ingestion_schedule_enabled ? "ENABLED" : "DISABLED"
}

resource "aws_cloudwatch_event_target" "run_lambda" {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ resource "aws_cloudwatch_event_rule" "lambda" {
name = lower("${var.identifier_prefix}${var.lambda_name}")
description = "Event rule for triggering lambda ${var.lambda_name}"
event_pattern = var.cloudwatch_event_pattern
is_enabled = true
state = "ENABLED"

tags = var.tags
}
Expand Down
2 changes: 2 additions & 0 deletions terraform/modules/s3-bucket/10-s3-bucket.tf
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ resource "aws_s3_bucket_lifecycle_configuration" "bucket" {
id = "expire-older-objects"
status = "Enabled"

filter {}

expiration {
days = var.expire_objects_days
}
Expand Down
2 changes: 1 addition & 1 deletion terraform/modules/set-budget-limit-amount/10-lambda.tf
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ resource "aws_cloudwatch_event_rule" "run_lambda_to_update_budget_once_a_month"
name = "run_lambda_to_update_budget_once_a_month"
description = "triggers the budget update lambda once per month"
schedule_expression = "cron(0 0 1 * ? *)"
is_enabled = false
state = "DISABLED"
}

resource "aws_cloudwatch_event_target" "run_lambda_to_update_budget_once_a_month" {
Expand Down
Loading