VED-1098 - MNS event automation tests

.github/workflows/quality-checks.yml

@@ -173,6 +173,17 @@ jobs:
           poetry run coverage run --source=src -m unittest discover || echo "mesh_processor tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../mesh_processor-coverage.xml
 
+      - name: Run unittest with mns_publisher
+        working-directory: lambdas/mns_publisher
+        id: mnspublisher
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/mns_publisher/src:${{ env.LAMBDA_PATH }}/mns_publisher/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "mns_publisher tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../mns_publisher-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription

.github/workflows/run-e2e-automation-tests.yml

@@ -234,7 +234,7 @@ jobs:
           SONAR_client_Secret: ${{ secrets.SONAR_client_Secret }}
           MEDICUS_client_Id: ${{ secrets.MEDICUS_client_Id }}
           MEDICUS_client_Secret: ${{ secrets.MEDICUS_client_Secret }}
-          aws_token_refresh: "False"
+          aws_account_id: ${{ vars.AWS_ACCOUNT_ID }}
           TEST_PATH: ${{ inputs.service_under_test == 'batch' && 'features/batchTests' || inputs.service_under_test == 'fhir_api' && 'features/APITests' || 'features' }}
           TEST_FILTER: ${{ inputs.suite_to_run == 'proxy_smoke' && 'Status_feature' || inputs.suite_to_run }}
         run: poetry run pytest "$TEST_PATH" -m "$TEST_FILTER" --junitxml=output/test-results.xml --alluredir=output/allure-results

infrastructure/instance/dynamodb.tf

@@ -68,6 +68,8 @@ resource "aws_dynamodb_table" "delta-dynamodb-table" {
   name                        = "imms-${local.resource_scope}-delta"
   billing_mode                = "PAY_PER_REQUEST"
   hash_key                    = "PK"
+  stream_enabled              = true
+  stream_view_type            = "NEW_IMAGE"
   deletion_protection_enabled = !local.is_temp
 
   attribute {

infrastructure/instance/environments/dev/internal-dev/variables.tfvars

@@ -2,6 +2,7 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
+mns_environment                   = "dev"
 error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true

infrastructure/instance/environments/dev/internal-qa/variables.tfvars

@@ -2,6 +2,7 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
+mns_environment                   = "dev"
 error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true

infrastructure/instance/environments/dev/pr/variables.tfvars

@@ -2,6 +2,7 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
+mns_environment                   = "dev"
 error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true

infrastructure/instance/environments/dev/ref/variables.tfvars

@@ -2,6 +2,7 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "ref"
+mns_environment                   = "dev"
 error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true

infrastructure/instance/environments/preprod/int-blue/variables.tfvars

@@ -2,6 +2,7 @@ environment                       = "preprod"
 immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
+mns_environment                   = "int"
 error_alarm_notifications_enabled = true
 
 # mesh no invocation period metric set to 3 days (in seconds) for preprod environment i.e 3 * 24 * 60 * 60

infrastructure/instance/environments/preprod/int-green/variables.tfvars

@@ -2,6 +2,7 @@ environment                       = "preprod"
 immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
+mns_environment                   = "int"
 error_alarm_notifications_enabled = true
 
 # mesh no invocation period metric set to 3 days (in seconds) for preprod environment i.e 3 * 24 * 60 * 60

infrastructure/instance/environments/prod/blue/variables.tfvars

@@ -3,6 +3,7 @@ immunisation_account_id           = "664418956997"
 dspp_core_account_id              = "232116723729"
 mns_account_id                    = "758334270304"
 pds_environment                   = "prod"
+mns_environment                   = "prod"
 error_alarm_notifications_enabled = true
 
 # mesh no invocation period metric set to 1 day (in seconds) for prod environment i.e 1 * 24 * 60 * 60

infrastructure/instance/environments/prod/green/variables.tfvars

@@ -3,6 +3,7 @@ immunisation_account_id           = "664418956997"
 dspp_core_account_id              = "232116723729"
 mns_account_id                    = "758334270304"
 pds_environment                   = "prod"
+mns_environment                   = "prod"
 error_alarm_notifications_enabled = true
 
 # mesh no invocation period metric set to 1 day (in seconds) for prod environment i.e 1 * 24 * 60 * 60

infrastructure/instance/mns_publisher.tf

@@ -0,0 +1,29 @@
+module "mns_publisher" {
+  source = "./modules/mns_publisher"
+
+  ddb_delta_stream_arn               = aws_dynamodb_table.delta-dynamodb-table.stream_arn
+  dynamo_kms_encryption_key_arn      = data.aws_kms_key.existing_dynamo_encryption_key.arn
+  enable_lambda_alarm                = var.error_alarm_notifications_enabled
+  immunisation_account_id            = var.immunisation_account_id
+  is_temp                            = local.is_temp
+  enable_mns_test_queue              = var.mns_environment == "dev"
+  resource_scope                     = local.resource_scope
+  imms_base_path                     = strcontains(var.sub_environment, "pr-") ? "immunisation-fhir-api/FHIR/R4-${var.sub_environment}" : "immunisation-fhir-api/FHIR/R4"
+  lambda_kms_encryption_key_arn      = data.aws_kms_key.existing_lambda_encryption_key.arn
+  mns_publisher_resource_name_prefix = "${local.resource_scope}-mns-outbound-events"
+  mns_test_notification_name_prefix  = "${local.resource_scope}-mns-test-notification"
+  secrets_manager_policy_path        = "${local.policy_path}/secret_manager.json"
+  account_id                         = data.aws_caller_identity.current.account_id
+  pds_environment                    = var.pds_environment
+  mns_environment                    = var.mns_environment
+
+  private_subnet_ids = local.private_subnet_ids
+  security_group_id  = data.aws_security_group.existing_securitygroup.id
+
+  shared_dir_sha              = local.shared_dir_sha
+  splunk_firehose_stream_name = module.splunk.firehose_stream_name
+
+  short_prefix = local.short_prefix
+
+  system_alarm_sns_topic_arn = data.aws_sns_topic.imms_system_alert_errors.arn
+}

infrastructure/instance/modules/mns_publisher/mns_outbound_events_eb_pipe.tf

@@ -0,0 +1,118 @@
+# IAM Role for EventBridge Pipe
+resource "aws_iam_role" "mns_outbound_events_eb_pipe" {
+  name = "${var.mns_publisher_resource_name_prefix}-eventbridge-pipe-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "pipes.amazonaws.com"
+        }
+        Condition = {
+          StringEquals = {
+            "aws:SourceAccount" = var.immunisation_account_id
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "mns_outbound_events_eb_pipe_source_policy" {
+  role = aws_iam_role.mns_outbound_events_eb_pipe.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "dynamodb:DescribeStream",
+          "dynamodb:GetRecords",
+          "dynamodb:GetShardIterator",
+          "dynamodb:ListStreams"
+        ],
+        "Resource" : var.ddb_delta_stream_arn
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "kms:Decrypt",
+          "kms:GenerateDataKey"
+        ],
+        "Resource" : var.dynamo_kms_encryption_key_arn
+      },
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "mns_outbound_events_eb_pipe_target_policy" {
+  role = aws_iam_role.mns_outbound_events_eb_pipe.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "sqs:GetQueueAttributes",
+          "sqs:SendMessage",
+        ],
+        Resource = [
+          aws_sqs_queue.mns_outbound_events.arn,
+        ]
+      },
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "mns_outbound_events_eb_pipe_cw_log_policy" {
+  role = aws_iam_role.mns_outbound_events_eb_pipe.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "logs:CreateLogGroup",
+          "logs:CreateLogStream",
+          "logs:PutLogEvents"
+        ],
+        Resource = [
+          "arn:aws:logs:${var.aws_region}:${var.immunisation_account_id}:log-group:/aws/vendedlogs/pipes/${var.mns_publisher_resource_name_prefix}-pipe-logs:*",
+        ]
+      },
+    ]
+  })
+}
+
+resource "aws_cloudwatch_log_group" "mns_outbound_events_eb_pipe" {
+  name              = "/aws/vendedlogs/pipes/${var.mns_publisher_resource_name_prefix}-pipe-logs"
+  retention_in_days = 30
+}
+
+resource "aws_pipes_pipe" "mns_outbound_events" {
+  depends_on = [
+    aws_iam_role_policy.mns_outbound_events_eb_pipe_source_policy,
+    aws_iam_role_policy.mns_outbound_events_eb_pipe_target_policy,
+    aws_iam_role_policy.mns_outbound_events_eb_pipe_cw_log_policy,
+  ]
+  name     = "${var.mns_publisher_resource_name_prefix}-pipe"
+  role_arn = aws_iam_role.mns_outbound_events_eb_pipe.arn
+  source   = var.ddb_delta_stream_arn
+  target   = aws_sqs_queue.mns_outbound_events.arn
+
+  source_parameters {
+    dynamodb_stream_parameters {
+      starting_position = "TRIM_HORIZON"
+    }
+  }
+
+  log_configuration {
+    include_execution_data = ["ALL"]
+    level                  = "ERROR"
+    cloudwatch_logs_log_destination {
+      log_group_arn = aws_cloudwatch_log_group.mns_outbound_events_eb_pipe.arn
+    }
+  }
+}

infrastructure/instance/modules/mns_publisher/mns_outbound_events_kms_key.tf

@@ -0,0 +1,87 @@
+resource "aws_kms_key" "mns_outbound_events" {
+  description         = "KMS key for encrypting MNS outbound immunisation events in SQS"
+  key_usage           = "ENCRYPT_DECRYPT"
+  enable_key_rotation = true
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "EnableRootPermissions"
+        Effect = "Allow"
+        Principal = {
+          AWS = "arn:aws:iam::${var.immunisation_account_id}:root"
+        },
+        Action = [
+          "kms:Create*",
+          "kms:Describe*",
+          "kms:Enable*",
+          "kms:List*",
+          "kms:Put*",
+          "kms:Update*",
+          "kms:Revoke*",
+          "kms:Disable*",
+          "kms:Get*",
+          "kms:Delete*",
+          "kms:ScheduleKeyDeletion",
+          "kms:CancelKeyDeletion",
+          "kms:GenerateDataKey*",
+          "kms:Decrypt",
+          "kms:Tag*"
+        ],
+        Resource = "*"
+      },
+      {
+        Sid    = "AllowSQSUseOfKey"
+        Effect = "Allow"
+        Principal = {
+          Service = "sqs.amazonaws.com"
+        }
+        Action = [
+          "kms:GenerateDataKey",
+          "kms:Decrypt"
+        ]
+        Resource = "*"
+        Condition = {
+          StringEquals = {
+            "kms:EncryptionContext:aws:sqs:queue_arn" = [
+              "arn:aws:sqs:${var.aws_region}:${var.immunisation_account_id}:${var.mns_publisher_resource_name_prefix}-queue",
+              "arn:aws:sqs:${var.aws_region}:${var.immunisation_account_id}:${var.mns_publisher_resource_name_prefix}-dead-letter-queue"
+            ]
+          }
+        }
+      },
+      {
+        Sid    = "AllowLambdaToDecrypt"
+        Effect = "Allow"
+        Principal = {
+          AWS = "arn:aws:iam::${var.immunisation_account_id}:role/${var.short_prefix}-mns-publisher-lambda-exec-role"
+        }
+        Action = [
+          "kms:Decrypt",
+          "kms:GenerateDataKey"
+        ]
+        Resource = "*"
+      },
+      {
+        Sid    = "AllowEventBridgePipesUseOfKey"
+        Effect = "Allow"
+        Principal = {
+          AWS = "arn:aws:iam::${var.immunisation_account_id}:role/${var.mns_publisher_resource_name_prefix}-eventbridge-pipe-role"
+        }
+        Action = [
+          "kms:GenerateDataKey",
+          "kms:Encrypt",
+          "kms:DescribeKey"
+
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_kms_alias" "mns_outbound_events_key" {
+  name          = "alias/${var.mns_publisher_resource_name_prefix}-key"
+  target_key_id = aws_kms_key.mns_outbound_events.id
+}