VED-1103: Refactor Batch Processor (recordprocessor) deployment to a dedicated pipeline stage

.github/workflows/continuous-deployment.yml

@@ -20,6 +20,9 @@ jobs:
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
+      recordprocessor_diff_base_sha: ${{ github.event.before }}
+      recordprocessor_diff_head_sha: ${{ github.sha }}
+      run_recordprocessor_diff_check: true
       create_mns_subscription: true
       environment: dev
       sub_environment: internal-dev
@@ -82,6 +85,9 @@ jobs:
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: ${{ matrix.sub_environment_name }}
+      recordprocessor_diff_base_sha: ${{ github.event.before }}
+      recordprocessor_diff_head_sha: ${{ github.sha }}
+      run_recordprocessor_diff_check: true
       create_mns_subscription: true
       environment: dev
       sub_environment: ${{ matrix.sub_environment_name }}

.github/workflows/deploy-backend.yml

@@ -6,6 +6,22 @@ on:
       apigee_environment:
         required: true
         type: string
+      build_recordprocessor_image:
+        required: false
+        type: boolean
+        default: false
+      recordprocessor_diff_base_sha:
+        required: false
+        type: string
+        default: ""
+      recordprocessor_diff_head_sha:
+        required: false
+        type: string
+        default: ""
+      run_recordprocessor_diff_check:
+        required: false
+        type: boolean
+        default: false
       create_mns_subscription:
         required: false
         type: boolean
@@ -39,6 +55,16 @@ on:
           - dev
           - preprod
           - prod
+      build_recordprocessor_image:
+        description: Build and push a new recordprocessor image for this deployment
+        required: true
+        type: boolean
+        default: true
+      recordprocessor_image_tag:
+        description: Existing recordprocessor image tag to deploy. Required when build_recordprocessor_image is false
+        required: false
+        type: string
+        default: ""
       sub_environment:
         type: string
         description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments
@@ -51,11 +77,146 @@ env: # Sonarcloud - do not allow direct usage of untrusted data
 run-name: Deploy Backend - ${{ inputs.environment }} ${{ inputs.sub_environment }}
 
 jobs:
+  detect-recordprocessor-changes:
+    if: ${{ inputs.run_recordprocessor_diff_check && inputs.recordprocessor_diff_base_sha != '' && inputs.recordprocessor_diff_head_sha != '' }}
+    runs-on: ubuntu-latest
+    outputs:
+      has_changes: ${{ steps.detect.outputs.has_changes }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          fetch-depth: 0
+
+      - name: Detect recordprocessor changes in commit range
+        id: detect
+        env:
+          RECORDPROCESSOR_DIFF_BASE_SHA: ${{ inputs.recordprocessor_diff_base_sha }}
+          RECORDPROCESSOR_DIFF_HEAD_SHA: ${{ inputs.recordprocessor_diff_head_sha }}
+        run: |
+          if git diff --quiet "${RECORDPROCESSOR_DIFF_BASE_SHA}" "${RECORDPROCESSOR_DIFF_HEAD_SHA}" -- lambdas/recordprocessor/ lambdas/shared/src/common/; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+          fi
+
+  build-and-push-recordprocessor:
+    needs: detect-recordprocessor-changes
+    if: ${{ always() && (inputs.build_recordprocessor_image || needs.detect-recordprocessor-changes.outputs.has_changes == 'true') }}
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      recordprocessor_image_tag: ${{ steps.build-image.outputs.recordprocessor_image_tag }}
+    name: Build and push recordprocessor image
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+    env:
+      AWS_REGION: eu-west-2
+      SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076
+
+      - name: Build and push Docker image
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        working-directory: lambdas
+        run: |
+          IMAGE_TAG="${SUB_ENVIRONMENT}-${GITHUB_SHA}"
+          REPOSITORY_NAME="imms-recordprocessor-repo"
+          IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}"
+
+          docker build -f recordprocessor/Dockerfile -t "${IMAGE_URI}" .
+          docker push "${IMAGE_URI}"
+          echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+
+  resolve-recordprocessor-image-tag:
+    needs: detect-recordprocessor-changes
+    if: ${{ always() && !inputs.build_recordprocessor_image && needs.detect-recordprocessor-changes.outputs.has_changes != 'true' }}
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      recordprocessor_image_tag: ${{ steps.resolve-image-tag.outputs.recordprocessor_image_tag }}
+    name: Resolve existing recordprocessor image tag
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+    env:
+      AWS_REGION: eu-west-2
+    steps:
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - name: Resolve or use recordprocessor image tag
+        id: resolve-image-tag
+        env:
+          REPOSITORY_NAME: imms-recordprocessor-repo
+          TAG_PREFIX: ${{ inputs.sub_environment }}-
+          PROVIDED_IMAGE_TAG: ${{ github.event.inputs.recordprocessor_image_tag }}
+        run: |
+          if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then
+            if [ -n "${PROVIDED_IMAGE_TAG}" ]; then
+              echo "Using provided recordprocessor image tag: ${PROVIDED_IMAGE_TAG}"
+              echo "recordprocessor_image_tag=${PROVIDED_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "When build_recordprocessor_image is false, recordprocessor_image_tag must be provided."
+            exit 1
+          fi
+
+          IMAGE_TAG=$(
+            aws ecr describe-images \
+              --repository-name "${REPOSITORY_NAME}" \
+              --region "${AWS_REGION}" \
+              --filter tagStatus=TAGGED \
+              --query 'sort_by(imageDetails,&imagePushedAt)[*].imageTags[*]' \
+              --output text \
+            | tr '\t' '\n' \
+            | grep "^${TAG_PREFIX}" \
+            | tail -n1 || true
+          )
+
+          if [ -z "${IMAGE_TAG}" ]; then
+            echo "No existing recordprocessor image found for prefix '${TAG_PREFIX}'."
+            echo "Trigger a run with build_recordprocessor_image=true to build one."
+            exit 1
+          fi
+
+          echo "Using existing recordprocessor image tag: ${IMAGE_TAG}"
+          echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+
   terraform-plan:
     permissions:
       id-token: write
       contents: read
+    needs:
+      - build-and-push-recordprocessor
+      - resolve-recordprocessor-image-tag
+    if: ${{ !cancelled() && (needs.build-and-push-recordprocessor.result == 'success' || needs.resolve-recordprocessor-image-tag.result == 'success') }}
+    outputs:
+      recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }}
     runs-on: ubuntu-latest
+    env:
+      TF_VAR_recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }}
     environment:
       name: ${{ inputs.environment }}
     steps:
@@ -95,7 +256,10 @@ jobs:
       id-token: write
       contents: read
     needs: terraform-plan
+    if: ${{ !cancelled() && needs.terraform-plan.result == 'success' }}
     runs-on: ubuntu-latest
+    env:
+      TF_VAR_recordprocessor_image_tag: ${{ needs.terraform-plan.outputs.recordprocessor_image_tag }}
     environment:
       name: ${{ inputs.environment }}
     steps:

.github/workflows/pr-deploy-and-test.yml

@@ -16,9 +16,14 @@ jobs:
 
   deploy-pr-backend:
     needs: [run-quality-checks]
+    if: ${{ always() && !failure() && !cancelled() }}
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
+      build_recordprocessor_image: ${{ github.event.action == 'opened' || github.event.action == 'reopened' }}
+      recordprocessor_diff_base_sha: ${{ github.event.before }}
+      recordprocessor_diff_head_sha: ${{ github.event.pull_request.head.sha }}
+      run_recordprocessor_diff_check: ${{ github.event.action == 'synchronize' }}
       create_mns_subscription: true
       environment: dev
       sub_environment: pr-${{github.event.pull_request.number}}

.github/workflows/pr-teardown.yml

@@ -75,3 +75,35 @@ jobs:
         working-directory: infrastructure/instance
         run: |
           make destroy apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
+
+      - name: Cleanup recordprocessor ECR images for PR
+        env:
+          AWS_REGION: eu-west-2
+          REPOSITORY_NAME: imms-recordprocessor-repo
+          IMAGE_TAG_PREFIX: ${{ env.BACKEND_SUB_ENVIRONMENT }}-
+        run: |
+          MATCHING_TAGS=$(
+            aws ecr list-images \
+              --repository-name "${REPOSITORY_NAME}" \
+              --region "${AWS_REGION}" \
+              --filter tagStatus=TAGGED \
+              --query "imageIds[?starts_with(imageTag, \`${IMAGE_TAG_PREFIX}\`)].imageTag" \
+              --output text
+          )
+
+          if [ -z "${MATCHING_TAGS}" ] || [ "${MATCHING_TAGS}" = "None" ]; then
+            echo "No recordprocessor images found for prefix '${IMAGE_TAG_PREFIX}'."
+            exit 0
+          fi
+
+          IMAGE_IDS_ARGS=""
+          for image_tag in ${MATCHING_TAGS}; do
+            echo "Queueing recordprocessor image tag '${image_tag}' for deletion..."
+            IMAGE_IDS_ARGS="${IMAGE_IDS_ARGS} imageTag=${image_tag}"
+          done
+
+          aws ecr batch-delete-image \
+            --repository-name "${REPOSITORY_NAME}" \
+            --region "${AWS_REGION}" \
+            --image-ids ${IMAGE_IDS_ARGS} \
+            --output json

infrastructure/account/recordprocessor_ecr_repo.tf

@@ -0,0 +1,30 @@
+resource "aws_ecr_repository" "recordprocessor_repository" {
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+  image_tag_mutability = "IMMUTABLE"
+  name                 = "imms-recordprocessor-repo"
+}
+
+resource "aws_ecr_lifecycle_policy" "recordprocessor_repository_lifecycle_policy" {
+  repository = aws_ecr_repository.recordprocessor_repository.name
+
+  policy = <<EOF
+{
+  "rules": [
+    {
+      "rulePriority": 1,
+      "description": "Keep last 10 images",
+      "selection": {
+        "tagStatus": "any",
+        "countType": "imageCountMoreThan",
+        "countNumber": 10
+      },
+      "action": {
+        "type": "expire"
+      }
+    }
+  ]
+}
+EOF
+}

infrastructure/instance/ecs_batch_processor_config.tf

@@ -1,4 +1,3 @@
-# Define the ECS Cluster
 resource "aws_ecs_cluster" "ecs_cluster" {
   name = "${local.short_prefix}-ecs-cluster"
 
@@ -11,59 +10,8 @@ resource "aws_ecs_cluster" "ecs_cluster" {
   }
 }
 
-# Locals for Lambda processing paths and hash
-locals {
-  processing_lambda_dir     = abspath("${path.root}/../../lambdas/recordprocessor")
-  processing_path_include   = ["**"]
-  processing_path_exclude   = ["**/__pycache__/**"]
-  processing_files_include  = setunion([for f in local.processing_path_include : fileset(local.processing_lambda_dir, f)]...)
-  processing_files_exclude  = setunion([for f in local.processing_path_exclude : fileset(local.processing_lambda_dir, f)]...)
-  processing_lambda_files   = sort(setsubtract(local.processing_files_include, local.processing_files_exclude))
-  processing_lambda_dir_sha = sha1(join("", [for f in local.processing_lambda_files : filesha1("${local.processing_lambda_dir}/${f}")]))
-  image_tag                 = "latest"
-}
-
-# Create ECR Repository for processing.
-resource "aws_ecr_repository" "processing_repository" {
-  image_scanning_configuration {
-    scan_on_push = true
-  }
-  name         = "${local.short_prefix}-processing-repo"
-  force_delete = local.is_temp
-}
-
-# Build and Push Docker Image to ECR (Reusing the existing module)
-module "processing_docker_image" {
-  source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
-  version = "8.7.0"
-
-  create_ecr_repo  = false
-  docker_file_path = "./recordprocessor/Dockerfile"
-  ecr_repo         = aws_ecr_repository.processing_repository.name
-  ecr_repo_lifecycle_policy = jsonencode({
-    "rules" : [
-      {
-        "rulePriority" : 1,
-        "description" : "Keep only the last 2 images",
-        "selection" : {
-          "tagStatus" : "any",
-          "countType" : "imageCountMoreThan",
-          "countNumber" : 2
-        },
-        "action" : {
-          "type" : "expire"
-        }
-      }
-    ]
-  })
-
-  platform      = "linux/amd64"
-  use_image_tag = false
-  source_path   = abspath("${path.root}/../../lambdas")
-  triggers = {
-    dir_sha        = local.processing_lambda_dir_sha
-    shared_dir_sha = local.shared_dir_sha
-  }
+data "aws_ecr_repository" "recordprocessor_repository" {
+  name = "${var.project_short_name}-recordprocessor-repo"
 }
 
 # Define the IAM Role for ECS Task Execution
@@ -157,7 +105,7 @@ resource "aws_iam_policy" "ecs_task_exec_policy" {
         Action = [
           "ecr:GetAuthorizationToken"
         ],
-        Resource = "arn:aws:ecr:${var.aws_region}:${var.immunisation_account_id}:repository/${local.short_prefix}-processing-repo"
+        Resource = "arn:aws:ecr:${var.aws_region}:${var.immunisation_account_id}:repository/${data.aws_ecr_repository.recordprocessor_repository.name}"
       },
       {
         "Effect" : "Allow",
@@ -197,7 +145,7 @@ resource "aws_ecs_task_definition" "ecs_task" {
 
   container_definitions = jsonencode([{
     name      = "${local.short_prefix}-process-records-container"
-    image     = "${aws_ecr_repository.processing_repository.repository_url}:${local.image_tag}"
+    image     = "${data.aws_ecr_repository.recordprocessor_repository.repository_url}:${var.recordprocessor_image_tag}"
     essential = true
     environment = [
       {

infrastructure/instance/variables.tf

@@ -99,6 +99,12 @@ variable "dynamodb_point_in_time_recovery_enabled" {
   default     = false
 }
 
+variable "recordprocessor_image_tag" {
+  description = "Tag of the recordprocessor (batch processor) container image in ECR"
+  type        = string
+  default     = "latest"
+}
+
 locals {
   prefix              = "${var.project_name}-${var.service}-${var.sub_environment}"
   short_prefix        = "${var.project_short_name}-${var.sub_environment}"