diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml index 1e2bebdfb..9eea92e0a 100644 --- a/.github/workflows/continuous-deployment.yml +++ b/.github/workflows/continuous-deployment.yml @@ -20,6 +20,9 @@ jobs: uses: ./.github/workflows/deploy-backend.yml with: apigee_environment: internal-dev + recordprocessor_diff_base_sha: ${{ github.event.before }} + recordprocessor_diff_head_sha: ${{ github.sha }} + run_recordprocessor_diff_check: true create_mns_subscription: true environment: dev sub_environment: internal-dev @@ -82,6 +85,9 @@ jobs: uses: ./.github/workflows/deploy-backend.yml with: apigee_environment: ${{ matrix.sub_environment_name }} + recordprocessor_diff_base_sha: ${{ github.event.before }} + recordprocessor_diff_head_sha: ${{ github.sha }} + run_recordprocessor_diff_check: true create_mns_subscription: true environment: dev sub_environment: ${{ matrix.sub_environment_name }} diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 6731fa17b..045a1814e 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -6,6 +6,22 @@ on: apigee_environment: required: true type: string + build_recordprocessor_image: + required: false + type: boolean + default: false + recordprocessor_diff_base_sha: + required: false + type: string + default: "" + recordprocessor_diff_head_sha: + required: false + type: string + default: "" + run_recordprocessor_diff_check: + required: false + type: boolean + default: false create_mns_subscription: required: false type: boolean @@ -39,6 +55,16 @@ on: - dev - preprod - prod + build_recordprocessor_image: + description: Build and push a new recordprocessor image for this deployment + required: true + type: boolean + default: true + recordprocessor_image_tag: + description: Existing recordprocessor image tag to deploy. Required when build_recordprocessor_image is false + required: false + type: string + default: "" sub_environment: type: string description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments @@ -51,11 +77,146 @@ env: # Sonarcloud - do not allow direct usage of untrusted data run-name: Deploy Backend - ${{ inputs.environment }} ${{ inputs.sub_environment }} jobs: + detect-recordprocessor-changes: + if: ${{ inputs.run_recordprocessor_diff_check && inputs.recordprocessor_diff_base_sha != '' && inputs.recordprocessor_diff_head_sha != '' }} + runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.detect.outputs.has_changes }} + steps: + - name: Checkout + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + fetch-depth: 0 + + - name: Detect recordprocessor changes in commit range + id: detect + env: + RECORDPROCESSOR_DIFF_BASE_SHA: ${{ inputs.recordprocessor_diff_base_sha }} + RECORDPROCESSOR_DIFF_HEAD_SHA: ${{ inputs.recordprocessor_diff_head_sha }} + run: | + if git diff --quiet "${RECORDPROCESSOR_DIFF_BASE_SHA}" "${RECORDPROCESSOR_DIFF_HEAD_SHA}" -- lambdas/recordprocessor/ lambdas/shared/src/common/; then + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "has_changes=true" >> "$GITHUB_OUTPUT" + fi + + build-and-push-recordprocessor: + needs: detect-recordprocessor-changes + if: ${{ always() && (inputs.build_recordprocessor_image || needs.detect-recordprocessor-changes.outputs.has_changes == 'true') }} + permissions: + id-token: write + contents: read + outputs: + recordprocessor_image_tag: ${{ steps.build-image.outputs.recordprocessor_image_tag }} + name: Build and push recordprocessor image + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + env: + AWS_REGION: eu-west-2 + SUB_ENVIRONMENT: ${{ inputs.sub_environment }} + steps: + - name: Checkout + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + + - name: Connect to AWS + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 + with: + aws-region: eu-west-2 + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops + role-session-name: github-actions + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 + + - name: Build and push Docker image + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + working-directory: lambdas + run: | + IMAGE_TAG="${SUB_ENVIRONMENT}-${GITHUB_SHA}" + REPOSITORY_NAME="imms-recordprocessor-repo" + IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}" + + docker build -f recordprocessor/Dockerfile -t "${IMAGE_URI}" . + docker push "${IMAGE_URI}" + echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + + resolve-recordprocessor-image-tag: + needs: detect-recordprocessor-changes + if: ${{ always() && !inputs.build_recordprocessor_image && needs.detect-recordprocessor-changes.outputs.has_changes != 'true' }} + permissions: + id-token: write + contents: read + outputs: + recordprocessor_image_tag: ${{ steps.resolve-image-tag.outputs.recordprocessor_image_tag }} + name: Resolve existing recordprocessor image tag + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + env: + AWS_REGION: eu-west-2 + steps: + - name: Connect to AWS + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 + with: + aws-region: eu-west-2 + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops + role-session-name: github-actions + + - name: Resolve or use recordprocessor image tag + id: resolve-image-tag + env: + REPOSITORY_NAME: imms-recordprocessor-repo + TAG_PREFIX: ${{ inputs.sub_environment }}- + PROVIDED_IMAGE_TAG: ${{ github.event.inputs.recordprocessor_image_tag }} + run: | + if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then + if [ -n "${PROVIDED_IMAGE_TAG}" ]; then + echo "Using provided recordprocessor image tag: ${PROVIDED_IMAGE_TAG}" + echo "recordprocessor_image_tag=${PROVIDED_IMAGE_TAG}" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "When build_recordprocessor_image is false, recordprocessor_image_tag must be provided." + exit 1 + fi + + IMAGE_TAG=$( + aws ecr describe-images \ + --repository-name "${REPOSITORY_NAME}" \ + --region "${AWS_REGION}" \ + --filter tagStatus=TAGGED \ + --query 'sort_by(imageDetails,&imagePushedAt)[*].imageTags[*]' \ + --output text \ + | tr '\t' '\n' \ + | grep "^${TAG_PREFIX}" \ + | tail -n1 || true + ) + + if [ -z "${IMAGE_TAG}" ]; then + echo "No existing recordprocessor image found for prefix '${TAG_PREFIX}'." + echo "Trigger a run with build_recordprocessor_image=true to build one." + exit 1 + fi + + echo "Using existing recordprocessor image tag: ${IMAGE_TAG}" + echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + terraform-plan: permissions: id-token: write contents: read + needs: + - build-and-push-recordprocessor + - resolve-recordprocessor-image-tag + if: ${{ !cancelled() && (needs.build-and-push-recordprocessor.result == 'success' || needs.resolve-recordprocessor-image-tag.result == 'success') }} + outputs: + recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }} runs-on: ubuntu-latest + env: + TF_VAR_recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }} environment: name: ${{ inputs.environment }} steps: @@ -95,7 +256,10 @@ jobs: id-token: write contents: read needs: terraform-plan + if: ${{ !cancelled() && needs.terraform-plan.result == 'success' }} runs-on: ubuntu-latest + env: + TF_VAR_recordprocessor_image_tag: ${{ needs.terraform-plan.outputs.recordprocessor_image_tag }} environment: name: ${{ inputs.environment }} steps: diff --git a/.github/workflows/pr-deploy-and-test.yml b/.github/workflows/pr-deploy-and-test.yml index 76db9846d..17d1e6d97 100644 --- a/.github/workflows/pr-deploy-and-test.yml +++ b/.github/workflows/pr-deploy-and-test.yml @@ -16,9 +16,14 @@ jobs: deploy-pr-backend: needs: [run-quality-checks] + if: ${{ always() && !failure() && !cancelled() }} uses: ./.github/workflows/deploy-backend.yml with: apigee_environment: internal-dev + build_recordprocessor_image: ${{ github.event.action == 'opened' || github.event.action == 'reopened' }} + recordprocessor_diff_base_sha: ${{ github.event.before }} + recordprocessor_diff_head_sha: ${{ github.event.pull_request.head.sha }} + run_recordprocessor_diff_check: ${{ github.event.action == 'synchronize' }} create_mns_subscription: true environment: dev sub_environment: pr-${{github.event.pull_request.number}} diff --git a/.github/workflows/pr-teardown.yml b/.github/workflows/pr-teardown.yml index 40734355b..489941627 100644 --- a/.github/workflows/pr-teardown.yml +++ b/.github/workflows/pr-teardown.yml @@ -75,3 +75,35 @@ jobs: working-directory: infrastructure/instance run: | make destroy apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT + + - name: Cleanup recordprocessor ECR images for PR + env: + AWS_REGION: eu-west-2 + REPOSITORY_NAME: imms-recordprocessor-repo + IMAGE_TAG_PREFIX: ${{ env.BACKEND_SUB_ENVIRONMENT }}- + run: | + MATCHING_TAGS=$( + aws ecr list-images \ + --repository-name "${REPOSITORY_NAME}" \ + --region "${AWS_REGION}" \ + --filter tagStatus=TAGGED \ + --query "imageIds[?starts_with(imageTag, \`${IMAGE_TAG_PREFIX}\`)].imageTag" \ + --output text + ) + + if [ -z "${MATCHING_TAGS}" ] || [ "${MATCHING_TAGS}" = "None" ]; then + echo "No recordprocessor images found for prefix '${IMAGE_TAG_PREFIX}'." + exit 0 + fi + + IMAGE_IDS_ARGS="" + for image_tag in ${MATCHING_TAGS}; do + echo "Queueing recordprocessor image tag '${image_tag}' for deletion..." + IMAGE_IDS_ARGS="${IMAGE_IDS_ARGS} imageTag=${image_tag}" + done + + aws ecr batch-delete-image \ + --repository-name "${REPOSITORY_NAME}" \ + --region "${AWS_REGION}" \ + --image-ids ${IMAGE_IDS_ARGS} \ + --output json diff --git a/infrastructure/account/recordprocessor_ecr_repo.tf b/infrastructure/account/recordprocessor_ecr_repo.tf new file mode 100644 index 000000000..9aaef8bfc --- /dev/null +++ b/infrastructure/account/recordprocessor_ecr_repo.tf @@ -0,0 +1,30 @@ +resource "aws_ecr_repository" "recordprocessor_repository" { + image_scanning_configuration { + scan_on_push = true + } + image_tag_mutability = "IMMUTABLE" + name = "imms-recordprocessor-repo" +} + +resource "aws_ecr_lifecycle_policy" "recordprocessor_repository_lifecycle_policy" { + repository = aws_ecr_repository.recordprocessor_repository.name + + policy = < int: incoming_message_body["encoder"] = encoder interim_message_body = file_level_validation(incoming_message_body=incoming_message_body) except Exception as e: # pylint: disable=broad-exception-caught - logger.error(f"File level validation failed: {e}") # If the file is invalid, processing should cease + logger.error(f"File level validation failed: {e}") # If the file is invalid, processing should cease. return 0 file_id = interim_message_body.get("message_id")