Fix: [AEA-6055] - debugging issues with post-dated updates

.github/scripts/deploy_api.sh

@@ -10,6 +10,7 @@ echo "Apigee environment: ${APIGEE_ENVIRONMENT}"
 echo "Proxygen private key name: ${PROXYGEN_PRIVATE_KEY_NAME}"
 echo "Proxygen KID: ${PROXYGEN_KID}"
 echo "Deploy Check Prescription Status Update: ${DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE}"
+echo "Expose Get Status Updates: ${EXPOSE_GET_STATUS_UPDATES}"
 echo "Dry run: ${DRY_RUN}"
 # shellcheck disable=SC2153
 echo "is_pull_request: ${IS_PULL_REQUEST}"
@@ -106,6 +107,14 @@ if [[ "${DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE}" == "false" ]]; then
     fi
 fi
 
+# Remove get-status-updates if not needed
+if [[ "${EXPOSE_GET_STATUS_UPDATES}" == "false" ]]; then
+    if [[ "${API_TYPE}" == "standard" ]]; then
+        echo "Removing get-status-updates endpoint"
+        jq 'del(.paths."/get-status-updates")' "$SPEC_PATH" > temp.json && mv temp.json "$SPEC_PATH"
+    fi
+fi
+
 # Find and replace the x-nhsd-apim.target.secret value
 jq --arg mtls_key "${MTLS_KEY}"  '.["x-nhsd-apim"].target.security.secret = "\($mtls_key)"' "${SPEC_PATH}" > temp.json && mv temp.json "${SPEC_PATH}"
 

.github/scripts/release_code.sh

@@ -61,6 +61,7 @@ sam deploy \
             LogRetentionInDays="$LOG_RETENTION_DAYS" \
             Environment="$TARGET_ENVIRONMENT" \
             DeployCheckPrescriptionStatusUpdate="$DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE" \
+            ExposeGetStatusUpdates="$EXPOSE_GET_STATUS_UPDATES" \
             EnableAlerts="$ENABLE_ALERTS" \
             StateMachineLogLevel="$STATE_MACHINE_LOG_LEVEL" \
             EnableNotificationsInternal="$ENABLE_NOTIFICATIONS_INTERNAL" \

.github/workflows/ci.yml

@@ -69,6 +69,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: true
       REQUIRE_APPLICATION_NAME: false
       RUN_REGRESSION_TEST: true
@@ -113,6 +114,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: false
       RUN_REGRESSION_TEST: false
       STATE_MACHINE_LOG_LEVEL: ALL
       LOG_LEVEL: DEBUG
@@ -151,6 +153,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: true
       RUN_REGRESSION_TEST: true
       STATE_MACHINE_LOG_LEVEL: ALL

.github/workflows/pull_request.yml

@@ -106,6 +106,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: false
       REQUIRE_APPLICATION_NAME: false
       RUN_REGRESSION_TEST: true
@@ -150,6 +151,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: false
       RUN_REGRESSION_TEST: false
       STATE_MACHINE_LOG_LEVEL: ALL
       ENABLE_BACKUP: "False"

.github/workflows/release.yml

@@ -72,6 +72,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: true
       REQUIRE_APPLICATION_NAME: false
       RUN_REGRESSION_TEST: true
@@ -119,6 +120,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: false
       RUN_REGRESSION_TEST: false
       STATE_MACHINE_LOG_LEVEL: ALL
       ENABLE_BACKUP: "False"
@@ -169,6 +171,7 @@ jobs:
       DYNAMODB_AUTOSCALE: true
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: true
       RUN_REGRESSION_TEST: false
       STATE_MACHINE_LOG_LEVEL: ERROR
@@ -220,6 +223,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: true
       RUN_REGRESSION_TEST: true
       STATE_MACHINE_LOG_LEVEL: ALL
@@ -263,6 +267,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: true
       ENABLE_ALERTS: true
       RUN_REGRESSION_TEST: true
       STATE_MACHINE_LOG_LEVEL: ALL
@@ -309,6 +314,7 @@ jobs:
       DYNAMODB_AUTOSCALE: false
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: true
+      EXPOSE_GET_STATUS_UPDATES: false
       RUN_REGRESSION_TEST: false
       STATE_MACHINE_LOG_LEVEL: ALL
       ENABLE_BACKUP: "False"
@@ -359,6 +365,7 @@ jobs:
       DYNAMODB_AUTOSCALE: true
       DEPLOY_APIGEE_CPSU: true
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: false
+      EXPOSE_GET_STATUS_UPDATES: false
       ENABLE_ALERTS: true
       RUN_REGRESSION_TEST: false
       STATE_MACHINE_LOG_LEVEL: ERROR

.github/workflows/run_regression_tests.yml

@@ -59,11 +59,11 @@ jobs:
         run: |
           if [[ "$TARGET_ENVIRONMENT" != "prod" && "$TARGET_ENVIRONMENT" != "ref" ]]; then
             # this should be the tag of the tests you want to run
-            REGRESSION_TEST_REPO_TAG=v3.6.2
+            REGRESSION_TEST_REPO_TAG=v3.11.0
 
             # this should be the tag of the regression test workflow you want to run
             # This will normally be the same as REGRESSION_TEST_REPO_TAG
-            REGRESSION_TEST_WORKFLOW_TAG=v3.6.2
+            REGRESSION_TEST_WORKFLOW_TAG=v3.11.0
 
             curl https://raw.githubusercontent.com/NHSDigital/electronic-prescription-service-api-regression-tests/refs/tags/${REGRESSION_TEST_WORKFLOW_TAG}/scripts/run_regression_tests.py -o run_regression_tests.py
             poetry install

.github/workflows/run_release_code_and_api.yml

@@ -49,6 +49,9 @@ on:
       DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE:
         type: boolean
         default: false
+      EXPOSE_GET_STATUS_UPDATES:
+        type: boolean
+        default: false
       ENABLE_ALERTS:
         type: boolean
         default: true
@@ -202,6 +205,7 @@ jobs:
           VERSION_NUMBER: ${{ inputs.VERSION_NUMBER }}
           DYNAMODB_AUTOSCALE: ${{ inputs.DYNAMODB_AUTOSCALE }}
           DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: ${{ inputs.DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE }}
+          EXPOSE_GET_STATUS_UPDATES: ${{ inputs.EXPOSE_GET_STATUS_UPDATES }}
           ENABLE_ALERTS: ${{ inputs.ENABLE_ALERTS }}
           STATE_MACHINE_LOG_LEVEL: ${{ inputs.STATE_MACHINE_LOG_LEVEL }}
           ENABLE_BACKUP: ${{ inputs.ENABLE_BACKUP }}
@@ -255,6 +259,7 @@ jobs:
           PROXYGEN_KID: "eps-cli-key-1"
           DRY_RUN: false
           DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: ${{ inputs.DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE }}
+          EXPOSE_GET_STATUS_UPDATES: ${{ inputs.EXPOSE_GET_STATUS_UPDATES }}
           IS_PULL_REQUEST: ${{ inputs.IS_PULL_REQUEST }}
           MTLS_KEY: ${{ inputs.MTLS_KEY }}
           ENABLE_MUTUAL_TLS: ${{ inputs.ENABLE_MUTUAL_TLS }}
@@ -276,6 +281,7 @@ jobs:
           PROXYGEN_KID: eps-cli-key-cpsu-1
           DRY_RUN: false
           DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE: ${{ inputs.DEPLOY_CHECK_PRESCRIPTION_STATUS_UPDATE }}
+          EXPOSE_GET_STATUS_UPDATES: ${{ inputs.EXPOSE_GET_STATUS_UPDATES }}
           IS_PULL_REQUEST: ${{ inputs.IS_PULL_REQUEST }}
           MTLS_KEY: ${{ inputs.MTLS_KEY }}
           ENABLE_MUTUAL_TLS: ${{ inputs.ENABLE_MUTUAL_TLS }}

SAMtemplates/apis/main.yaml

@@ -54,6 +54,14 @@ Parameters:
     Type: String
     Default: none
 
+  GetStatusUpdatesFunctionName:
+    Type: String
+    Default: none
+
+  GetStatusUpdatesFunctionArn:
+    Type: String
+    Default: none
+
   NHSNotifyUpdateCallbackFunctionName:
     Type: String
     Default: none
@@ -71,6 +79,10 @@ Parameters:
   DeployCheckPrescriptionStatusUpdate:
     Type: String
 
+  ExposeGetStatusUpdates:
+    Type: String
+    Default: false
+
   ForwardCsocLogs:
     Type: String
     Default: false
@@ -81,6 +93,9 @@ Conditions:
   ShouldDeployCheckPrescriptionStatusUpdate: !Equals
     - true
     - !Ref DeployCheckPrescriptionStatusUpdate
+  ShouldExposeGetStatusUpdates: !Equals
+    - true
+    - !Ref ExposeGetStatusUpdates
 
 Resources:
   GenerateCertificate:
@@ -526,6 +541,58 @@ Resources:
         IntegrationHttpMethod: POST
         Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${CheckPrescriptionStatusUpdatesFunctionArn}/invocations
 
+  GetStatusUpdatesResource:
+    Condition: ShouldExposeGetStatusUpdates
+    Type: AWS::ApiGateway::Resource
+    Properties:
+      RestApiId: !Ref RestApiGateway
+      ParentId: !GetAtt RestApiGateway.RootResourceId
+      PathPart: get-status-updates
+
+  GetStatusUpdatesMethod:
+    Condition: ShouldExposeGetStatusUpdates
+    Type: AWS::ApiGateway::Method
+    Properties:
+      RestApiId: !Ref RestApiGateway
+      ResourceId: !Ref GetStatusUpdatesResource
+      HttpMethod: POST
+      AuthorizationType: NONE
+      Integration:
+        Type: AWS
+        Credentials: !GetAtt RestApiGatewayResources.Outputs.ApiGwRoleArn
+        IntegrationHttpMethod: POST
+        Uri: !Sub arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${GetStatusUpdatesFunctionArn}/invocations
+        RequestTemplates:
+          application/json: |-
+            $input.json('$')
+        IntegrationResponses:
+          - StatusCode: 200
+            ResponseTemplates:
+              application/json: |-
+                $input.body
+      MethodResponses:
+        - StatusCode: "200"
+
+  GetStatusUpdatesHandle:
+    Condition: ShouldExposeGetStatusUpdates
+    DependsOn: GetStatusUpdatesMethod
+    Type: "AWS::CloudFormation::WaitConditionHandle"
+
+  GetStatusUpdatesWaitHandle:
+    Type: "AWS::CloudFormation::WaitConditionHandle"
+
+  GetStatusUpdatesWaitCondition:
+    Type: "AWS::CloudFormation::WaitCondition"
+    Properties:
+      Handle:
+        !If [
+          ShouldExposeGetStatusUpdates,
+          !Ref GetStatusUpdatesHandle,
+          !Ref GetStatusUpdatesWaitHandle,
+        ]
+      Timeout: "1"
+      Count: 0
+
   # we can not use a conditional in depends on in ApiGateway::Deployment
   # so instead we use a wait condition that has the conditional in it and use that as a dependency
   # taken from https://garbe.io/blog/2017/07/17/cloudformation-hacks/
@@ -553,7 +620,7 @@ Resources:
   # if you add a new endpoint, then change the name of this resource
   # also need to change it in RestApiGatewayStage.Properties.DeploymentId
   # *********************************************************************
-  RestApiGatewayDeploymentV2f:
+  RestApiGatewayDeploymentV2h:
     Type: AWS::ApiGateway::Deployment
     DependsOn:
       # see note above if you add something in here when you add a new endpoint
@@ -563,6 +630,7 @@ Resources:
       - Format1UpdatePrescriptionStatusMethod
       - CheckPrescriptionStatusUpdatesWaitCondition
       - NotificationDeliveryStatusCallbackMethod
+      - GetStatusUpdatesWaitCondition
       # see note above if you add something in here when you add a new endpoint
     Properties:
       RestApiId: !Ref RestApiGateway
@@ -571,7 +639,7 @@ Resources:
     Type: AWS::ApiGateway::Stage
     Properties:
       RestApiId: !Ref RestApiGateway
-      DeploymentId: !Ref RestApiGatewayDeploymentV2f
+      DeploymentId: !Ref RestApiGatewayDeploymentV2h
       StageName: prod
       TracingEnabled: true
       AccessLogSetting:
@@ -595,6 +663,10 @@ Resources:
             - Fn::ImportValue: !Sub ${StackName}:functions:${StatusFunctionName}:ExecuteLambdaPolicyArn
             - Fn::ImportValue: !Sub ${StackName}:functions:${CapabilityStatementFunctionName}:ExecuteLambdaPolicyArn
             - Fn::ImportValue: !Sub ${StackName}:functions:${NHSNotifyUpdateCallbackFunctionName}:ExecuteLambdaPolicyArn
+            - !If
+              - ShouldExposeGetStatusUpdates
+              - Fn::ImportValue: !Sub ${StackName}:functions:${GetStatusUpdatesFunctionName}:ExecuteLambdaPolicyArn
+              - !Ref AWS::NoValue
             - Fn::ImportValue: !Sub ${StackName}:state-machines:${Format1UpdatePrescriptionsStatusStateMachineName}:ExecuteStateMachinePolicy
             - !If
               - ShouldDeployCheckPrescriptionStatusUpdate

SAMtemplates/main_template.yaml

@@ -71,6 +71,13 @@ Parameters:
   DeployCheckPrescriptionStatusUpdate:
     Type: String
 
+  ExposeGetStatusUpdates:
+    Type: String
+    Default: false
+    AllowedValues:
+      - true
+      - false
+
   Environment:
     Type: String
 
@@ -223,11 +230,14 @@ Resources:
         CapabilityStatementFunctionArn: !GetAtt Functions.Outputs.CapabilityStatementFunctionArn
         CheckPrescriptionStatusUpdatesFunctionName: !GetAtt Functions.Outputs.CheckPrescriptionStatusUpdatesFunctionName
         CheckPrescriptionStatusUpdatesFunctionArn: !GetAtt Functions.Outputs.CheckPrescriptionStatusUpdatesFunctionArn
+        GetStatusUpdatesFunctionName: !GetAtt Functions.Outputs.GetStatusUpdatesFunctionName
+        GetStatusUpdatesFunctionArn: !GetAtt Functions.Outputs.GetStatusUpdatesFunctionArn
         NHSNotifyUpdateCallbackFunctionName: !GetAtt Functions.Outputs.NHSNotifyUpdateCallbackFunctionName
         NHSNotifyUpdateCallbackFunctionArn: !GetAtt Functions.Outputs.NHSNotifyUpdateCallbackFunctionArn
         LogRetentionInDays: !Ref LogRetentionInDays
         EnableSplunk: !Ref EnableSplunk
         DeployCheckPrescriptionStatusUpdate: !Ref DeployCheckPrescriptionStatusUpdate
+        ExposeGetStatusUpdates: !Ref ExposeGetStatusUpdates
         ForwardCsocLogs: !Ref ForwardCsocLogs
 
   Functions:

SAMtemplates/tables/main.yaml

@@ -184,6 +184,8 @@ Resources:
               - TerminalStatus
               - LastModified
               - Status
+                # TODO: Remove this when we deprecate post dated prescriptions.
+              - PostDatedLastModifiedSetAt
             ProjectionType: INCLUDE
           ProvisionedThroughput: !If
             - EnableDynamoDBAutoScalingCondition
@@ -201,7 +203,7 @@ Resources:
             - ReadCapacityUnits: 1
               WriteCapacityUnits: !Ref MinWritePrescriptionStatusUpdatesCapacity
             - !Ref "AWS::NoValue"
-        # TODO: Remove this when we deprecate post modified prescriptions.
+        # TODO: Remove this when we deprecate post dated prescriptions.
         - IndexName: PrescriptionIDPostDatedIndex
           KeySchema:
             - AttributeName: PrescriptionID

packages/gsul/src/getStatusUpdates.ts

@@ -51,6 +51,11 @@ export const filterOutFutureReduceToLatestUpdates = (
     const updateTime = Date.parse(item.lastUpdateDateTime)
     return updateTime <= currentTime
   })
+  logger.debug("filtered out future updates (should only be post-dated ones)", {
+    prescriptionID: inputPrescription.prescriptionID,
+    count_dropped: (items.length - validTimeUpdates.length),
+    count_received: items.length
+  })
 
   // group by itemId and separate post-dated from regular updates
   const itemGroups: Record<string, {regular: itemType | null, postDated: itemType | null}> = {}
@@ -79,6 +84,9 @@ export const filterOutFutureReduceToLatestUpdates = (
       }
     }
   })
+  logger.debug("grouped updates by itemId and type", {
+    itemGroupCount: Object.entries(itemGroups).length
+  })
 
   // flatten both regular and post-dated updates into single array
   // but exclude post-dated updates if they have been revoked by a subsequent regular update
@@ -96,12 +104,20 @@ export const filterOutFutureReduceToLatestUpdates = (
       }
     }
   })
+  logger.debug("flattened updates into unique items", {
+    validTimeCount: validTimeUpdates.length, uniqueItemsCount: uniqueItems.length
+  })
 
   const result: outputPrescriptionType = {
     prescriptionID: inputPrescription.prescriptionID,
     onboarded: items.length > 0, // consider onboarded even if all updates were post-dated
     items: uniqueItems
   }
+  logger.info("returning updates result", {
+    prescriptionID: result.prescriptionID,
+    onboarded: result.onboarded,
+    itemCount: result.items.length
+  })
   return result
 }