Merge pull request #1112 from NHSDigital/NRL-1666-point-perftest-at-baseline-table

NRL-1866 Point the perftest environment at the baseline table to cond…

Author: mattdean3-nhs

scripts/seed_nft_tables.py

@@ -11,18 +11,20 @@
 # import json
 import numpy as np
 
-from nrlf.consumer.fhir.r4.model import DocumentReference
 from nrlf.core.constants import (
     CATEGORY_ATTRIBUTES,
     SNOMED_SYSTEM_URL,
     TYPE_ATTRIBUTES,
     TYPE_CATEGORIES,
-    Categories,
-    PointerTypes,
 )
 from nrlf.core.dynamodb.model import DocumentPointer
 from nrlf.core.logger import logger
 from nrlf.tests.data import load_document_reference
+from tests.performance.seed_data_constants import (  # DEFAULT_COUNT_DISTRIBUTIONS,
+    CHECKSUM_WEIGHTS,
+    DEFAULT_CUSTODIAN_DISTRIBUTIONS,
+    DEFAULT_TYPE_DISTRIBUTIONS,
+)
 
 dynamodb = boto3.client("dynamodb")
 resource = boto3.resource("dynamodb")
@@ -31,69 +33,6 @@
 
 DOC_REF_TEMPLATE = load_document_reference("NFT-template")
 
-CHECKSUM_WEIGHTS = [i for i in range(10, 1, -1)]
-
-# These are based on the Nov 7th 2025 pointer stats report
-DEFAULT_TYPE_DISTRIBUTIONS = {
-    "736253002": 65,  # mental health crisis plan
-    "1382601000000107": 5,  # respect form
-    "887701000000100": 15,  # emergency healthcare plan
-    "861421000000109": 5,  # eol care coordination summary
-    "735324008": 5,  # treatment escalation plan
-    "824321000000109": 5,  # summary record
-}
-
-DEFAULT_CUSTODIAN_DISTRIBUTIONS = {
-    "736253002": {
-        "TRPG": 9,
-        "TRHA": 1,
-        "TRRE": 20,
-        "TRAT": 10,
-        "TWR4": 4,
-        "TRKL": 9,
-        "TRW1": 5,
-        "TRH5": 1,
-        "TRP7": 13,
-        "TRWK": 8,
-        "TRQY": 3,
-        "TRV5": 3,
-        "TRJ8": 2,
-        "TRXA": 4,
-        "T11X": 1,
-        "TG6V": 2,
-    },
-    "1382601000000107": {"T8GX8": 3, "TQUY": 2},  # respect form
-    "887701000000100": {
-        "TV1": 1,
-        "TV2": 2,
-        "TV3": 1,
-        "TV4": 1,
-        "TV5": 3,
-        "TV6": 1,
-    },  # emergency healthcare plan
-    "861421000000109": {
-        "TV1": 2,
-        "TV2": 2,
-        "TV3": 1,
-        "TV4": 1,
-        "TV5": 3,
-        "TV6": 1,
-    },  # eol care coordination summary
-    "735324008": {
-        "TV1": 1,
-        "TV2": 1,
-        "TV3": 1,
-        "TV4": 2,
-        "TV5": 2,
-        "TV6": 1,
-    },  # treatment escalation plan
-    "824321000000109": {
-        "TRXT": 1,
-    },  # summary record currently has only one supplier
-}
-
-DEFAULT_COUNT_DISTRIBUTIONS = {"1": 91, "2": 8, "3": 1}
-
 
 class TestNhsNumbersIterator:
     def __iter__(self):
@@ -262,7 +201,7 @@ def _get_pointer_count_poisson_distributions(
 
 
 def _set_up_custodian_iterators(
-    custodian_dists: dict[str, dict[str, int]]
+    custodian_dists: dict[str, dict[str, int]],
 ) -> dict[str, Iterator[str]]:
     custodian_iters: dict[str, Iterator[str]] = {}
     for pointer_type in custodian_dists:

terraform/account-wide-infrastructure/modules/pointers-table/vars.tf

@@ -23,6 +23,6 @@ variable "kms_deletion_window_in_days" {
 
 variable "enable_backups" {
   type        = bool
-  description = "Enable AwS cloud backup"
+  description = "Enable AWS cloud backup"
   default     = false
 }

terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf

@@ -33,5 +33,5 @@ module "ref-pointers-table" {
 
 module "perftest-pointers-table" {
   source      = "../modules/pointers-table"
-  name_prefix = "nhsd-nrlf--perftest"
+  name_prefix = "nhsd-nrlf--perftest-baseline"
 }

terraform/infrastructure/README.md

@@ -116,6 +116,8 @@ replacing `{ENV_NAME}` with the environment name (e.g. `dev`, `qa`, `qa-sandbox`
 
 To tear down the infrastructure, you need to use Terraform to destroy the resources in your Terraform workspace.
 
+First `make build-artifacts`. Then assume management and run `make get-s3-perms ENV={ENV_NAME}` in the project root.
+
 To teardown the infrastructure, do the following:
 
 ```

terraform/infrastructure/etc/perftest.tfvars

@@ -1,7 +1,7 @@
 account_name     = "perftest"
 aws_account_name = "test"
 
-dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest"
+dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest-baseline"
 
 domain        = "perftest.record-locator.national.nhs.uk"
 public_domain = "perftest.api.service.nhs.uk"

tests/performance/README.md

@@ -1,20 +1,44 @@
 # Performance Testing
 
-some high level context short
+We have performance tests which give us a benchmark of how NRLF performs under load for consumers and producers.
 
-## Run perf tests
+## Run performance tests
 
 ### Prep the environment
 
 Perf tests are generally conducted in the perftest env. There's a selection of tables in the perftest env representing different pointer volume scenarios e.g. perftest-baseline vs perftest-1million (todo: update with real names!).
 
-To reset this table to the expected state for perftests, restore the table from a backup.
+#### Point perftest at a different pointers table
 
-In the steps below, make sure the table name is the table your environment is pointing at. You might need to redeploy NRLF lambdas to point at the desired table.
+We (will) have multiple tables representing different states of NRLF in the future e.g. all patients receiving an IPS (International Patient Summary), onboarding particular high-volume suppliers.
+
+In order to run performance tests to get figures for these different states, we can point the perftest environment at one of these tables.
+
+Currently, this requires tearing down the existing environment and restoring from scratch:
+
+1. Follow instructions in terraform/infrastructure/readme.md to tear down the perf test environment.
+   - Do **not** tear down shared account-wide infrastructure
+2. Update `perftest-pointers-table.name_prefix` in `terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf` to be the table name you want, minus "-pointers-table"
+   - e.g. to use the baseline table `nhsd-nrlf--perftest-baseline-pointers-table`, set `name_prefix = "nhsd-nrlf--perftest-baseline"`
+3. Update `dynamodb_pointers_table_prefix` in `terraform/infrastructure/etc/perftest.tfvars` same as above.
+   - e.g. to use the baseline table `dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest-baseline"`
+4. Commit changes to a branch & push
+5. Run the [Deploy Account-wide infrastructure](https://github.com/NHSDigital/NRLF/actions/workflows/deploy-account-wide-infra.yml) workflow against your branch & `account-test`.
+   - If you get a terraform failure like "tried to create table but it already exists", you will need to do some fanangaling:
+     1. make sure there is a backup of your chosen table or create one if not. In the AWS console: dynamodb > tables > your perftest table > backups > create backup > Create on-demand backup > leave all settings as defaults > create backup. This might take up to an hour to complete.
+     2. once backed up, delete your table. In the AWS console: dynamodb > tables > your perftest table > actions > delete table
+     3. Rerun the Deploy Account-wide infrastructure action.
+     4. Terraform will create an empty table with the correct name & (most importantly!) read/write IAM policies.
+     5. Delete the empty table created by terraform and restore from the backup, specifying the same table name you've defined in code.
+6. Run the [Persistent Environment Deploy](https://github.com/NHSDigital/NRLF/actions/workflows/persistent-environment.yml) workflow against your branch & `perftest` to restore the environment with lambdas pointed at your chosen table.
+7. You can check this has been successful by checking the table name in the lambdas.
+   - In the AWS console: Lambda > functions > pick any perftest-1 lambda > Configuration > Environment variables > `TABLE_NAME` should be your desired pointer table e.g. `nhsd-nrlf--perftest-baseline-pointers-table`
+
+If you've followed these steps, you will also need to [generate permissions](#generate-permissions) as the organisation permissions will have been lost when the environment was torn down.
 
 ### Prepare to run tests
 
-#### Pull certs for env
+#### Pull certs for perftest
 
 ```sh
 assume management
@@ -26,19 +50,21 @@ make truststore-pull-all ENV=perftest
 You will need to generate pointer permissions the first time performance tests are run in an environment e.g. if the perftest environment is destroyed & recreated.
 
 ```sh
-make generate permissions   # makes a bunch of json permission files
+# In project root
+make generate permissions   # makes a bunch of json permission files for test organisations
 make build  # will take all permissions & create nrlf_permissions.zip file
 
 # apply this new permissions zip file to your environment
 cd ./terraform/infrastructure
-assume test # needed?
+assume nhsd-nrlf-test
 make init TF_WORKSPACE_NAME=perftest-1 ENV=perftest
-tf apply
+make ENV=perftest USE_SHARED_RESOURCES=true apply
 ```
 
 #### Generate input files
 
 ```sh
+assume nhsd-nrlf-test
 # creates 2 csv files and a json file
 make perftest-prepare PERFTEST_TABLE_NAME=perftest-baseline
 ```
@@ -49,3 +75,13 @@ make perftest-prepare PERFTEST_TABLE_NAME=perftest-baseline
 make perftest-consumer ENV_TYPE=perftest PERFTEST_HOST=perftest-1.perftest.record-locator.national.nhs.uk
 make perftest-producer ENV_TYPE=perftest PERFTEST_HOST=perftest-1.perftest.record-locator.national.nhs.uk
 ```
+
+## Assumptions / Caveats
+
+- Run performance tests in the perftest environment only\*
+- Both producer & consumer tests are repeatable
+- These tests work on the assumption that all nhs numbers in the test data are serial and lie within a fixed range i.e. picking any number between NHS_NUMBER_MINIMUM & NHS_NUMBER_MAXIMUM will yield a patient with pointer(s).
+- Configure scenarios in the `consumer/perftest.config.json` & `producer/perftest.config.json` files. This does not alter the number of stages per scenario, that's fixed in `perftest.js`.
+- Consider running these tests multiple times to get figures for a warm environment - perftest, unlike prod, is not well-used so you will get cold-start figures on your first run
+
+\*These performance tests are tightly coupled to the seed scripts that populate test data. This means these tests can only be run in an environment containing solely test data created by the seed data scripts. `perftest` is a dedicated environment to do this in, but in theory any environment could be populated with the seed data and used.

tests/performance/perftest_environment.py

@@ -5,6 +5,7 @@
 import re
 
 import boto3
+from seed_data_constants import CHECKSUM_WEIGHTS
 
 DYNAMODB = boto3.resource("dynamodb", region_name="eu-west-2")
 
@@ -55,10 +56,6 @@ def extract_consumer_data(output_dir="."):
     print(f"Consumer data written to {out}")  # noqa: T201
 
 
-# Semi-deterministic NHS number generator (duplicated from seed_nft_tables.py)
-CHECKSUM_WEIGHTS = [i for i in range(10, 1, -1)]
-
-
 class TestNhsNumbersIterator:
     def __iter__(self):
         self.first9 = 900000000

tests/performance/producer/delete_permissions.py

@@ -1,7 +1,8 @@
 from pathlib import Path
 
 import fire
-from seed_nft_tables import DEFAULT_CUSTODIAN_DISTRIBUTIONS
+
+from tests.performance.seed_data_constants import DEFAULT_CUSTODIAN_DISTRIBUTIONS
 
 
 def main(permissions_dir="../../dist/nrlf_permissions/K6PerformanceTest"):

tests/performance/producer/generate_distributions.py

@@ -1,8 +1,10 @@
 import json
 from pathlib import Path
 
-# Import the constants from your seed_nft_tables.py
-from seed_nft_tables import DEFAULT_CUSTODIAN_DISTRIBUTIONS, DEFAULT_TYPE_DISTRIBUTIONS
+from tests.performance.seed_data_constants import (
+    DEFAULT_CUSTODIAN_DISTRIBUTIONS,
+    DEFAULT_TYPE_DISTRIBUTIONS,
+)
 
 
 def expand_distribution(dist):

tests/performance/producer/generate_permissions.py

@@ -2,7 +2,8 @@
 from pathlib import Path
 
 import fire
-from seed_nft_tables import DEFAULT_CUSTODIAN_DISTRIBUTIONS
+
+from tests.performance.seed_data_constants import DEFAULT_CUSTODIAN_DISTRIBUTIONS
 
 
 def main(output_dir="../../dist/nrlf_permissions/K6PerformanceTest"):