NRL-1884 Fix tests to use default location for input files rather than relying on prepare step to copy around. Include table name in info file rather than input file name

Author: anjalitrace2-nhs

Makefile

@@ -270,7 +270,7 @@ perftest-prepare:	## Prepare input files for producer & consumer perf tests
 	mkdir -p "${DIST_PATH}/nft"
 	aws s3 cp "s3://nhsd-nrlf--${ENV}-metadata/performance/seed-pointers-extract-${PERFTEST_TABLE_NAME}.zip" "${DIST_PATH}/pointer_extract-${PERFTEST_TABLE_NAME}.zip"
 	unzip "${DIST_PATH}/pointer_extract-${PERFTEST_TABLE_NAME}.zip"
-	cp "${DIST_PATH}/nft/seed-pointers-extract-${PERFTEST_TABLE_NAME}.csv" "${DIST_PATH}/seed-pointers-extract.csv"
+# 	cp "${DIST_PATH}/nft/seed-pointers-extract-${PERFTEST_TABLE_NAME}.csv" "${DIST_PATH}/seed-pointers-extract.csv"
 	PYTHONPATH=. poetry run python ./tests/performance/generate_producer_distributions.py
 
 perftest-producer:	## Run producer perf tests

scripts/seed_nft_tables.py

@@ -8,8 +8,6 @@
 
 import boto3
 import fire
-
-# import json
 import numpy as np
 
 from nrlf.core.boto import get_s3_client
@@ -22,6 +20,7 @@
 from nrlf.core.dynamodb.model import DocumentPointer
 from nrlf.core.logger import logger
 from nrlf.tests.data import load_document_reference
+from tests.performance.perftest_environment import create_extract_metadata_file
 from tests.performance.seed_data_constants import (  # DEFAULT_COUNT_DISTRIBUTIONS,
     CHECKSUM_WEIGHTS,
     CUSTODIAN_DISTRIBUTION_PROFILES,
@@ -89,7 +88,7 @@ def _make_seed_pointer(
 
 
 def _write_pointer_extract_to_file(table_name, pointer_data):
-    local_csv_out = f"{nft_dist_path}/seed-pointers-extract-{table_name}.csv"
+    local_csv_out = f"{nft_dist_path}/seed-pointers-extract.csv"
     local_meta_out = f"{nft_dist_path}/info.json"
 
     print(f"writing pointer extract to files {local_csv_out} {local_meta_out}")
@@ -100,8 +99,7 @@ def _write_pointer_extract_to_file(table_name, pointer_data):
         writer.writerows(pointer_data)
     print(f"Pointer data saved to {local_csv_out}")
 
-    os.system(f"./scripts/get-current-info.sh > {local_meta_out}")
-    print(f"Pointer extract metadata saved to {local_meta_out}")
+    create_extract_metadata_file(table_name, nft_dist_path)
 
 
 def _populate_seed_table(

tests/performance/README.md

@@ -36,7 +36,7 @@ Currently, this requires tearing down the existing environment and restoring fro
      2. once backed up, delete your table. In the AWS console: dynamodb > tables > your perftest table > actions > delete table
      3. Rerun the Deploy Account-wide infrastructure action.
      4. Terraform will create an empty table with the correct name & (most importantly!) read/write IAM policies.
-     5. Delete the empty table created by terraform and restore from the backup, specifying the same table name you've defined in code.
+     5. Delete the empty table created by terraform and restore from the backup, specifying the same table name you've defined in code & selecting the matching customer managed encryption key.
 6. Run the [Persistent Environment Deploy](https://github.com/NHSDigital/NRLF/actions/workflows/persistent-environment.yml) workflow against your branch & `perftest` to restore the environment with lambdas pointed at your chosen table.
 7. You can check this has been successful by checking the table name in the lambdas.
    - In the AWS console: Lambda > functions > pick any perftest-1 lambda > Configuration > Environment variables > `TABLE_NAME` should be your desired pointer table e.g. `nhsd-nrlf--perftest-baseline-pointers-table`

tests/performance/consumer/client_perftest.js

@@ -3,9 +3,8 @@ import { check } from "k6";
 import exec from "k6/execution";
 import { CATEGORY_TYPE_GROUPS } from "../type-category-mappings.js";
 
-const csvPath = __ENV.DIST_PATH
-  ? `../../../${__ENV.DIST_PATH}/seed-pointers-extract.csv`
-  : "../seed-pointers-extract.csv";
+const distPath = __ENV.DIST_PATH || "./dist";
+const csvPath = `../../../${distPath}/nft/seed-pointers-extract.csv`;
 const csv = open(csvPath);
 const lines = csv.trim().split("\n");
 // Skip header

tests/performance/generate_producer_distributions.py

@@ -1,7 +1,7 @@
 import json
 from pathlib import Path
 
-from seed_data_constants import (
+from tests.performance.seed_data_constants import (
     DEFAULT_CUSTODIAN_DISTRIBUTIONS,
     DEFAULT_TYPE_DISTRIBUTIONS,
 )

tests/performance/perftest_environment.py

@@ -1,9 +1,11 @@
 import csv
+import json
 import os
 import re
 
 import boto3
-from seed_data_constants import CHECKSUM_WEIGHTS
+
+from tests.performance.seed_data_constants import CHECKSUM_WEIGHTS
 
 DYNAMODB = boto3.resource("dynamodb", region_name="eu-west-2")
 
@@ -19,6 +21,26 @@ def _get_pointers_table_name():
     return f"nhsd-nrlf--{perftest_table_name}-pointers-table"
 
 
+def create_extract_metadata_file(
+    table_name: str,
+    output_dir=".",
+):
+    meta_out = output_dir + f"/info.json"
+
+    os.system(f"./scripts/get-current-info.sh > {meta_out}.temp")
+
+    with open(f"{meta_out}.temp", "r") as temp_f:
+        metadata = json.load(temp_f)
+        # Set any additional metadata fields (e.g. table name) as needed
+        metadata["table_name"] = table_name
+        with open(f"{meta_out}", "w+") as f:
+            json.dump(metadata, f, indent=2)
+
+    os.system(f"rm {meta_out}.temp")
+
+    print(f"Pointer extract metadata saved to {meta_out}")  # noqa: T201
+
+
 class TestNhsNumbersIterator:
     def __iter__(self):
         self.first9 = 900000000
@@ -50,7 +72,7 @@ def generate_pointer_table_extract(
     Generate a CSV file containing all pointer IDs, pointer type, custodian, and nhs_number (patient).
     """
     table_name = _get_pointers_table_name()
-    out = output_dir + f"/seed-pointers-extract-{table_name}.csv"
+    out = output_dir + f"/seed-pointers-extract.csv"
     table = DYNAMODB.Table(table_name)
     scan_kwargs = {}
     done = False
@@ -95,6 +117,8 @@ def generate_pointer_table_extract(
             writer.writerows(buffer)
     print(f"Pointer extract CSV data written to {out}")  # noqa: T201
 
+    create_extract_metadata_file(table_name, output_dir)
+
 
 if __name__ == "__main__":
     import fire

tests/performance/producer/client_perftest.js

@@ -6,9 +6,8 @@ import { crypto } from "k6/experimental/webcrypto";
 import { createRecord } from "../setup.js";
 import exec from "k6/execution";
 
-const csvPath = __ENV.DIST_PATH
-  ? `../../../${__ENV.DIST_PATH}/seed-pointers-extract.csv`
-  : "../seed-pointers-extract.csv";
+const distPath = __ENV.DIST_PATH || "./dist";
+const csvPath = `../../../${distPath}/nft/seed-pointers-extract.csv`;
 const csv = open(csvPath);
 const lines = csv.trim().split("\n");
 // Skip header