diff --git a/Dockerfile b/Dockerfile index 454e924..131753c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use a Python image with uv pre-installed -FROM ghcr.io/astral-sh/uv:python3.13-alpine +FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim # Install the project into `/app` WORKDIR /app diff --git a/README.md b/README.md index 1561d0d..f994606 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # sds-loader +![Version](https://ons-badges-752336435892.europe-west2.run.app/api/badge/custom?left=Python&right=3.13) + A microservice for loading and modifying data into SDS @@ -108,4 +110,7 @@ docker run \ spine3/firebase-emulator & ``` +## Performance testing + +The directory `performance_tests` contains a script for generating large datasets, that can be used to performance test the application diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 06382ef..0000000 --- a/TODO.md +++ /dev/null @@ -1,2 +0,0 @@ - -- Integration tests diff --git a/app/profiles.py b/app/profiles.py index cd48ef4..9dc061a 100644 --- a/app/profiles.py +++ b/app/profiles.py @@ -91,8 +91,8 @@ def build_pubsub_broadcaster() -> PubsubBroadcaster: container[DatasetDeletionRepositoryInterface] = FirestoreDatasetDeletionRepository container[DatasetBroadcastInterface] = PubsubBroadcaster container[SchemaService] = SchemaService( - bucket_publisher=GcsSchemaPublisher, - repository_publisher=GithubSchemaPublisher, + bucket_publisher=GcsSchemaPublisher(), + repository_publisher=GithubSchemaPublisher(), ) diff --git a/app/repositories/dataset_deletion/firestore_dataset_deletion_repository.py b/app/repositories/dataset_deletion/firestore_dataset_deletion_repository.py index 5834c67..be95124 100644 --- a/app/repositories/dataset_deletion/firestore_dataset_deletion_repository.py +++ b/app/repositories/dataset_deletion/firestore_dataset_deletion_repository.py @@ -64,7 +64,7 @@ def mark_record_status(self, guid: Guid, status: DeleteStatus) -> None: document_reference.update({"status": status.value}) # If the status is deleted, then mark a timestamp - if status == DeleteStatus.DELETED: + if status == "deleted": utc_dt = datetime.now(timezone.utc) # UTC time dt = utc_dt.astimezone() # local time timestamp = dt.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -74,7 +74,7 @@ def mark_record_status(self, guid: Guid, status: DeleteStatus) -> None: def get_dataset_to_delete(self) -> Guid | None: # First, try to fetch a PROCESSING dataset - processing = self.mark_deletion_collection.where("status", "==", DeleteStatus.PROCESSING.value).limit(1).stream() + processing = self.mark_deletion_collection.where("status", "==", "processing").limit(1).stream() processing_list = list(processing) @@ -98,7 +98,7 @@ def get_dataset_to_delete(self) -> Guid | None: return guid # If no "Processing" results found, fetch a PENDING dataset - pending = self.mark_deletion_collection.where("status", "==", DeleteStatus.PENDING.value).limit(1).stream() + pending = self.mark_deletion_collection.where("status", "==", "pending").limit(1).stream() pending_list = list(pending) diff --git a/app/repositories/dataset_storage/firestore_dataset_storage_repository.py b/app/repositories/dataset_storage/firestore_dataset_storage_repository.py index f7a0833..99ffff2 100644 --- a/app/repositories/dataset_storage/firestore_dataset_storage_repository.py +++ b/app/repositories/dataset_storage/firestore_dataset_storage_repository.py @@ -1,6 +1,7 @@ from typing import Protocol from google.cloud import firestore +from google.cloud.firestore_v1.bulk_writer import BulkWriterOptions from app import get_logger from app.interfaces.dataset_storage_repository_interface import DatasetStorageRepositoryInterface @@ -13,7 +14,6 @@ class FirestoreSettings(Protocol): project_id: str firestore_database: str - should_batch: bool class FirestoreDatasetStorageRepository(DatasetStorageRepositoryInterface): @@ -35,10 +35,6 @@ def __init__(self, settings: FirestoreSettings): f"Connected to Firestore with project_id: {settings.project_id} and database: {settings.firestore_database}" ) - # Max size in bytes we can upload in one batch - self.MAX_BATCH_SIZE_BYTES = 9 * 1024 * 1024 - self.MAX_NUMBER_OF_WRITES_PER_BATCH = 500 - # Initialize Firestore collections self.dataset_collection = self.client.collection("datasets") @@ -88,107 +84,53 @@ def store_dataset( unit_data_collection_with_metadata: list[UnitDataset], unit_data_identifiers: list[str], ): - # Write to firebase in batches or not depends on the settings - - if self.settings.should_batch: - logger.info("Writing to Firestore in BATCH mode") - self._store_dataset_with_batching( - dataset_id=dataset_id, - dataset_metadata=dataset_metadata, - unit_data_collection_with_metadata=unit_data_collection_with_metadata, - unit_data_identifiers=unit_data_identifiers, - ) - else: - logger.info("Writing to Firestore in NORMAL mode") - self._store_dataset_without_batching( - dataset_id=dataset_id, - dataset_metadata=dataset_metadata, - unit_data_collection_with_metadata=unit_data_collection_with_metadata, - unit_data_identifiers=unit_data_identifiers, - ) - - def _store_dataset_without_batching( - self, - dataset_id: Guid, - dataset_metadata: DatasetMetadataWithoutId, - unit_data_collection_with_metadata: list[UnitDataset], - unit_data_identifiers: list[str], - ): - """ - Write the data to firestore without batching - """ - # Create a new document for this dataset - new_dataset_document = self.dataset_collection.document(dataset_id) - - # Store the core data first - new_dataset_document.set(dataset_metadata.model_dump(), merge=True) - # Create a new collection for the units - units_collection = new_dataset_document.collection("units") - - # Go through unit data - for unit_data, unit_identifier in zip(unit_data_collection_with_metadata, unit_data_identifiers): - # Create and save the unit data as a new sub document - units_collection.document(unit_identifier).set(unit_data.model_dump()) + logger.info("Writing to Firestore in BATCH mode") + self._store_dataset_with_bulk_writer( + dataset_id=dataset_id, + dataset_metadata=dataset_metadata, + unit_data_collection_with_metadata=unit_data_collection_with_metadata, + unit_data_identifiers=unit_data_identifiers, + ) - def _store_dataset_with_batching( + def _store_dataset_with_bulk_writer( self, dataset_id: Guid, dataset_metadata: DatasetMetadataWithoutId, unit_data_collection_with_metadata: list[UnitDataset], unit_data_identifiers: list[str], ): - """ - Use batches to write the data to firestore - """ - - # Create a new document for this dataset new_dataset_document = self.dataset_collection.document(dataset_id) - - # Store the core data first - new_dataset_document.set(dataset_metadata.model_dump(), merge=True) - - # Create a new collection for the units units_collection = new_dataset_document.collection("units") - # Initialise a batch - batch = self.client.batch() - batch_size_bytes = 0 - batch_num_records = 0 - - # Go through unit data - for unit_data, unit_identifier in zip(unit_data_collection_with_metadata, unit_data_identifiers): - """ - Add this unit to the current batch if ... - - 1. adding it does not exceed the batch size limits - 2. adding it does not exceed the batch record limits - """ + bulk_writer = self.client.bulk_writer( + options=BulkWriterOptions( + max_ops_per_second=2500 + ) + ) - # Work out the size of this unit - unit_size = len(unit_data.model_dump_json().encode("utf-8")) + try: + # Include metadata in bulk + bulk_writer.set( + new_dataset_document, + dataset_metadata.model_dump(), + merge=False, + ) - # Work out if the current batch is too big already - if (batch_size_bytes + unit_size >= self.MAX_BATCH_SIZE_BYTES) or ( - batch_num_records + 1 >= self.MAX_NUMBER_OF_WRITES_PER_BATCH + for unit, unit_identifier in zip( + unit_data_collection_with_metadata, + unit_data_identifiers, ): - # Commit the current batch - batch.commit() - - # Start a new batch - batch = self.client.batch() - batch_size_bytes = 0 - batch_num_records = 0 + bulk_writer.set( + units_collection.document(unit_identifier), + unit.model_dump(), + merge=False, + ) - # Add the unit to the new batch - new_unit = units_collection.document(unit_identifier) - batch.set(new_unit, unit_data.model_dump(), merge=True) - batch_size_bytes += unit_size - batch_num_records += 1 + bulk_writer.flush() - # If we never exceeded batch limit we still need to commit - if batch_size_bytes > 0: - batch.commit() + finally: + bulk_writer.close() def delete_dataset_version(self, survey_id: str, period_id: str, version: int): dataset_metadata = self._get_dataset_metadata(survey_id, period_id, version) @@ -206,7 +148,7 @@ def delete_dataset_by_guid(self, guid: Guid): # Delete each collection for collection in collections: - collection.recursive_delete() + self.client.recursive_delete(collection) # Delete the dataset itself self.dataset_collection.document(guid).delete() diff --git a/app/routes.py b/app/routes.py index 5ab5c23..9fec411 100644 --- a/app/routes.py +++ b/app/routes.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Request from fastapi.params import Query from lagom.integrations.fast_api import FastApiIntegration -from sdx_base.models.pubsub import get_message, Message, get_data +from sdx_base.models.pubsub import get_message, Message from starlette.responses import JSONResponse from app import get_logger @@ -12,6 +12,7 @@ from app.services.dataset_service import DatasetService from app.services.schema_service import SchemaService from app.settings import get_instance +from app.util.file_getters import get_file_path_from_bucket_notification, get_file_paths_from_github_notification logger = get_logger() router = APIRouter() @@ -47,19 +48,24 @@ async def version(): async def publish_schemas( request: Request, source: Annotated[ - Literal["github", "bucket"], Query(description="The source of the files specified in this request.") + Literal["github", "bucket"], Query(description="The source of the file specified in this request.") ] = "github", schema_service: SchemaService = DEPS.depends(SchemaService), ): """ - This endpoint handles a publishing schemas from a given - location. + This endpoint handles a publishing schemas from a given location + + - If the source is "bucket", this will always be a single file (the one added to the bucket) + - The body of the message Must contain a "name" field in the json payload, this specifies the name of the file in the bucket + - If the source is "github", this could be multiple files (the new additions to the repo) + - The body of the message must contain a comma separated list of file names """ try: # Fetch the message from pubsub message: Message = await get_message(request) except Exception as e: + logger.exception("Exception fetching message from request") return JSONResponse( status_code=500, content={"success": False, "message": "Invalid message body received: " + str(e)}, @@ -67,7 +73,10 @@ async def publish_schemas( try: # Publish the new schemas - schema_service.publish_new_schemas(source=source, file_list=get_data(message).split("\n")) + schema_service.publish_new_schemas( + source=source, + file_list=[get_file_path_from_bucket_notification(message)] if source.lower() == "bucket" else get_file_paths_from_github_notification(message), + ) except NonCriticalException as e: # Return a status 200 (non-critical exception) return JSONResponse( @@ -76,6 +85,8 @@ async def publish_schemas( ) except (SchemaException, Exception) as e: + logger.exception("Exception publishing schemas") + return JSONResponse( status_code=500, content={"success": False, "message": "Exception publishing schema: " + str(e)}, diff --git a/app/services/schema_service.py b/app/services/schema_service.py index 18d45f1..7e916df 100644 --- a/app/services/schema_service.py +++ b/app/services/schema_service.py @@ -36,10 +36,10 @@ def _publish_single_file(self, file_name: str, publisher: PublisherProtocol): # :param publisher: publisher - the publishing protocol to use to publish the file """ try: - publisher.publish_schema(file_name) + publisher.publish_schema(file_name=file_name) logger.info(f"Successfully published schema: {file_name}") except Exception as e: - logger.error(f"Failed to publish schema {file_name}: {e}") + logger.exception(f"Failed to publish schema {file_name}: {e}") def _filter_github_files(self, files: list[str]) -> list[str]: # noqa """ diff --git a/app/settings.py b/app/settings.py index 445f586..c301a75 100644 --- a/app/settings.py +++ b/app/settings.py @@ -24,7 +24,6 @@ class Settings(AppSettings): project_id: The GCP project ID autodelete_dataset: Whether to automatically delete datasets from the source repo (bucket) after publishing retain_old_dataset: Whether to retain old versions of an updated dataset in the target repo (firestore) - should_batch: Whether to batch write data to firestore to avoid memory limits dataset_bucket_name: The bucket name to pick up datasets from firestore_database: The Firestore database to publish datasets to publish_dataset_topic_id: The Pub/Sub topic ID to publish dataset updates to @@ -34,8 +33,7 @@ class Settings(AppSettings): project_id: str = "ons-sds-sandbox" autodelete_dataset: bool = True - retain_old_dataset: bool = True - should_batch: bool = True + retain_old_datasets: bool = True dataset_bucket_name: str firestore_database: str publish_dataset_topic_id: str diff --git a/app/util/__init__.py b/app/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/util/file_getters.py b/app/util/file_getters.py new file mode 100644 index 0000000..985b459 --- /dev/null +++ b/app/util/file_getters.py @@ -0,0 +1,24 @@ +import json + +from sdx_base.models.pubsub import get_data + +from app import get_logger + +logger = get_logger() + + +def get_file_path_from_bucket_notification(message) -> str: + """ + Extract the file name from the bucket notification message + """ + raw_data = get_data(message) + raw_dict = json.loads(raw_data) + logger.debug("Bucket notification message: ", raw_dict) + return raw_dict["name"] + + +def get_file_paths_from_github_notification(message) -> list[str]: + """ + Extract the file names from the github notification message + """ + return get_data(message).split("\n") diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 0000000..80d49d4 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,63 @@ +steps: + - name: gcr.io/cloud-builders/docker + id: "docker build and push" + entrypoint: sh + args: + - "-c" + - | + docker build --cache-from "europe-west2-docker.pkg.dev/${PROJECT_ID}/sds-loader/sds-loader:latest" -t "europe-west2-docker.pkg.dev/${PROJECT_ID}/sds-loader/sds-loader:${SHORT_SHA}" -t "europe-west2-docker.pkg.dev/${PROJECT_ID}/sds-loader/sds-loader:latest" . + docker push "europe-west2-docker.pkg.dev/${PROJECT_ID}/sds-loader/sds-loader:${SHORT_SHA}" + docker push "europe-west2-docker.pkg.dev/${PROJECT_ID}/sds-loader/sds-loader:latest" + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "Run container" + entrypoint: gcloud + args: + [ + "run", + "deploy", + "sds-loader", + "--no-invoker-iam-check", + "--image", + "europe-west2-docker.pkg.dev/${PROJECT_ID}/sds-loader/sds-loader:${SHORT_SHA}", + "--region", + "europe-west2", + "--allow-unauthenticated", + "--ingress", + "internal-and-cloud-load-balancing", + "--set-env-vars", + "DATASET_BUCKET_NAME=ons-sds-angus-sds-europe-west2-dataset", + "--set-env-vars", + "FIRESTORE_DATABASE=ons-sds-angus-sds", + "--set-env-vars", + "PUBLISH_DATASET_TOPIC_ID=projects/ons-sds-angus/topics/ons-sds-publish-dataset", + "--set-env-vars", + "PROJECT_ID=${PROJECT_ID}", + "--set-env-vars", + "RETAIN_OLD_DATASETS=true", + "--set-env-vars", + "SECRET_ID=iap-secret", + "--set-env-vars", + "SCHEMA_PUBLISH_BUCKET_NAME=ons-sds-angus-europe-west2-schema-publish", + "--set-env-vars", + "SDS_URL=https://34.98.110.64.nip.io", + "--cpu-boost", + "--session-affinity", + "--min-instances", + "1", + "--max-instances", + "100", + "--concurrency", + "1", + "--cpu", + "4", + "--memory", + "16Gi", + "--timeout", + "3600s", + "--service-account", + "cloudbuild-sa@ons-sds-angus.iam.gserviceaccount.com" + ] + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..f7b39bc --- /dev/null +++ b/compose.yaml @@ -0,0 +1,20 @@ +services: + sds-loader: + build: + context: . + dockerfile: Dockerfile + + container_name: sds-loader + + ports: + - "5006:5006" + + environment: + PROJECT_ID: ons-sdx-bob + PROFILE: dev + DATASET_BUCKET_NAME: ons-sdx-bob-datasets + FIRESTORE_DATABASE: test + PUBLISH_DATASET_TOPIC_ID: projects/ons-sds-sandbox/topics/publish-dataset + PORT: 5006 + + restart: unless-stopped diff --git a/performance_tests/BRES_20_unit_data_dummy_encrypted.txt b/performance_tests/BRES_20_unit_data_dummy_encrypted.txt new file mode 100644 index 0000000..7206efe --- /dev/null +++ b/performance_tests/BRES_20_unit_data_dummy_encrypted.txt @@ -0,0 +1 @@ +eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkEyNTZHQ00iLCJraWQiOiJjZGRmZWRjOWU1OTQ2MmU0MmViNjAxY2ZhYzE5YmNjNGViYjkzMzIzIn0.HJwLJKTe6YeieGDVF3Glf_xE_qqnoo18FBMKcuHF5leOJrCX2dqLBGIxK-FeSSsq3Z0ce08wcFJo5DauMTweZM5vJCCOaybiNwx2KykfhLaQaMUo6y2Sf_LNhUX07HX1LcB6mXG9S2xUKmjxGV8r_tC6nL52sdMNvczmdXrQUKSl8n7yMjPD-LkxUazuDe5hNVsO-1JMkIByVutFP3CyYtKWhFVXX58i9m2x1MM0dWigARrP138JZV2mttRiuJFV4R8jSeK2PrhE4RtWLm4RCMgDcxu3M7E5KG0C1gmpra1eOVzK3LX1XBEPgYf7cJ3pc3TrGNvcLdWq8CZPHR_8M3FZVQ5Jm2AJ0IGa70xjWdxofwhpPP-2cKOJUfxSRptHHd1_pJUqH0uNYzVYbs3PncPIJfabTzfI89HBPymtMV8XmIQNdcAqlJwljFM1VyD8kBqNrR4o5k0RT5VyS4RXodV_8aT7fyWywdTiO7et6cBsOMpWjCWSeJxJ2_1LFmB8LwphZJTnzGbuwPivCjJt0DTVEBngQSQzqTU2waAA8n-H0BBA7LdhzusgDd5oka9yk6vednjGcW_hmrbPuwmtxa8yIYdSVf1vZO9pvGxnhDyXm6apVtWmvLZM1C40K3BZEMQRo2WTOoCGiVn7afFHI-lJd4SaAmjA6ymjhXFTfKE.hSEH3_pAnLf4x1Ij.RZDyCbkdSa8YekpqWI4nipeBbbZ72GkZ2_S7jOr9b3tYkERm4cqQqPwknBHeSotwn5K-VzZQytzsGbuOCSEiOUnyhnQIT3CkulrBR_3pd7oOyJhODW7CBXs9TD6QfrvSFDH-VVjyNBhneP820y0cSt04viK55x3E9BlkbIw4ZQ8C8PXgHYWm_8o3P8pS0h3zxCj0qmLWV1Mcagdruvyareqa102CJen55tZfpKTi_P5DiI9DB4Pm1rcadORzxGffqWa1vH5VL4QqCIw7E6h3I5zsgNXyv22JlQtn5XcUCPhegYNiSpecR8aalSc69-wBBWPReW7N7bzGX_-A7yIGFqxT_wvIt8SQDGeZZokU6KL2EyKEklWeZClnESSDPBpmX7BTih7eTaJitiAwKlNvPo6QuAQFp5kYuBK7FHtXTVDNmA6DeQVVLqmY2872Q67A5kyJ4LW49ytgbue0P2uotDcOwuDKV3TfkiaHYuC4NMLWhwuJNfvOuoZtjI0AgfjsKIxp3vjCfHiEsIHgQNY3xxiE_jcKzlrWYPgCNF8r1KO210rHRi3SXiu-pY0KlHsD6yvdOYmj05RFLShbLv_skFYIhfkijR1FHKXGD6wPEJH0JDPDrB31P2B_b2AE6AiOKp0lzpfXmKMcpq7x_RUzkc_Ix0vbmEF7kkEatZyfWwkc4nr7PyCOs398OSkuDbpbBJXti2vC_MRWl638qPSEGbV2hxYhj00CpLunOTbAS6RG_918b2KHJ-MIUOsYaSEgL2yo0-TL1gvXl7rhFLHFQ3g3mPw7hKHcjNPN67C2fu7slESltWJ-f1Y-J2KL5nSdz8yc72NGncupTvgJ4zWUE_F_kZPkxhJB643se7oGu9_fitUvD6PDrUSwRsaVIEe3jxTGxfY2hWXOBSUUNycIUq_8hGN3OxjhYu9_VPuI2444rUZmVyyZPYtuwf2KGlmQ97egQ2MBC2Cse46-fmC1YL0ouJPrrwPdQvEVV2joukMEKUzllheF31RZJwRx1eg5JoF4RLulOwgGLRwOTYVhrmz2RBn5LUeDECt15Go_ZVkjCIVtv2UR2-JU9kxc3RCpv9lLRCZc_YTC0Z-GtooPC3XwMF6G4ASFKtf-p2RiOT4Ye-EiQ3T5Ok4zDfsHn124vGNYOUNSYKqNzZsZcAwkC8IQ6AIr8YK3rlAqQ8w1K4-20tNCdFKTc8kx-_dY8cbCReIoHk6JBVn9XygUAUXE8GU5QRZ3HQmMK-DQHvEOiL0Eg40Lq4axjqtS5oVNLSM-kQpLy9rbK9ieRE47KjHuAU7r2UqZcy9FHwYUwFJyIdQLzdWFwCNTeDUjnibIU2shLATb5b9rDVvIxcWNL7DtZy5ZfacyZ-2M_tlXUKTiC21VKHQZ0wcgoQhTArv5LJh0F2xqrhhasppFFlxIMip7LFiAzWuwOnZV_Ewk6VWeHLv8vk6DzMLXsjeJFoXgYOECX2s3FNV9M57_gPBhEeD2D5PChkOzpBFerdPkW5dhYFpqZAKsoA7vSrU3HCAACb6itqlAIcvd-xkGg5JwQpDVQ0ia57xFK8JUfDR9CKkG6iyZ-LEvJH3-MgKPkovFPWsaLUMD9JH-WbITy7LeGrYfL5TB4wNVj8EnTlGa_q-jf8BWYB8PVWJzorVDmO1XyK0y9fk4QdjO0OYF2ZDfgPKQz94IUuqAXx3nAmVowfns8yH-BDKzjQ4ojPu9BYB80u8Wf-WmhHf348biUY1uYx4IyBP8ATY6YQr9wobrf1-u4WVyMX7SlZv5oJWbaNzSnJavSI4xkH4vRa-FicO8mTDkk-yMcrvxlJkM9YlB3j_27WGC2kd3zP3w8tGz3hpTlJCZY52fouKh_2E8T3c8z088Zj3MfsDnMd4KZfLONhfhpfYvFODgYAxOW5HsDgpndPUMk4CzdzfaeD1RKCNu5gEb4wxsEj061tyqgx-eN_J0rKtkQ4wiNYTZoQhqdCRpiUaO5OnLE4cELq7jxd7zc2OAwbhSNCUDSFXZZtojoNVYmvTmtwTJaKEVSnvwsdCy6aHLBSdEgXawXBIrRK5zN31UwzQl99dCKtlMPx_Bsle5jOhP7T6U4Cr87NU4N8a71JA0XI-EvnQJQibFfpQ-KZrjc6i_5bVQDEU7abgpq_GylTjUYpDKRQSWmJAxiff6SamvQ8SE5q45bM59ocWCSMHCQcr7-GBYgbMTku39QoXjeaUVFqd2Looond6sff59Ido52HvS2KyWianJYvaveklStWkaXtSi_X4XNr67KZs3xO5j0CdD4wMbc_rFlaxGoVHI0rsvul3uSHr2riaM5jgJKAWl7_QWqZ3kYLug0md58usErv1pI67wXoYxLcMksZ8uSQo1JWibNTMWTFgIJnzzHiUo6aJWXvPcb6mp2-0Duy4XVzR_riUg-EbgptIScwODIrpAG_HT9jzfrb8lxjsVT7-UGwZNCsIVLLTVWonY-FalNSiaOq7dXSeL65XLOQK85mPL-xzKMqd7_NawfJOXvui3LPszVXG24uZd2fA-HYlzHGsYOa5cxmhBCdj3l8tlJq6MTbLubzbApYmdJsz-gVo_eDQNXghOkGbt7YXiL9QomlGzSrx2MLUU_dXOGTjWVdatJzlUITnXfrYwHTd7TX1NMh134h_8XvfsYX6sDV1q1AAKMrDxJ2zcAMWLAdnw8F9_9Ix4QIqKVMXk_4PgDM10Kbh-QUdFdsWUNFFXEeYcvRjh34EyFe8DFIGutbRCVDsawp7E7dgsHUygM1aIRX4hcVM-tnF1DN-baddVao2qBprVwfaCegdBOP7VWEY0ru19WCIITlBiHD7fE5pbH45hw_ciQGgnf9mTF_QbMZI6YovXnus4d2L4BQVQjIXikohRQCLmnl_bzSDl8VJzR7ARiegpqnn-98yAJ4XzaVnfm-zpTC6tAhZ39ZDdwp5xNPZrbPXd03j1MFIoefUr_JAc6nluPCCvMhwlaWO1Iur96tfiD9IYeG6awhDPZIs84ze2VbKsSPqC-nLvYJv0y57c47yo5SPzVL4OTWoRH9szGRSSmwNxua_9d4sYS--D5BLQmo_VntviTN9C4oL0ThXi8CBb3YoH-eAZibv7-uO8pmOC6j2sdo3XeqdaVjxiqE1b2b8FvSywk_iu7o_UrSSPRaFjYz8u9Q7Q_04nwkdg9xHGneuhmL4L1HWAb8atp1tcrO04oZUf3wl8zXRjLFk1bgcm1XKkr7JCBQG5HxsP-c6CyHfxptWvfnEsMjhoU7I153A59nkcI0CQZBRCdcXzX6uUeRFtC12TG5tA1R2QApOyKow9Jyna3yaQKmPee2UePRzx4SvwQFNmpucHfgibQ9OrFp6ArzQVENyEYPXYLLqXME3LxZubsiMsCPQNB3WY_UQ0Dvua8sY8goU8GY0V3sU5WQZ0ngCokLAmi9hbtdTr5e2uIGhcCLKoQSRHJiXOl8dxrK5meccbfxA1uKU9TtCLdi9W9uGcSzaBxgi7RfyjnYK7XdUYYD51rgAmmNfuCv2l94Mys1NdGc-sWCASrpru0wL8nSLlo3HuIDWzcCPJUmGulFAKG1NxHqrJLNU0LCPAXACBSTeVrvdV-bK3V1rUf49-4d52fxGWUEBjLJ3zIY9kZHEFQ7jT0ksVMKBKGnZT5dcLad_08wTes1M7-W8qn-WfL_-wBUulG6mj2SH366ITHQgT-_aL10yt3lSsEwUQ4I5_g6Endn8fffBLdbRTADHdy0BPddX80e1-6zGkpSw2AFwwjlXFiWlyjVQLdfydX3mUxSDyvAeZVMSben7C2nvvLq1wA8sCgg9jOh9WPJ0KZnOz2wfKho1hVud89sjlkwK0fvkkwzwsf6kzCnlsXy0xFKG2PR1Kgm6RAwT1TbcLpy8VCltrk7snpr9KDMUkOwISQeZJxjoH6yKqWcJ40D5UHU9xTXDuZRB472PxuHSW_hZwMD-SsCT9IcPxPmtmHJdc7q30YmhPezkDzVfYBVJxtRzoAHdtW1q0dQixNjg9I2UFe6bO-SXjLDW6FYPEUMVzcn4JuXVtpNSi1Rq3hFDUd3_0-s3u11c9pbB07dP_2ySHDzD9d0OCWzsOl40F_uUFN5yNxiLF_Wrgb8mu29DQBTa5JNUWdMf0lJZAquxDULugXaHCO6vv8zud8PDADSh0LKMynrQq5LLB7rYD1-q2DXUsCeIomm5-ui5sKHWUrpHT5ZeGSjaBMmn1pRb0hPTfGfYpRhhOzvPnw0HGReroIl6IoFONTAKZ2K75FuC5KKv4zCHMNGd7uDaMlIFI3yEKaopK6HHe_QWv0s6CSZ7LevmUPK1dDnAMyb3zBi0cfA2DrtpEFa7IweoYEn9dhr4ufAPWWwqWEg8_VpHADl_QYVoycKTa-lUrM_FjhtwxwUjquUYlVZ5bZr1kH3PUpI61UvJmP0JZFGHhK8GhPTXElf5zR3S753Siff-rImgeusYFjkhYo6EyyCIW5Pku4-YxjrPeFzP26DVIArfK31VNZ6NEnlqAHT-olq5UOQNccVnay6e_tv6FflqssyRYVdZ6IR3zDTqdwAKgicznLStj11IQytVGIsuyURsOP07UJhYk4RIrnlmT71rp5HcSy8_mSi4H_12y_syt1rfqrssAa7ICPzhtRDJoH8WN1UEWT2_zFi06tbwEPD_PVvG_ujzyAymHHiBcg8x6F70NYdU_RzVu5wF3uToN3b9qQ4nE-DD9mCU_NhWXo84zVX9dLdfoA9AbenEEIixHIFWbpNNcV5nScYrbfIgeDrCQBDnd-KcK9kBLZa5iiQbi-uKNPkudVTchwu-lOx5plshOClgT8MjJ72EPOMztsh5HjlrqD8m70EeuTE_H5JvSAFDKfItkERB1RcEfGvOD1-U9uVdpkSlS2YUlmFxqpZN-3whlB8_HugjV-zyViy71lQpl_1g9dj4vilQt8T-lOEJ4hDqLpmXG_THaVFlGEAVEu1pyTVEz34i2thTNbjwlePdyqcKBxocdwRl0q7xy4XUDmKyuVkzcMr2a-psoycfolqvFQLLKmhEsiwBb-hnmac6m4Hqy89nw8CALwVl8OP8n9CeHjrvSifZGDFxDZ7rAVjv8UbEAHx-quMdIpEHz04_uJC6F5mNVMsxUGdgvrPbdX-2bJVsJkSqUfG8AHccPwYERWSD9U41ciajdK4mDwVGnIVNfirvp_Qt6r8D0VpsxkrN85lDv0zhrM4dsGKQ_2DkXKgN6k5DJeIwdMFdzT-prZ6fHxhol3-BtS5QqkKE2J_JbQCbj13NsZGfbrILTSJngzY2yxhxsCZ-XqKwbCzFpCxHWQCQ-QzKN-O2puLTSg4uWqa_8i29itNRBON1usVhwxtpmuIiQWNWzj7Bqrg5mwPvBKH-8f50OtpOZG7bj1y3e-tEtGoN4Si6m7ZlPesIyXhNefXtdvDPg_qBWaUCE7YM-y052smqW8RGITFMKQXfZfVQ9U116YqdcUKk59FSD7NP25F3WDA_Gsb2fXNayq6mCq4N5ubAtoDIpheaNAcGUp81VhQ260zmzMozrLvNJ3RAXl6EaXKmNDIFMhLQ4MWRr5SivMhauzez-aIX343UBXoZopCQfrYsSLN1WsWOV4Kkhh8INj1JwG7bF0o0wWBylICXzt36vQIZU6PIebmxFXCjuDnCYHBTETJ2kbigtIhlOLN5omSWhYubQYun9Adi6muLcVzHqXxp0fS_008kauEYgsYO2-XWISNm4civPMChp7Ubj4uXDA2TVdeaz8031n5EqwYiuTzoCxbhty3RyQdA5dZnRHWBelsTsmfSYzLAuf0_wkyVyRXBulLPVVxFjc3VzvlzuRW6KCE18k4K7XYdWKxyC4qYefelssIbpX4glke2bzOFgk0iK1LaHZoFrmoOBNospzylBYTE4o4NRCHCuJ5XGj0-__P3lFRFxiBBQ5hnC8mP3XSF8CoD9tCIr6iqjVFMpwbryU8GmG2V8EQYC2hyLDEAMRooTbN44KvclFM3vM9-IXAD7zm0tMEWSBu-wlnlYOib19IbYMgDePgyjxXY3ZdOHgGsxWa0o42ltsMuJepmgtJ_763J9iAGMqSYhNq-eKCnSY5SakedmNhcEnHTgjXKoA_58GaNgjlq_ggfHwF6gm9WCJjrWGKrrVOdmSlosU00_CdAZdhFmzzUk3jsU7UYVuNn2NXe3DudM8aC4Gy64q6kcVIoUgzY3bpbriywLgStsX5MLvtVUsjjSrk3CmZV3x4UhqQOTOXYRYmIkxgED8zx5QdPHteXYjgYmRwHCWk6Fj0VK7yyV7oXvtBl0PN4Gn1-j0otp54I_mgRYBTtG526sUGHnTMpbdtKuRa0voESQEJ4k9mxQLLtudiZDPI70ZLBmJmmSqqSodBdtwOxormlbaWTYIjNhCAMqFH7z3mFuwngfy5QjZGOQKqSqFzKRYJpjVA8X8kJ2KLzwbMPcceNsnT0BAZs4bev95gG_JE4aD9P1Ly-UDp8XtYcIDZKGLL4RsH7Hn_vzuHvJGLmE2fDmOJc0FL0P51xKMB6zAf181BkazzYtQRLoJ3VLX4OSOqZ4XN3MgJBB1sA7vsofqa8XmT12zKuyHdgvX7B8D-6PF_fQ0IWIiiukpkHziQZ_ivOuzeAex2sk6WVx20T0Sl-kNQtAqKrxQqSDehjbHcfUYks853iFEVT-NcXQBd4NIBkEZkdIFnidkV7TKfxPLZhqOFjPfcSrBHpjJTbRQVoK4h6LNIw2SCjkFX60O3DS4rKaGRxXNsPYYcQj28CO1TJFTuW-aMYkoRac26RkFe_IKpmhIwoAxX5OeuyQiXTmtGkYMSbfxpj-HS6itIZ4jmzESyI1Xvba6uDC5BOgdHEL1iVOMgkLvcpY6Pci7QUwL6UAmBeCmxOi5Mnmh8uS42EGR2y4g5cP8oohM5PZzBeandBsB7_oVjeqbtzOQVwIdn3fSpJhbf-3NeXX_XPaf1sH-iAtRU5YW_L5IEz1ji_FcSpw1ydky9qZKVAnfKgEg5Q5yyrUBf8109WCmGVriT-EmkR_7ytFXHwLH_Bf3M-akqVNLPL_mVSjfU18ZP42ZpXGy8CO_90aIukW6i7viAz0nH2-LnlD5KEphvZED3l_-D9mRd7uLr9N7znVH8ZXXwG04rdmC6RSXehTq619sgzpU0HUFwl-2GEaYBgAE9JVEoI5r5FKLK9JriAhbgghBgbCUeirP-ou6ntSQyifNgu4MvmEfDLdUn0luVB19A7arm8AKywtO4mxxbon2hPyqbpaizZX2YoX8BcbFB8iJZ8u15qhGK6yJWzruhuU7Gi_llXVHC_IuQczUvbVlzezzROR0zLoBwqRx1c32UoH6vVdSoEj0kzAFNrybd3ga6q1ZVke2KejDF3-X-e6zAhAwllaF3a6QnJ2KG6zPVvXL2vtLPViTpuR26XzsKPfkUUEiPWJjBcDzluw8brWliJ3rmr1Nd0-oyBeb1dXm1dshengB-wdjLZ1BUAfkwsEcZtq6qguzPl5XEiCTACVAekmp3nDXb3NEesxDnjRWz08q37E17w3O6O2Usj3lK1apifsGdUmvY98yoDHhQwT05__sPSRnhqrzuiYksl2qx6N8cWqwvP0FxrgKJUOct6nHNk6ht8rmnTvyPnHrhULwqfjbbUO31Cmiq44rPbP-BG_QWU6sS0TQHdRtlCzCYMceUUS8C8zNT7S-8GbVPp5JIAKxDpiQpBTPnTT_w_Jayht_F5VLh2wk28BFxanlMfp6hbC_A3_WUX3nsddDCdfl-N0RFToz9HOn9Q3xlLYEVb-ODp9J4vDfkofLVOVG8ciwxe7J1ChslTNrkhkzAA5Odz7Ixfon4mWfwdyuogRvNPm0H07hu2eqmHLaPdGTcK6nCKss_kBPpzJZSDt_30vP-MesCYJWELEzst-uwUdx0ongFJqP3hslMINTLGaTN4nBObYae48PPf328JstQ_Nszlf-PU6qhnQCsn9Y4Q7FVp-XA0wLrt0kvsyklGBVOORuNMNs1Th9LuwIz4RSojHvO1-edYznC4yC8vGI8omY_HU7n_BBHMOGtMV29xpKS604x8L6vaehXCDjk8f8bw_gGxbajUTN1bYsIqP_PmiT1dZb4aoVQk57slJ0-D2gTopwIt2UrGIn9ImoT9L9LON-5hCSh87r_tAM8BhC3c_XNoz1lhaXctMs6cEZgP8PiHfQYArd3DnHPZOPRzvRavvsDngC1t_Ln16JuTBNIanpZetXDuo144qwr2P-J_Mhwl5Mi4hTvLYeCKpiVANNiP2FkXBIbFF_tg1ZGqKashPIDZ3Hn9JgcXM5ovhQZEItqozj8BFSSXpLfT_pj_kVrhKYJtDsSOL30vPyh_Kf_wc_cfc6iaRUb4aW-sM2rOt3LeEpB113SJnXQvarE43u1SCRVO1G49blFM1uMUmX_KRSfyxwjy9tzcrJOFGx-98s-qCRJi1rDTpbKlJo90Q8CNed3LnQm34rRnec_uOls13s_kilXBlD5cq8-g-wil5SDb0DSP8rlLTMqMMeMEQflf7U7IQJT2RiNNH3AFubw9g47YRG4IyDGlEJRfn62UQn1WZQueqzpyRzaTTtlr7upFRQEjhMcTojDiFzXKr31cKT8qQ1bDYnaTI6tijODOt3AalCiF6wBMW-3zm77dvfYSxZLn8RkQ_wllQ4j317mh8zMeiFl6Ydsowld94XZx6ZBxcfHdVhiv5jRQhrzRcdeZHHmsDEzPpKoLdLCdUXBqvcjUMvNzcpJZOgT8Y-VQSoiv_xWK1jrGqwIZ0WuZrHOyByFmsWM1JfDXqVfwcgTugdcFdzo5DE2u8jzxola4tvoKJFHm1dNjrHSFsMAftwtbaphb8a53B-iAIjSU43XFlUXACqsuryLuuHY8fvR3kG8QjvlAuzp5YJO1LK-BtpxXIQaZOewGsjPxDaEQkbxqA-cOsM2rM3BDguXXOS06Z6vsFUJURjb7ag0mzD9nF-ZeeMWL7Ytjpx0xkz6NIdl_7-xb5uh9RLu91XRuzOE1HLjeXk9LOsnNoNddOFf-nGD_IB1-2Cdrh0OiM2LyE9wvSO8ffuGAMjUVuR53CAtJn2QNl5ktSwXYmYe7QTG5Da6271dZSEMOH0d4dAm5LH88ifyp3ibR8ZZ3E1PI7zXP7wTr8XfIm7rC62EttNHyknxhP9riPY7SVI-Kn-rw5LRsFoREd1SFoeHkmgLUJzxmZgmIm00Ls0-ipiaTH5nZQYAHUZCvC_qcNBATf8QRPm15yfAoUh_2hh4-qKiDIu600cM9k9jXVdl3iAVz1nTmMmgaVH7YLNikc83ssPhBKiz8DPU3NzSI_4sMN5HNz_CblvIRKzCfsJUZPTiGk-cqZTQWAPjRMJvpN3DVij-wvpE6FwSTzcz1yjs7AyG4THBHGYv9gd23C8vBTJU_VLK5FfYEDAp7RXpyjdFUkcpAXiFwPe7H5CNSEGIMMjMqtXarehRCvlDmlIvRTXgGCtu8OHuKyq6NbrC9qkPPfY_3H7wZUy0M_ssuXDJOTaPCU595gZNRY4WP4EPob_Nd8_NMV-jyFl4vTgoFLz9jPKAmWzyPabTs3IFXURF7TlqoGcyVjhDbi34-upqCoQkrmntA1xhMr-Ssc4Ca9PEIofMAi_6ZHJHmdbj3BIgGO69iW5IFeQJgDd7-OI8oM6x6ROI1kKyJ77sKfsSl1SBDuyFiRc2POB0wG8SkXxmtgNAKnEWjzy20mXclwTb2GEetmj_bXgV41GuX1ICnCeTa0in3MOg3EMiP0dlKAt2sS7l2Q0i-ZMWfMyibU6dttzpXzzyux94arqFzft0_VSuMMNUo7qCaTIi4RXOQ5zu9e-esqQRa-VHreC5NkJqf7tSzKCK4tgUqpIUqIUYBHCoPuyn7vDW8ONU0kcUWx9Q9WnTMzKCZoBZ5SwdWiwMZVq3Djk8t9bRaqhZxTbtNCW2fHAqP7KGOTmoEkEkCnK7nhQ2G6rp6thyNZNKwVESrhDHEln-e2glYl_u-86UiJHpWd-1su8gPKM05BVMn0OGjxDZz39yLz-c4DTIegZe8HywG1VMq131Va7edLM7Ws0cp-IlHuDcJDF-s5D0TRaX82TPFO1u8oBpsMlmko6dYCZuBMFVBoTuSMfB20KxZrocqvtvX3RMY-BmlGGvUYspQFJtzHdM90m8cojdNr6TEESb0oUSMInHopxGFXBztoqHMrcFGAu-qyIFi2tKphHuYq5SKM2fnt39WsMCyh6hBu_XuMs0f99rm6C4WqUffTKLGNREHZCIbDGESdSXm8MLmmLDBO0wQIfd3lmynnucG-vpdOd9jeWDiIP_Ezt5SencXZSXpLhp-fBkSCI_dHIhxkAXLcxG7M25LFHGd17_Oi2pAN94MjqcK-DXXm8DpdAIvcgUbDIufNx3JDuQj-hmg4zBhJ-ZSLM0vHyZYTYIRus_niI2Hfv627KopVsMpRt72JcKNxaZYLXGtd8yy33W_C0k-oAAaEWg00bQ_blOquvfMlAlo8kJhU7co1j6hbF1Pohtz_zHtDJMlE0JgBaiYPpA7WdADaNhlqfsuNZOT6zAIhA0PVu66G_LFTMgW2AL0ISMZDYuvWKBifWy8Vsn4zJRTBEKBGxJk_S64ZlgAkXorE3U2DteO-2KbG-aQ4CMfuVP9Q2_Vs2v5lUfrbjRoq16fTPkPc9RsWkWC7vw90BtzAPVJThuIE-U7-eN97IxYLjB0zOKzcfTqK5U23n0NZ0UVxtAhfEyxZbPPsbeaK0WiJrmlTlCKxcSpvFJBIdX6uOtBAWrakVosxRf3-nJwQaJSNC9GHK6mN8P7JLccsVAruB_51Zb9h6ijEUREG7tveR6FK2mrRb7FNiyaMRaaJti_oVZGVyuRp-BT005r2LOhryp3w4Rd3m_5VuC4absQ4jB3sCetE-pD9Ax-Krc6Xc2kt-QBzzux_nPJi0Vr5VhS5voJ-ThddHygo4SUKsVS4S-dBglfEEZUB40Lc8Ml-0TNgtRaa61Kt7w5BQ2MGoHo2jmse5Bq4JQ_DqgW8R_Gxu_zxHLv1sMKnhDPa-rC2JAxkXRLUrH_rB-1dQqvLRno18I2ZipOezzG5U-GZUmAHMzJrvWcTP5kz9mkv1uSrlrT-K7oliI7aS2nRqpVCTNpHvXnVFC3rWaCkQeowHfQW5GgK-U8LS5qJES9dWmnwuImQPSlGYOZIh_9O8lkozXDDR5wMxd56a8WdcZC2TO_jKbVwIGJ_aDGpxQoUEcJjGDDmk8qLEB0dlGAOVvOrzN15aUF-ZPPUtxFh3BO3Ym-V_XsGEq_0JX5AaIzZuDnerm58cE9UQy8fnVnT0epoFbI3BQrPQKaSMtKF-pYhuGapB9qEcfQTRJbd7O3tpjMq1azxRjcWnpVdMrLdqEYDOUiC-arguK29aDYDAeQmZVfT-MKgwA9q8aVdQuAQqwTo4Jqb_kJ8R3h9YSMxVPlQhgMqhhlPjOdbmEyCHQjVp76AZn5T0dPSiOgBL7mNbubgKZ7CTIEpjyrAwnyYwM5PKqjmNiV-OpNdFTUgg6Q2-BIyvQVVvpbeBZbBOZVEa7QSlR-syke2da2VbR6bt-P4hSYuA3tn8FQBusiWlg7ey_4Rck0TUURlOCQLMbnDMBiFhHP7OnYeUUpwBNbR8jSCjXcjlIf6FVtD23zJ2JJuLvG3yokZRC2csakipRM7uqM6RzUI9KixRVXr6Gd-sgjOunSwbemc5qSLWfKv3VG1z_6UdFe3YOM5x8a-8Hqx6UCnVzdg3nKZ8CsaTzKZ2Zjb4I5IyGhGIiRVEptCO784VC7KJK1BxIsoIsLpnJMcD7pr6c_ArAxTtXCGVBEb7Vngzqx5I-F89xBjGtc0Jg75C7Th9x5MnSHID3k0SFGklEYhBgA58AHDSf17BGgJ2bqlpDK_mcpmQoGcCRsbg32I4CSTbh3uCovhjMI_SrOkWV3KRawmnCrdt0NH2YnmqAipwWGRygfoxzbgf-7wKmChoMKZ5erjOd1csthFVSTAAPGBYNlj92CTs1gvRz2rZ4dOYmFyCMC2WldJCzd4gLTaoBBVAEmn_dAtvYvYL9ItH7HCI10U18Of7VvMzKHgMbx9zj-OOiuHs_4JEbMkpllraNW2HvEI89gtVC4BlB5BxwMbgkliA5kUQI4DD9fopqH6NpEU4HmZSlJpvVFrP4fu0PHxUNV-q8r0RsLPv955CEGacTS10hHhFbfi86mASV-j4Rxr68JvJ4uv34vEsR8dtH2R7J3BSstoTdry3oIP8cxJahzUGJsS3RXeke77PekIjKNcKU7ToxJljvshKfU20ZeWfqkbMNnq9PhEWVo1o5sKUtvZ58fz5thspiWNMQe8JlK-sSuCONHsw5Ae5f2RcezDBm3gM1yGY89XeRNu81dc8BLrxmcjm0LAl-eYXJLPhsDp1pmSPzBmqmNZZ4AdHz3w1qdzbLRIx0wuNIJPVtEnfmFTpdAEohAFJNay-RQOG5lL0mES-104CP82snlgE_OXp16uuivhoTHHGmElwENO2kXKATT4dyCuvSCLa7i7UZ1hdzcIpcV5PqVm0vTOyx3lA-0Dp8it3OHqnoSDZB3NXlQUPi4D2FqNS0aFfMyHEjTdIkrQhar_Ma0u4pe3rGIFHmvmrozUDtxf5hGgP8UiOZcDXufLByCN4yWjtNBlD2R7aeuKsFfjbwM0K5T6i4EFgvreisbZpeeJDgeOE4A4xQg_7rUzppR1jlrY1yXKoJF5txHrrazsUo2QlRoFNlTMa57Lf03YmISJZ7CnGHKaY-NfBEz7ad51c52Qq1OsRZat6gJ6KKpvaTKNXO8DpGMDBpErhijhLTiHPq1WUG5i94654j3xW0eHt-WmlTm5PpTQIX7oySZBHxqw4XWnpebOgf6YZVnRdQEiqtfmBqvUXmi8Q5poRmZYMSg19FHamgECiQoMgpZZvvD-jGKRx1Qtt5JnCFiUlik30OKdk6CgR1XKS50tYgbHEnEa5TleT6zY7Cf_-wigENbH060HdSXRzzGteSHHfDUASUih1W9-4ZkPLQdXVLtwfVNfHBGuaufochvk7rN3dkHbJI1p2dIP_bgPQSdiAvU_ayiuVr5VWE5RUi0XPnE8crw0T1zlvXuRsveOIwgdez8x6Kg28PkyjT4YdyE1FzDDxw3_Pf-7VcRFQ43B8uExr0ePab4Nh-ehC-OZw7weC0m0x1qvLhD-EsGgFHhplw8lN5wBU70LZ114cW9AHp6EQfUlBykgiU6jlndCOYCVVtREBXoErkSFbwSUS1aJ-R9qsRXGW15k3Gu2pJpjukrooTq0woT4dR-ZZgcGKvGYuIS572Z39nOnJjRTO-rHEyhZM8AcMrkG7cSmmNlLNWuvyfE1CrzKuh0j0GA1Y_Iwj2Rw9lca-baCO0GIp_6WYxdEmWjDSwneo6BaBOiIksAL8AGe2WM9R1lm4WJVb2we0b4-kpQvrSWWGln25kGNlt0-GiMp_weIrUBMKpSVrRFeB-2Zc5ntffIlPp_Z2mGmf2s7Wby4lwdHLbggIsK6cScM22AlAMdK_gjjBuTeoqxbWsCgZWA_hivSoCeX1oMmssmT1Ek3tb0zJWrU_V5v9x_Eoz66N6QRwODZtIKXEsFTQpq-7e6c_qEo9A4Z0DuIpDQN9ioLlEsIybJYWWLWIh5UG8yd0LhwPABH1jiY6dU93YBXyN9KyU5VvOvYFnzqFz8CNR2yPuPety-y9hfL5qdPtwTozJhBvaM30C05rxLyfyphLjkezkjumyHGmQddRC6whhmcGEBP5GnMAMmYkbJeDZrKoUMlv15RSIjRg4TeKUzMuxs7iPLE3Z0cEYeUKPqb4lZn2bJ4sWlAIkpdtpdb-hbibu6pD6n2sNMKlBjkmIOAOs82EsPvsiY3fX5AVrjQTlr2fg6H-T68SbhjLINBFCYxmlMqx4sr50gUdiXTHliYfWq0kdo6-e1Bbzwv2WgqlG13sZpotdu4cdUdWDAKC8NqlE9wWTce_5LsJZMn-QBAzdXKkIVA18LdqwrmYhFZYIdnFHHmBXL96_56QHAXv_pxgxCP4XByxigGeKZNJiL6jjt-XnqmFimkOKrfZLVkebUJSy4cKjYItvhXFZDg9ehVMpP94QIRGHHYZTnK7TlUfs-Y8y7daS2j7qiPlvoHS-m0S8fBxZq2QioMB3ZS7DtaQWUAmq_z3d96tL8O6OZVKVP-8bSggEF57Zqk6EdTiJlP2YHh3Mt8LPJuoe1NyDCkvb4oBWRgm-QomDTz9wNPaLXpmdpw7dgzP3jrHZ_vS3nX-lUo5BQeOFvV97o2R-x_KyNwOjRgw5M8bc4pZ9lUHBVVwCAxP4IprXD-4sglGSYh9Vw0x1hb9NELCNnE0_Nm3BpiDr3fgr-mnPP1i9wTg2JITL9kOIChaPWciJiLUunr5s-yPDEkEsKav0V8wFH-OlxBvh5vyOOsj7IsfYXfZGNjq_pM8DgvekTWe2nNWetkA4qfo9vgyyacpuPzUxMKRt3rMdcdj8Wjn3ylClvm_R2a8oneGUrKfPgm9hBxgjnS-ewA6yx6ZbWoKZb8idvcXQh2tefgWvRnjlyjaHpAVCGPrxef5AYZXxeJvllYw_sDQhm-6qVch49WpN1JfOqrXqMZABaB9eEcC3NlVNkbuzPiB16LVeAc08Wf6Z82Q_MMANqbHTEP68fCF2VM1SnP3cGbwuYnfE1Jg0jxhUCm-xazJfpsI1Gw9ikqb3DNQTq-904eC1ocYlTBihz-T_LTXEzNZVAgV9xDGM-zdFTw_XysgP1DCwZzRLoS0YHyDTHfXgA5LzaNx-UcYuJcjxtkyqW7qg5HZ7ZaeTuoxXzh15qE6Qies7wRi0KwVxew1EpMknbgpR_Fx54xxoNc88h85QbrY52Ft_HTYTUx9rRV4yH-PI_jjzqEqCz6dggHnMsAaYRsyPbSBbFfuzli0gk8OtrwVtVglM69_saNBUQU59VhCwt5Mqu1GIhTyfEbHF5KIZ1DsgbQyGByK4HXGlJfxr5gizCLa_nKP3ylZC96B82vHvvfB8OESF5IgSXB4yi43-l0Vw54jSq5vTMzUWKMi1bjqXuL7BUKY3XASHPnuIdQsDSKtONqx6gMgZiOQWjiuRiNqDHDU1FB8KVe9_oh901ivJ5Jq1SR5AnAjaNdXmhQZ3DEXgENOE1XIXofzN5qUB8nU09uQKEIjFSaH0rD6SJ-yDqe6sE8Gai04U4ri7VWPjv33S5L9Nx8rjDZbQsuQWKiqz2Y2XMl6u4n1f85yHMsU6XjD7J5kTDj1spVcHpfIBei_TzvqFSBVVB45w30bCxXJ3jOdXHSgeemUHRjQcMA4OAanxGEiIvbhzsrAtutO3IY2ORIqy4j-Q3pZn2DrlOP-tM1L22oFwKLhiAyp6Rs2m74AsiP4NXZrbOk2wIhAnLgmTO7PA2-k1HHzswRQ6b3IgHRzem3PK74NRmejtU7oY_tBEFkbe64Zs_2lR8-D93_jVU3XFXc6irzw3SrjYH5flpYLLJcfXDJjP0PGqL5Ws3ar0bVEuOLpB477opRv4yApB9spnwPqpsvGPRTE1kRBoDCFTQygNiXWHoNUHKJwov8U5CL-xdigNdr4ZkCIvA-B3ByB1hN6DqCEBnblnVhLlNefkmyTnqwkopnvUYeRFQVi-po3KeDkhgJ9JCXZgbtVJnItL5egAYIq1HQXo5Ww2v54x9dGmi6T41rsdrXBFBxO-L2_w4DUy4rJojcI6NwHMGS948RPFTwcfbsqPegdpSQ56nek5yrOceG--PHTwGE1BAoTQWwrc2RcQ-AhD6jJKTzVxdByoXV1cfLsTEalLCMmdRjdw_6iaaBK-p3CbjEVeN8VRQDtHrf6AH52aRL_5Uep8au0yKbaARC3sdzU7rQJexgjFEOOyF5lFZDN2MSxZVLr1tJYWevJdYMEOEgfXBtmZNTn_ImgnhPKYPMZ4j46w97chcTEGVgm4wkIn2D1JnuUsC83OrvXgba4XjGkE8zBTScu_N12KEle8kLaA4rbvSSB_5ciGhxHUy0FbXawdDQDmnxfXvSEXrf7hs8vWuSJ3UK2FnYSlJVMv5MCqbdJAA0R706H7HXoaEjetN_ZSU1mVdvv1nXZS712w5ynYEM2sJX-8Gm6Hwksug3yscJNxMx7uPF3sDKvhoU29N7zqUKz27Fk4Pl31Q8D9wE8FtXHzspgmj1mqhg8Quah8chI3pj3Ml6VGXuHpTtQJ1p8EbIGqx9DeJ9CDWheqgNkrAWFWCIJO6JeDHC1jrt5upu0W3mGvY0HgpNlN6AcWHWNuhi.DvOxrlglw7kKv5KRoK4G_w \ No newline at end of file diff --git a/performance_tests/BRES_20_unit_data_dummy_unencrypted.txt b/performance_tests/BRES_20_unit_data_dummy_unencrypted.txt new file mode 100644 index 0000000..10cab4d --- /dev/null +++ b/performance_tests/BRES_20_unit_data_dummy_unencrypted.txt @@ -0,0 +1,12 @@ +{ + "identifier": "12345678", + "name": "This can be up to 105 characters in length 35 + 35 + 35 58 61 64 67 70 73 76 79 82 85 88 91 94 97 101 105", + "trading_name": "This can be up to 105 characters in length 35 + 35 + 35 58 61 64 67 70 73 76 79 82 85 88 91 94 97 101 105", + "business_description": "This can be up to 100 characters in length 46 49 52 55 58 61 64 67 70 73 76 79 82 85 88 91 94 97 100" + "address": ["Up 2 30 Characters 21 24 27 30", + "Up 2 30 Characters 21 24 27 30", + "Up 2 30 Characters 21 24 27 30", + "Up 2 30 Characters 21 24 27 30", + "Up 2 30 Characters 21 24 27 30", + "HHO1 1AA"] + } diff --git a/performance_tests/__init__.py b/performance_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/performance_tests/generate_dataset.py b/performance_tests/generate_dataset.py new file mode 100644 index 0000000..603a795 --- /dev/null +++ b/performance_tests/generate_dataset.py @@ -0,0 +1,169 @@ +import json +import random +from pathlib import Path +from typing import Any + + +class JsonGenerator: + def __init__( + self, + survey_id: str, + file_name: str, + dataset_entries: int, + fixed_identifiers: list[str], + write_unit_data_from_file: bool, + write_unit_data_file: str + ): + self.survey_id = survey_id + self.file_name = file_name + self.dataset_entries = int(dataset_entries) + self.fixed_identifiers = fixed_identifiers + self.write_unit_data_from_file = write_unit_data_from_file + self.write_unit_data_file = write_unit_data_file + + def generate_dataset_file(self) -> None: + """ + Generate the dataset file. + """ + try: + if not Path(self.file_name).is_file(): + json_data = self._generate_json_data( + self.dataset_entries, self.survey_id, self.fixed_identifiers + ) + + # Specify the output file name + output_file_name = self.file_name + + # Write the JSON data to a file + with open(output_file_name, "w") as json_file: + json.dump(json_data, json_file, indent=2) + + print(f"Data successfully written to {output_file_name}") + except Exception as e: + print(f"Error generating dataset file: {e}") + + def _generate_json_data( + self, entries_count: int, survey_id: str, fixed_identifiers: list[str] + ) -> dict[str, Any]: + """ + Generate the JSON data for the dataset file. + + Args: + entries_count (int): the number of unit data entries to generate + survey_id (str): the survey id (locust test id) + + Returns: + dict[str, any]: the JSON data for the dataset file + """ + data = { + "survey_id": survey_id, + "period_id": "test_period_id", + "form_types": ["0001"], + "schema_version": "v1.0.0", + "data": [], + } + + unique_ids = set() + working_fixed_identifiers = fixed_identifiers.copy() + + for _ in range(entries_count): + working_fixed_identifiers, identifier = self._generate_unique_identifier( + unique_ids, working_fixed_identifiers + ) + unique_ids.add(identifier) + + if self.write_unit_data_from_file: + unit_data = self._generate_unit_data_from_file(identifier) + else: + unit_data = self._generate_unit_data() + + data["data"].append({"identifier": identifier, "unit_data": unit_data}) + + return data + + def _generate_unique_identifier( + self, existing_ids: set[str], working_fixed_identifiers: list[str] + ) -> tuple[list[str], str]: + """ + Generate a unique identifier for the dataset file. + + Args: + existing_ids (set[str]): the set of existing identifiers + fixed_identifiers (list[str]): the list of fixed identifiers + + Returns: + tuple[list[str], str]: the updated list of fixed identifiers and the generated identifier + """ + while True: + identifier = ( + working_fixed_identifiers.pop() + if working_fixed_identifiers + # else str(random.randint(10000, 999999)) + else str(random.randint(1000000000, 9999999999)) + ) + if identifier not in existing_ids: + return working_fixed_identifiers, identifier + + def _generate_unit_data_from_file(self, identifier: str) -> str | dict: + """ + Get the unit data content for the dataset file. + """ + return_unit_data = None + + if self.write_unit_data_file.endswith(".json"): + unit_data = self._get_unit_data_from_json(self.write_unit_data_file) + return_unit_data = unit_data.copy() + return_unit_data["identifier"] = identifier + else: + return_unit_data = self._get_unit_data_from_str(self.write_unit_data_file) + + return return_unit_data + + def _generate_unit_data(self) -> str: + """ + Generate the unit data content for the dataset file. + """ + # Customize this function to generate whatever unit data you need + return "Example data " + str(random.randint(1, self.dataset_entries)) + + def _get_unit_data_from_str(self, filename: str) -> str: + """ + Generate the unit data content for the dataset file from a file. + """ + # Customize this function to generate unit data from a file + with open(filename, "r") as file: + txt = file.read() + file.close() + + return txt + + def _get_unit_data_from_json(self, filename: str) -> dict: + """ + Generate the unit data content for the dataset file from a file. + """ + # Customize this function to generate unit data from a file + with open(filename, "r") as file: + txt = file.read() + file.close() + + return json.loads(txt) + + +if __name__ == "__main__": + # Instantiate JsonGenerator with desired parameters + survey_id = "test_survey_id" + file_name = "dummy_dataset_400k_encrypt.json" + dataset_entries = 400000 # Number of entries of unit_data you want in your dataset + fixed_identifiers = [] # List of fixed unit_data identifiers you want to use + write_unit_data_from_file = True # Set to True if you want to write unit_data from a file + write_unit_data_file = "performance_tests/BRES_20_unit_data_dummy_encrypted.txt" # Path to file of unit_data to write + + json_generator = JsonGenerator( + survey_id, + file_name, + dataset_entries, + fixed_identifiers, + write_unit_data_from_file, + write_unit_data_file + ) + json_generator.generate_dataset_file() diff --git a/pyproject.toml b/pyproject.toml index 739c7af..2f1af45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [project] name = "sds-loader" -version = "0.1.5" +version = "0.1.17" description = "A service for loading new SDS schemas and datasets" requires-python = "==3.13.*" dependencies = [ "sdx-base>=0.2.9", - "sds-common==1.0.17", + "sds-common==1.0.18", "lagom>=2.7.7", "polyfactory>=3.3.0", "pydantic-settings>=2.13.1", diff --git a/run.py b/run.py index b093785..2db43cd 100644 --- a/run.py +++ b/run.py @@ -1,11 +1,9 @@ from fastapi import FastAPI from pathlib import Path -from requests import Request -from sdx_base.models.pubsub import Envelope from sdx_base.run import initialise from sdx_base.server.server import RouterConfig from sdx_base.server.servers import default_server -from sdx_base.server.tx_id import txid_not_applicable, txid_from_request +from sdx_base.server.tx_id import txid_not_applicable from app.middleware.timing import TimingMiddleware from app.routes import router @@ -32,20 +30,8 @@ def load_startup_banner() -> str: return "sds-loader" -async def smart_txid(request: Request) -> str: - """ - Extract the tx_id from the request - """ - if request.method == "GET": - return await txid_not_applicable(request) - envelope: Envelope = await request.json() - if "message" in envelope: - return await txid_from_request(request) - return await txid_not_applicable(request) - - # Basic router configuration -router_1 = RouterConfig(router, tx_id_getter=smart_txid) +router_1 = RouterConfig(router, tx_id_getter=txid_not_applicable) # Initialize the FastAPI app with the specified settings, routers, and project root. diff --git a/tests/integration/test_publish_schemas_endpoint.py b/tests/integration/test_publish_schemas_endpoint.py index eb79314..59d1b92 100644 --- a/tests/integration/test_publish_schemas_endpoint.py +++ b/tests/integration/test_publish_schemas_endpoint.py @@ -89,7 +89,7 @@ def test_publish_schemas_to_bucket_with_all_valid_schemas( ) # Create fake files to simulate new added schemas sent to loader - received_filenames = "gcp/ons-sdx-bob/buckets/v1.json" + received_filenames = '{"name": "v1.json"}' # Create a TestClient instance client = TestClient(test_app) @@ -103,7 +103,7 @@ def test_publish_schemas_to_bucket_with_all_valid_schemas( assert response.status_code == 200 # Assert the file was published by the mock_bucket_publisher - assert "gcp/ons-sdx-bob/buckets/v1.json" in mock_bucket_publisher.published_schemas + assert "v1.json" in mock_bucket_publisher.published_schemas # Assert the mock_repo_publisher is empty assert len(mock_repo_publisher.published_schemas) == 0 diff --git a/uv.lock b/uv.lock index 89bcbaa..c4bda70 100644 --- a/uv.lock +++ b/uv.lock @@ -1302,7 +1302,7 @@ wheels = [ [[package]] name = "sds-common" -version = "1.0.17" +version = "1.0.18" source = { registry = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/simple/" } dependencies = [ { name = "cloudevents" }, @@ -1321,14 +1321,14 @@ dependencies = [ { name = "requests" }, { name = "setuptools" }, ] -sdist = { url = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/sds-common/sds_common-1.0.17.tar.gz", hash = "sha256:7d667f714046d9667239f2120b5c15dbca394c9c5b7eda92cb27fee7f7b59d29" } +sdist = { url = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/sds-common/sds_common-1.0.18.tar.gz", hash = "sha256:cb21a6c60efd72e69b0822419e708a526947a9990cc1e050ce56562189e01ded" } wheels = [ - { url = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/sds-common/sds_common-1.0.17-py3-none-any.whl", hash = "sha256:650b0bc3893693fc772ff040e44868313b1d7d87bd9cc154f0f7d780414f37d2" }, + { url = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/sds-common/sds_common-1.0.18-py3-none-any.whl", hash = "sha256:3db57733f0e2f33ae954d0469debc13a4800bc0fe175ac11292707e64a37d72f" }, ] [[package]] name = "sds-loader" -version = "0.1.5" +version = "0.1.17" source = { virtual = "." } dependencies = [ { name = "fastapi" }, @@ -1362,7 +1362,7 @@ requires-dist = [ { name = "polyfactory", specifier = ">=3.3.0" }, { name = "pydantic-settings", specifier = ">=2.13.1" }, { name = "requests", specifier = ">=2.32.3" }, - { name = "sds-common", specifier = "==1.0.17", index = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/simple/" }, + { name = "sds-common", specifier = "==1.0.18", index = "https://europe-west2-python.pkg.dev/ons-sds-ci/sds-python-packages/simple/" }, { name = "sdx-base", specifier = ">=0.2.9", index = "https://europe-west2-python.pkg.dev/ons-sdx-ci/sdx-python-packages/simple/" }, { name = "starlette", specifier = ">=0.48.0" }, ]