diff --git a/notebook/scripts/planning/load-table-from-tascomi-API-endpoint.ipynb b/notebook/scripts/planning/load-table-from-tascomi-API-endpoint.ipynb index 1d1648ffd..bc9ce2711 100644 --- a/notebook/scripts/planning/load-table-from-tascomi-API-endpoint.ipynb +++ b/notebook/scripts/planning/load-table-from-tascomi-API-endpoint.ipynb @@ -58,7 +58,7 @@ "private_key = PRIVATE_KEY.encode('utf-8')\n", "\n", "table_to_read = \"\"\n", - "request_uri = f'https://hackney-planning.idoxcloud.com/rest/v1/{table_to_read}'\n", + "request_uri = f'https://hackney-planning.tascomi.com/rest/v1/{table_to_read}'\n", "request_method = 'GET'" ] }, diff --git a/scripts/jobs/planning/tascomi_api_ingestion.py b/scripts/jobs/planning/tascomi_api_ingestion.py index e5c1352a9..b758b0d5b 100644 --- a/scripts/jobs/planning/tascomi_api_ingestion.py +++ b/scripts/jobs/planning/tascomi_api_ingestion.py @@ -87,7 +87,7 @@ def get_number_of_pages(resource, query=""): headers = authenticate_tascomi(headers, public_key, private_key) - url = f'https://hackney-planning.idoxcloud.com/rest/v1/{resource}{query}' + url = f'https://hackney-planning.tascomi.com/rest/v1/{resource}{query}' res = requests.get(url, data="", headers=headers) if res.status_code == 202: logger.info(f"received status code 202, whilst getting number of pages for {resource}, with query {query}") @@ -144,7 +144,7 @@ def get_requests_since_last_import(resource, last_import_date): number_of_pages = number_of_pages_reponse["number_of_pages"] logger.info(f"Number of pages to retrieve for {day}: {number_of_pages}") requests_list += [RequestRow(page_number, - f'https://hackney-planning.idoxcloud.com/rest/v1/{resource}?page={page_number}&last_updated={day}', + f'https://hackney-planning.tascomi.com/rest/v1/{resource}?page={page_number}&last_updated={day}', "") for page_number in range(1, number_of_pages + 1)] number_of_requests = len(requests_list) if number_of_requests == 0: @@ -162,7 +162,7 @@ def get_requests_for_full_load(resource): number_of_pages = number_of_pages_reponse["number_of_pages"] logger.info(f"Number of pages to retrieve: {number_of_pages}") requests_list = [ - RequestRow(page_number, f'https://hackney-planning.idoxcloud.com/rest/v1/{resource}?page={page_number}', "") for + RequestRow(page_number, f'https://hackney-planning.tascomi.com/rest/v1/{resource}?page={page_number}', "") for page_number in range(1, number_of_pages + 1)] number_of_requests = len(requests_list) requests_list = sc.parallelize(requests_list) diff --git a/scripts/tests/planning/test_tascomi_parse_tables.py b/scripts/tests/planning/test_tascomi_parse_tables.py index fcaf294b4..d0306237c 100644 --- a/scripts/tests/planning/test_tascomi_parse_tables.py +++ b/scripts/tests/planning/test_tascomi_parse_tables.py @@ -53,7 +53,7 @@ def test_parsed_row_data(self, spark): "creation_user_id": None, "title_id": "4", "page_number": 691, - "import_api_url_requested": "https://hackney-planning.idoxcloud.com/rest/v1/contacts?page=691", + "import_api_url_requested": "https://hackney-planning.tascomi.com/rest/v1/contacts?page=691", "import_api_status_code": 200, "import_exception_thrown": "", "import_datetime": datetime(2021, 9, 16, 13, 10), @@ -69,7 +69,7 @@ def parse_json_into_dataframe(self, spark, column, data): data_with_imports = [ { "page_number": 691, - "import_api_url_requested": "https://hackney-planning.idoxcloud.com/rest/v1/contacts?page=691", + "import_api_url_requested": "https://hackney-planning.tascomi.com/rest/v1/contacts?page=691", "import_api_status_code": 200, "import_exception_thrown": "", "import_datetime": datetime(2021, 9, 16, 13, 10), diff --git a/terraform/core/15-unrestricted-geolive-database-ingestion.tf b/terraform/core/15-unrestricted-geolive-database-ingestion.tf index 05d87c399..b05fc095a 100644 --- a/terraform/core/15-unrestricted-geolive-database-ingestion.tf +++ b/terraform/core/15-unrestricted-geolive-database-ingestion.tf @@ -280,6 +280,7 @@ module "llpg_geolive_ingestion_job" { jdbc_connections = [module.llpg_geolive_database_ingestion[0].jdbc_connection_name] triggered_by_crawler = module.llpg_geolive_database_ingestion[0].crawler_name workflow_name = module.llpg_geolive_database_ingestion[0].workflow_name + max_retries = 1 job_parameters = { "--s3_ingestion_bucket_target" = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/" "--s3_ingestion_details_target" = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/ingestion-details/" diff --git a/terraform/etl/24-aws-glue-tascomi-data.tf b/terraform/etl/24-aws-glue-tascomi-data.tf index aeda54a37..64de561be 100644 --- a/terraform/etl/24-aws-glue-tascomi-data.tf +++ b/terraform/etl/24-aws-glue-tascomi-data.tf @@ -142,7 +142,7 @@ resource "aws_glue_trigger" "tascomi_tables_weekly_ingestion_triggers" { name = "${local.short_identifier_prefix}Tascomi ${title(replace(each.value, "_", " "))} Ingestion Trigger" type = "SCHEDULED" - schedule = "cron(0 21 ? * MON *)" + schedule = "cron(0 16 ? * SUN *)" enabled = local.is_production_environment actions {