From 9631dfb999f289a55d24ab446cb2193d9c9eedec Mon Sep 17 00:00:00 2001 From: Tian Chen <38001883+Tian-2017@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:11:57 +0000 Subject: [PATCH 1/2] upgrade lambda python version to 3.11 --- lambdas/kafka_test/Makefile | 2 +- terraform/core/13-mssql-ingestion.tf | 2 +- terraform/core/22-sagemaker.tf | 4 ++-- terraform/core/38-api-ingestion.tf | 4 ++-- terraform/modules/api-ingestion-lambda/01-inputs-required.tf | 4 ++-- terraform/modules/api-ingestion-lambda/10-lambda.tf | 2 +- .../modules/aws-lambda-folder-ingestion/02-inputs-optional.tf | 4 ++-- terraform/modules/aws-lambda/02-inputs-optional.tf | 2 +- terraform/modules/g-drive-to-s3/10-lambda.tf | 2 +- terraform/modules/glue-failure-alert-notifications/10-main.tf | 2 +- terraform/modules/kafka-test-lambda/10-lambda.tf | 4 ++-- terraform/modules/lambda-alarms-handler/10-aws-lambda.tf | 4 ++-- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lambdas/kafka_test/Makefile b/lambdas/kafka_test/Makefile index 5f52f49c6..11cce1ec9 100644 --- a/lambdas/kafka_test/Makefile +++ b/lambdas/kafka_test/Makefile @@ -7,6 +7,6 @@ install-requirements: . venv/bin/activate && sudo pipenv lock --requirements > requirements.txt #. venv/bin/activate && sudo pip install --target ./lib -r requirements.txt - . venv/bin/activate && sudo docker run -v $(PWD):/var/task "lambci/lambda:build-python3.8" /bin/sh -c "pip install --target ./lib -r requirements.txt; exit" + . venv/bin/activate && sudo docker run -v $(PWD):/var/task "lambci/lambda:build-python3.11" /bin/sh -c "pip install --target ./lib -r requirements.txt; exit" sudo cp ../../terraform/modules/kafka-schema-registry/schemas/* ./lib/ rm -rf venv/ diff --git a/terraform/core/13-mssql-ingestion.tf b/terraform/core/13-mssql-ingestion.tf index 6654697b1..e74f51294 100644 --- a/terraform/core/13-mssql-ingestion.tf +++ b/terraform/core/13-mssql-ingestion.tf @@ -190,7 +190,7 @@ module "max_concurrency_lambda" { s3_key = "academy-revs-and-bens-housing-needs-database-ingestion-max-concurrency.zip" lambda_source_dir = "../../lambdas/calculate_max_concurrency" lambda_output_path = "../../lambdas/calculate_max_concurrency/max-concurrency.zip" - runtime = "python3.8" + runtime = "python3.11" } resource "aws_iam_role" "academy_step_functions_role" { diff --git a/terraform/core/22-sagemaker.tf b/terraform/core/22-sagemaker.tf index c90a07929..cd5e65c59 100644 --- a/terraform/core/22-sagemaker.tf +++ b/terraform/core/22-sagemaker.tf @@ -110,7 +110,7 @@ resource "aws_iam_role_policy_attachment" "shutdown_notebooks" { policy_arn = aws_iam_policy.shutdown_notebooks[0].arn } -# Lambda function +# Lambda function data "archive_file" "shutdown_notebooks" { type = "zip" source_dir = "../../lambdas/shutdown_notebooks" @@ -138,7 +138,7 @@ resource "aws_lambda_function" "shutdown_notebooks" { role = aws_iam_role.shutdown_notebooks[0].arn handler = "main.shutdown_notebooks" - runtime = "python3.8" + runtime = "python3.11" function_name = "${local.short_identifier_prefix}shutdown-notebooks" s3_bucket = module.lambda_artefact_storage.bucket_id s3_key = aws_s3_object.shutdown_notebooks.key diff --git a/terraform/core/38-api-ingestion.tf b/terraform/core/38-api-ingestion.tf index bb3d8fa17..e061ef847 100644 --- a/terraform/core/38-api-ingestion.tf +++ b/terraform/core/38-api-ingestion.tf @@ -16,7 +16,7 @@ module "icaseworks_api_ingestion" { lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id lambda_name = "icaseworks-api-ingestion" lambda_handler = "main.lambda_handler" - runtime_language = "python3.8" + runtime_language = "python3.11" secrets_manager_kms_key = aws_kms_key.secrets_manager_key s3_target_bucket_arn = module.landing_zone.bucket_arn s3_target_bucket_name = local.s3_target_bucket_name @@ -44,7 +44,7 @@ module "vonage_api_ingestion" { lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id lambda_name = "vonage-api-ingestion" lambda_handler = "main.lambda_handler" - runtime_language = "python3.8" + runtime_language = "python3.11" secrets_manager_kms_key = aws_kms_key.secrets_manager_key s3_target_bucket_arn = module.landing_zone.bucket_arn s3_target_bucket_name = local.s3_target_bucket_name diff --git a/terraform/modules/api-ingestion-lambda/01-inputs-required.tf b/terraform/modules/api-ingestion-lambda/01-inputs-required.tf index 138e3905d..faecd2295 100644 --- a/terraform/modules/api-ingestion-lambda/01-inputs-required.tf +++ b/terraform/modules/api-ingestion-lambda/01-inputs-required.tf @@ -65,11 +65,11 @@ variable "runtime_language" { validation { condition = ( contains([ - "python3.8", + "python3.11", "nodejs14.x" ], var.runtime_language) ) - error_message = "The value cannot be a blank string, and must be one of the following: 'python3.8' or 'nodejs14.x'" + error_message = "The value cannot be a blank string, and must be one of the following: 'python3.11' or 'nodejs14.x'" } } diff --git a/terraform/modules/api-ingestion-lambda/10-lambda.tf b/terraform/modules/api-ingestion-lambda/10-lambda.tf index 9f3f9dc8c..d78a8d27e 100644 --- a/terraform/modules/api-ingestion-lambda/10-lambda.tf +++ b/terraform/modules/api-ingestion-lambda/10-lambda.tf @@ -13,7 +13,7 @@ data "aws_iam_policy_document" "lambda_assume_role" { } locals { - command = var.runtime_language == "python3.8" ? "make install-requirements" : (var.runtime_language == "nodejs14.x" ? "npm install" : 0) + command = var.runtime_language == "python3.11" ? "make install-requirements" : (var.runtime_language == "nodejs14.x" ? "npm install" : 0) # This ensures that this data resource will not be evaluated until # after the null_resource has been created. lambda_exporter_id = null_resource.run_install_requirements.id diff --git a/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf b/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf index e47a439f7..dc7e14513 100644 --- a/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf +++ b/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf @@ -19,9 +19,9 @@ variable "tags" { variable "runtime" { type = string description = "Runtime to use for the Lambda Function" - default = "python3.8" + default = "python3.11" validation { - condition = can(regex("python3[.]([7-9]|10)", var.runtime)) + condition = contains(["python3.9", "python3.10", "python3.11"], var.runtime) error_message = "Runtime must be a valid Python runtime" } diff --git a/terraform/modules/aws-lambda/02-inputs-optional.tf b/terraform/modules/aws-lambda/02-inputs-optional.tf index a06f63c63..2f4dce476 100644 --- a/terraform/modules/aws-lambda/02-inputs-optional.tf +++ b/terraform/modules/aws-lambda/02-inputs-optional.tf @@ -19,7 +19,7 @@ variable "tags" { variable "runtime" { type = string description = "Runtime to use for the Lambda Function" - default = "python3.8" + default = "python3.11" validation { condition = can(regex("python3[.]([7-9]|10)", var.runtime)) error_message = "Runtime must be a valid Python runtime" diff --git a/terraform/modules/g-drive-to-s3/10-lambda.tf b/terraform/modules/g-drive-to-s3/10-lambda.tf index e555e995b..2d78f570f 100644 --- a/terraform/modules/g-drive-to-s3/10-lambda.tf +++ b/terraform/modules/g-drive-to-s3/10-lambda.tf @@ -133,7 +133,7 @@ resource "aws_lambda_function" "g_drive_to_s3_copier_lambda" { role = aws_iam_role.g_drive_to_s3_copier_lambda.arn handler = "main.lambda_handler" - runtime = "python3.8" + runtime = "python3.11" function_name = lower("${var.identifier_prefix}g-drive-${var.lambda_name}") s3_bucket = var.lambda_artefact_storage_bucket s3_key = aws_s3_object.g_drive_to_s3_copier_lambda.key diff --git a/terraform/modules/glue-failure-alert-notifications/10-main.tf b/terraform/modules/glue-failure-alert-notifications/10-main.tf index 77e68e3c5..2704250b9 100644 --- a/terraform/modules/glue-failure-alert-notifications/10-main.tf +++ b/terraform/modules/glue-failure-alert-notifications/10-main.tf @@ -89,7 +89,7 @@ resource "aws_lambda_function" "lambda" { function_name = lower("${var.identifier_prefix}${var.lambda_name}") role = aws_iam_role.lambda.arn handler = "main.lambda_handler" - runtime = "python3.8" + runtime = "python3.11" source_code_hash = filebase64sha256(data.archive_file.lambda.output_path) s3_bucket = var.lambda_artefact_storage_bucket s3_key = "${local.lambda_name_underscore}.zip" diff --git a/terraform/modules/kafka-test-lambda/10-lambda.tf b/terraform/modules/kafka-test-lambda/10-lambda.tf index b00db83ca..11bd76487 100644 --- a/terraform/modules/kafka-test-lambda/10-lambda.tf +++ b/terraform/modules/kafka-test-lambda/10-lambda.tf @@ -15,7 +15,7 @@ data "aws_iam_policy_document" "lambda_assume_role" { locals { command = "make install-requirements" - confluent_kafka_command = "docker run -v \"$PWD\":/var/task \"lambci/lambda:build-python3.8\" /bin/sh -c \"pip install -r requirements.txt -t python/lib/python3.8/site-packages/; exit\"" + confluent_kafka_command = "docker run -v \"$PWD\":/var/task \"lambci/lambda:build-python3.11\" /bin/sh -c \"pip install -r requirements.txt -t python/lib/python3.11/site-packages/; exit\"" # This ensures that this data resource will not be evaluated until # after the null_resource has been created. lambda_exporter_id = null_resource.run_install_requirements.id @@ -151,7 +151,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda.arn handler = "main.lambda_handler" - runtime = "python3.8" + runtime = "python3.11" function_name = lower("${var.identifier_prefix}${var.lambda_name}") s3_bucket = var.lambda_artefact_storage_bucket s3_key = aws_s3_object.lambda.key diff --git a/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf b/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf index 6cd653e25..a5e3c0d2d 100644 --- a/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf +++ b/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf @@ -78,7 +78,7 @@ resource "aws_iam_role_policy_attachment" "lambda" { } #lambda -#packaging +#packaging resource "null_resource" "run_install_requirements" { triggers = { dir_sha1 = sha1(join("", [for f in fileset(path.module, "../../../lambdas/${local.lambda_name_underscore}/*") : filesha1("${path.module}/${f}")])) @@ -110,7 +110,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda.arn handler = "main.lambda_handler" - runtime = "python3.8" + runtime = "python3.11" function_name = lower("${var.identifier_prefix}${var.lambda_name}") s3_bucket = var.lambda_artefact_storage_bucket s3_key = aws_s3_object.lambda.key From dbf9514b0dfaaff9aca54022d21191eef4ad1c7d Mon Sep 17 00:00:00 2001 From: Tian Chen <38001883+Tian-2017@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:32:33 +0000 Subject: [PATCH 2/2] downgrade to 3.9 --- lambdas/g_drive_folder_to_s3/Dockerfile | 6 +----- lambdas/kafka_test/Makefile | 2 +- terraform/core/13-mssql-ingestion.tf | 2 +- terraform/core/22-sagemaker.tf | 2 +- terraform/core/38-api-ingestion.tf | 4 ++-- .../modules/api-ingestion-lambda/01-inputs-required.tf | 4 ++-- terraform/modules/api-ingestion-lambda/10-lambda.tf | 2 +- .../aws-lambda-folder-ingestion/02-inputs-optional.tf | 2 +- terraform/modules/aws-lambda/02-inputs-optional.tf | 2 +- terraform/modules/g-drive-to-s3/10-lambda.tf | 2 +- .../modules/glue-failure-alert-notifications/10-main.tf | 2 +- terraform/modules/kafka-test-lambda/10-lambda.tf | 4 ++-- terraform/modules/lambda-alarms-handler/10-aws-lambda.tf | 2 +- 13 files changed, 16 insertions(+), 20 deletions(-) diff --git a/lambdas/g_drive_folder_to_s3/Dockerfile b/lambdas/g_drive_folder_to_s3/Dockerfile index c03cea259..9711daf3f 100644 --- a/lambdas/g_drive_folder_to_s3/Dockerfile +++ b/lambdas/g_drive_folder_to_s3/Dockerfile @@ -18,7 +18,7 @@ COPY Pipfile Pipfile.lock /app/ COPY main.py ./source/ # Install Python dependencies using pipenv -RUN pipenv install +RUN pipenv install RUN pipenv requirements > requirements.txt RUN pip install -t ./source/lib -r requirements.txt @@ -27,7 +27,3 @@ WORKDIR /app/source RUN zip -r g_drive_folder_to_s3.zip . CMD "pyhon3", "main.py" - - - - diff --git a/lambdas/kafka_test/Makefile b/lambdas/kafka_test/Makefile index 11cce1ec9..ef9d98db2 100644 --- a/lambdas/kafka_test/Makefile +++ b/lambdas/kafka_test/Makefile @@ -7,6 +7,6 @@ install-requirements: . venv/bin/activate && sudo pipenv lock --requirements > requirements.txt #. venv/bin/activate && sudo pip install --target ./lib -r requirements.txt - . venv/bin/activate && sudo docker run -v $(PWD):/var/task "lambci/lambda:build-python3.11" /bin/sh -c "pip install --target ./lib -r requirements.txt; exit" + . venv/bin/activate && sudo docker run -v $(PWD):/var/task "lambci/lambda:build-python3.9" /bin/sh -c "pip install --target ./lib -r requirements.txt; exit" sudo cp ../../terraform/modules/kafka-schema-registry/schemas/* ./lib/ rm -rf venv/ diff --git a/terraform/core/13-mssql-ingestion.tf b/terraform/core/13-mssql-ingestion.tf index e74f51294..09fcc401c 100644 --- a/terraform/core/13-mssql-ingestion.tf +++ b/terraform/core/13-mssql-ingestion.tf @@ -190,7 +190,7 @@ module "max_concurrency_lambda" { s3_key = "academy-revs-and-bens-housing-needs-database-ingestion-max-concurrency.zip" lambda_source_dir = "../../lambdas/calculate_max_concurrency" lambda_output_path = "../../lambdas/calculate_max_concurrency/max-concurrency.zip" - runtime = "python3.11" + runtime = "python3.9" } resource "aws_iam_role" "academy_step_functions_role" { diff --git a/terraform/core/22-sagemaker.tf b/terraform/core/22-sagemaker.tf index cd5e65c59..6f80eedbb 100644 --- a/terraform/core/22-sagemaker.tf +++ b/terraform/core/22-sagemaker.tf @@ -138,7 +138,7 @@ resource "aws_lambda_function" "shutdown_notebooks" { role = aws_iam_role.shutdown_notebooks[0].arn handler = "main.shutdown_notebooks" - runtime = "python3.11" + runtime = "python3.9" function_name = "${local.short_identifier_prefix}shutdown-notebooks" s3_bucket = module.lambda_artefact_storage.bucket_id s3_key = aws_s3_object.shutdown_notebooks.key diff --git a/terraform/core/38-api-ingestion.tf b/terraform/core/38-api-ingestion.tf index e061ef847..fdf7ff232 100644 --- a/terraform/core/38-api-ingestion.tf +++ b/terraform/core/38-api-ingestion.tf @@ -16,7 +16,7 @@ module "icaseworks_api_ingestion" { lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id lambda_name = "icaseworks-api-ingestion" lambda_handler = "main.lambda_handler" - runtime_language = "python3.11" + runtime_language = "python3.9" secrets_manager_kms_key = aws_kms_key.secrets_manager_key s3_target_bucket_arn = module.landing_zone.bucket_arn s3_target_bucket_name = local.s3_target_bucket_name @@ -44,7 +44,7 @@ module "vonage_api_ingestion" { lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id lambda_name = "vonage-api-ingestion" lambda_handler = "main.lambda_handler" - runtime_language = "python3.11" + runtime_language = "python3.9" secrets_manager_kms_key = aws_kms_key.secrets_manager_key s3_target_bucket_arn = module.landing_zone.bucket_arn s3_target_bucket_name = local.s3_target_bucket_name diff --git a/terraform/modules/api-ingestion-lambda/01-inputs-required.tf b/terraform/modules/api-ingestion-lambda/01-inputs-required.tf index faecd2295..3eac36ef6 100644 --- a/terraform/modules/api-ingestion-lambda/01-inputs-required.tf +++ b/terraform/modules/api-ingestion-lambda/01-inputs-required.tf @@ -65,11 +65,11 @@ variable "runtime_language" { validation { condition = ( contains([ - "python3.11", + "python3.9", "nodejs14.x" ], var.runtime_language) ) - error_message = "The value cannot be a blank string, and must be one of the following: 'python3.11' or 'nodejs14.x'" + error_message = "The value cannot be a blank string, and must be one of the following: 'python3.9' or 'nodejs14.x'" } } diff --git a/terraform/modules/api-ingestion-lambda/10-lambda.tf b/terraform/modules/api-ingestion-lambda/10-lambda.tf index d78a8d27e..c63c43326 100644 --- a/terraform/modules/api-ingestion-lambda/10-lambda.tf +++ b/terraform/modules/api-ingestion-lambda/10-lambda.tf @@ -13,7 +13,7 @@ data "aws_iam_policy_document" "lambda_assume_role" { } locals { - command = var.runtime_language == "python3.11" ? "make install-requirements" : (var.runtime_language == "nodejs14.x" ? "npm install" : 0) + command = var.runtime_language == "python3.9" ? "make install-requirements" : (var.runtime_language == "nodejs14.x" ? "npm install" : 0) # This ensures that this data resource will not be evaluated until # after the null_resource has been created. lambda_exporter_id = null_resource.run_install_requirements.id diff --git a/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf b/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf index dc7e14513..30365e2fe 100644 --- a/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf +++ b/terraform/modules/aws-lambda-folder-ingestion/02-inputs-optional.tf @@ -19,7 +19,7 @@ variable "tags" { variable "runtime" { type = string description = "Runtime to use for the Lambda Function" - default = "python3.11" + default = "python3.9" validation { condition = contains(["python3.9", "python3.10", "python3.11"], var.runtime) error_message = "Runtime must be a valid Python runtime" diff --git a/terraform/modules/aws-lambda/02-inputs-optional.tf b/terraform/modules/aws-lambda/02-inputs-optional.tf index 2f4dce476..aac526a2d 100644 --- a/terraform/modules/aws-lambda/02-inputs-optional.tf +++ b/terraform/modules/aws-lambda/02-inputs-optional.tf @@ -19,7 +19,7 @@ variable "tags" { variable "runtime" { type = string description = "Runtime to use for the Lambda Function" - default = "python3.11" + default = "python3.9" validation { condition = can(regex("python3[.]([7-9]|10)", var.runtime)) error_message = "Runtime must be a valid Python runtime" diff --git a/terraform/modules/g-drive-to-s3/10-lambda.tf b/terraform/modules/g-drive-to-s3/10-lambda.tf index 2d78f570f..e0603a295 100644 --- a/terraform/modules/g-drive-to-s3/10-lambda.tf +++ b/terraform/modules/g-drive-to-s3/10-lambda.tf @@ -133,7 +133,7 @@ resource "aws_lambda_function" "g_drive_to_s3_copier_lambda" { role = aws_iam_role.g_drive_to_s3_copier_lambda.arn handler = "main.lambda_handler" - runtime = "python3.11" + runtime = "python3.9" function_name = lower("${var.identifier_prefix}g-drive-${var.lambda_name}") s3_bucket = var.lambda_artefact_storage_bucket s3_key = aws_s3_object.g_drive_to_s3_copier_lambda.key diff --git a/terraform/modules/glue-failure-alert-notifications/10-main.tf b/terraform/modules/glue-failure-alert-notifications/10-main.tf index 2704250b9..c88329e1c 100644 --- a/terraform/modules/glue-failure-alert-notifications/10-main.tf +++ b/terraform/modules/glue-failure-alert-notifications/10-main.tf @@ -89,7 +89,7 @@ resource "aws_lambda_function" "lambda" { function_name = lower("${var.identifier_prefix}${var.lambda_name}") role = aws_iam_role.lambda.arn handler = "main.lambda_handler" - runtime = "python3.11" + runtime = "python3.9" source_code_hash = filebase64sha256(data.archive_file.lambda.output_path) s3_bucket = var.lambda_artefact_storage_bucket s3_key = "${local.lambda_name_underscore}.zip" diff --git a/terraform/modules/kafka-test-lambda/10-lambda.tf b/terraform/modules/kafka-test-lambda/10-lambda.tf index 11bd76487..9ee538573 100644 --- a/terraform/modules/kafka-test-lambda/10-lambda.tf +++ b/terraform/modules/kafka-test-lambda/10-lambda.tf @@ -15,7 +15,7 @@ data "aws_iam_policy_document" "lambda_assume_role" { locals { command = "make install-requirements" - confluent_kafka_command = "docker run -v \"$PWD\":/var/task \"lambci/lambda:build-python3.11\" /bin/sh -c \"pip install -r requirements.txt -t python/lib/python3.11/site-packages/; exit\"" + confluent_kafka_command = "docker run -v \"$PWD\":/var/task \"lambci/lambda:build-python3.9\" /bin/sh -c \"pip install -r requirements.txt -t python/lib/python3.9/site-packages/; exit\"" # This ensures that this data resource will not be evaluated until # after the null_resource has been created. lambda_exporter_id = null_resource.run_install_requirements.id @@ -151,7 +151,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda.arn handler = "main.lambda_handler" - runtime = "python3.11" + runtime = "python3.9" function_name = lower("${var.identifier_prefix}${var.lambda_name}") s3_bucket = var.lambda_artefact_storage_bucket s3_key = aws_s3_object.lambda.key diff --git a/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf b/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf index a5e3c0d2d..b70575170 100644 --- a/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf +++ b/terraform/modules/lambda-alarms-handler/10-aws-lambda.tf @@ -110,7 +110,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda.arn handler = "main.lambda_handler" - runtime = "python3.11" + runtime = "python3.9" function_name = lower("${var.identifier_prefix}${var.lambda_name}") s3_bucket = var.lambda_artefact_storage_bucket s3_key = aws_s3_object.lambda.key