From 8579c5bf0ef8b136c5dbd925717976d2072e17dc Mon Sep 17 00:00:00 2001 From: Daniel Woste Date: Mon, 4 Jul 2022 22:23:08 +0200 Subject: [PATCH 1/5] Adds AWS CDK config (intial) --- .gitignore | 6 ++++++ cdk.json | 35 +++++++++++++++++++++++++++++++++++ cdk/.gitignore | 10 ++++++++++ cdk/app.py | 28 ++++++++++++++++++++++++++++ cdk/ons/__init__.py | 0 cdk/ons/ons_serverless.py | 39 +++++++++++++++++++++++++++++++++++++++ requirements/cdk.txt | 4 ++++ 7 files changed, 122 insertions(+) create mode 100644 cdk.json create mode 100644 cdk/.gitignore create mode 100644 cdk/app.py create mode 100644 cdk/ons/__init__.py create mode 100644 cdk/ons/ons_serverless.py create mode 100644 requirements/cdk.txt diff --git a/.gitignore b/.gitignore index c9ab9bb..f0d07a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.swp +package-lock.json .idea .venv .envrc @@ -27,3 +29,7 @@ docs/_images/need_pie_*.png # Ignore dynaconf secret files .secrets.* + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cdk.json b/cdk.json new file mode 100644 index 0000000..93ace43 --- /dev/null +++ b/cdk.json @@ -0,0 +1,35 @@ +{ + "app": "python3 cdk/app.py", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "requirements*.txt", + "source.bat", + "**/__init__.py", + "python/__pycache__", + "tests" + ] + }, + "context": { + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, + "@aws-cdk/aws-lambda:recognizeVersionProps": true, + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ] + } +} diff --git a/cdk/.gitignore b/cdk/.gitignore new file mode 100644 index 0000000..37833f8 --- /dev/null +++ b/cdk/.gitignore @@ -0,0 +1,10 @@ +*.swp +package-lock.json +__pycache__ +.pytest_cache +.venv +*.egg-info + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cdk/app.py b/cdk/app.py new file mode 100644 index 0000000..a5cdbba --- /dev/null +++ b/cdk/app.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import os + +import aws_cdk as cdk + +from ons.ons_serverless import OnsServerless + + +app = cdk.App() +OnsServerless(app, "OnsServerless", + # If you don't specify 'env', this stack will be environment-agnostic. + # Account/Region-dependent features and context lookups will not work, + # but a single synthesized template can be deployed anywhere. + + # Uncomment the next line to specialize this stack for the AWS Account + # and Region that are implied by the current CLI configuration. + + #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), + + # Uncomment the next line if you know exactly what Account and Region you + # want to deploy the stack to. */ + + #env=cdk.Environment(account='123456789012', region='us-east-1'), + + # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html + ) + +app.synth() diff --git a/cdk/ons/__init__.py b/cdk/ons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cdk/ons/ons_serverless.py b/cdk/ons/ons_serverless.py new file mode 100644 index 0000000..5b1463d --- /dev/null +++ b/cdk/ons/ons_serverless.py @@ -0,0 +1,39 @@ +from aws_cdk import ( + Stack, + aws_ec2 as ec2, + aws_efs as efs, + aws_lambda as lambda_ +) +from constructs import Construct + + +class OnsServerless(Stack): + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + vpc = ec2.Vpc(self, 'ONS-VPC') + filesystem = efs.FileSystem(self, 'ONS-Efs', vpc=vpc) + + access_point = filesystem.add_access_point( + 'AccessPoint', + path='/export/lambda', + create_acl={ + 'owner_uid': '1001', + 'owner_gid': '1001', + 'permissions': '750' + }, + posix_user={ + 'uid': '1001', + 'gid': '1001' + }) + + lambda_.Function( + self, + "ONS_Start", + filesystem=lambda_.FileSystem.from_efs_access_point(access_point, '/mnt/ons'), + vpc=vpc, + runtime=lambda_.Runtime.PYTHON_3_8, # required + code=lambda_.Code.from_asset('open_needs_server'), + handler="my_exported_func" + ) diff --git a/requirements/cdk.txt b/requirements/cdk.txt new file mode 100644 index 0000000..9178ed1 --- /dev/null +++ b/requirements/cdk.txt @@ -0,0 +1,4 @@ +# Requirements used to deploy the Server to AWS + +aws-cdk-lib==2.30.0 +constructs>=10.0.0,<11.0.0 From 6ead05be143ddb20600ff6370c9a1e27899fd4e7 Mon Sep 17 00:00:00 2001 From: Daniel Woste Date: Tue, 5 Jul 2022 20:59:20 +0200 Subject: [PATCH 2/5] Adding aws config --- .gitignore | 1 + cdk/ons/ons_serverless.py | 54 ++++++++++++++++++++++++++++++++++++--- open_needs_server/aws.py | 11 ++++++++ requirements/aws.txt | 3 +++ 4 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 open_needs_server/aws.py create mode 100644 requirements/aws.txt diff --git a/.gitignore b/.gitignore index f0d07a2..d149843 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ _build .doctrees docs/github_images/ .eggs/ +.build/ docs/_images/need_pie_*.png .devcontainer/ diff --git a/cdk/ons/ons_serverless.py b/cdk/ons/ons_serverless.py index 5b1463d..60ebe6a 100644 --- a/cdk/ons/ons_serverless.py +++ b/cdk/ons/ons_serverless.py @@ -1,4 +1,9 @@ +import os +import subprocess +import shutil + from aws_cdk import ( + BundlingOptions, Stack, aws_ec2 as ec2, aws_efs as efs, @@ -12,7 +17,9 @@ class OnsServerless(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) - vpc = ec2.Vpc(self, 'ONS-VPC') + self.clean_build_folder() + + vpc = ec2.Vpc(self, 'ONS-VPC', nat_gateways=0) filesystem = efs.FileSystem(self, 'ONS-Efs', vpc=vpc) access_point = filesystem.add_access_point( @@ -28,12 +35,51 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: 'gid': '1001' }) + entrypoint_name = 'ons_layer' + lambda_.Function( self, "ONS_Start", filesystem=lambda_.FileSystem.from_efs_access_point(access_point, '/mnt/ons'), vpc=vpc, - runtime=lambda_.Runtime.PYTHON_3_8, # required - code=lambda_.Code.from_asset('open_needs_server'), - handler="my_exported_func" + runtime=lambda_.Runtime.PYTHON_3_9, # required + code=lambda_.Code.from_asset(self.create_sources()), + handler="open_needs_server.aws.handler", + layers=[ + self.create_dependencies_layer(self.stack_name, entrypoint_name) + ] ) + + def clean_build_folder(self): + print('Cleaning .build') + shutil.rmtree('.build') + + def create_sources(self): + print('Copying sources') + source_path = 'open_needs_server' + target_folder = 'open_needs_server' + target_path = '.build/src/' + + target_final = f'{target_path}/{target_folder}' + shutil.copytree(source_path, target_final, dirs_exist_ok=True) + print(f' {target_path}') + return target_path + + def create_dependencies_layer(self, project_name, function_name: str) -> lambda_.LayerVersion: + print('Copying dependencies') + requirements_files = [ + 'requirements/server.txt', + 'requirements/aws.txt' + ] + output_dir = f'.build/deps' + + if not os.environ.get('SKIP_PIP'): + subprocess.check_call( + f'pip install -r {" -r".join(requirements_files)} -t {output_dir}/python'.split() + ) + + layer_id = f'{project_name}-{function_name}-dependencies' + layer_code = lambda_.Code.from_asset(output_dir) + + print(f' {output_dir}') + return lambda_.LayerVersion(self, layer_id, code=layer_code) diff --git a/open_needs_server/aws.py b/open_needs_server/aws.py new file mode 100644 index 0000000..f0c1dd3 --- /dev/null +++ b/open_needs_server/aws.py @@ -0,0 +1,11 @@ +""" +Provides a special AWS lambda handler to start the Open-Needs server. + +Ideas from https://www.deadbear.io/simple-serverless-fastapi-with-aws-lambda/ +""" +from mangum import Mangum + +from main import ons_app + + +handler = Mangum(ons_app) diff --git a/requirements/aws.txt b/requirements/aws.txt new file mode 100644 index 0000000..846d97c --- /dev/null +++ b/requirements/aws.txt @@ -0,0 +1,3 @@ +# These dependencies are need to install Open-Needs Server on AWS + +mangum \ No newline at end of file From 116ed86f2724148993ef19a7b767803a1e4d3bf9 Mon Sep 17 00:00:00 2001 From: Daniel Woste Date: Tue, 5 Jul 2022 21:46:51 +0200 Subject: [PATCH 3/5] =?UTF-8?q?Forcing=20greenlet.=20But=20py-deps=20shoul?= =?UTF-8?q?d=20be=20build=20on=20=C3=B6ambda-docker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cdk/ons/ons_serverless.py | 6 ++++++ open_needs_server/aws.py | 2 +- requirements/server.txt | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cdk/ons/ons_serverless.py b/cdk/ons/ons_serverless.py index 60ebe6a..ffea8f0 100644 --- a/cdk/ons/ons_serverless.py +++ b/cdk/ons/ons_serverless.py @@ -57,12 +57,18 @@ def clean_build_folder(self): def create_sources(self): print('Copying sources') source_path = 'open_needs_server' + config_path = 'settings.toml' + target_folder = 'open_needs_server' target_path = '.build/src/' target_final = f'{target_path}/{target_folder}' shutil.copytree(source_path, target_final, dirs_exist_ok=True) print(f' {target_path}') + + print('Copying config') + shutil.copy(config_path, target_path) + return target_path def create_dependencies_layer(self, project_name, function_name: str) -> lambda_.LayerVersion: diff --git a/open_needs_server/aws.py b/open_needs_server/aws.py index f0c1dd3..8c82f76 100644 --- a/open_needs_server/aws.py +++ b/open_needs_server/aws.py @@ -5,7 +5,7 @@ """ from mangum import Mangum -from main import ons_app +from open_needs_server.main import ons_app handler = Mangum(ons_app) diff --git a/requirements/server.txt b/requirements/server.txt index 17633d6..56f193a 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -2,6 +2,7 @@ rich uvicorn pydantic fastapi +sqlalchemy[asyncio] fastapi-users[sqlalchemy]>=10.0.0 aiosqlite dynaconf From db94928f807c502cc876a08586bdb8548516dc69 Mon Sep 17 00:00:00 2001 From: Daniel Woste Date: Mon, 11 Jul 2022 21:54:59 +0200 Subject: [PATCH 4/5] Adds aws dockr build --- cdk/ons/ons_serverless.py | 55 +++++++++++++++++++++++++++------------ requirements/aws.txt | 2 +- requirements/cdk.txt | 3 ++- requirements/server.txt | 2 +- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/cdk/ons/ons_serverless.py b/cdk/ons/ons_serverless.py index ffea8f0..46a5354 100644 --- a/cdk/ons/ons_serverless.py +++ b/cdk/ons/ons_serverless.py @@ -7,7 +7,8 @@ Stack, aws_ec2 as ec2, aws_efs as efs, - aws_lambda as lambda_ + aws_lambda_python_alpha as pylambda, + aws_lambda as _lambda ) from constructs import Construct @@ -36,15 +37,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: }) entrypoint_name = 'ons_layer' + self.create_sources() - lambda_.Function( + # https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_lambda_python_alpha/PythonFunction.html + pylambda.PythonFunction( self, "ONS_Start", - filesystem=lambda_.FileSystem.from_efs_access_point(access_point, '/mnt/ons'), + filesystem=_lambda.FileSystem.from_efs_access_point(access_point, '/mnt/ons'), vpc=vpc, - runtime=lambda_.Runtime.PYTHON_3_9, # required - code=lambda_.Code.from_asset(self.create_sources()), - handler="open_needs_server.aws.handler", + runtime=_lambda.Runtime.PYTHON_3_7, # required + entry=".build/src/", + index="open_needs_server/aws.py", + handler="handler", layers=[ self.create_dependencies_layer(self.stack_name, entrypoint_name) ] @@ -52,9 +56,9 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: def clean_build_folder(self): print('Cleaning .build') - shutil.rmtree('.build') + shutil.rmtree('.build', ignore_errors=True) - def create_sources(self): + def create_sources(self) -> str: print('Copying sources') source_path = 'open_needs_server' config_path = 'settings.toml' @@ -71,21 +75,40 @@ def create_sources(self): return target_path - def create_dependencies_layer(self, project_name, function_name: str) -> lambda_.LayerVersion: + def create_dependencies_layer(self, project_name, function_name: str) -> pylambda.PythonLayerVersion: + """ + https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_lambda/LayerVersion.html + + :param project_name: + :param function_name: + :return: + """ print('Copying dependencies') requirements_files = [ 'requirements/server.txt', 'requirements/aws.txt' ] - output_dir = f'.build/deps' + output_dir = f'.build/deps/' + output_req = f'.build/deps/requirements.txt' + os.makedirs(output_dir, exist_ok=True) + + # if not os.environ.get('SKIP_PIP'): + # subprocess.check_call( + # f'pip install -r {" -r".join(requirements_files)} -t {output_dir}/python'.split() + # ) - if not os.environ.get('SKIP_PIP'): - subprocess.check_call( - f'pip install -r {" -r".join(requirements_files)} -t {output_dir}/python'.split() - ) + with open(output_req, 'w') as req_output: + for req_file in requirements_files: + with open(req_file) as req_input: + req_output.write(req_input.read()) + req_output.write('\n') layer_id = f'{project_name}-{function_name}-dependencies' - layer_code = lambda_.Code.from_asset(output_dir) + # layer_code = lambda_.Code.from_asset(output_dir) + + # Uses docker to build deps + layer = pylambda.PythonLayerVersion(self, layer_id, + entry=output_dir) print(f' {output_dir}') - return lambda_.LayerVersion(self, layer_id, code=layer_code) + return layer diff --git a/requirements/aws.txt b/requirements/aws.txt index 846d97c..a95d60a 100644 --- a/requirements/aws.txt +++ b/requirements/aws.txt @@ -1,3 +1,3 @@ # These dependencies are need to install Open-Needs Server on AWS -mangum \ No newline at end of file +mangum diff --git a/requirements/cdk.txt b/requirements/cdk.txt index 9178ed1..85d615d 100644 --- a/requirements/cdk.txt +++ b/requirements/cdk.txt @@ -1,4 +1,5 @@ # Requirements used to deploy the Server to AWS -aws-cdk-lib==2.30.0 +# aws-cdk-lib==2.30.0 +aws-cdk.aws-lambda-python-alpha constructs>=10.0.0,<11.0.0 diff --git a/requirements/server.txt b/requirements/server.txt index 56f193a..e1bb769 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -7,4 +7,4 @@ fastapi-users[sqlalchemy]>=10.0.0 aiosqlite dynaconf sqladmin -jsonschema \ No newline at end of file +jsonschema From 12ccc31a5257e34c0b01d914d2570bae579b0d2e Mon Sep 17 00:00:00 2001 From: Daniel Woste Date: Thu, 14 Jul 2022 16:16:07 +0200 Subject: [PATCH 5/5] Lambda setup with own Docker-Image (Py3.10) --- Dockerfile | 53 +++++++++++++++++++++++++++++++++++++++ cdk/ons/ons_serverless.py | 21 ++++++---------- docs/contributing.rst | 9 ++++++- settings.toml | 3 ++- 4 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce21900 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ + +# Define function directory +ARG FUNCTION_DIR="/function" + +FROM python:3.10-buster as build-image + + +# Install aws-lambda-cpp build dependencies +RUN apt-get update && \ + apt-get install -y \ + g++ \ + make \ + cmake \ + unzip \ + libcurl4-openssl-dev \ + tree + +## Include global arg in this stage of the build +ARG FUNCTION_DIR +# Create function directory +RUN mkdir -p ${FUNCTION_DIR} + +# Copy function code +ADD open_needs_server/ ${FUNCTION_DIR}/open_needs_server + +# Install the runtime interface client +RUN pip3 install \ + --target ${FUNCTION_DIR} \ + awslambdaric + +COPY requirements/ ${FUNCTION_DIR}/requirements/ +COPY settings.toml ${FUNCTION_DIR} + +# RUN yum install python3.10 + +# Install the function's dependencies using file requirements.txt +# from your project folder.COPY requirements.txt . +RUN pip3 install -r ${FUNCTION_DIR}/requirements/server.txt --target "${FUNCTION_DIR}" +RUN pip3 install -r ${FUNCTION_DIR}/requirements/aws.txt --target "${FUNCTION_DIR}" + +# Multi-stage build: grab a fresh copy of the base image +FROM python:3.10-buster + +# Include global arg in this stage of the build +ARG FUNCTION_DIR +# Set working directory to function root directory +WORKDIR ${FUNCTION_DIR} + +# Copy in the build image dependencies +COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} + +ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +CMD [ "open_needs_server.aws.handler" ] \ No newline at end of file diff --git a/cdk/ons/ons_serverless.py b/cdk/ons/ons_serverless.py index 46a5354..79293e0 100644 --- a/cdk/ons/ons_serverless.py +++ b/cdk/ons/ons_serverless.py @@ -3,6 +3,7 @@ import shutil from aws_cdk import ( + Duration, BundlingOptions, Stack, aws_ec2 as ec2, @@ -39,20 +40,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: entrypoint_name = 'ons_layer' self.create_sources() - # https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_lambda_python_alpha/PythonFunction.html - pylambda.PythonFunction( - self, - "ONS_Start", - filesystem=_lambda.FileSystem.from_efs_access_point(access_point, '/mnt/ons'), - vpc=vpc, - runtime=_lambda.Runtime.PYTHON_3_7, # required - entry=".build/src/", - index="open_needs_server/aws.py", - handler="handler", - layers=[ - self.create_dependencies_layer(self.stack_name, entrypoint_name) - ] - ) + docker_lambda = _lambda.DockerImageFunction(self, 'ONS_Start_Docker', + code=_lambda.DockerImageCode.from_image_asset( + '.'), + timeout=Duration.seconds(30), # Default is only 3 seconds + memory_size=512 # If your docker code is pretty complex + ) def clean_build_folder(self): print('Cleaning .build') diff --git a/docs/contributing.rst b/docs/contributing.rst index b192238..f181875 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -13,4 +13,11 @@ Documentation build pip install -r requirements/docs.txt cd docs - make html \ No newline at end of file + make html + +Docker +~~~~~~ + +Build container via: ``docker build -t ons . `` + +Start bash into container via: ``docker run -it --entrypoint /bin/bash ons`` diff --git a/settings.toml b/settings.toml index a80e88c..6452831 100644 --- a/settings.toml +++ b/settings.toml @@ -21,7 +21,8 @@ extensions = [ ] [database] -sql_string = "sqlite+aiosqlite:///./ons.db" +# sql_string = "sqlite+aiosqlite:///./ons.db" +sql_string = "sqlite+aiosqlite:////tmp/ons.db" [admin]