diff --git a/.gitignore b/.gitignore index c9ab9bb..d149843 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.swp +package-lock.json .idea .venv .envrc @@ -16,6 +18,7 @@ _build .doctrees docs/github_images/ .eggs/ +.build/ docs/_images/need_pie_*.png .devcontainer/ @@ -27,3 +30,7 @@ docs/_images/need_pie_*.png # Ignore dynaconf secret files .secrets.* + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce21900 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ + +# Define function directory +ARG FUNCTION_DIR="/function" + +FROM python:3.10-buster as build-image + + +# Install aws-lambda-cpp build dependencies +RUN apt-get update && \ + apt-get install -y \ + g++ \ + make \ + cmake \ + unzip \ + libcurl4-openssl-dev \ + tree + +## Include global arg in this stage of the build +ARG FUNCTION_DIR +# Create function directory +RUN mkdir -p ${FUNCTION_DIR} + +# Copy function code +ADD open_needs_server/ ${FUNCTION_DIR}/open_needs_server + +# Install the runtime interface client +RUN pip3 install \ + --target ${FUNCTION_DIR} \ + awslambdaric + +COPY requirements/ ${FUNCTION_DIR}/requirements/ +COPY settings.toml ${FUNCTION_DIR} + +# RUN yum install python3.10 + +# Install the function's dependencies using file requirements.txt +# from your project folder.COPY requirements.txt . +RUN pip3 install -r ${FUNCTION_DIR}/requirements/server.txt --target "${FUNCTION_DIR}" +RUN pip3 install -r ${FUNCTION_DIR}/requirements/aws.txt --target "${FUNCTION_DIR}" + +# Multi-stage build: grab a fresh copy of the base image +FROM python:3.10-buster + +# Include global arg in this stage of the build +ARG FUNCTION_DIR +# Set working directory to function root directory +WORKDIR ${FUNCTION_DIR} + +# Copy in the build image dependencies +COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} + +ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +CMD [ "open_needs_server.aws.handler" ] \ No newline at end of file diff --git a/cdk.json b/cdk.json new file mode 100644 index 0000000..93ace43 --- /dev/null +++ b/cdk.json @@ -0,0 +1,35 @@ +{ + "app": "python3 cdk/app.py", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "requirements*.txt", + "source.bat", + "**/__init__.py", + "python/__pycache__", + "tests" + ] + }, + "context": { + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, + "@aws-cdk/aws-lambda:recognizeVersionProps": true, + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ] + } +} diff --git a/cdk/.gitignore b/cdk/.gitignore new file mode 100644 index 0000000..37833f8 --- /dev/null +++ b/cdk/.gitignore @@ -0,0 +1,10 @@ +*.swp +package-lock.json +__pycache__ +.pytest_cache +.venv +*.egg-info + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cdk/app.py b/cdk/app.py new file mode 100644 index 0000000..a5cdbba --- /dev/null +++ b/cdk/app.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import os + +import aws_cdk as cdk + +from ons.ons_serverless import OnsServerless + + +app = cdk.App() +OnsServerless(app, "OnsServerless", + # If you don't specify 'env', this stack will be environment-agnostic. + # Account/Region-dependent features and context lookups will not work, + # but a single synthesized template can be deployed anywhere. + + # Uncomment the next line to specialize this stack for the AWS Account + # and Region that are implied by the current CLI configuration. + + #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), + + # Uncomment the next line if you know exactly what Account and Region you + # want to deploy the stack to. */ + + #env=cdk.Environment(account='123456789012', region='us-east-1'), + + # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html + ) + +app.synth() diff --git a/cdk/ons/__init__.py b/cdk/ons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cdk/ons/ons_serverless.py b/cdk/ons/ons_serverless.py new file mode 100644 index 0000000..79293e0 --- /dev/null +++ b/cdk/ons/ons_serverless.py @@ -0,0 +1,107 @@ +import os +import subprocess +import shutil + +from aws_cdk import ( + Duration, + BundlingOptions, + Stack, + aws_ec2 as ec2, + aws_efs as efs, + aws_lambda_python_alpha as pylambda, + aws_lambda as _lambda +) +from constructs import Construct + + +class OnsServerless(Stack): + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + self.clean_build_folder() + + vpc = ec2.Vpc(self, 'ONS-VPC', nat_gateways=0) + filesystem = efs.FileSystem(self, 'ONS-Efs', vpc=vpc) + + access_point = filesystem.add_access_point( + 'AccessPoint', + path='/export/lambda', + create_acl={ + 'owner_uid': '1001', + 'owner_gid': '1001', + 'permissions': '750' + }, + posix_user={ + 'uid': '1001', + 'gid': '1001' + }) + + entrypoint_name = 'ons_layer' + self.create_sources() + + docker_lambda = _lambda.DockerImageFunction(self, 'ONS_Start_Docker', + code=_lambda.DockerImageCode.from_image_asset( + '.'), + timeout=Duration.seconds(30), # Default is only 3 seconds + memory_size=512 # If your docker code is pretty complex + ) + + def clean_build_folder(self): + print('Cleaning .build') + shutil.rmtree('.build', ignore_errors=True) + + def create_sources(self) -> str: + print('Copying sources') + source_path = 'open_needs_server' + config_path = 'settings.toml' + + target_folder = 'open_needs_server' + target_path = '.build/src/' + + target_final = f'{target_path}/{target_folder}' + shutil.copytree(source_path, target_final, dirs_exist_ok=True) + print(f' {target_path}') + + print('Copying config') + shutil.copy(config_path, target_path) + + return target_path + + def create_dependencies_layer(self, project_name, function_name: str) -> pylambda.PythonLayerVersion: + """ + https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_lambda/LayerVersion.html + + :param project_name: + :param function_name: + :return: + """ + print('Copying dependencies') + requirements_files = [ + 'requirements/server.txt', + 'requirements/aws.txt' + ] + output_dir = f'.build/deps/' + output_req = f'.build/deps/requirements.txt' + os.makedirs(output_dir, exist_ok=True) + + # if not os.environ.get('SKIP_PIP'): + # subprocess.check_call( + # f'pip install -r {" -r".join(requirements_files)} -t {output_dir}/python'.split() + # ) + + with open(output_req, 'w') as req_output: + for req_file in requirements_files: + with open(req_file) as req_input: + req_output.write(req_input.read()) + req_output.write('\n') + + layer_id = f'{project_name}-{function_name}-dependencies' + # layer_code = lambda_.Code.from_asset(output_dir) + + # Uses docker to build deps + layer = pylambda.PythonLayerVersion(self, layer_id, + entry=output_dir) + + print(f' {output_dir}') + return layer diff --git a/docs/contributing.rst b/docs/contributing.rst index b192238..f181875 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -13,4 +13,11 @@ Documentation build pip install -r requirements/docs.txt cd docs - make html \ No newline at end of file + make html + +Docker +~~~~~~ + +Build container via: ``docker build -t ons . `` + +Start bash into container via: ``docker run -it --entrypoint /bin/bash ons`` diff --git a/open_needs_server/aws.py b/open_needs_server/aws.py new file mode 100644 index 0000000..8c82f76 --- /dev/null +++ b/open_needs_server/aws.py @@ -0,0 +1,11 @@ +""" +Provides a special AWS lambda handler to start the Open-Needs server. + +Ideas from https://www.deadbear.io/simple-serverless-fastapi-with-aws-lambda/ +""" +from mangum import Mangum + +from open_needs_server.main import ons_app + + +handler = Mangum(ons_app) diff --git a/requirements/aws.txt b/requirements/aws.txt new file mode 100644 index 0000000..a95d60a --- /dev/null +++ b/requirements/aws.txt @@ -0,0 +1,3 @@ +# These dependencies are need to install Open-Needs Server on AWS + +mangum diff --git a/requirements/cdk.txt b/requirements/cdk.txt new file mode 100644 index 0000000..85d615d --- /dev/null +++ b/requirements/cdk.txt @@ -0,0 +1,5 @@ +# Requirements used to deploy the Server to AWS + +# aws-cdk-lib==2.30.0 +aws-cdk.aws-lambda-python-alpha +constructs>=10.0.0,<11.0.0 diff --git a/requirements/server.txt b/requirements/server.txt index 17633d6..e1bb769 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -2,8 +2,9 @@ rich uvicorn pydantic fastapi +sqlalchemy[asyncio] fastapi-users[sqlalchemy]>=10.0.0 aiosqlite dynaconf sqladmin -jsonschema \ No newline at end of file +jsonschema diff --git a/settings.toml b/settings.toml index a80e88c..6452831 100644 --- a/settings.toml +++ b/settings.toml @@ -21,7 +21,8 @@ extensions = [ ] [database] -sql_string = "sqlite+aiosqlite:///./ons.db" +# sql_string = "sqlite+aiosqlite:///./ons.db" +sql_string = "sqlite+aiosqlite:////tmp/ons.db" [admin]