From 1dfe82275d2c84f373e635dc1481592b094d99c3 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 13:34:54 -0700 Subject: [PATCH 01/42] initial work, hopefully this exists when merging over the weekend --- .github/workflows/dev-branch.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/dev-branch.yaml diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml new file mode 100644 index 0000000..212bd1a --- /dev/null +++ b/.github/workflows/dev-branch.yaml @@ -0,0 +1,22 @@ +name: dev-branch-deployment +on: + push: + branches-ignore: + - main +jobs: + dev-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Set up python + with: + python-version: 3.11 + - name: Install python dependencies + run: pip install -r requirements.txt + + dev-deploy: + runs-on: ubuntu-latest + steps: + + From 807f5c7fc78f976f38be7fe842fcf39e522eab4d Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 13:36:09 -0700 Subject: [PATCH 02/42] file whitespace cleanup --- .github/workflows/dev-branch.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 212bd1a..8efd0ae 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -18,5 +18,3 @@ jobs: dev-deploy: runs-on: ubuntu-latest steps: - - From deec723d5559a6c6680a96ca387780db0930020b Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 18:50:01 -0700 Subject: [PATCH 03/42] almost done --- .github/workflows/dev-branch.yaml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 8efd0ae..8b77cae 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -3,8 +3,9 @@ on: push: branches-ignore: - main + - master jobs: - dev-build: + dev-deploy-to-aws: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -14,7 +15,20 @@ jobs: python-version: 3.11 - name: Install python dependencies run: pip install -r requirements.txt - - dev-deploy: - runs-on: ubuntu-latest - steps: + - name: Configure AWS profile + uses: aws-actions/configure-aws-credentialsW@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Install CDK CLI with nodejs + uses: actions/setup-node@v4 + with: + node-version: 20 + run: | + + + - name: Build the CDK stack using the aws profile + run: | + cdk bootsrap + From b3d7435dce50c78527d8c8fbe400454d437fb8a2 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 19:03:43 -0700 Subject: [PATCH 04/42] Made a separate folder for dev and prod stacks. Need to get rid of some unneeded services that may run. --- .github/workflows/dev-branch.yaml | 5 +- immersion/dev-stack/app.py | 7 + immersion/{ => dev-stack}/immersion_stack.py | 0 immersion/prod-stack/app.py | 7 + immersion/prod-stack/immersion_stack.py | 243 +++++++++++++++++++ 5 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 immersion/dev-stack/app.py rename immersion/{ => dev-stack}/immersion_stack.py (100%) create mode 100644 immersion/prod-stack/app.py create mode 100644 immersion/prod-stack/immersion_stack.py diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 8b77cae..14231a8 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -26,9 +26,10 @@ jobs: with: node-version: 20 run: | - + npm install -g aws-cdk - name: Build the CDK stack using the aws profile run: | - cdk bootsrap + cd immersion/ + cdk deploy --require-approval never diff --git a/immersion/dev-stack/app.py b/immersion/dev-stack/app.py new file mode 100644 index 0000000..84069bd --- /dev/null +++ b/immersion/dev-stack/app.py @@ -0,0 +1,7 @@ +from immersion_stack import ImmersionStack +import aws_cdk as cdk + +app = cdk.App() +ImmersionStack(app, "ImmersionStackDev") + +app.synth() \ No newline at end of file diff --git a/immersion/immersion_stack.py b/immersion/dev-stack/immersion_stack.py similarity index 100% rename from immersion/immersion_stack.py rename to immersion/dev-stack/immersion_stack.py diff --git a/immersion/prod-stack/app.py b/immersion/prod-stack/app.py new file mode 100644 index 0000000..75cb550 --- /dev/null +++ b/immersion/prod-stack/app.py @@ -0,0 +1,7 @@ +from immersion_stack import ImmersionStack +import aws_cdk as cdk + +app = cdk.App() +ImmersionStack(app, "ImmersionStackProd") + +app.synth() \ No newline at end of file diff --git a/immersion/prod-stack/immersion_stack.py b/immersion/prod-stack/immersion_stack.py new file mode 100644 index 0000000..9b52827 --- /dev/null +++ b/immersion/prod-stack/immersion_stack.py @@ -0,0 +1,243 @@ +from aws_cdk import ( + # Duration, + Stack, + Duration, + aws_dynamodb as dynamodb, + aws_ec2 as ec2, + aws_ecs as ecs, + aws_sqs as sqs, + aws_cloudwatch as cw, + aws_applicationautoscaling as appautoscaling, + aws_lambda as _lambda, + aws_lambda_python_alpha as lambda_python, + aws_ssm as ssm +) +from aws_cdk.aws_ecr_assets import DockerImageAsset +from aws_cdk.aws_cloudwatch_actions import ApplicationScalingAction +from aws_cdk.aws_lambda import Runtime +from constructs import Construct +from dotenv import load_dotenv +import os + +load_dotenv() + +class ImmersionStack(Stack): + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + # DynamoDB Table Definitions + serverTable = dynamodb.TableV2( + self, + f'{os.getenv('APP_NAME')}ServerTable', + partition_key=dynamodb.Attribute(name='serverId', type=dynamodb.AttributeType.STRING), + ) + + onboardingTable = dynamodb.TableV2( + self, + f'{os.getenv('APP_NAME')}OrganizationTable', + partition_key=dynamodb.Attribute(name='organizationId', type=dynamodb.AttributeType.NUMBER) + ) + + cacheTable = dynamodb.TableV2( + self, + f'{os.getenv('APP_NAME')}APICacheTable', + partition_key=dynamodb.Attribute(name='clubId', type=dynamodb.AttributeType.STRING), + ) + + eventTable = dynamodb.TableV2( + self, + f'{os.getenv('APP_NAME')}EventTable', + partition_key=dynamodb.Attribute(name='eventId', type=dynamodb.AttributeType.STRING), + ) + + # SQS Queue and Size Metric Defintion + queue = sqs.Queue( + self, + f'{os.getenv('APP_NAME')}DataQueue', + queue_name=f'{os.getenv('APP_NAME')}_Data_Queue', + ) + + scale_metric = queue.metric_approximate_number_of_messages_visible( + period=Duration.minutes(1), + statistic="Average", + ) + + scale_out_alarm = scale_metric.create_alarm( + self, + f'{os.getenv('APP_NAME')}DataParserScaleOutAlarm', + alarm_name=f'{os.getenv('APP_NAME')}DataProcessScaleOutAlarm', + threshold=1, + evaluation_periods=1, + comparison_operator=cw.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=cw.TreatMissingData.NOT_BREACHING + ) + + scale_in_alarm = scale_metric.create_alarm( + self, + f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', + alarm_name=f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', + threshold=0, + evaluation_periods=1, + comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=cw.TreatMissingData.NOT_BREACHING + ) + + # Container Cluster Component Defintion + vpc = ec2.Vpc.from_lookup( + self, + 'VPC', + vpc_name='Club-VPC' + ) + + cluster = ecs.Cluster( + self, + f'{os.getenv('APP_NAME')}ServiceCluster', + vpc=vpc + ) + + # Discord App Container Definition + app_task_defintion = ecs.FargateTaskDefinition( + self, + f'{os.getenv('APP_NAME')}DiscordAppTaskDefinition', + memory_limit_mib=1024, # 1 GB + cpu=512, # 0.5 vCPU + ) + + app_task_defintion.add_container( + f'{os.getenv('APP_NAME')}DiscordApp', + image=ecs.ContainerImage.from_docker_image_asset( + DockerImageAsset( + self, + f'{os.getenv('APP_NAME')}DiscordAppDockerImage', + directory='src/discordapp/' + ) + ), + environment={ + 'DISCORD_TOKEN': os.getenv('DISCORD_TOKEN') + } + ) + + app_service = ecs.FargateService( + self, + f'{os.getenv('APP_NAME')}DiscordAppService', + cluster=cluster, + task_definition=app_task_defintion, + assign_public_ip=True + ) + + parser_logging = ecs.AwsLogDriver( + stream_prefix='parserlogs' + ) + + # Data Parser Task Definition + parser_task_definition = ecs.FargateTaskDefinition( + self, + f'{os.getenv('APP_NAME')}DataParserTask', + memory_limit_mib=1024, # 1 GB + cpu=512, # 0.5 vCPU + ) + + parser_task_definition.add_container( + f'{os.getenv('APP_NAME')}DataParser', + # TODO: REFACTOR TO USE GITHUB CONTAINER REGISTRY! + image=ecs.ContainerImage.from_docker_image_asset( + DockerImageAsset( + self, + f'{os.getenv('APP_NAME')}DataParserImage', + directory='src/data_parser/', + ) + ), + environment={ + 'QUEUE_URL': queue.queue_url, + 'ORGANIZATION_TABLE': onboardingTable.table_name, + }, + logging=parser_logging, + ) + queue.grant_consume_messages(parser_task_definition.task_role) + onboardingTable.grant_write_data(parser_task_definition.task_role) + + parser_service = ecs.FargateService( + self, + f'{os.getenv('APP_NAME')}DataParserService', + cluster=cluster, + task_definition=parser_task_definition, + assign_public_ip=True + ) + + # Scale metrics for data parser service + scaling_target = appautoscaling.ScalableTarget( + self, + id=f'{os.getenv('APP_NAME')}ParserScalingTarget', + service_namespace=appautoscaling.ServiceNamespace.ECS, + scalable_dimension='ecs:service:DesiredCount', + min_capacity=0, + max_capacity=1, + resource_id=f'service/{parser_service.cluster.cluster_name}/{parser_service.service_name}' + ) + + scale_out_action = appautoscaling.StepScalingAction( + self, + 'ParserScaleOutAction', + scaling_target=scaling_target, + adjustment_type=appautoscaling.AdjustmentType.EXACT_CAPACITY, + ) + + scale_out_action.add_adjustment( + adjustment=1, + lower_bound=0 + ) + + + scale_out_action.add_adjustment( + adjustment=0, + upper_bound=0 + ) + + scale_out_alarm.add_alarm_action(ApplicationScalingAction(scale_out_action)) + + scale_in_action = appautoscaling.StepScalingAction( + self, + 'ParserScaleInAction', + scaling_target=scaling_target, + adjustment_type=appautoscaling.AdjustmentType.EXACT_CAPACITY, + ) + + scale_in_action.add_adjustment( + adjustment=0, + lower_bound=0, + ) + + scale_in_action.add_adjustment( + adjustment=1, + upper_bound=0, + ) + + scale_in_alarm.add_alarm_action(ApplicationScalingAction(scale_in_action)) + + # TODO: figure out how to make a layer for python dependencies + # filter_layer = _lambda.LayerVersion( + # self, + # f'{os.getenv('APP_NAME')}FilterLayer', + + # ) + + # Data Filter Lambda Functions + engage_api_key_param = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + f'{os.getenv('APP_NAME')}APIKEY', + parameter_name=f'{os.getenv('SSM_PARAMETER_NAME_API')}' + ) + + club_information_lambda = lambda_python.PythonFunction( + self, + f'{os.getenv('APP_NAME')}ClubInformationLambda', + runtime=Runtime.PYTHON_3_13, + entry='src/data_filters/onboarding', + handler='lambda_handler', + environment={ + 'QUEUE_URL': queue.queue_url + }, + ) + queue.grant_send_messages(club_information_lambda) + engage_api_key_param.grant_read(club_information_lambda) \ No newline at end of file From 2699b1ca53b3894125743cd035fda6dd2ed67726 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 19:09:07 -0700 Subject: [PATCH 05/42] Change into right directory for CI/CD --- .github/workflows/dev-branch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 14231a8..b948163 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -30,6 +30,6 @@ jobs: - name: Build the CDK stack using the aws profile run: | - cd immersion/ + cd immersion/dev-stack cdk deploy --require-approval never From ad16a7dee5fa65e6e18cb4146a60b9dafbcedb70 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 19:11:37 -0700 Subject: [PATCH 06/42] Can't use run and uses on the same step. Fixed that issue. --- .github/workflows/dev-branch.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index b948163..f6c3d6a 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -21,10 +21,11 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - - name: Install CDK CLI with nodejs - uses: actions/setup-node@v4 + - uses: actions/setup-node@v4 + name: Install nodejs with: node-version: 20 + - name: Install CDK CLI with nodejs run: | npm install -g aws-cdk From 895f34aaa7bff4971f70e595b217fbb07fd23157 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 19:12:23 -0700 Subject: [PATCH 07/42] Typo on one of the actions. --- .github/workflows/dev-branch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index f6c3d6a..5ee850c 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -16,7 +16,7 @@ jobs: - name: Install python dependencies run: pip install -r requirements.txt - name: Configure AWS profile - uses: aws-actions/configure-aws-credentialsW@v4 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 4687e59124ee3cef0e395aad70c89751c1b02492 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 25 Jun 2025 19:16:07 -0700 Subject: [PATCH 08/42] file cleanup --- .github/workflows/dev-branch.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 5ee850c..a1dc67b 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -33,4 +33,3 @@ jobs: run: | cd immersion/dev-stack cdk deploy --require-approval never - From 73a467c761e931e4a65f19fe10846995a09be33e Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 11:29:53 -0700 Subject: [PATCH 09/42] Added pipeline for master branch --- .github/workflows/master-branch.yaml | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/master-branch.yaml diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml new file mode 100644 index 0000000..3796441 --- /dev/null +++ b/.github/workflows/master-branch.yaml @@ -0,0 +1,35 @@ +name: dev-branch-deployment +on: + pull_request: + branches: + - main + - master +jobs: + dev-deploy-to-aws: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Set up python + with: + python-version: 3.11 + - name: Install python dependencies + run: pip install -r requirements.txt + - name: Configure AWS profile + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - uses: actions/setup-node@v4 + name: Install nodejs + with: + node-version: 20 + - name: Install CDK CLI with nodejs + run: | + npm install -g aws-cdk + + - name: Build the CDK stack using the aws profile + run: | + cd immersion/dev-stack + cdk deploy --require-approval never From 7aaaeb367051accf585be7fc0a19fa394c3c276f Mon Sep 17 00:00:00 2001 From: Gurpranked <93165798+Gurpranked@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:48:21 -0400 Subject: [PATCH 10/42] Update master-branch.yaml Fix AWS Region error on deployment configuration --- .github/workflows/master-branch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 3796441..772d092 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -20,7 +20,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs with: From a9b139ef12eafab7cb465267d1591239b9ba7b00 Mon Sep 17 00:00:00 2001 From: Gurpranked <93165798+Gurpranked@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:48:48 -0400 Subject: [PATCH 11/42] Update dev-branch.yaml Fix AWS Region issue on deployment configuration (dev) --- .github/workflows/dev-branch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index a1dc67b..4b2caf2 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -20,7 +20,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs with: From 66f389bf06899f2fd35d5f1b70145cfd996db2c5 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 13:18:54 -0700 Subject: [PATCH 12/42] Added STS Assume Role ARN --- .github/workflows/master-branch.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 3796441..2de5343 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -18,8 +18,7 @@ jobs: - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.STS_ASSUME_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs From 595614a35e05b6ca5a0728bbbb8cd7c1c1b67d31 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 13:24:16 -0700 Subject: [PATCH 13/42] Added STS assume role for dev branch pipeline --- .github/workflows/dev-branch.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 4b2caf2..7f444cf 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -18,8 +18,7 @@ jobs: - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.STS_ASSUME_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs From 60cd3ab089e2a8d99e7dfa76d08e83199e0bf8e9 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 13:27:16 -0700 Subject: [PATCH 14/42] Changed role-to-assume to secrets. --- .github/workflows/dev-branch.yaml | 2 +- .github/workflows/master-branch.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 7f444cf..68aba05 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -18,7 +18,7 @@ jobs: - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ env.STS_ASSUME_ROLE_ARN }} + role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 2de5343..ff0a2da 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -18,8 +18,8 @@ jobs: - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ env.STS_ASSUME_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs with: From d7e42a27d1e374477f595e36b41a042937a37737 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 13:35:57 -0700 Subject: [PATCH 15/42] Added permissions for id-token in the pipeline --- .github/workflows/dev-branch.yaml | 4 ++++ .github/workflows/master-branch.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 68aba05..e5d8fb9 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -7,6 +7,9 @@ on: jobs: dev-deploy-to-aws: runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -19,6 +22,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} + role-session-name: github-actions-cfn-deploy aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index ff0a2da..04fb2db 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -7,6 +7,9 @@ on: jobs: dev-deploy-to-aws: runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -19,6 +22,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} + role-session-name: github-actions-cfn-deploy aws-region: ${{ vars.AWS_REGION }} - uses: actions/setup-node@v4 name: Install nodejs From 25f0ecf33698a96c126f824501755e2e3375f34a Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 13:43:55 -0700 Subject: [PATCH 16/42] Added correct commands to deploy to AWS --- .github/workflows/dev-branch.yaml | 4 ++-- .github/workflows/master-branch.yaml | 4 ++-- immersion/{dev-stack => environments/dev}/app.py | 0 immersion/{dev-stack => environments/dev}/immersion_stack.py | 0 immersion/{prod-stack => environments/prod}/app.py | 0 .../{prod-stack => environments/prod}/immersion_stack.py | 0 6 files changed, 4 insertions(+), 4 deletions(-) rename immersion/{dev-stack => environments/dev}/app.py (100%) rename immersion/{dev-stack => environments/dev}/immersion_stack.py (100%) rename immersion/{prod-stack => environments/prod}/app.py (100%) rename immersion/{prod-stack => environments/prod}/immersion_stack.py (100%) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index e5d8fb9..6c10649 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -34,5 +34,5 @@ jobs: - name: Build the CDK stack using the aws profile run: | - cd immersion/dev-stack - cdk deploy --require-approval never + cd immersion/environments/dev + cdk deploy --app "python3 app.py" --require-approval never diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 04fb2db..8da5653 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -34,5 +34,5 @@ jobs: - name: Build the CDK stack using the aws profile run: | - cd immersion/dev-stack - cdk deploy --require-approval never + cd immersion/environments/prod + cdk deploy --app "python3 app.py" --require-approval never diff --git a/immersion/dev-stack/app.py b/immersion/environments/dev/app.py similarity index 100% rename from immersion/dev-stack/app.py rename to immersion/environments/dev/app.py diff --git a/immersion/dev-stack/immersion_stack.py b/immersion/environments/dev/immersion_stack.py similarity index 100% rename from immersion/dev-stack/immersion_stack.py rename to immersion/environments/dev/immersion_stack.py diff --git a/immersion/prod-stack/app.py b/immersion/environments/prod/app.py similarity index 100% rename from immersion/prod-stack/app.py rename to immersion/environments/prod/app.py diff --git a/immersion/prod-stack/immersion_stack.py b/immersion/environments/prod/immersion_stack.py similarity index 100% rename from immersion/prod-stack/immersion_stack.py rename to immersion/environments/prod/immersion_stack.py From 0b940a004c305eb93bb2c46a41ab2669a90ccf89 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 15:26:32 -0700 Subject: [PATCH 17/42] changed single quote to double quote --- immersion/environments/dev/immersion_stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 9b52827..9cdc23c 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -29,7 +29,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}ServerTable', + f"{os.getenv('APP_NAME')}ServerTable", partition_key=dynamodb.Attribute(name='serverId', type=dynamodb.AttributeType.STRING), ) From 4f27a34c8b66830763ded3b1a07b09c0fa0e0350 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 15:32:29 -0700 Subject: [PATCH 18/42] changed single quote to double quote in all of the formatted strings that use os.getenv function --- immersion/environments/dev/immersion_stack.py | 38 +++++++++--------- .../environments/prod/immersion_stack.py | 40 +++++++++---------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 9cdc23c..5e8cb1e 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -35,26 +35,26 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: onboardingTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}OrganizationTable', + f"{os.getenv('APP_NAME')}OrganizationTable", partition_key=dynamodb.Attribute(name='organizationId', type=dynamodb.AttributeType.NUMBER) ) cacheTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}APICacheTable', + f"{os.getenv('APP_NAME')}APICacheTable", partition_key=dynamodb.Attribute(name='clubId', type=dynamodb.AttributeType.STRING), ) eventTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}EventTable', + f"{os.getenv('APP_NAME')}EventTable", partition_key=dynamodb.Attribute(name='eventId', type=dynamodb.AttributeType.STRING), ) # SQS Queue and Size Metric Defintion queue = sqs.Queue( self, - f'{os.getenv('APP_NAME')}DataQueue', + f"{os.getenv('APP_NAME')}DataQueue", queue_name=f'{os.getenv('APP_NAME')}_Data_Queue', ) @@ -65,7 +65,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, - f'{os.getenv('APP_NAME')}DataParserScaleOutAlarm', + f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", alarm_name=f'{os.getenv('APP_NAME')}DataProcessScaleOutAlarm', threshold=1, evaluation_periods=1, @@ -75,7 +75,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, - f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', + f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", alarm_name=f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', threshold=0, evaluation_periods=1, @@ -92,24 +92,24 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cluster = ecs.Cluster( self, - f'{os.getenv('APP_NAME')}ServiceCluster', + f"{os.getenv('APP_NAME')}ServiceCluster", vpc=vpc ) # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f'{os.getenv('APP_NAME')}DiscordAppTaskDefinition', + f"{os.getenv('APP_NAME')}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) app_task_defintion.add_container( - f'{os.getenv('APP_NAME')}DiscordApp', + f"{os.getenv('APP_NAME')}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f'{os.getenv('APP_NAME')}DiscordAppDockerImage', + f"{os.getenv('APP_NAME')}DiscordAppDockerImage", directory='src/discordapp/' ) ), @@ -120,7 +120,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: app_service = ecs.FargateService( self, - f'{os.getenv('APP_NAME')}DiscordAppService', + f"{os.getenv('APP_NAME')}DiscordAppService", cluster=cluster, task_definition=app_task_defintion, assign_public_ip=True @@ -133,18 +133,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f'{os.getenv('APP_NAME')}DataParserTask', + f"{os.getenv('APP_NAME')}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) parser_task_definition.add_container( - f'{os.getenv('APP_NAME')}DataParser', + f"{os.getenv('APP_NAME')}DataParser", # TODO: REFACTOR TO USE GITHUB CONTAINER REGISTRY! image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f'{os.getenv('APP_NAME')}DataParserImage', + f"{os.getenv('APP_NAME')}DataParserImage", directory='src/data_parser/', ) ), @@ -159,7 +159,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parser_service = ecs.FargateService( self, - f'{os.getenv('APP_NAME')}DataParserService', + f"{os.getenv('APP_NAME')}DataParserService", cluster=cluster, task_definition=parser_task_definition, assign_public_ip=True @@ -168,7 +168,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Scale metrics for data parser service scaling_target = appautoscaling.ScalableTarget( self, - id=f'{os.getenv('APP_NAME')}ParserScalingTarget', + id=f"{os.getenv('APP_NAME')}ParserScalingTarget", service_namespace=appautoscaling.ServiceNamespace.ECS, scalable_dimension='ecs:service:DesiredCount', min_capacity=0, @@ -225,13 +225,13 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Filter Lambda Functions engage_api_key_param = ssm.StringParameter.from_secure_string_parameter_attributes( self, - f'{os.getenv('APP_NAME')}APIKEY', - parameter_name=f'{os.getenv('SSM_PARAMETER_NAME_API')}' + f"{os.getenv('APP_NAME')}APIKEY", + parameter_name=f"{os.getenv('SSM_PARAMETER_NAME_API')}" ) club_information_lambda = lambda_python.PythonFunction( self, - f'{os.getenv('APP_NAME')}ClubInformationLambda', + f"{os.getenv('APP_NAME')}ClubInformationLambda", runtime=Runtime.PYTHON_3_13, entry='src/data_filters/onboarding', handler='lambda_handler', diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 9b52827..5e8cb1e 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -29,32 +29,32 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}ServerTable', + f"{os.getenv('APP_NAME')}ServerTable", partition_key=dynamodb.Attribute(name='serverId', type=dynamodb.AttributeType.STRING), ) onboardingTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}OrganizationTable', + f"{os.getenv('APP_NAME')}OrganizationTable", partition_key=dynamodb.Attribute(name='organizationId', type=dynamodb.AttributeType.NUMBER) ) cacheTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}APICacheTable', + f"{os.getenv('APP_NAME')}APICacheTable", partition_key=dynamodb.Attribute(name='clubId', type=dynamodb.AttributeType.STRING), ) eventTable = dynamodb.TableV2( self, - f'{os.getenv('APP_NAME')}EventTable', + f"{os.getenv('APP_NAME')}EventTable", partition_key=dynamodb.Attribute(name='eventId', type=dynamodb.AttributeType.STRING), ) # SQS Queue and Size Metric Defintion queue = sqs.Queue( self, - f'{os.getenv('APP_NAME')}DataQueue', + f"{os.getenv('APP_NAME')}DataQueue", queue_name=f'{os.getenv('APP_NAME')}_Data_Queue', ) @@ -65,7 +65,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, - f'{os.getenv('APP_NAME')}DataParserScaleOutAlarm', + f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", alarm_name=f'{os.getenv('APP_NAME')}DataProcessScaleOutAlarm', threshold=1, evaluation_periods=1, @@ -75,7 +75,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, - f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', + f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", alarm_name=f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', threshold=0, evaluation_periods=1, @@ -92,24 +92,24 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cluster = ecs.Cluster( self, - f'{os.getenv('APP_NAME')}ServiceCluster', + f"{os.getenv('APP_NAME')}ServiceCluster", vpc=vpc ) # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f'{os.getenv('APP_NAME')}DiscordAppTaskDefinition', + f"{os.getenv('APP_NAME')}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) app_task_defintion.add_container( - f'{os.getenv('APP_NAME')}DiscordApp', + f"{os.getenv('APP_NAME')}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f'{os.getenv('APP_NAME')}DiscordAppDockerImage', + f"{os.getenv('APP_NAME')}DiscordAppDockerImage", directory='src/discordapp/' ) ), @@ -120,7 +120,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: app_service = ecs.FargateService( self, - f'{os.getenv('APP_NAME')}DiscordAppService', + f"{os.getenv('APP_NAME')}DiscordAppService", cluster=cluster, task_definition=app_task_defintion, assign_public_ip=True @@ -133,18 +133,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f'{os.getenv('APP_NAME')}DataParserTask', + f"{os.getenv('APP_NAME')}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) parser_task_definition.add_container( - f'{os.getenv('APP_NAME')}DataParser', + f"{os.getenv('APP_NAME')}DataParser", # TODO: REFACTOR TO USE GITHUB CONTAINER REGISTRY! image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f'{os.getenv('APP_NAME')}DataParserImage', + f"{os.getenv('APP_NAME')}DataParserImage", directory='src/data_parser/', ) ), @@ -159,7 +159,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parser_service = ecs.FargateService( self, - f'{os.getenv('APP_NAME')}DataParserService', + f"{os.getenv('APP_NAME')}DataParserService", cluster=cluster, task_definition=parser_task_definition, assign_public_ip=True @@ -168,7 +168,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Scale metrics for data parser service scaling_target = appautoscaling.ScalableTarget( self, - id=f'{os.getenv('APP_NAME')}ParserScalingTarget', + id=f"{os.getenv('APP_NAME')}ParserScalingTarget", service_namespace=appautoscaling.ServiceNamespace.ECS, scalable_dimension='ecs:service:DesiredCount', min_capacity=0, @@ -225,13 +225,13 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Filter Lambda Functions engage_api_key_param = ssm.StringParameter.from_secure_string_parameter_attributes( self, - f'{os.getenv('APP_NAME')}APIKEY', - parameter_name=f'{os.getenv('SSM_PARAMETER_NAME_API')}' + f"{os.getenv('APP_NAME')}APIKEY", + parameter_name=f"{os.getenv('SSM_PARAMETER_NAME_API')}" ) club_information_lambda = lambda_python.PythonFunction( self, - f'{os.getenv('APP_NAME')}ClubInformationLambda', + f"{os.getenv('APP_NAME')}ClubInformationLambda", runtime=Runtime.PYTHON_3_13, entry='src/data_filters/onboarding', handler='lambda_handler', From a407b92b2a2b4858adb80e780cc23389c5278e21 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 15:33:58 -0700 Subject: [PATCH 19/42] changed single quote to double quote on line 58 --- immersion/environments/dev/immersion_stack.py | 2 +- immersion/environments/prod/immersion_stack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 5e8cb1e..0518edd 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -55,7 +55,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: queue = sqs.Queue( self, f"{os.getenv('APP_NAME')}DataQueue", - queue_name=f'{os.getenv('APP_NAME')}_Data_Queue', + queue_name=f"{os.getenv('APP_NAME')}_Data_Queue", ) scale_metric = queue.metric_approximate_number_of_messages_visible( diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 5e8cb1e..0518edd 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -55,7 +55,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: queue = sqs.Queue( self, f"{os.getenv('APP_NAME')}DataQueue", - queue_name=f'{os.getenv('APP_NAME')}_Data_Queue', + queue_name=f"{os.getenv('APP_NAME')}_Data_Queue", ) scale_metric = queue.metric_approximate_number_of_messages_visible( From 5fd03d7e8905c3a340cb28c198b8787d2e2ed26f Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 15:36:18 -0700 Subject: [PATCH 20/42] changed single quote to double quote on lines 69 and 79 --- immersion/environments/dev/immersion_stack.py | 4 ++-- immersion/environments/prod/immersion_stack.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 0518edd..e9422fd 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -66,7 +66,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", - alarm_name=f'{os.getenv('APP_NAME')}DataProcessScaleOutAlarm', + alarm_name=f"{os.getenv('APP_NAME')}DataProcessScaleOutAlarm", threshold=1, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, @@ -76,7 +76,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", - alarm_name=f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', + alarm_name=f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", threshold=0, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 0518edd..e9422fd 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -66,7 +66,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", - alarm_name=f'{os.getenv('APP_NAME')}DataProcessScaleOutAlarm', + alarm_name=f"{os.getenv('APP_NAME')}DataProcessScaleOutAlarm", threshold=1, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, @@ -76,7 +76,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", - alarm_name=f'{os.getenv('APP_NAME')}DataParserScaleInAlarm', + alarm_name=f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", threshold=0, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, From 61d7d69ea2e5d2278b0cbd6ddec256b4cc670979 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 17:59:38 -0700 Subject: [PATCH 21/42] replace os.getenv functions with the environment var name --- immersion/environments/dev/immersion_stack.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index e9422fd..b8165e1 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -26,6 +26,20 @@ class ImmersionStack(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) + if os.getenv('CI') == "true": + APP_NAME = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + "CI_APP_NAME", + parameter_name="/immersion/app_name" + ) + + DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + "DISCORD_TOKEN", + parameter_name="/immersion/discord_token" + ) + os.exit() + # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, From 16d0e69fcc5e431c1a5f319785d5d4695897731e Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Fri, 27 Jun 2025 21:54:45 -0700 Subject: [PATCH 22/42] Need to replace all the os.getenv with the variable names at the top. --- immersion/environments/dev/immersion_stack.py | 12 +++++++++- .../environments/prod/immersion_stack.py | 24 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index b8165e1..f0a2157 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -38,7 +38,17 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: "DISCORD_TOKEN", parameter_name="/immersion/discord_token" ) - os.exit() + else: + APP_NAME = os.getenv('APP_NAME') + DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') + + SSM_PARAMETER_NAME_API = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + "SSM_PARAMETER_NAME_API", + parameter_name="engage_api_key_test" + ) + + os.exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index e9422fd..f0a2157 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -26,6 +26,30 @@ class ImmersionStack(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) + if os.getenv('CI') == "true": + APP_NAME = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + "CI_APP_NAME", + parameter_name="/immersion/app_name" + ) + + DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + "DISCORD_TOKEN", + parameter_name="/immersion/discord_token" + ) + else: + APP_NAME = os.getenv('APP_NAME') + DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') + + SSM_PARAMETER_NAME_API = ssm.StringParameter.from_secure_string_parameter_attributes( + self, + "SSM_PARAMETER_NAME_API", + parameter_name="engage_api_key_test" + ) + + os.exit() # for now + # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, From e6f02a41b8834c1806177250a9152930da073427 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Sun, 29 Jun 2025 23:19:44 -0700 Subject: [PATCH 23/42] Replaced all the os.getenv functions with just the variable names defined at the beginning of the CF stack function. --- immersion/environments/dev/immersion_stack.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index f0a2157..fc46dc8 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -53,33 +53,33 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}ServerTable", + f"{APP_NAME}ServerTable", partition_key=dynamodb.Attribute(name='serverId', type=dynamodb.AttributeType.STRING), ) onboardingTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}OrganizationTable", + f"{APP_NAME}OrganizationTable", partition_key=dynamodb.Attribute(name='organizationId', type=dynamodb.AttributeType.NUMBER) ) cacheTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}APICacheTable", + f"{APP_NAME}APICacheTable", partition_key=dynamodb.Attribute(name='clubId', type=dynamodb.AttributeType.STRING), ) eventTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}EventTable", + f"{APP_NAME}EventTable", partition_key=dynamodb.Attribute(name='eventId', type=dynamodb.AttributeType.STRING), ) # SQS Queue and Size Metric Defintion queue = sqs.Queue( self, - f"{os.getenv('APP_NAME')}DataQueue", - queue_name=f"{os.getenv('APP_NAME')}_Data_Queue", + f"{APP_NAME}DataQueue", + queue_name=f"{APP_NAME}_Data_Queue", ) scale_metric = queue.metric_approximate_number_of_messages_visible( @@ -89,8 +89,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, - f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", - alarm_name=f"{os.getenv('APP_NAME')}DataProcessScaleOutAlarm", + f"{APP_NAME}DataParserScaleOutAlarm", + alarm_name=f"{APP_NAME}DataProcessScaleOutAlarm", threshold=1, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, @@ -99,8 +99,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, - f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", - alarm_name=f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", + f"{APP_NAME}DataParserScaleInAlarm", + alarm_name=f"{APP_NAME}DataParserScaleInAlarm", threshold=0, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, @@ -116,35 +116,35 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cluster = ecs.Cluster( self, - f"{os.getenv('APP_NAME')}ServiceCluster", + f"{APP_NAME}ServiceCluster", vpc=vpc ) # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f"{os.getenv('APP_NAME')}DiscordAppTaskDefinition", + f"{APP_NAME}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) app_task_defintion.add_container( - f"{os.getenv('APP_NAME')}DiscordApp", + f"{APP_NAME}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f"{os.getenv('APP_NAME')}DiscordAppDockerImage", + f"{APP_NAME}DiscordAppDockerImage", directory='src/discordapp/' ) ), environment={ - 'DISCORD_TOKEN': os.getenv('DISCORD_TOKEN') + 'DISCORD_TOKEN': DISCORD_TOKEN } ) app_service = ecs.FargateService( self, - f"{os.getenv('APP_NAME')}DiscordAppService", + f"{APP_NAME}DiscordAppService", cluster=cluster, task_definition=app_task_defintion, assign_public_ip=True @@ -157,18 +157,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f"{os.getenv('APP_NAME')}DataParserTask", + f"{APP_NAME}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) parser_task_definition.add_container( - f"{os.getenv('APP_NAME')}DataParser", + f"{APP_NAME}DataParser", # TODO: REFACTOR TO USE GITHUB CONTAINER REGISTRY! image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f"{os.getenv('APP_NAME')}DataParserImage", + f"{APP_NAME}DataParserImage", directory='src/data_parser/', ) ), @@ -183,7 +183,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parser_service = ecs.FargateService( self, - f"{os.getenv('APP_NAME')}DataParserService", + f"{APP_NAME}DataParserService", cluster=cluster, task_definition=parser_task_definition, assign_public_ip=True @@ -192,7 +192,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Scale metrics for data parser service scaling_target = appautoscaling.ScalableTarget( self, - id=f"{os.getenv('APP_NAME')}ParserScalingTarget", + id=f"{APP_NAME}ParserScalingTarget", service_namespace=appautoscaling.ServiceNamespace.ECS, scalable_dimension='ecs:service:DesiredCount', min_capacity=0, @@ -242,20 +242,20 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # TODO: figure out how to make a layer for python dependencies # filter_layer = _lambda.LayerVersion( # self, - # f'{os.getenv('APP_NAME')}FilterLayer', + # f'{APP_NAME}FilterLayer', # ) # Data Filter Lambda Functions engage_api_key_param = ssm.StringParameter.from_secure_string_parameter_attributes( self, - f"{os.getenv('APP_NAME')}APIKEY", + f"{APP_NAME}APIKEY", parameter_name=f"{os.getenv('SSM_PARAMETER_NAME_API')}" ) club_information_lambda = lambda_python.PythonFunction( self, - f"{os.getenv('APP_NAME')}ClubInformationLambda", + f"{APP_NAME}ClubInformationLambda", runtime=Runtime.PYTHON_3_13, entry='src/data_filters/onboarding', handler='lambda_handler', From 1fcdb2026e788b75b742e89746a1e2bc52ae0435 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Sun, 29 Jun 2025 23:22:23 -0700 Subject: [PATCH 24/42] os.exit() -> os._exit() --- immersion/environments/dev/immersion_stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index fc46dc8..18dc533 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -48,7 +48,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parameter_name="engage_api_key_test" ) - os.exit() # for now + os._exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( From cd027c6c6aaf6e575e471cb2212c1bb7b53e3bbe Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Sun, 29 Jun 2025 23:27:49 -0700 Subject: [PATCH 25/42] Copied contents of dev CF Stack to prod CF Stack, need to change the dev accordingly. --- .../environments/prod/immersion_stack.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index f0a2157..18dc533 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -48,38 +48,38 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parameter_name="engage_api_key_test" ) - os.exit() # for now + os._exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}ServerTable", + f"{APP_NAME}ServerTable", partition_key=dynamodb.Attribute(name='serverId', type=dynamodb.AttributeType.STRING), ) onboardingTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}OrganizationTable", + f"{APP_NAME}OrganizationTable", partition_key=dynamodb.Attribute(name='organizationId', type=dynamodb.AttributeType.NUMBER) ) cacheTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}APICacheTable", + f"{APP_NAME}APICacheTable", partition_key=dynamodb.Attribute(name='clubId', type=dynamodb.AttributeType.STRING), ) eventTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}EventTable", + f"{APP_NAME}EventTable", partition_key=dynamodb.Attribute(name='eventId', type=dynamodb.AttributeType.STRING), ) # SQS Queue and Size Metric Defintion queue = sqs.Queue( self, - f"{os.getenv('APP_NAME')}DataQueue", - queue_name=f"{os.getenv('APP_NAME')}_Data_Queue", + f"{APP_NAME}DataQueue", + queue_name=f"{APP_NAME}_Data_Queue", ) scale_metric = queue.metric_approximate_number_of_messages_visible( @@ -89,8 +89,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, - f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", - alarm_name=f"{os.getenv('APP_NAME')}DataProcessScaleOutAlarm", + f"{APP_NAME}DataParserScaleOutAlarm", + alarm_name=f"{APP_NAME}DataProcessScaleOutAlarm", threshold=1, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, @@ -99,8 +99,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, - f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", - alarm_name=f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", + f"{APP_NAME}DataParserScaleInAlarm", + alarm_name=f"{APP_NAME}DataParserScaleInAlarm", threshold=0, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, @@ -116,35 +116,35 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cluster = ecs.Cluster( self, - f"{os.getenv('APP_NAME')}ServiceCluster", + f"{APP_NAME}ServiceCluster", vpc=vpc ) # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f"{os.getenv('APP_NAME')}DiscordAppTaskDefinition", + f"{APP_NAME}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) app_task_defintion.add_container( - f"{os.getenv('APP_NAME')}DiscordApp", + f"{APP_NAME}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f"{os.getenv('APP_NAME')}DiscordAppDockerImage", + f"{APP_NAME}DiscordAppDockerImage", directory='src/discordapp/' ) ), environment={ - 'DISCORD_TOKEN': os.getenv('DISCORD_TOKEN') + 'DISCORD_TOKEN': DISCORD_TOKEN } ) app_service = ecs.FargateService( self, - f"{os.getenv('APP_NAME')}DiscordAppService", + f"{APP_NAME}DiscordAppService", cluster=cluster, task_definition=app_task_defintion, assign_public_ip=True @@ -157,18 +157,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f"{os.getenv('APP_NAME')}DataParserTask", + f"{APP_NAME}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) parser_task_definition.add_container( - f"{os.getenv('APP_NAME')}DataParser", + f"{APP_NAME}DataParser", # TODO: REFACTOR TO USE GITHUB CONTAINER REGISTRY! image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f"{os.getenv('APP_NAME')}DataParserImage", + f"{APP_NAME}DataParserImage", directory='src/data_parser/', ) ), @@ -183,7 +183,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parser_service = ecs.FargateService( self, - f"{os.getenv('APP_NAME')}DataParserService", + f"{APP_NAME}DataParserService", cluster=cluster, task_definition=parser_task_definition, assign_public_ip=True @@ -192,7 +192,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Scale metrics for data parser service scaling_target = appautoscaling.ScalableTarget( self, - id=f"{os.getenv('APP_NAME')}ParserScalingTarget", + id=f"{APP_NAME}ParserScalingTarget", service_namespace=appautoscaling.ServiceNamespace.ECS, scalable_dimension='ecs:service:DesiredCount', min_capacity=0, @@ -242,20 +242,20 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # TODO: figure out how to make a layer for python dependencies # filter_layer = _lambda.LayerVersion( # self, - # f'{os.getenv('APP_NAME')}FilterLayer', + # f'{APP_NAME}FilterLayer', # ) # Data Filter Lambda Functions engage_api_key_param = ssm.StringParameter.from_secure_string_parameter_attributes( self, - f"{os.getenv('APP_NAME')}APIKEY", + f"{APP_NAME}APIKEY", parameter_name=f"{os.getenv('SSM_PARAMETER_NAME_API')}" ) club_information_lambda = lambda_python.PythonFunction( self, - f"{os.getenv('APP_NAME')}ClubInformationLambda", + f"{APP_NAME}ClubInformationLambda", runtime=Runtime.PYTHON_3_13, entry='src/data_filters/onboarding', handler='lambda_handler', From 08e217115a598c5bb31ac269b09b8bce7d602c95 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Sun, 29 Jun 2025 23:39:17 -0700 Subject: [PATCH 26/42] File cleanup & will make a PR if CI/CD works for the dev CF stack --- immersion/environments/dev/app.py | 2 +- immersion/environments/dev/immersion_stack.py | 6 +++--- immersion/environments/prod/app.py | 2 +- immersion/environments/prod/immersion_stack.py | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/immersion/environments/dev/app.py b/immersion/environments/dev/app.py index 84069bd..0a20763 100644 --- a/immersion/environments/dev/app.py +++ b/immersion/environments/dev/app.py @@ -4,4 +4,4 @@ app = cdk.App() ImmersionStack(app, "ImmersionStackDev") -app.synth() \ No newline at end of file +app.synth() diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 18dc533..ed6c9b6 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -48,8 +48,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parameter_name="engage_api_key_test" ) - os._exit() # for now - + # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, @@ -264,4 +263,5 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: }, ) queue.grant_send_messages(club_information_lambda) - engage_api_key_param.grant_read(club_information_lambda) \ No newline at end of file + engage_api_key_param.grant_read(club_information_lambda) + \ No newline at end of file diff --git a/immersion/environments/prod/app.py b/immersion/environments/prod/app.py index 75cb550..ea30267 100644 --- a/immersion/environments/prod/app.py +++ b/immersion/environments/prod/app.py @@ -4,4 +4,4 @@ app = cdk.App() ImmersionStack(app, "ImmersionStackProd") -app.synth() \ No newline at end of file +app.synth() diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 18dc533..a588da4 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -264,4 +264,5 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: }, ) queue.grant_send_messages(club_information_lambda) - engage_api_key_param.grant_read(club_information_lambda) \ No newline at end of file + engage_api_key_param.grant_read(club_information_lambda) + \ No newline at end of file From 0df217d3bfe1ee4682ded7a584b67caeef2b3e5b Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 09:17:48 -0700 Subject: [PATCH 27/42] Hopefully this provides the right context for AWS CDK deploy commands --- immersion/environments/dev/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/immersion/environments/dev/app.py b/immersion/environments/dev/app.py index 0a20763..503db4d 100644 --- a/immersion/environments/dev/app.py +++ b/immersion/environments/dev/app.py @@ -2,6 +2,11 @@ import aws_cdk as cdk app = cdk.App() -ImmersionStack(app, "ImmersionStackDev") +ImmersionStack(app, "ImmersionStackDev", { + "env": { + "account": "default", + "region": "us-east-1" + } +}) app.synth() From e9114f87e3a082aa61f18d349f9cbf69e081f797 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 09:49:56 -0700 Subject: [PATCH 28/42] Added aws account id as a secret to gh actions --- .github/workflows/dev-branch.yaml | 2 ++ .github/workflows/master-branch.yaml | 4 +++- immersion/environments/dev/app.py | 9 +++++---- immersion/environments/prod/app.py | 8 +++++++- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 6c10649..d7ba8f3 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -7,6 +7,8 @@ on: jobs: dev-deploy-to-aws: runs-on: ubuntu-latest + env: + CICD_ACCOUNT_ID: ${{ secrets.CICD_ACCOUNT_ID }} permissions: id-token: write contents: read diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 8da5653..4938f7e 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -7,6 +7,8 @@ on: jobs: dev-deploy-to-aws: runs-on: ubuntu-latest + env: + CICD_ACCOUNT_ID: ${{ secrets.CICD_ACCOUNT_ID }} permissions: id-token: write contents: read @@ -34,5 +36,5 @@ jobs: - name: Build the CDK stack using the aws profile run: | - cd immersion/environments/prod + cd immersion/environments/dev cdk deploy --app "python3 app.py" --require-approval never diff --git a/immersion/environments/dev/app.py b/immersion/environments/dev/app.py index 503db4d..348598b 100644 --- a/immersion/environments/dev/app.py +++ b/immersion/environments/dev/app.py @@ -1,12 +1,13 @@ from immersion_stack import ImmersionStack import aws_cdk as cdk +import os app = cdk.App() -ImmersionStack(app, "ImmersionStackDev", { - "env": { - "account": "default", +ImmersionStack(app, "ImmersionStackDev", + env = { + "account": os.getenv('CICD_ACCOUNT_ID'), # Assuming it's only being deployed with GH Actions "region": "us-east-1" } -}) +) app.synth() diff --git a/immersion/environments/prod/app.py b/immersion/environments/prod/app.py index ea30267..348598b 100644 --- a/immersion/environments/prod/app.py +++ b/immersion/environments/prod/app.py @@ -1,7 +1,13 @@ from immersion_stack import ImmersionStack import aws_cdk as cdk +import os app = cdk.App() -ImmersionStack(app, "ImmersionStackProd") +ImmersionStack(app, "ImmersionStackDev", + env = { + "account": os.getenv('CICD_ACCOUNT_ID'), # Assuming it's only being deployed with GH Actions + "region": "us-east-1" + } +) app.synth() From 1d0a77bda0356befd938190f42c49591dd4e2eaa Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 10:00:35 -0700 Subject: [PATCH 29/42] Fixed failure to find Docker image (wrong directory specified) --- immersion/environments/dev/immersion_stack.py | 8 ++++---- immersion/environments/prod/immersion_stack.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index ed6c9b6..3419dfa 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -48,7 +48,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parameter_name="engage_api_key_test" ) - + os._exit() # for now + # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, @@ -133,7 +134,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DockerImageAsset( self, f"{APP_NAME}DiscordAppDockerImage", - directory='src/discordapp/' + directory='../../../src/discordapp/' ) ), environment={ @@ -168,7 +169,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DockerImageAsset( self, f"{APP_NAME}DataParserImage", - directory='src/data_parser/', + directory='../../../src/data_parser/', ) ), environment={ @@ -264,4 +265,3 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ) queue.grant_send_messages(club_information_lambda) engage_api_key_param.grant_read(club_information_lambda) - \ No newline at end of file diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index a588da4..3419dfa 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -134,7 +134,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DockerImageAsset( self, f"{APP_NAME}DiscordAppDockerImage", - directory='src/discordapp/' + directory='../../../src/discordapp/' ) ), environment={ @@ -169,7 +169,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DockerImageAsset( self, f"{APP_NAME}DataParserImage", - directory='src/data_parser/', + directory='../../../src/data_parser/', ) ), environment={ @@ -265,4 +265,3 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ) queue.grant_send_messages(club_information_lambda) engage_api_key_param.grant_read(club_information_lambda) - \ No newline at end of file From 1b7542a5ba1c704a2b8aa2120cf264084cfdb9fb Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 10:03:29 -0700 Subject: [PATCH 30/42] Removed the os._exit function --- immersion/environments/dev/immersion_stack.py | 1 - immersion/environments/prod/immersion_stack.py | 1 - 2 files changed, 2 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 3419dfa..41dcea2 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -48,7 +48,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parameter_name="engage_api_key_test" ) - os._exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 3419dfa..41dcea2 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -48,7 +48,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parameter_name="engage_api_key_test" ) - os._exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( From 40e8eae86692a358037cdfe26e77ae54432cb002 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 12:05:23 -0700 Subject: [PATCH 31/42] Perhaps this fixes the unable to deserialize error? --- immersion/environments/dev/immersion_stack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 41dcea2..d31a678 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -127,7 +127,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cpu=512, # 0.5 vCPU ) - app_task_defintion.add_container( + app_task_defintion.add_container(aws_cdk.aws_ecs.ContainerDefinitionOptions( f"{APP_NAME}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( @@ -139,7 +139,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: environment={ 'DISCORD_TOKEN': DISCORD_TOKEN } - ) + )) app_service = ecs.FargateService( self, From 8971f227f75a85f9bbd38a7cb671da3f2f037865 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 19:38:22 -0700 Subject: [PATCH 32/42] Extracts the string value from the secure string from SSM parameter store --- immersion/environments/dev/immersion_stack.py | 6 +++--- immersion/environments/prod/immersion_stack.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index d31a678..520cb89 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -31,13 +31,13 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: self, "CI_APP_NAME", parameter_name="/immersion/app_name" - ) + ).string_value DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( self, "DISCORD_TOKEN", parameter_name="/immersion/discord_token" - ) + ).string_value else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') @@ -46,7 +46,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: self, "SSM_PARAMETER_NAME_API", parameter_name="engage_api_key_test" - ) + ).string_value # DynamoDB Table Definitions diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 41dcea2..e1f99ad 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -31,13 +31,13 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: self, "CI_APP_NAME", parameter_name="/immersion/app_name" - ) + ).string_value DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( self, "DISCORD_TOKEN", parameter_name="/immersion/discord_token" - ) + ).string_value else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') @@ -46,7 +46,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: self, "SSM_PARAMETER_NAME_API", parameter_name="engage_api_key_test" - ) + ).string_value # DynamoDB Table Definitions From e8d18c4e47e934f4998a4f3eace908008a77031f Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 19:53:45 -0700 Subject: [PATCH 33/42] ecs task definitions have a constant string for the name, hoping that fixes the unresolved tokens error --- immersion/environments/dev/immersion_stack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 520cb89..96af32a 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -122,7 +122,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f"{APP_NAME}DiscordAppTaskDefinition", + "ImmersionDiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) @@ -156,7 +156,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f"{APP_NAME}DataParserTask", + "ImmersionDataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) From 26a2b6c703b23def11516aeb2892f5405edfd7e4 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 19:55:37 -0700 Subject: [PATCH 34/42] aws_cdk not found, so removed that --- immersion/environments/dev/immersion_stack.py | 4 ++-- immersion/environments/prod/immersion_stack.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 96af32a..3713c43 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -127,7 +127,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cpu=512, # 0.5 vCPU ) - app_task_defintion.add_container(aws_cdk.aws_ecs.ContainerDefinitionOptions( + app_task_defintion.add_container( f"{APP_NAME}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( @@ -139,7 +139,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: environment={ 'DISCORD_TOKEN': DISCORD_TOKEN } - )) + ) app_service = ecs.FargateService( self, diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index e1f99ad..3713c43 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -122,7 +122,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f"{APP_NAME}DiscordAppTaskDefinition", + "ImmersionDiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) @@ -156,7 +156,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f"{APP_NAME}DataParserTask", + "ImmersionDataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) From 5d6f1e5488a57ea9780ad0c5a5a6036905aceb0e Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 19:58:06 -0700 Subject: [PATCH 35/42] Runtime python3.13 not found, changed to python_3_10 as suggested by the error log --- immersion/environments/dev/immersion_stack.py | 2 +- immersion/environments/prod/immersion_stack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 3713c43..b253f35 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -255,7 +255,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: club_information_lambda = lambda_python.PythonFunction( self, f"{APP_NAME}ClubInformationLambda", - runtime=Runtime.PYTHON_3_13, + runtime=Runtime.PYTHON_3_10, entry='src/data_filters/onboarding', handler='lambda_handler', environment={ diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 3713c43..b253f35 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -255,7 +255,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: club_information_lambda = lambda_python.PythonFunction( self, f"{APP_NAME}ClubInformationLambda", - runtime=Runtime.PYTHON_3_13, + runtime=Runtime.PYTHON_3_10, entry='src/data_filters/onboarding', handler='lambda_handler', environment={ From dad4200e65308c0b3fa022178728dddf66e32d2f Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Mon, 30 Jun 2025 20:01:52 -0700 Subject: [PATCH 36/42] Fixed directory issues --- immersion/environments/dev/immersion_stack.py | 2 +- immersion/environments/prod/immersion_stack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index b253f35..3c5073a 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -256,7 +256,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: self, f"{APP_NAME}ClubInformationLambda", runtime=Runtime.PYTHON_3_10, - entry='src/data_filters/onboarding', + entry='../../../src/data_filters/onboarding', handler='lambda_handler', environment={ 'QUEUE_URL': queue.queue_url diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index b253f35..3c5073a 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -256,7 +256,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: self, f"{APP_NAME}ClubInformationLambda", runtime=Runtime.PYTHON_3_10, - entry='src/data_filters/onboarding', + entry='../../../src/data_filters/onboarding', handler='lambda_handler', environment={ 'QUEUE_URL': queue.queue_url From c02baac290120c09f40227da54126c5f586d7b57 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Tue, 1 Jul 2025 10:32:34 -0700 Subject: [PATCH 37/42] Wrong functions were used to retrieve SSM parameter values --- .github/workflows/dev-branch.yaml | 7 +++++++ .github/workflows/master-branch.yaml | 9 ++++++++- immersion/environments/dev/immersion_stack.py | 8 ++++---- immersion/environments/prod/immersion_stack.py | 8 ++++---- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index d7ba8f3..7b7d46c 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -9,27 +9,34 @@ jobs: runs-on: ubuntu-latest env: CICD_ACCOUNT_ID: ${{ secrets.CICD_ACCOUNT_ID }} + permissions: id-token: write contents: read + steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Set up python with: python-version: 3.11 + - name: Install python dependencies run: pip install -r requirements.txt + - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} role-session-name: github-actions-cfn-deploy aws-region: ${{ vars.AWS_REGION }} + - uses: actions/setup-node@v4 name: Install nodejs with: node-version: 20 + - name: Install CDK CLI with nodejs run: | npm install -g aws-cdk diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 4938f7e..d62b8aa 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -5,31 +5,38 @@ on: - main - master jobs: - dev-deploy-to-aws: + prod-deploy-to-aws: runs-on: ubuntu-latest env: CICD_ACCOUNT_ID: ${{ secrets.CICD_ACCOUNT_ID }} + permissions: id-token: write contents: read + steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Set up python with: python-version: 3.11 + - name: Install python dependencies run: pip install -r requirements.txt + - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} role-session-name: github-actions-cfn-deploy aws-region: ${{ vars.AWS_REGION }} + - uses: actions/setup-node@v4 name: Install nodejs with: node-version: 20 + - name: Install CDK CLI with nodejs run: | npm install -g aws-cdk diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 3c5073a..b0bc4ee 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -27,7 +27,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) if os.getenv('CI') == "true": - APP_NAME = ssm.StringParameter.from_secure_string_parameter_attributes( + APP_NAME = ssm.StringParameter.from_string_parameter_attributes( self, "CI_APP_NAME", parameter_name="/immersion/app_name" @@ -36,7 +36,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( self, "DISCORD_TOKEN", - parameter_name="/immersion/discord_token" + parameter_name="/immersion/discord-token-secure" ).string_value else: APP_NAME = os.getenv('APP_NAME') @@ -122,7 +122,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - "ImmersionDiscordAppTaskDefinition", + f"{APP_NAME}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) @@ -156,7 +156,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - "ImmersionDataParserTask", + f"{APP_NAME}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 3c5073a..b0bc4ee 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -27,7 +27,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) if os.getenv('CI') == "true": - APP_NAME = ssm.StringParameter.from_secure_string_parameter_attributes( + APP_NAME = ssm.StringParameter.from_string_parameter_attributes( self, "CI_APP_NAME", parameter_name="/immersion/app_name" @@ -36,7 +36,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( self, "DISCORD_TOKEN", - parameter_name="/immersion/discord_token" + parameter_name="/immersion/discord-token-secure" ).string_value else: APP_NAME = os.getenv('APP_NAME') @@ -122,7 +122,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - "ImmersionDiscordAppTaskDefinition", + f"{APP_NAME}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) @@ -156,7 +156,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - "ImmersionDataParserTask", + f"{APP_NAME}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) From 5bf4f37ca058ecf273629ac0e7f29b4bc08be672 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Tue, 1 Jul 2025 17:01:27 -0700 Subject: [PATCH 38/42] Changed to value_from_lookup --- immersion/environments/dev/immersion_stack.py | 9 +++------ immersion/environments/prod/immersion_stack.py | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index b0bc4ee..53f6b29 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -27,24 +27,21 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) if os.getenv('CI') == "true": - APP_NAME = ssm.StringParameter.from_string_parameter_attributes( + APP_NAME = ssm.StringParameter.value_from_lookup( self, - "CI_APP_NAME", parameter_name="/immersion/app_name" ).string_value - DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( + DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, - "DISCORD_TOKEN", parameter_name="/immersion/discord-token-secure" ).string_value else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') - SSM_PARAMETER_NAME_API = ssm.StringParameter.from_secure_string_parameter_attributes( + SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( self, - "SSM_PARAMETER_NAME_API", parameter_name="engage_api_key_test" ).string_value diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index b0bc4ee..53f6b29 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -27,24 +27,21 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) if os.getenv('CI') == "true": - APP_NAME = ssm.StringParameter.from_string_parameter_attributes( + APP_NAME = ssm.StringParameter.value_from_lookup( self, - "CI_APP_NAME", parameter_name="/immersion/app_name" ).string_value - DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( + DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, - "DISCORD_TOKEN", parameter_name="/immersion/discord-token-secure" ).string_value else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') - SSM_PARAMETER_NAME_API = ssm.StringParameter.from_secure_string_parameter_attributes( + SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( self, - "SSM_PARAMETER_NAME_API", parameter_name="engage_api_key_test" ).string_value From 385483eb73fad24715e8be3ca8e0d895a0a70b57 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Tue, 1 Jul 2025 17:02:59 -0700 Subject: [PATCH 39/42] Removed .string_value attribute --- immersion/environments/dev/immersion_stack.py | 6 +++--- immersion/environments/prod/immersion_stack.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 53f6b29..39dfb40 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -30,12 +30,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: APP_NAME = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/app_name" - ).string_value + ) DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/discord-token-secure" - ).string_value + ) else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') @@ -43,7 +43,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( self, parameter_name="engage_api_key_test" - ).string_value + ) # DynamoDB Table Definitions diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 53f6b29..39dfb40 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -30,12 +30,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: APP_NAME = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/app_name" - ).string_value + ) DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/discord-token-secure" - ).string_value + ) else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') @@ -43,7 +43,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( self, parameter_name="engage_api_key_test" - ).string_value + ) # DynamoDB Table Definitions From b7d6a1c60b3d41f8ba74bb35e5c5c121e12af5b3 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Sun, 29 Jun 2025 23:22:23 -0700 Subject: [PATCH 40/42] Created a CI/CD pipeline for deploying to AWS CDK. --- .github/workflows/dev-branch.yaml | 9 +++ .github/workflows/master-branch.yaml | 13 +++- immersion/environments/dev/app.py | 10 ++- immersion/environments/dev/immersion_stack.py | 22 +++--- immersion/environments/prod/app.py | 10 ++- .../environments/prod/immersion_stack.py | 70 +++++++++---------- 6 files changed, 78 insertions(+), 56 deletions(-) diff --git a/.github/workflows/dev-branch.yaml b/.github/workflows/dev-branch.yaml index 6c10649..7b7d46c 100644 --- a/.github/workflows/dev-branch.yaml +++ b/.github/workflows/dev-branch.yaml @@ -7,27 +7,36 @@ on: jobs: dev-deploy-to-aws: runs-on: ubuntu-latest + env: + CICD_ACCOUNT_ID: ${{ secrets.CICD_ACCOUNT_ID }} + permissions: id-token: write contents: read + steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Set up python with: python-version: 3.11 + - name: Install python dependencies run: pip install -r requirements.txt + - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} role-session-name: github-actions-cfn-deploy aws-region: ${{ vars.AWS_REGION }} + - uses: actions/setup-node@v4 name: Install nodejs with: node-version: 20 + - name: Install CDK CLI with nodejs run: | npm install -g aws-cdk diff --git a/.github/workflows/master-branch.yaml b/.github/workflows/master-branch.yaml index 8da5653..d62b8aa 100644 --- a/.github/workflows/master-branch.yaml +++ b/.github/workflows/master-branch.yaml @@ -5,34 +5,43 @@ on: - main - master jobs: - dev-deploy-to-aws: + prod-deploy-to-aws: runs-on: ubuntu-latest + env: + CICD_ACCOUNT_ID: ${{ secrets.CICD_ACCOUNT_ID }} + permissions: id-token: write contents: read + steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Set up python with: python-version: 3.11 + - name: Install python dependencies run: pip install -r requirements.txt + - name: Configure AWS profile uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.STS_ASSUME_ROLE_ARN }} role-session-name: github-actions-cfn-deploy aws-region: ${{ vars.AWS_REGION }} + - uses: actions/setup-node@v4 name: Install nodejs with: node-version: 20 + - name: Install CDK CLI with nodejs run: | npm install -g aws-cdk - name: Build the CDK stack using the aws profile run: | - cd immersion/environments/prod + cd immersion/environments/dev cdk deploy --app "python3 app.py" --require-approval never diff --git a/immersion/environments/dev/app.py b/immersion/environments/dev/app.py index 84069bd..348598b 100644 --- a/immersion/environments/dev/app.py +++ b/immersion/environments/dev/app.py @@ -1,7 +1,13 @@ from immersion_stack import ImmersionStack import aws_cdk as cdk +import os app = cdk.App() -ImmersionStack(app, "ImmersionStackDev") +ImmersionStack(app, "ImmersionStackDev", + env = { + "account": os.getenv('CICD_ACCOUNT_ID'), # Assuming it's only being deployed with GH Actions + "region": "us-east-1" + } +) -app.synth() \ No newline at end of file +app.synth() diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index fc46dc8..39dfb40 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -27,28 +27,24 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) if os.getenv('CI') == "true": - APP_NAME = ssm.StringParameter.from_secure_string_parameter_attributes( + APP_NAME = ssm.StringParameter.value_from_lookup( self, - "CI_APP_NAME", parameter_name="/immersion/app_name" ) - DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( + DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, - "DISCORD_TOKEN", - parameter_name="/immersion/discord_token" + parameter_name="/immersion/discord-token-secure" ) else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') - SSM_PARAMETER_NAME_API = ssm.StringParameter.from_secure_string_parameter_attributes( + SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( self, - "SSM_PARAMETER_NAME_API", parameter_name="engage_api_key_test" ) - os.exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( @@ -134,7 +130,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DockerImageAsset( self, f"{APP_NAME}DiscordAppDockerImage", - directory='src/discordapp/' + directory='../../../src/discordapp/' ) ), environment={ @@ -169,7 +165,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: DockerImageAsset( self, f"{APP_NAME}DataParserImage", - directory='src/data_parser/', + directory='../../../src/data_parser/', ) ), environment={ @@ -256,12 +252,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: club_information_lambda = lambda_python.PythonFunction( self, f"{APP_NAME}ClubInformationLambda", - runtime=Runtime.PYTHON_3_13, - entry='src/data_filters/onboarding', + runtime=Runtime.PYTHON_3_10, + entry='../../../src/data_filters/onboarding', handler='lambda_handler', environment={ 'QUEUE_URL': queue.queue_url }, ) queue.grant_send_messages(club_information_lambda) - engage_api_key_param.grant_read(club_information_lambda) \ No newline at end of file + engage_api_key_param.grant_read(club_information_lambda) diff --git a/immersion/environments/prod/app.py b/immersion/environments/prod/app.py index 75cb550..348598b 100644 --- a/immersion/environments/prod/app.py +++ b/immersion/environments/prod/app.py @@ -1,7 +1,13 @@ from immersion_stack import ImmersionStack import aws_cdk as cdk +import os app = cdk.App() -ImmersionStack(app, "ImmersionStackProd") +ImmersionStack(app, "ImmersionStackDev", + env = { + "account": os.getenv('CICD_ACCOUNT_ID'), # Assuming it's only being deployed with GH Actions + "region": "us-east-1" + } +) -app.synth() \ No newline at end of file +app.synth() diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index f0a2157..39dfb40 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -27,59 +27,55 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) if os.getenv('CI') == "true": - APP_NAME = ssm.StringParameter.from_secure_string_parameter_attributes( + APP_NAME = ssm.StringParameter.value_from_lookup( self, - "CI_APP_NAME", parameter_name="/immersion/app_name" ) - DISCORD_TOKEN = ssm.StringParameter.from_secure_string_parameter_attributes( + DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, - "DISCORD_TOKEN", - parameter_name="/immersion/discord_token" + parameter_name="/immersion/discord-token-secure" ) else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') - SSM_PARAMETER_NAME_API = ssm.StringParameter.from_secure_string_parameter_attributes( + SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( self, - "SSM_PARAMETER_NAME_API", parameter_name="engage_api_key_test" ) - os.exit() # for now # DynamoDB Table Definitions serverTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}ServerTable", + f"{APP_NAME}ServerTable", partition_key=dynamodb.Attribute(name='serverId', type=dynamodb.AttributeType.STRING), ) onboardingTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}OrganizationTable", + f"{APP_NAME}OrganizationTable", partition_key=dynamodb.Attribute(name='organizationId', type=dynamodb.AttributeType.NUMBER) ) cacheTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}APICacheTable", + f"{APP_NAME}APICacheTable", partition_key=dynamodb.Attribute(name='clubId', type=dynamodb.AttributeType.STRING), ) eventTable = dynamodb.TableV2( self, - f"{os.getenv('APP_NAME')}EventTable", + f"{APP_NAME}EventTable", partition_key=dynamodb.Attribute(name='eventId', type=dynamodb.AttributeType.STRING), ) # SQS Queue and Size Metric Defintion queue = sqs.Queue( self, - f"{os.getenv('APP_NAME')}DataQueue", - queue_name=f"{os.getenv('APP_NAME')}_Data_Queue", + f"{APP_NAME}DataQueue", + queue_name=f"{APP_NAME}_Data_Queue", ) scale_metric = queue.metric_approximate_number_of_messages_visible( @@ -89,8 +85,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_out_alarm = scale_metric.create_alarm( self, - f"{os.getenv('APP_NAME')}DataParserScaleOutAlarm", - alarm_name=f"{os.getenv('APP_NAME')}DataProcessScaleOutAlarm", + f"{APP_NAME}DataParserScaleOutAlarm", + alarm_name=f"{APP_NAME}DataProcessScaleOutAlarm", threshold=1, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, @@ -99,8 +95,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: scale_in_alarm = scale_metric.create_alarm( self, - f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", - alarm_name=f"{os.getenv('APP_NAME')}DataParserScaleInAlarm", + f"{APP_NAME}DataParserScaleInAlarm", + alarm_name=f"{APP_NAME}DataParserScaleInAlarm", threshold=0, evaluation_periods=1, comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, @@ -116,35 +112,35 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: cluster = ecs.Cluster( self, - f"{os.getenv('APP_NAME')}ServiceCluster", + f"{APP_NAME}ServiceCluster", vpc=vpc ) # Discord App Container Definition app_task_defintion = ecs.FargateTaskDefinition( self, - f"{os.getenv('APP_NAME')}DiscordAppTaskDefinition", + f"{APP_NAME}DiscordAppTaskDefinition", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) app_task_defintion.add_container( - f"{os.getenv('APP_NAME')}DiscordApp", + f"{APP_NAME}DiscordApp", image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f"{os.getenv('APP_NAME')}DiscordAppDockerImage", - directory='src/discordapp/' + f"{APP_NAME}DiscordAppDockerImage", + directory='../../../src/discordapp/' ) ), environment={ - 'DISCORD_TOKEN': os.getenv('DISCORD_TOKEN') + 'DISCORD_TOKEN': DISCORD_TOKEN } ) app_service = ecs.FargateService( self, - f"{os.getenv('APP_NAME')}DiscordAppService", + f"{APP_NAME}DiscordAppService", cluster=cluster, task_definition=app_task_defintion, assign_public_ip=True @@ -157,19 +153,19 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Data Parser Task Definition parser_task_definition = ecs.FargateTaskDefinition( self, - f"{os.getenv('APP_NAME')}DataParserTask", + f"{APP_NAME}DataParserTask", memory_limit_mib=1024, # 1 GB cpu=512, # 0.5 vCPU ) parser_task_definition.add_container( - f"{os.getenv('APP_NAME')}DataParser", + f"{APP_NAME}DataParser", # TODO: REFACTOR TO USE GITHUB CONTAINER REGISTRY! image=ecs.ContainerImage.from_docker_image_asset( DockerImageAsset( self, - f"{os.getenv('APP_NAME')}DataParserImage", - directory='src/data_parser/', + f"{APP_NAME}DataParserImage", + directory='../../../src/data_parser/', ) ), environment={ @@ -183,7 +179,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: parser_service = ecs.FargateService( self, - f"{os.getenv('APP_NAME')}DataParserService", + f"{APP_NAME}DataParserService", cluster=cluster, task_definition=parser_task_definition, assign_public_ip=True @@ -192,7 +188,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Scale metrics for data parser service scaling_target = appautoscaling.ScalableTarget( self, - id=f"{os.getenv('APP_NAME')}ParserScalingTarget", + id=f"{APP_NAME}ParserScalingTarget", service_namespace=appautoscaling.ServiceNamespace.ECS, scalable_dimension='ecs:service:DesiredCount', min_capacity=0, @@ -242,26 +238,26 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # TODO: figure out how to make a layer for python dependencies # filter_layer = _lambda.LayerVersion( # self, - # f'{os.getenv('APP_NAME')}FilterLayer', + # f'{APP_NAME}FilterLayer', # ) # Data Filter Lambda Functions engage_api_key_param = ssm.StringParameter.from_secure_string_parameter_attributes( self, - f"{os.getenv('APP_NAME')}APIKEY", + f"{APP_NAME}APIKEY", parameter_name=f"{os.getenv('SSM_PARAMETER_NAME_API')}" ) club_information_lambda = lambda_python.PythonFunction( self, - f"{os.getenv('APP_NAME')}ClubInformationLambda", - runtime=Runtime.PYTHON_3_13, - entry='src/data_filters/onboarding', + f"{APP_NAME}ClubInformationLambda", + runtime=Runtime.PYTHON_3_10, + entry='../../../src/data_filters/onboarding', handler='lambda_handler', environment={ 'QUEUE_URL': queue.queue_url }, ) queue.grant_send_messages(club_information_lambda) - engage_api_key_param.grant_read(club_information_lambda) \ No newline at end of file + engage_api_key_param.grant_read(club_information_lambda) From 0a90ad9b479f5be7764587ffc5c6f2701c876133 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 2 Jul 2025 17:57:04 -0700 Subject: [PATCH 41/42] Fixed indentation --- immersion/environments/dev/immersion_stack.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/immersion/environments/dev/immersion_stack.py b/immersion/environments/dev/immersion_stack.py index 39dfb40..61bc293 100644 --- a/immersion/environments/dev/immersion_stack.py +++ b/immersion/environments/dev/immersion_stack.py @@ -30,20 +30,20 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: APP_NAME = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/app_name" - ) + ) DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/discord-token-secure" - ) + ) else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( - self, - parameter_name="engage_api_key_test" - ) + self, + parameter_name="engage_api_key_test" + ) # DynamoDB Table Definitions @@ -259,5 +259,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: 'QUEUE_URL': queue.queue_url }, ) + queue.grant_send_messages(club_information_lambda) engage_api_key_param.grant_read(club_information_lambda) From 6e3994c359ab0549f448f3908af2922fbde22c22 Mon Sep 17 00:00:00 2001 From: HoustonBoston Date: Wed, 2 Jul 2025 17:58:47 -0700 Subject: [PATCH 42/42] Fixed indenting in the prod stack env --- immersion/environments/prod/immersion_stack.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/immersion/environments/prod/immersion_stack.py b/immersion/environments/prod/immersion_stack.py index 39dfb40..61bc293 100644 --- a/immersion/environments/prod/immersion_stack.py +++ b/immersion/environments/prod/immersion_stack.py @@ -30,20 +30,20 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: APP_NAME = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/app_name" - ) + ) DISCORD_TOKEN = ssm.StringParameter.value_from_lookup( self, parameter_name="/immersion/discord-token-secure" - ) + ) else: APP_NAME = os.getenv('APP_NAME') DISCORD_TOKEN = os.getenv('DISCORD_TOKEN') SSM_PARAMETER_NAME_API = ssm.StringParameter.value_from_lookup( - self, - parameter_name="engage_api_key_test" - ) + self, + parameter_name="engage_api_key_test" + ) # DynamoDB Table Definitions @@ -259,5 +259,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: 'QUEUE_URL': queue.queue_url }, ) + queue.grant_send_messages(club_information_lambda) engage_api_key_param.grant_read(club_information_lambda)