From 523fa59c335fe2715cb36ca394de0a32eb4670da Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 01:11:34 +0100 Subject: [PATCH 01/68] Improve deploy workflow --- .github/workflows/deploy.yml | 79 ++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c71bd61353..6aeb607e59 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,8 +12,12 @@ env: TF_WORKSPACE: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} jobs: - build-pretix: - runs-on: [self-hosted] + check-new-pretix-version: + name: Check pretix needs building + runs-on: ubuntu-24.04 + outputs: + image_exists: ${{ steps.image.outputs.image_exists }} + githash: ${{ steps.git.outputs.githash }} steps: - uses: actions/checkout@v4 with: @@ -40,21 +44,34 @@ jobs: else echo "image_exists=0" >> $GITHUB_OUTPUT fi + + build-pretix: + name: Build pretix + runs-on: [self-hosted] + needs: [check-new-pretix-version] + if: ${{ needs.check-new-pretix-version.outputs.image_exists == 0 }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 - uses: actions/checkout@v4 - if: ${{ steps.image.outputs.image_exists == 0 }} with: repository: pretix/pretix ref: v2024.10.0 path: ./pretix-clone - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} uses: aws-actions/amazon-ecr-login@v2 - name: Set up Docker Buildx id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-buildx-action@v3 - name: Build pretix base - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 id: build-pretix-base with: @@ -64,12 +81,11 @@ jobs: provenance: false push: true tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ steps.git.outputs.githash }} + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ needs.check-new-pretix-version.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 - name: Build and push pretix - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 with: context: ./pretix @@ -78,32 +94,24 @@ jobs: provenance: false push: true tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ steps.git.outputs.githash }} + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ needs.check-new-pretix-version.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 build-args: | - PRETIX_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ steps.git.outputs.githash }} + PRETIX_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ needs.check-new-pretix-version.outputs.githash }} - build-be: - runs-on: [self-hosted] - permissions: - packages: write - contents: read + check-new-backend-version: + name: Check backend needs building + runs-on: ubuntu-24.04 outputs: + image_exists: ${{ steps.image.outputs.image_exists }} githash: ${{ steps.git.outputs.githash }} - steps: - uses: actions/checkout@v4 with: ref: ${{ github.ref }} fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - name: Get service githash id: git run: | @@ -119,18 +127,27 @@ jobs: else echo "image_exists=0" >> $GITHUB_OUTPUT fi + + build-be: + name: Build backend + runs-on: [self-hosted] + needs: [check-new-backend-version] + if: ${{ needs.check-new-backend-version.outputs.image_exists == 0 }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-qemu-action@v3 - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} uses: aws-actions/amazon-ecr-login@v2 - name: Set up Docker Buildx id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-buildx-action@v3 - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 with: context: ./backend @@ -139,12 +156,13 @@ jobs: provenance: false push: true tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ steps.git.outputs.githash }} + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ needs.check-new-backend-version.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 deploy-be: + name: Deploy backend runs-on: ubuntu-24.04 needs: [build-be, build-pretix] environment: @@ -169,7 +187,6 @@ jobs: - name: Terraform Validate id: validate run: terraform validate -no-color - - name: Terraform apply run: terraform apply -target module.pretix -target module.pycon_backend -target module.clamav -target module.database -target module.emails -target module.cluster -no-color -auto-approve &> /dev/null env: @@ -180,7 +197,7 @@ jobs: wait-be-update: runs-on: ubuntu-24.04 - needs: [deploy-be, build-be] + needs: [deploy-be, build-be, check-new-backend-version] steps: - uses: actions/checkout@v4 with: @@ -190,9 +207,10 @@ jobs: uses: ./.github/actions/wait-for-deployment with: url: https://${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it/health/ - githash: ${{ needs.build-be.outputs.githash }} + githash: ${{ needs.check-new-backend-version.outputs.githash }} build-fe: + name: Build frontend needs: [wait-be-update] runs-on: [self-hosted] permissions: @@ -273,6 +291,7 @@ jobs: "sentry_auth_token=${{ steps.vars.outputs.sentry_auth_token }}" deploy-fe: + name: Deploy frontend runs-on: ubuntu-24.04 needs: [build-fe] environment: From bcaba6d314c99d81ca2d057d2777100ef8d6ffa7 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 01:24:28 +0100 Subject: [PATCH 02/68] changes --- .github/workflows/build-pretix.yml | 60 ++++++++++++++ .github/workflows/deploy.yml | 129 +++++++++++++---------------- 2 files changed, 117 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/build-pretix.yml diff --git a/.github/workflows/build-pretix.yml b/.github/workflows/build-pretix.yml new file mode 100644 index 0000000000..39b90958a4 --- /dev/null +++ b/.github/workflows/build-pretix.yml @@ -0,0 +1,60 @@ +on: + workflow_call: + inputs: + githash: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - uses: actions/checkout@v4 + with: + repository: pretix/pretix + ref: v2024.10.0 + path: ./pretix-clone + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Build pretix base + uses: docker/build-push-action@v6 + id: build-pretix-base + with: + context: ./pretix-clone + file: ./pretix-clone/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ inputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 + - name: Build and push pretix + uses: docker/build-push-action@v6 + with: + context: ./pretix + file: ./pretix/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ inputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 + build-args: | + PRETIX_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ inputs.githash }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6aeb607e59..b5845284bd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,7 +12,7 @@ env: TF_WORKSPACE: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} jobs: - check-new-pretix-version: + check-pretix-build: name: Check pretix needs building runs-on: ubuntu-24.04 outputs: @@ -48,60 +48,13 @@ jobs: build-pretix: name: Build pretix runs-on: [self-hosted] - needs: [check-new-pretix-version] - if: ${{ needs.check-new-pretix-version.outputs.image_exists == 0 }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - uses: actions/checkout@v4 - with: - repository: pretix/pretix - ref: v2024.10.0 - path: ./pretix-clone - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - name: Build pretix base - uses: docker/build-push-action@v6 - id: build-pretix-base - with: - context: ./pretix-clone - file: ./pretix-clone/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ needs.check-new-pretix-version.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 - - name: Build and push pretix - uses: docker/build-push-action@v6 - with: - context: ./pretix - file: ./pretix/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ needs.check-new-pretix-version.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 - build-args: | - PRETIX_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ needs.check-new-pretix-version.outputs.githash }} + needs: [check-pretix-build] + if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} + uses: ./.github/actions/build-pretix.yml + with: + githash: ${{ needs.check-pretix-build.outputs.githash }} - check-new-backend-version: + check-backend-build: name: Check backend needs building runs-on: ubuntu-24.04 outputs: @@ -112,6 +65,12 @@ jobs: with: ref: ${{ github.ref }} fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 - name: Get service githash id: git run: | @@ -131,8 +90,8 @@ jobs: build-be: name: Build backend runs-on: [self-hosted] - needs: [check-new-backend-version] - if: ${{ needs.check-new-backend-version.outputs.image_exists == 0 }} + needs: [check-backend-build] + if: ${{ needs.check-backend-build.outputs.image_exists == 0 }} steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -156,7 +115,7 @@ jobs: provenance: false push: true tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ needs.check-new-backend-version.outputs.githash }} + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ needs.check-backend-build.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 @@ -196,8 +155,9 @@ jobs: AWS_DEFAULT_REGION: eu-central-1 wait-be-update: + name: Wait backend deployment runs-on: ubuntu-24.04 - needs: [deploy-be, build-be, check-new-backend-version] + needs: [deploy-be, build-be, check-backend-build] steps: - uses: actions/checkout@v4 with: @@ -207,17 +167,46 @@ jobs: uses: ./.github/actions/wait-for-deployment with: url: https://${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it/health/ - githash: ${{ needs.check-new-backend-version.outputs.githash }} + githash: ${{ needs.check-backend-build.outputs.githash }} + + check-frontend-build: + name: Check frontend needs building + runs-on: ubuntu-24.04 + outputs: + image_exists: ${{ steps.image.outputs.image_exists }} + githash: ${{ steps.git.outputs.githash }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- frontend) + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend --image-ids=imageTag=${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi build-fe: name: Build frontend - needs: [wait-be-update] + needs: [wait-be-update, check-frontend-build] runs-on: [self-hosted] - permissions: - packages: write - contents: read - outputs: - githash: ${{ steps.git.outputs.githash }} + if: ${{ needs.check-frontend-build.outputs.image_exists == 0 }} steps: - uses: actions/checkout@v4 with: @@ -245,18 +234,14 @@ jobs: echo "image_exists=0" >> $GITHUB_OUTPUT fi - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-qemu-action@v3 - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} uses: aws-actions/amazon-ecr-login@v2 - name: Set up Docker Buildx id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-buildx-action@v3 - name: Get vars id: vars - if: ${{ steps.image.outputs.image_exists == 0 }} run: | cms_hostname=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/cms-hostname) echo "CMS_HOSTNAME=$cms_hostname" >> "$GITHUB_OUTPUT" @@ -268,7 +253,6 @@ jobs: echo "::add-mask::$sentry_auth_token" echo "SENTRY_AUTH_TOKEN=$sentry_auth_token" >> "$GITHUB_OUTPUT" - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 with: context: ./frontend @@ -326,8 +310,9 @@ jobs: AWS_DEFAULT_REGION: eu-central-1 wait-fe-update: + name: Wait frontend deployment runs-on: ubuntu-24.04 - needs: [deploy-fe, build-fe] + needs: [deploy-fe, check-frontend-build] steps: - uses: actions/checkout@v4 with: @@ -337,4 +322,4 @@ jobs: uses: ./.github/actions/wait-for-deployment with: url: https://${{ fromJSON('["pastaporto-frontend", "frontend"]')[github.ref == 'refs/heads/main'] }}.pycon.it/api/health - githash: ${{ needs.build-fe.outputs.githash }} + githash: ${{ needs.check-frontend-build.outputs.githash }} From dba3e9623450b1eda50a9caf3d845ba61b4cf566 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 01:25:50 +0100 Subject: [PATCH 03/68] fix --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b5845284bd..d1ed9c7256 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,7 +50,7 @@ jobs: runs-on: [self-hosted] needs: [check-pretix-build] if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} - uses: ./.github/actions/build-pretix.yml + uses: ./.github/workflows/build-pretix.yml with: githash: ${{ needs.check-pretix-build.outputs.githash }} From 6d3bc838ea227b41e1d1500b92b290d16bd46553 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 01:38:19 +0100 Subject: [PATCH 04/68] split --- .github/workflows/build-backend.yml | 42 ++++++++++ .github/workflows/build-frontend.yml | 62 +++++++++++++++ .github/workflows/build-pretix.yml | 3 +- .github/workflows/deploy.yml | 113 +++------------------------ 4 files changed, 117 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/build-backend.yml create mode 100644 .github/workflows/build-frontend.yml diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..1daf56e228 --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,42 @@ +on: + workflow_call: + inputs: + githash: + required: true + type: string + +jobs: + build: + name: Build backend + runs-on: [self-hosted] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Set up QEMU dependency + uses: docker/setup-qemu-action@v3 + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./backend + file: ./backend/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 0000000000..6fd48d3eb5 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,62 @@ +on: + workflow_call: + inputs: + githash: + required: true + type: string + +jobs: + build: + name: Build frontend + runs-on: [self-hosted] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Set up QEMU dependency + uses: docker/setup-qemu-action@v3 + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Get vars + id: vars + run: | + cms_hostname=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/cms-hostname) + echo "CMS_HOSTNAME=$cms_hostname" >> "$GITHUB_OUTPUT" + + conference_code=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/conference-code) + echo "CONFERENCE_CODE=$conference_code" >> "$GITHUB_OUTPUT" + + sentry_auth_token=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/common/sentry-auth-token) + echo "::add-mask::$sentry_auth_token" + echo "SENTRY_AUTH_TOKEN=$sentry_auth_token" >> "$GITHUB_OUTPUT" + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./frontend + file: ./frontend/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend:${{ inputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 + build-args: | + API_URL_SERVER=https://${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it + NEXT_PUBLIC_SITE_URL=${{ fromJSON('["pastaporto-frontend.", ""]')[github.ref == 'refs/heads/main'] }}pycon.it + CMS_ADMIN_HOST=${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it + CMS_HOSTNAME=${{ steps.vars.outputs.cms_hostname }} + CONFERENCE_CODE=${{ steps.vars.outputs.conference_code }} + GIT_HASH=${{ inputs.githash }} + secrets: | + "sentry_auth_token=${{ steps.vars.outputs.sentry_auth_token }}" diff --git a/.github/workflows/build-pretix.yml b/.github/workflows/build-pretix.yml index 39b90958a4..45e73c3b42 100644 --- a/.github/workflows/build-pretix.yml +++ b/.github/workflows/build-pretix.yml @@ -7,7 +7,8 @@ on: jobs: build: - runs-on: ubuntu-latest + name: Build pretix + runs-on: [self-hosted] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d1ed9c7256..df420bb3e4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -46,13 +46,12 @@ jobs: fi build-pretix: - name: Build pretix - runs-on: [self-hosted] needs: [check-pretix-build] - if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} uses: ./.github/workflows/build-pretix.yml with: githash: ${{ needs.check-pretix-build.outputs.githash }} + secrets: inherit + if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} check-backend-build: name: Check backend needs building @@ -88,37 +87,12 @@ jobs: fi build-be: - name: Build backend - runs-on: [self-hosted] needs: [check-backend-build] + uses: ./.github/workflows/build-backend.yml + with: + githash: ${{ needs.check-backend-build.outputs.githash }} + secrets: inherit if: ${{ needs.check-backend-build.outputs.image_exists == 0 }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Set up QEMU dependency - uses: docker/setup-qemu-action@v3 - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: ./backend - file: ./backend/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ needs.check-backend-build.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 deploy-be: name: Deploy backend @@ -203,76 +177,12 @@ jobs: fi build-fe: - name: Build frontend - needs: [wait-be-update, check-frontend-build] - runs-on: [self-hosted] + needs: [check-frontend-build] + uses: ./.github/workflows/build-frontend.yml + with: + githash: ${{ needs.check-frontend-build.outputs.githash }} + secrets: inherit if: ${{ needs.check-frontend-build.outputs.image_exists == 0 }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- frontend) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend --image-ids=imageTag=${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - - name: Set up QEMU dependency - uses: docker/setup-qemu-action@v3 - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - name: Get vars - id: vars - run: | - cms_hostname=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/cms-hostname) - echo "CMS_HOSTNAME=$cms_hostname" >> "$GITHUB_OUTPUT" - - conference_code=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/conference-code) - echo "CONFERENCE_CODE=$conference_code" >> "$GITHUB_OUTPUT" - - sentry_auth_token=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/common/sentry-auth-token) - echo "::add-mask::$sentry_auth_token" - echo "SENTRY_AUTH_TOKEN=$sentry_auth_token" >> "$GITHUB_OUTPUT" - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: ./frontend - file: ./frontend/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend:${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 - build-args: | - API_URL_SERVER=https://${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it - NEXT_PUBLIC_SITE_URL=${{ fromJSON('["pastaporto-frontend.", ""]')[github.ref == 'refs/heads/main'] }}pycon.it - CMS_ADMIN_HOST=${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it - CMS_HOSTNAME=${{ steps.vars.outputs.cms_hostname }} - CONFERENCE_CODE=${{ steps.vars.outputs.conference_code }} - GIT_HASH=${{ steps.git.outputs.githash }} - secrets: | - "sentry_auth_token=${{ steps.vars.outputs.sentry_auth_token }}" deploy-fe: name: Deploy frontend @@ -300,7 +210,6 @@ jobs: - name: Terraform Validate id: validate run: terraform validate -no-color - - name: Terraform apply run: terraform apply -no-color -auto-approve &> /dev/null env: From 2851bcf3f744d64a517c9e49278a3158972a7965 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 01:39:22 +0100 Subject: [PATCH 05/68] fix --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index df420bb3e4..a9a2ac7242 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -177,7 +177,7 @@ jobs: fi build-fe: - needs: [check-frontend-build] + needs: [wait-be-update, check-frontend-build] uses: ./.github/workflows/build-frontend.yml with: githash: ${{ needs.check-frontend-build.outputs.githash }} From 9279dad6571c8f180342629ca148c86a19feed6d Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 01:40:47 +0100 Subject: [PATCH 06/68] test --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a9a2ac7242..edff530949 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -97,7 +97,7 @@ jobs: deploy-be: name: Deploy backend runs-on: ubuntu-24.04 - needs: [build-be, build-pretix] + needs: [build-be, build-pretix, check-frontend-build] environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: From 0b670e1aab7ae7d9e4ddbf02a3de7615b4c6ac6a Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 12:05:06 +0100 Subject: [PATCH 07/68] change --- .github/workflows/deploy.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index edff530949..7385e5513b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -97,7 +97,13 @@ jobs: deploy-be: name: Deploy backend runs-on: ubuntu-24.04 - needs: [build-be, build-pretix, check-frontend-build] + needs: [ + build-be, + build-pretix, + check-backend-build, + check-pretix-build, + check-frontend-build + ] environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: From 7d7d6f6ddcfd82c89fb3d26a57d99c361a5b3ad8 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 12:06:24 +0100 Subject: [PATCH 08/68] always --- .github/workflows/deploy.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7385e5513b..84f96ee8cc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -97,13 +97,8 @@ jobs: deploy-be: name: Deploy backend runs-on: ubuntu-24.04 - needs: [ - build-be, - build-pretix, - check-backend-build, - check-pretix-build, - check-frontend-build - ] + needs: [build-be, build-pretix, check-frontend-build] + if: ${{ always()}} environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: From 40ae02a2fcd48cc8ff721542a94c25c6ea790a1c Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 12:13:09 +0100 Subject: [PATCH 09/68] Updates --- .github/workflows/deploy.yml | 84 ++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 84f96ee8cc..77f4987304 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -45,13 +45,38 @@ jobs: echo "image_exists=0" >> $GITHUB_OUTPUT fi - build-pretix: - needs: [check-pretix-build] - uses: ./.github/workflows/build-pretix.yml - with: - githash: ${{ needs.check-pretix-build.outputs.githash }} - secrets: inherit - if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} + check-frontend-build: + name: Check frontend needs building + runs-on: ubuntu-24.04 + outputs: + image_exists: ${{ steps.image.outputs.image_exists }} + githash: ${{ steps.git.outputs.githash }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- frontend) + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend --image-ids=imageTag=${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi check-backend-build: name: Check backend needs building @@ -86,6 +111,14 @@ jobs: echo "image_exists=0" >> $GITHUB_OUTPUT fi + build-pretix: + needs: [check-pretix-build] + uses: ./.github/workflows/build-pretix.yml + with: + githash: ${{ needs.check-pretix-build.outputs.githash }} + secrets: inherit + if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} + build-be: needs: [check-backend-build] uses: ./.github/workflows/build-backend.yml @@ -122,7 +155,7 @@ jobs: id: validate run: terraform validate -no-color - name: Terraform apply - run: terraform apply -target module.pretix -target module.pycon_backend -target module.clamav -target module.database -target module.emails -target module.cluster -no-color -auto-approve &> /dev/null + run: terraform apply -target module.pretix -target module.pycon_backend -target module.clamav -target module.database -target module.emails -target module.cluster -target module.vpc -no-color -auto-approve &> /dev/null env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -132,7 +165,7 @@ jobs: wait-be-update: name: Wait backend deployment runs-on: ubuntu-24.04 - needs: [deploy-be, build-be, check-backend-build] + needs: [deploy-be, check-backend-build] steps: - uses: actions/checkout@v4 with: @@ -144,39 +177,6 @@ jobs: url: https://${{ fromJSON('["pastaporto-", ""]')[github.ref == 'refs/heads/main'] }}admin.pycon.it/health/ githash: ${{ needs.check-backend-build.outputs.githash }} - check-frontend-build: - name: Check frontend needs building - runs-on: ubuntu-24.04 - outputs: - image_exists: ${{ steps.image.outputs.image_exists }} - githash: ${{ steps.git.outputs.githash }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- frontend) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend --image-ids=imageTag=${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - build-fe: needs: [wait-be-update, check-frontend-build] uses: ./.github/workflows/build-frontend.yml From 785a22c2e5cc97c7caa82e6a544f7222e39e78c8 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 13:14:49 +0100 Subject: [PATCH 10/68] changes --- .github/workflows/build-backend.yml | 2 +- .github/workflows/deploy.yml | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 1daf56e228..4a2ddea91e 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -7,7 +7,7 @@ on: jobs: build: - name: Build backend + name: Build runs-on: [self-hosted] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 77f4987304..03df23dcee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -112,6 +112,7 @@ jobs: fi build-pretix: + name: Build pretix needs: [check-pretix-build] uses: ./.github/workflows/build-pretix.yml with: @@ -120,6 +121,7 @@ jobs: if: ${{ needs.check-pretix-build.outputs.image_exists == 0 }} build-be: + name: Build backend needs: [check-backend-build] uses: ./.github/workflows/build-backend.yml with: @@ -131,7 +133,7 @@ jobs: name: Deploy backend runs-on: ubuntu-24.04 needs: [build-be, build-pretix, check-frontend-build] - if: ${{ always()}} + if: always() && !cancelled() && !failure() environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: @@ -167,10 +169,6 @@ jobs: runs-on: ubuntu-24.04 needs: [deploy-be, check-backend-build] steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - name: Wait stable deployment uses: ./.github/actions/wait-for-deployment with: @@ -224,10 +222,6 @@ jobs: runs-on: ubuntu-24.04 needs: [deploy-fe, check-frontend-build] steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - name: Wait stable deployment uses: ./.github/actions/wait-for-deployment with: From 4ac7ed02242e5ff62e26eeeb2853a6904f5ac5b5 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 13:27:55 +0100 Subject: [PATCH 11/68] change --- .github/workflows/deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 03df23dcee..fdc72b5a6d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -133,7 +133,7 @@ jobs: name: Deploy backend runs-on: ubuntu-24.04 needs: [build-be, build-pretix, check-frontend-build] - if: always() && !cancelled() && !failure() + if: always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: @@ -168,6 +168,7 @@ jobs: name: Wait backend deployment runs-on: ubuntu-24.04 needs: [deploy-be, check-backend-build] + if: always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') steps: - name: Wait stable deployment uses: ./.github/actions/wait-for-deployment From 3db342d3a465f371293b43fc11081b0d2506ab97 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 13:30:32 +0100 Subject: [PATCH 12/68] change --- .github/workflows/deploy.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fdc72b5a6d..7d8559647c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -170,6 +170,10 @@ jobs: needs: [deploy-be, check-backend-build] if: always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 - name: Wait stable deployment uses: ./.github/actions/wait-for-deployment with: @@ -223,6 +227,10 @@ jobs: runs-on: ubuntu-24.04 needs: [deploy-fe, check-frontend-build] steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 - name: Wait stable deployment uses: ./.github/actions/wait-for-deployment with: From afed24ff1b7c0ee2bd6b36518dfa4faec2ddb435 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 13:33:20 +0100 Subject: [PATCH 13/68] change --- .github/workflows/deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7d8559647c..32e2558f35 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -186,12 +186,13 @@ jobs: with: githash: ${{ needs.check-frontend-build.outputs.githash }} secrets: inherit - if: ${{ needs.check-frontend-build.outputs.image_exists == 0 }} + if: ${{ needs.check-frontend-build.outputs.image_exists == 0 }} && always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') deploy-fe: name: Deploy frontend runs-on: ubuntu-24.04 needs: [build-fe] + if: always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: @@ -226,6 +227,7 @@ jobs: name: Wait frontend deployment runs-on: ubuntu-24.04 needs: [deploy-fe, check-frontend-build] + if: always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') steps: - uses: actions/checkout@v4 with: From 3e71189d2819180beb53cc53ff38a4e32da65adc Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 13:43:50 +0100 Subject: [PATCH 14/68] change --- .github/workflows/deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 32e2558f35..c4a674deda 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -181,12 +181,13 @@ jobs: githash: ${{ needs.check-backend-build.outputs.githash }} build-fe: + name: Build frontend needs: [wait-be-update, check-frontend-build] uses: ./.github/workflows/build-frontend.yml with: githash: ${{ needs.check-frontend-build.outputs.githash }} secrets: inherit - if: ${{ needs.check-frontend-build.outputs.image_exists == 0 }} && always() && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + if: always() && ${{ needs.check-frontend-build.outputs.image_exists == 0 }} && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') deploy-fe: name: Deploy frontend From f977798c7d96d3749d52a88802871907025fbda7 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 13:53:22 +0100 Subject: [PATCH 15/68] change --- .github/workflows/build-frontend.yml | 2 +- .github/workflows/deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml index 6fd48d3eb5..d06a343b49 100644 --- a/.github/workflows/build-frontend.yml +++ b/.github/workflows/build-frontend.yml @@ -7,7 +7,7 @@ on: jobs: build: - name: Build frontend + name: Build runs-on: [self-hosted] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c4a674deda..e516d72a1a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -187,7 +187,7 @@ jobs: with: githash: ${{ needs.check-frontend-build.outputs.githash }} secrets: inherit - if: always() && ${{ needs.check-frontend-build.outputs.image_exists == 0 }} && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + if: always() && needs.check-frontend-build.outputs.image_exists == 0 && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') deploy-fe: name: Deploy frontend From 6f167f56e81952df9449711f00fc2b69f933d99f Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 14:03:54 +0100 Subject: [PATCH 16/68] change change change change change change pdates updates --- .editorconfig | 2 +- .github/workflows/build-backend.yml | 2 +- .github/workflows/build-frontend.yml | 13 ++-- .github/workflows/build-pretix.yml | 2 +- .github/workflows/deploy.yml | 1 + .gitignore | 1 + backend/Dockerfile | 1 + backend/README.md | 1 + infrastructure/tools/.terraform.lock.hcl | 77 +++++++++++++++++++ infrastructure/tools/.tool-versions | 1 + infrastructure/tools/github_repo.tf | 3 + infrastructure/tools/github_runner_lambda.tf | 62 +++++++++++++++ infrastructure/tools/github_webhook.tf | 17 ++++ .../tools/lambdas/github_runner_webhook.py | 71 +++++++++++++++++ infrastructure/tools/main.tf | 26 +++++++ infrastructure/tools/runners-cluster.tf | 3 + 16 files changed, 275 insertions(+), 8 deletions(-) create mode 100644 infrastructure/tools/.terraform.lock.hcl create mode 100644 infrastructure/tools/.tool-versions create mode 100644 infrastructure/tools/github_repo.tf create mode 100644 infrastructure/tools/github_runner_lambda.tf create mode 100644 infrastructure/tools/github_webhook.tf create mode 100644 infrastructure/tools/lambdas/github_runner_webhook.py create mode 100644 infrastructure/tools/main.tf create mode 100644 infrastructure/tools/runners-cluster.tf diff --git a/.editorconfig b/.editorconfig index c8559687a6..b62fcf63b0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -30,5 +30,5 @@ indent_size = 2 [Makefile] indent_style = tab -[*.tf] +[{*.tf,*.tofu}] indent_size = 2 diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 4a2ddea91e..3d946704d4 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -8,7 +8,7 @@ on: jobs: build: name: Build - runs-on: [self-hosted] + runs-on: [self-hosted, arm64-fargate] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml index d06a343b49..a521f9f4f6 100644 --- a/.github/workflows/build-frontend.yml +++ b/.github/workflows/build-frontend.yml @@ -4,11 +4,14 @@ on: githash: required: true type: string + tf_environment: + required: true + type: string jobs: build: name: Build - runs-on: [self-hosted] + runs-on: [self-hosted, arm64-fargate] steps: - uses: actions/checkout@v4 with: @@ -30,13 +33,13 @@ jobs: - name: Get vars id: vars run: | - cms_hostname=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/cms-hostname) + cms_hostname=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ inputs.tf_environment }}/pycon-frontend/cms-hostname) echo "CMS_HOSTNAME=$cms_hostname" >> "$GITHUB_OUTPUT" - conference_code=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/pycon-frontend/conference-code) + conference_code=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ inputs.tf_environment }}/pycon-frontend/conference-code) echo "CONFERENCE_CODE=$conference_code" >> "$GITHUB_OUTPUT" - sentry_auth_token=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ env.TF_WORKSPACE }}/common/sentry-auth-token) + sentry_auth_token=$(aws ssm get-parameter --output text --query Parameter.Value --with-decryption --name /pythonit/${{ inputs.tf_environment }}/common/sentry-auth-token) echo "::add-mask::$sentry_auth_token" echo "SENTRY_AUTH_TOKEN=$sentry_auth_token" >> "$GITHUB_OUTPUT" - name: Build and push @@ -47,7 +50,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false push: true - tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }}-pycon-frontend:${{ inputs.githash }} + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ inputs.tf_environment }}-pycon-frontend:${{ inputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 diff --git a/.github/workflows/build-pretix.yml b/.github/workflows/build-pretix.yml index 45e73c3b42..677acd3c91 100644 --- a/.github/workflows/build-pretix.yml +++ b/.github/workflows/build-pretix.yml @@ -8,7 +8,7 @@ on: jobs: build: name: Build pretix - runs-on: [self-hosted] + runs-on: [self-hosted, arm64-fargate] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e516d72a1a..6d08c31845 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -186,6 +186,7 @@ jobs: uses: ./.github/workflows/build-frontend.yml with: githash: ${{ needs.check-frontend-build.outputs.githash }} + tf_environment: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} secrets: inherit if: always() && needs.check-frontend-build.outputs.image_exists == 0 && !cancelled() && !failure() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') diff --git a/.gitignore b/.gitignore index 4e5e99d433..a59e8280b2 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,4 @@ backend/__pypackages__/ backend/custom_admin/.astro/ backend/custom_admin/core.* core.* +**/.archive_files/*.zip diff --git a/backend/Dockerfile b/backend/Dockerfile index 32a237e27d..9a4c81780b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -87,3 +87,4 @@ RUN mkdir -p assets && .venv/bin/python manage.py collectstatic --noinput ENTRYPOINT ["/home/app/.venv/bin/gunicorn"] CMD [ "pycon.wsgi" ] +#test diff --git a/backend/README.md b/backend/README.md index e1edd09758..2f9518f566 100644 --- a/backend/README.md +++ b/backend/README.md @@ -63,3 +63,4 @@ And for the admin go to: http://localhost:8000/admin You should be able to login with the user we create a few moments ago. +#testchange diff --git a/infrastructure/tools/.terraform.lock.hcl b/infrastructure/tools/.terraform.lock.hcl new file mode 100644 index 0000000000..e43e02d9f9 --- /dev/null +++ b/infrastructure/tools/.terraform.lock.hcl @@ -0,0 +1,77 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/archive" { + version = "2.7.0" + hashes = [ + "h1:54u+Kfg2ITHB1k/6tC2Y/0//J66lseBIz2ydMJHvAB8=", + "zh:351318d6caad0b03c28b30362e9d8b801f94b0bfd11b08566ad2b37f63671f8a", + "zh:56b70c49a2b60cb6a4f42bed921c30863481eb0422bb180e50dde271104731d9", + "zh:7781aac1366a5ac0226f83331f152921c58b155fb91f506b0854b8267b870432", + "zh:7aabcaccacfcaa2fb84fdea93453552617e0efecc9869e88b63607c459fb99aa", + "zh:7bcd07d40cf33f64c19f033d3ee64a243f09e039a2516ab3707aceb51669a532", + "zh:a423791fae7a51f1235f47ba13baa61d50714b10d84bbf391a6c0b55fe2e1a9a", + "zh:c8af7913de581bf2ca2dbf0906e604123a2a8f86a087c3b40a79cdacd5e867f5", + "zh:ee6c2995e7d462a0e769d62e5345e04acddff8af38422e7dd3925d1769408e7b", + "zh:f5c2311af4ee6ad8f1bd7eb4a1d38e371d1cbe90abf812797f6863e420f1c463", + "zh:fb3c41fa9fe5181db85c58d4893474082a2c7f14ba5a69e38c609b1fda9ba741", + ] +} + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.82.2" + constraints = "5.82.2" + hashes = [ + "h1:65B6GjFmKobkuaO58AcDPUu/otnTR7qRTynWTgU5A7Q=", + "zh:0c662b1f27119941e2329155070c8a81f6979e3d5e98296949f373c46487d84e", + "zh:56cbaeaf050174bb9ba9a00f31fe441365ffece6ad39254885c9d4e6517e27e3", + "zh:6e7f66c9f9955deb9ce0a2ae4b6df366b3b6bc3e23bca1fcdd4351ecab3ec907", + "zh:7971a22df020391ca20936584282d01f3380bcfc79260afbdf04d6b4d50cf75a", + "zh:7b8f2175447e64c9a9c9624818a597cd6c1609f379f7f8e3e179ee7394a467b7", + "zh:a805e26eb7f58e46b130decab79c0871dfd73206d7bd6c1793e27c2dc35fd8b6", + "zh:b6163e746c77d47269c0e3014bdc1f805152b3d94471dd6443373d91b848cf45", + "zh:c46872fd0c304852bd506c036f31dbe047a97036992319a3c594949538651680", + "zh:d586f1ef412f336ede54d720a78f11a64952a39bfe401e141d24826530cb2449", + "zh:ddaf2e1de0f1660ac31c5f1bebd40cc2b29e19f64bf41591c3b5df00dc268367", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.6.3" + hashes = [ + "h1:Ry0Lr0zaoicslZlcUR4rAySPpl/a7QupfMfuAxhW3fw=", + "zh:1bfd2e54b4eee8c761a40b6d99d45880b3a71abc18a9a7a5319204da9c8363b2", + "zh:21a15ac74adb8ba499aab989a4248321b51946e5431219b56fc827e565776714", + "zh:221acfac3f7a5bcd6cb49f79a1fca99da7679bde01017334bad1f951a12d85ba", + "zh:3026fcdc0c1258e32ab519df878579160b1050b141d6f7883b39438244e08954", + "zh:50d07a7066ea46873b289548000229556908c3be746059969ab0d694e053ee4c", + "zh:54280cdac041f2c2986a585f62e102bc59ef412cad5f4ebf7387c2b3a357f6c0", + "zh:632adf40f1f63b0c5707182853c10ae23124c00869ffff05f310aef2ed26fcf3", + "zh:b8c2876cce9a38501d14880a47e59a5182ee98732ad7e576e9a9ce686a46d8f5", + "zh:f27e6995e1e9fe3914a2654791fc8d67cdce44f17bf06e614ead7dfd2b13d3ae", + "zh:f423f2b7e5c814799ad7580b5c8ae23359d8d342264902f821c357ff2b3c6d3d", + ] +} + +provider "registry.opentofu.org/integrations/github" { + version = "6.4.0" + constraints = "~> 6.0" + hashes = [ + "h1:sJvuRMYWJ/ykZXTuoCuocHvx06hTwDVrXVVXq1814bw=", + "zh:00f431c2a2510efcb1115442dda5e90815bcb16e1a3301679ade0139fa963d3b", + "zh:12a862f4317b3cb65682c1b687650cd91eeee99e63774bdcfa8bcfc64bad097b", + "zh:226d5e09ff27f94cb9336089181d26f85cb30219b863a579597f2e107f37de49", + "zh:402ecaa5add568a52ee01d816810f3b90f693be35c680fcdc9b6284bf55326f1", + "zh:60e3bdd9fbefb3c1d790bc08889c1dc0e83636b82284faaa709411aa4f96bb9f", + "zh:625099eeff2f8aaecd22a24a451b326828435c8f9de86f2e5e99872e7b467fa7", + "zh:79e8b665421009df2260f50e10da1f7a7863b557ece96e2b07dfd2fad1e86fcd", + "zh:98e471fefc93dcfedeec750c694110db7d3331dc3a256191d30b9d2f70d12157", + "zh:a17702765e1fa92d1c288ddfd97075819ad61b344b341be7e09c554c841a6d9e", + "zh:ca72ccf40624ae26bf4660d8dd84a51638f0a1e78d5f19fdfaafaef97f838af6", + "zh:d009ab5527d45c44c424d26cd2eb51a5a6a6448f3fb1023b675789588cc08d64", + "zh:e5811be1e942a75b14dfcd3e03523d8df60cfbde0d7e24d75e78480a02a58949", + "zh:e6008ad28225ad6996b06bcd7f3070863329df406a56754e7fb9c31d6301ace4", + "zh:f1d93f56ea4f87183a5de4780704907605851d95a2d285a9ec755bf784c5569c", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/infrastructure/tools/.tool-versions b/infrastructure/tools/.tool-versions new file mode 100644 index 0000000000..33dbae16aa --- /dev/null +++ b/infrastructure/tools/.tool-versions @@ -0,0 +1 @@ +opentofu 1.8.8 diff --git a/infrastructure/tools/github_repo.tf b/infrastructure/tools/github_repo.tf new file mode 100644 index 0000000000..6bedb2e402 --- /dev/null +++ b/infrastructure/tools/github_repo.tf @@ -0,0 +1,3 @@ +data "github_repository" "pycon" { + full_name = "pythonitalia/pycon" +} diff --git a/infrastructure/tools/github_runner_lambda.tf b/infrastructure/tools/github_runner_lambda.tf new file mode 100644 index 0000000000..0555fdbe5f --- /dev/null +++ b/infrastructure/tools/github_runner_lambda.tf @@ -0,0 +1,62 @@ +data "aws_iam_policy_document" "github_runner_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "github_runner_iam" { + name = "github_runner_iam" + assume_role_policy = data.aws_iam_policy_document.github_runner_assume_role.json +} + +resource "aws_iam_role_policy" "github_runner_lambda_policy" { + name = "github_runner_lambda_policy" + role = aws_iam_role.github_runner_iam.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "*" + } + ] + }) +} + +data "archive_file" "github_runner_webhook_artifact" { + type = "zip" + source_file = "${path.root}/lambdas/github_runner_webhook.py" + output_path = "${path.root}/.archive_files/github_runner_webhook.zip" +} + +resource "aws_lambda_function" "github_runner_webhook" { + function_name = "github_runner_webhook" + role = aws_iam_role.github_runner_iam.arn + handler = "github_runner_webhook.handler" + runtime = "python3.13" + filename = data.archive_file.github_runner_webhook_artifact.output_path + source_code_hash = data.archive_file.github_runner_webhook_artifact.output_base64sha256 + environment { + variables = { + WEBHOOK_SECRET = random_password.webhook_secret.result + } + } +} + +resource "aws_lambda_function_url" "github_runner_webhook" { + function_name = aws_lambda_function.github_runner_webhook.function_name + authorization_type = "NONE" +} diff --git a/infrastructure/tools/github_webhook.tf b/infrastructure/tools/github_webhook.tf new file mode 100644 index 0000000000..aa7d4c51c5 --- /dev/null +++ b/infrastructure/tools/github_webhook.tf @@ -0,0 +1,17 @@ +resource "random_password" "webhook_secret" { + length = 64 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + +resource "github_repository_webhook" "github_runner_notify" { + repository = data.github_repository.pycon.name + events = ["workflow_job"] + active = true + + configuration { + url = aws_lambda_function_url.github_runner_webhook.function_url + secret = random_password.webhook_secret.result + content_type = "json" + } +} diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py new file mode 100644 index 0000000000..b46b29ca22 --- /dev/null +++ b/infrastructure/tools/lambdas/github_runner_webhook.py @@ -0,0 +1,71 @@ +import json +import os +import hashlib +import hmac + +WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"] + + +def handler(event, context): + body = event.get("body").encode("utf-8") + + if not verify_signature( + body, WEBHOOK_SECRET, event["headers"]["x-hub-signature-256"] + ): + return {"statusCode": 401, "body": ""} + + github_event = event["headers"]["x-github-event"] + body = json.loads(body) + + match github_event: + case "workflow_job": + handle_workflow_job(body, context) + case _: + ... + + return {"statusCode": 200, "body": ""} + + +def handle_workflow_job(body, context): + action = body["action"] + + if action != "queued": + return + + workflow_job = body["workflow_job"] + workflow_name = workflow_job["workflow_name"] + + if workflow_name != "Deploy": + return + + labels = workflow_job["labels"] + if labels != ["self-hosted", "arm64-fargate"]: + return + + print("Handling workflow job - start?") + print("Body:", body) + print("Context:", context) + + +def verify_signature(payload_body, secret_token, signature_header): + """Verify that the payload was sent from GitHub by validating SHA256. + + Raise and return 403 if not authorized. + + Args: + payload_body: original request body to verify (request.body()) + secret_token: GitHub app webhook token (WEBHOOK_SECRET) + signature_header: header received from GitHub (x-hub-signature-256) + """ + if not signature_header: + return False + + hash_object = hmac.new( + secret_token.encode("utf-8"), msg=payload_body, digestmod=hashlib.sha256 + ) + expected_signature = "sha256=" + hash_object.hexdigest() + + if not hmac.compare_digest(expected_signature, signature_header): + return False + + return True diff --git a/infrastructure/tools/main.tf b/infrastructure/tools/main.tf new file mode 100644 index 0000000000..5a23606d26 --- /dev/null +++ b/infrastructure/tools/main.tf @@ -0,0 +1,26 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.82.2" + } + github = { + source = "integrations/github" + version = "~> 6.0" + } + } + + backend "s3" { + bucket = "xujw087uco-infrastructure-tools-opentofu" + key = "opentofu.tfstate" + region = "eu-central-1" + } +} + +provider "aws" { + region = "eu-central-1" +} + +provider "github" { + owner = "pythonitalia" +} diff --git a/infrastructure/tools/runners-cluster.tf b/infrastructure/tools/runners-cluster.tf new file mode 100644 index 0000000000..33f3aac2bd --- /dev/null +++ b/infrastructure/tools/runners-cluster.tf @@ -0,0 +1,3 @@ +resource "aws_ecs_cluster" "cluster" { + name = "github-actions-runners" +} From fa698959293aa225992355a50838af27c1337299 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 19:12:01 +0100 Subject: [PATCH 17/68] change --- .github/workflows/build-backend.yml | 2 +- infrastructure/tools/github_runner_cluster.tf | 15 ++++ infrastructure/tools/github_runner_lambda.tf | 31 +++++-- infrastructure/tools/github_runner_task.tf | 86 +++++++++++++++++++ .../tools/lambdas/github_runner_webhook.py | 38 +++++++- infrastructure/tools/runners-cluster.tf | 3 - 6 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 infrastructure/tools/github_runner_cluster.tf create mode 100644 infrastructure/tools/github_runner_task.tf delete mode 100644 infrastructure/tools/runners-cluster.tf diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 3d946704d4..ee033e8539 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -8,7 +8,7 @@ on: jobs: build: name: Build - runs-on: [self-hosted, arm64-fargate] + runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} steps: - uses: actions/checkout@v4 with: diff --git a/infrastructure/tools/github_runner_cluster.tf b/infrastructure/tools/github_runner_cluster.tf new file mode 100644 index 0000000000..d7739d0646 --- /dev/null +++ b/infrastructure/tools/github_runner_cluster.tf @@ -0,0 +1,15 @@ +resource "aws_ecs_cluster" "github_runners" { + name = "github-actions-runners" +} + +resource "aws_ecs_cluster_capacity_providers" "github_runners" { + cluster_name = aws_ecs_cluster.github_runners.name + + capacity_providers = ["FARGATE_SPOT"] + + default_capacity_provider_strategy { + base = 1 + weight = 100 + capacity_provider = "FARGATE_SPOT" + } +} diff --git a/infrastructure/tools/github_runner_lambda.tf b/infrastructure/tools/github_runner_lambda.tf index 0555fdbe5f..ee3219e671 100644 --- a/infrastructure/tools/github_runner_lambda.tf +++ b/infrastructure/tools/github_runner_lambda.tf @@ -1,4 +1,4 @@ -data "aws_iam_policy_document" "github_runner_assume_role" { +data "aws_iam_policy_document" "github_runner_webhook_assume_role" { statement { effect = "Allow" @@ -11,14 +11,18 @@ data "aws_iam_policy_document" "github_runner_assume_role" { } } -resource "aws_iam_role" "github_runner_iam" { - name = "github_runner_iam" - assume_role_policy = data.aws_iam_policy_document.github_runner_assume_role.json +resource "aws_iam_role" "github_runner_webhook_role" { + name = "github_runner_webhook_role" + assume_role_policy = data.aws_iam_policy_document.github_runner_webhook_assume_role.json } -resource "aws_iam_role_policy" "github_runner_lambda_policy" { - name = "github_runner_lambda_policy" - role = aws_iam_role.github_runner_iam.id +data "aws_ssm_parameter" "github_token" { + name = "/github-runner/github-token" +} + +resource "aws_iam_role_policy" "github_runner_webhook_lambda_policy" { + name = "github_runner_webhook_lambda_policy" + role = aws_iam_role.github_runner_webhook_role.id policy = jsonencode({ Version = "2012-10-17" @@ -31,6 +35,15 @@ resource "aws_iam_role_policy" "github_runner_lambda_policy" { "logs:PutLogEvents" ] Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ssm:GetParameter" + ] + Resource = [ + data.aws_ssm_parameter.github_token.arn + ] } ] }) @@ -44,14 +57,16 @@ data "archive_file" "github_runner_webhook_artifact" { resource "aws_lambda_function" "github_runner_webhook" { function_name = "github_runner_webhook" - role = aws_iam_role.github_runner_iam.arn + role = aws_iam_role.github_runner_webhook_role.arn handler = "github_runner_webhook.handler" runtime = "python3.13" filename = data.archive_file.github_runner_webhook_artifact.output_path source_code_hash = data.archive_file.github_runner_webhook_artifact.output_base64sha256 + timeout = 60 environment { variables = { WEBHOOK_SECRET = random_password.webhook_secret.result + GITHUB_TOKEN_SSM_NAME = data.aws_ssm_parameter.github_token.name } } } diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf new file mode 100644 index 0000000000..2cda667cce --- /dev/null +++ b/infrastructure/tools/github_runner_task.tf @@ -0,0 +1,86 @@ +data "aws_iam_policy_document" "github_runner_execution_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "github_runner_execution_role" { + name = "github_runner_execution_role" + assume_role_policy = data.aws_iam_policy_document.github_runner_execution_assume_role.json +} + +resource "aws_iam_role_policy" "github_runner_execution_role_policy" { + name = "github_runner_execution_role_policy" + role = aws_iam_role.github_runner_execution_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = [ + aws_cloudwatch_log_group.github_runner.arn, + "${aws_cloudwatch_log_group.github_runner.arn}*" + ] + }, + { + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + ] + Resource = "*" + } + ] + }) +} + + +resource "aws_cloudwatch_log_group" "github_runner" { + name = "/github-runner/" + retention_in_days = 1 +} + +resource "aws_ecs_task_definition" "github_runner" { + family = "github-runner" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = 1024 + memory = 2048 + execution_role_arn = aws_iam_role.github_runner_execution_role.arn + + container_definitions = jsonencode([ + { + name = "runner" + image = "ghcr.io/actions/actions-runner:2.321.0" + essential = true + portMappings = [] + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.github_runner.name + "awslogs-region" = "eu-central-1" + "awslogs-stream-prefix" = "runner" + } + } + }, + ]) + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } +} diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py index b46b29ca22..25dc02dfc0 100644 --- a/infrastructure/tools/lambdas/github_runner_webhook.py +++ b/infrastructure/tools/lambdas/github_runner_webhook.py @@ -1,9 +1,12 @@ +import boto3 import json import os import hashlib import hmac +from urllib import request WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"] +GITHUB_TOKEN_SSM_NAME = os.environ["GITHUB_TOKEN_SSM_NAME"] def handler(event, context): @@ -42,7 +45,40 @@ def handle_workflow_job(body, context): if labels != ["self-hosted", "arm64-fargate"]: return - print("Handling workflow job - start?") + ssm_client = boto3.client("ssm") + github_token = ssm_client.get_parameter(Name=GITHUB_TOKEN_SSM_NAME)["Parameter"][ + "Value" + ] + + payload = { + "name": "Test from Lambda", + "runner_group_id": 3, + "labels": [ + "lambda-test" + # 'self-hosted', + # 'arm64-fargate', + ], + } + payload_encoded = json.dumps(payload).encode("utf-8") + print("sending payload:", payload_encoded) + req = request.Request( + "https://api.github.com/orgs/pythonitalia/actions/runners/generate-jitconfig", + data=payload_encoded, + method="POST", + headers={ + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github.v3+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + + with request.urlopen(req) as response: + response_data = response.read().decode("utf-8") + print(response_data) + + jit_config = json.loads(response_data)["encoded_jit_config"] + + print("Handling workflow job - start?", jit_config) print("Body:", body) print("Context:", context) diff --git a/infrastructure/tools/runners-cluster.tf b/infrastructure/tools/runners-cluster.tf deleted file mode 100644 index 33f3aac2bd..0000000000 --- a/infrastructure/tools/runners-cluster.tf +++ /dev/null @@ -1,3 +0,0 @@ -resource "aws_ecs_cluster" "cluster" { - name = "github-actions-runners" -} From 9e6072be4cef58f6749e9a030851547c58c70727 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 19:29:40 +0100 Subject: [PATCH 18/68] change --- infrastructure/tools/github_runner_lambda.tf | 8 ++ .../tools/lambdas/github_runner_webhook.py | 22 ++++-- infrastructure/tools/vpc.tf | 76 +++++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 infrastructure/tools/vpc.tf diff --git a/infrastructure/tools/github_runner_lambda.tf b/infrastructure/tools/github_runner_lambda.tf index ee3219e671..e3fc381e56 100644 --- a/infrastructure/tools/github_runner_lambda.tf +++ b/infrastructure/tools/github_runner_lambda.tf @@ -63,10 +63,18 @@ resource "aws_lambda_function" "github_runner_webhook" { filename = data.archive_file.github_runner_webhook_artifact.output_path source_code_hash = data.archive_file.github_runner_webhook_artifact.output_base64sha256 timeout = 60 + environment { variables = { WEBHOOK_SECRET = random_password.webhook_secret.result GITHUB_TOKEN_SSM_NAME = data.aws_ssm_parameter.github_token.name + NETWORK_CONFIGURATION = jsonencode({ + "awsvpcConfiguration": { + "subnets": [aws_subnet.public["eu-central-1a"].id], + "securityGroups": [], + "assignPublicIp": "ENABLED" + } + }) } } } diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py index 25dc02dfc0..db59734dc8 100644 --- a/infrastructure/tools/lambdas/github_runner_webhook.py +++ b/infrastructure/tools/lambdas/github_runner_webhook.py @@ -7,6 +7,7 @@ WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"] GITHUB_TOKEN_SSM_NAME = os.environ["GITHUB_TOKEN_SSM_NAME"] +NETWORK_CONFIGURATION = os.environ["NETWORK_CONFIGURATION"] def handler(event, context): @@ -42,22 +43,23 @@ def handle_workflow_job(body, context): return labels = workflow_job["labels"] - if labels != ["self-hosted", "arm64-fargate"]: + arm64_fargate_label = next( + (label for label in labels if "arm64-fargate-" in label), None + ) + if not arm64_fargate_label: return + unique_run_id = arm64_fargate_label.replace("arm64-fargate-", "") + ssm_client = boto3.client("ssm") github_token = ssm_client.get_parameter(Name=GITHUB_TOKEN_SSM_NAME)["Parameter"][ "Value" ] payload = { - "name": "Test from Lambda", + "name": f"Runner for run #{unique_run_id}", "runner_group_id": 3, - "labels": [ - "lambda-test" - # 'self-hosted', - # 'arm64-fargate', - ], + "labels": [arm64_fargate_label], } payload_encoded = json.dumps(payload).encode("utf-8") print("sending payload:", payload_encoded) @@ -82,6 +84,12 @@ def handle_workflow_job(body, context): print("Body:", body) print("Context:", context) + ecs_client = boto3.client("ecs") + ecs_client.start_task( + cluster="github-actions-runners", + networkConfiguration=json.loads(NETWORK_CONFIGURATION), + ) + def verify_signature(payload_body, secret_token, signature_header): """Verify that the payload was sent from GitHub by validating SHA256. diff --git a/infrastructure/tools/vpc.tf b/infrastructure/tools/vpc.tf new file mode 100644 index 0000000000..7f94027976 --- /dev/null +++ b/infrastructure/tools/vpc.tf @@ -0,0 +1,76 @@ +locals { + public_azs_cidr = { + "eu-central-1a" : "10.0.1.0/24", + "eu-central-1b" : "10.0.2.0/24", + "eu-central-1c" : "10.0.3.0/24", + } + private_azs_cidr = { + "eu-central-1a" : "10.0.4.0/24", + "eu-central-1b" : "10.0.5.0/24", + "eu-central-1c" : "10.0.6.0/24", + } +} + +resource "aws_vpc" "default" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + + tags = { + Name = "main-vpc" + } +} + +resource "aws_subnet" "private" { + for_each = local.private_azs_cidr + vpc_id = aws_vpc.default.id + availability_zone = each.key + cidr_block = each.value + + tags = { + Name = "main-vpc-private-subnet-${each.key}" + Type = "private" + AZ = each.key + } +} + +resource "aws_subnet" "public" { + for_each = local.public_azs_cidr + vpc_id = aws_vpc.default.id + availability_zone = each.key + cidr_block = each.value + map_public_ip_on_launch = true + + tags = { + Name = "main-vpc-public-subnet-${each.key}" + Type = "public" + AZ = each.key + } +} + +resource "aws_route_table" "public" { + for_each = toset(keys(local.public_azs_cidr)) + vpc_id = aws_vpc.default.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.default.id + } + + tags = { + Name = "main-vpc-public-route-${each.value}" + } + + depends_on = [ + aws_internet_gateway.default + ] +} + +resource "aws_route_table_association" "public_subnet_to_public_route" { + for_each = toset(keys(local.public_azs_cidr)) + route_table_id = aws_route_table.public[each.value].id + subnet_id = aws_subnet.public[each.value].id +} + +resource "aws_internet_gateway" "default" { + vpc_id = aws_vpc.default.id +} From b905ea3ba51c6c19aebffb407adea72d681da502 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 20:47:46 +0100 Subject: [PATCH 19/68] change --- .github/workflows/build-backend.yml | 21 +++++-------------- infrastructure/tools/github_runner_lambda.tf | 20 ++++++++++++++++++ infrastructure/tools/github_runner_task.tf | 1 + .../tools/lambdas/github_runner_webhook.py | 16 +++++++++++--- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index ee033e8539..2d4983c4cb 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -20,23 +20,12 @@ jobs: aws-access-key-id: ${{ secrets.aws_access_key_id }} aws-secret-access-key: ${{ secrets.aws_secret_access_key }} aws-region: eu-central-1 - - name: Set up QEMU dependency - uses: docker/setup-qemu-action@v3 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - name: Build and push - uses: docker/build-push-action@v6 + - name: Run kaniko + uses: pythonitalia/kaniko-action@main with: - context: ./backend - file: ./backend/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + cache: true + cache-repository: ghcr.io/${{ github.repository }}/cache diff --git a/infrastructure/tools/github_runner_lambda.tf b/infrastructure/tools/github_runner_lambda.tf index e3fc381e56..f3f7560448 100644 --- a/infrastructure/tools/github_runner_lambda.tf +++ b/infrastructure/tools/github_runner_lambda.tf @@ -44,6 +44,24 @@ resource "aws_iam_role_policy" "github_runner_webhook_lambda_policy" { Resource = [ data.aws_ssm_parameter.github_token.arn ] + }, + { + Effect = "Allow" + Action = [ + "ecs:RunTask" + ] + Resource = [ + "${aws_ecs_task_definition.github_runner.arn}*", + ] + }, + { + Effect = "Allow" + Action = [ + "iam:PassRole" + ] + Resource = [ + aws_iam_role.github_runner_execution_role.arn + ] } ] }) @@ -75,6 +93,8 @@ resource "aws_lambda_function" "github_runner_webhook" { "assignPublicIp": "ENABLED" } }) + ECS_CLUSTER_NAME = aws_ecs_cluster.github_runners.name + ECS_TASK_DEFINITION = aws_ecs_task_definition.github_runner.arn } } } diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf index 2cda667cce..3905ce41d8 100644 --- a/infrastructure/tools/github_runner_task.tf +++ b/infrastructure/tools/github_runner_task.tf @@ -67,6 +67,7 @@ resource "aws_ecs_task_definition" "github_runner" { name = "runner" image = "ghcr.io/actions/actions-runner:2.321.0" essential = true + entrypoint = ["bash", "-c"] portMappings = [] logConfiguration = { logDriver = "awslogs" diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py index db59734dc8..7a3c045f07 100644 --- a/infrastructure/tools/lambdas/github_runner_webhook.py +++ b/infrastructure/tools/lambdas/github_runner_webhook.py @@ -8,6 +8,8 @@ WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"] GITHUB_TOKEN_SSM_NAME = os.environ["GITHUB_TOKEN_SSM_NAME"] NETWORK_CONFIGURATION = os.environ["NETWORK_CONFIGURATION"] +ECS_CLUSTER_NAME = os.environ["ECS_CLUSTER_NAME"] +ECS_TASK_DEFINITION = os.environ["ECS_TASK_DEFINITION"] def handler(event, context): @@ -62,7 +64,7 @@ def handle_workflow_job(body, context): "labels": [arm64_fargate_label], } payload_encoded = json.dumps(payload).encode("utf-8") - print("sending payload:", payload_encoded) + req = request.Request( "https://api.github.com/orgs/pythonitalia/actions/runners/generate-jitconfig", data=payload_encoded, @@ -85,9 +87,17 @@ def handle_workflow_job(body, context): print("Context:", context) ecs_client = boto3.client("ecs") - ecs_client.start_task( - cluster="github-actions-runners", + ecs_client.run_task( + cluster=ECS_CLUSTER_NAME, + taskDefinition=ECS_TASK_DEFINITION, networkConfiguration=json.loads(NETWORK_CONFIGURATION), + count=1, + launchType="FARGATE", + overrides={ + "containerOverrides": [ + {"name": "runner", "command": [f"./run.sh --jitconfig {jit_config}"]} + ] + }, ) From 1dff25d2add8a49db752019294dcfd43cb99619a Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 21:00:28 +0100 Subject: [PATCH 20/68] change --- .github/workflows/build-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 2d4983c4cb..3a34b62483 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -23,7 +23,7 @@ jobs: - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - name: Run kaniko - uses: pythonitalia/kaniko-action@main + uses: pythonitalia/kaniko-action@v1.0 with: push: true tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} From 7dd113a8c4bfdf65dc5102313ad293ebab7a8206 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 21:49:07 +0100 Subject: [PATCH 21/68] change --- .github/workflows/build-backend.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 3a34b62483..3047f3791c 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -9,6 +9,8 @@ jobs: build: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} + container: + image: gcr.io/kaniko-project/executor:debug steps: - uses: actions/checkout@v4 with: From b4a8ae15ecba551b6f0e85e7d2962d260c8ad83f Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 22:02:45 +0100 Subject: [PATCH 22/68] change --- infrastructure/tools/github_runner_task.tf | 4 ++-- infrastructure/tools/images/kaniko/Dockerfile | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 infrastructure/tools/images/kaniko/Dockerfile diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf index 3905ce41d8..2f3da02718 100644 --- a/infrastructure/tools/github_runner_task.tf +++ b/infrastructure/tools/github_runner_task.tf @@ -65,7 +65,7 @@ resource "aws_ecs_task_definition" "github_runner" { container_definitions = jsonencode([ { name = "runner" - image = "ghcr.io/actions/actions-runner:2.321.0" + image = "ghcr.io/pythonitalia/pycon/kaniko:latest" essential = true entrypoint = ["bash", "-c"] portMappings = [] @@ -77,7 +77,7 @@ resource "aws_ecs_task_definition" "github_runner" { "awslogs-stream-prefix" = "runner" } } - }, + } ]) runtime_platform { diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile new file mode 100644 index 0000000000..9f22825592 --- /dev/null +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -0,0 +1,11 @@ +FROM gcr.io/kaniko-project/executor:debug AS kaniko + +FROM ghcr.io/actions/actions-runner:2.321.0 + +COPY --from=kaniko /kaniko/ /kaniko/ + +ENV SSL_CERT_DIR /kaniko/ssl/certs +ENV PATH $PATH:/usr/local/bin:/kaniko +ENV DOCKER_CONFIG /kaniko/.docker/ + +USER root From fdc832c8d1b12d8d5952537dbc1c88cca96b9cc2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 22:03:43 +0100 Subject: [PATCH 23/68] change --- .github/workflows/build-backend.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 3047f3791c..c8b9b6388d 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -9,8 +9,6 @@ jobs: build: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} - container: - image: gcr.io/kaniko-project/executor:debug steps: - uses: actions/checkout@v4 with: @@ -24,10 +22,12 @@ jobs: aws-region: eu-central-1 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - - name: Run kaniko - uses: pythonitalia/kaniko-action@v1.0 - with: - push: true - tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} - cache: true - cache-repository: ghcr.io/${{ github.repository }}/cache + - name: Run ls + run: ls + # - name: Run kaniko + # uses: pythonitalia/kaniko-action@v1.0 + # with: + # push: true + # tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + # cache: true + # cache-repository: ghcr.io/${{ github.repository }}/cache From e5f5a693d6143e6b7f4148bd029d9f7dda7941f7 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 22:08:51 +0100 Subject: [PATCH 24/68] change --- .github/workflows/build-backend.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index c8b9b6388d..606dbaecd4 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -22,12 +22,13 @@ jobs: aws-region: eu-central-1 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - - name: Run ls - run: ls - # - name: Run kaniko - # uses: pythonitalia/kaniko-action@v1.0 - # with: - # push: true - # tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} - # cache: true - # cache-repository: ghcr.io/${{ github.repository }}/cache + - name: Run kaniko + working-directory: ./backend + run: | + /kaniko/executor \ + --dockerfile=./Dockerfile \ + --verbosity debug \ + --insecure \ + --skip-tls-verify \ + --force \ + --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} From 74fb86a9c265eec2e8f86945e9bb3042dfa465b2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 22:47:24 +0100 Subject: [PATCH 25/68] change --- .github/workflows/build-backend.yml | 4 +--- infrastructure/tools/images/kaniko/Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 606dbaecd4..a19c44a28e 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -27,8 +27,6 @@ jobs: run: | /kaniko/executor \ --dockerfile=./Dockerfile \ + --context=. \ --verbosity debug \ - --insecure \ - --skip-tls-verify \ - --force \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile index 9f22825592..abe8c5860c 100644 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -2,10 +2,10 @@ FROM gcr.io/kaniko-project/executor:debug AS kaniko FROM ghcr.io/actions/actions-runner:2.321.0 -COPY --from=kaniko /kaniko/ /kaniko/ +USER runner + +COPY --chown=runner:docker --from=kaniko /kaniko/ /kaniko/ ENV SSL_CERT_DIR /kaniko/ssl/certs ENV PATH $PATH:/usr/local/bin:/kaniko ENV DOCKER_CONFIG /kaniko/.docker/ - -USER root From 8573af5a94418b758b2f779365043614699bb6b2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 23:05:37 +0100 Subject: [PATCH 26/68] change --- .github/workflows/build-backend.yml | 2 +- infrastructure/tools/images/kaniko/Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index a19c44a28e..2d8b165936 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -27,6 +27,6 @@ jobs: run: | /kaniko/executor \ --dockerfile=./Dockerfile \ - --context=. \ + --context=./ \ --verbosity debug \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile index abe8c5860c..86facf4e48 100644 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -2,10 +2,10 @@ FROM gcr.io/kaniko-project/executor:debug AS kaniko FROM ghcr.io/actions/actions-runner:2.321.0 -USER runner - -COPY --chown=runner:docker --from=kaniko /kaniko/ /kaniko/ +COPY --from=kaniko /kaniko/ /kaniko/ ENV SSL_CERT_DIR /kaniko/ssl/certs ENV PATH $PATH:/usr/local/bin:/kaniko ENV DOCKER_CONFIG /kaniko/.docker/ + +ENV RUNNER_ALLOW_RUNASROOT 1 From 4cef3fb05beb792eb245e090e7f608e42f70918b Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 23:20:55 +0100 Subject: [PATCH 27/68] change --- infrastructure/tools/images/kaniko/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile index 86facf4e48..88b4e29399 100644 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -9,3 +9,4 @@ ENV PATH $PATH:/usr/local/bin:/kaniko ENV DOCKER_CONFIG /kaniko/.docker/ ENV RUNNER_ALLOW_RUNASROOT 1 +USER root From ffdfa285b4e9a2c82504b77f9cd98a50505fddc8 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 23:36:48 +0100 Subject: [PATCH 28/68] change --- infrastructure/tools/images/kaniko/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile index 88b4e29399..66b4846080 100644 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -10,3 +10,4 @@ ENV DOCKER_CONFIG /kaniko/.docker/ ENV RUNNER_ALLOW_RUNASROOT 1 USER root +RUN groupadd messagebus || true From 22ec79bfa188b74bb3bf19ba1791a28ebacec1a3 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 23:39:20 +0100 Subject: [PATCH 29/68] change --- infrastructure/tools/images/kaniko/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile index 66b4846080..9444002e00 100644 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -10,4 +10,4 @@ ENV DOCKER_CONFIG /kaniko/.docker/ ENV RUNNER_ALLOW_RUNASROOT 1 USER root -RUN groupadd messagebus || true +RUN sed -i '/messagebus/d' /var/lib/dpkg/statoverride From 7384f6abb27a46c9c95b04883f6359df16f18c05 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 23:54:40 +0100 Subject: [PATCH 30/68] change --- .github/workflows/build-backend.yml | 4 ++++ infrastructure/tools/lambdas/github_runner_webhook.py | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 2d8b165936..2bc1da9e2e 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -30,3 +30,7 @@ jobs: --context=./ \ --verbosity debug \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + - name: Test + if: always() + run: | + sleep diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py index 7a3c045f07..faa30200d1 100644 --- a/infrastructure/tools/lambdas/github_runner_webhook.py +++ b/infrastructure/tools/lambdas/github_runner_webhook.py @@ -91,6 +91,7 @@ def handle_workflow_job(body, context): cluster=ECS_CLUSTER_NAME, taskDefinition=ECS_TASK_DEFINITION, networkConfiguration=json.loads(NETWORK_CONFIGURATION), + enableExecuteCommand=True, count=1, launchType="FARGATE", overrides={ From bd6253735b7da4acea1e463a7fff9db42cabdfd9 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 4 Jan 2025 23:59:07 +0100 Subject: [PATCH 31/68] change --- infrastructure/tools/github_runner_task.tf | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf index 2f3da02718..8ef1839b4c 100644 --- a/infrastructure/tools/github_runner_task.tf +++ b/infrastructure/tools/github_runner_task.tf @@ -48,6 +48,44 @@ resource "aws_iam_role_policy" "github_runner_execution_role_policy" { }) } +data "aws_iam_policy_document" "github_runner_task_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + + +resource "aws_iam_role" "github_runner_task_role" { + name = "github_runner_task_role" + assume_role_policy = data.aws_iam_policy_document.github_runner_task_assume_role.json +} + +resource "aws_iam_role_policy" "github_runner_task_role_policy" { + name = "github_runner_task_role_policy" + role = aws_iam_role.github_runner_task_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "ecs:ExecuteCommand", + "ssmmessages:*" + ] + Resource = "*" + } + ] + }) +} + resource "aws_cloudwatch_log_group" "github_runner" { name = "/github-runner/" @@ -61,6 +99,7 @@ resource "aws_ecs_task_definition" "github_runner" { cpu = 1024 memory = 2048 execution_role_arn = aws_iam_role.github_runner_execution_role.arn + task_role_arn = aws_iam_role.github_runner_task_role.arn container_definitions = jsonencode([ { From 2d9eb72345404668c26b060011900cf1cc7192b2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 00:01:39 +0100 Subject: [PATCH 32/68] change --- infrastructure/tools/github_runner_lambda.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/tools/github_runner_lambda.tf b/infrastructure/tools/github_runner_lambda.tf index f3f7560448..2955f8ba6c 100644 --- a/infrastructure/tools/github_runner_lambda.tf +++ b/infrastructure/tools/github_runner_lambda.tf @@ -60,7 +60,8 @@ resource "aws_iam_role_policy" "github_runner_webhook_lambda_policy" { "iam:PassRole" ] Resource = [ - aws_iam_role.github_runner_execution_role.arn + aws_iam_role.github_runner_execution_role.arn, + aws_iam_role.github_runner_task_role.arn, ] } ] From 311f36baa44c52e3e10fb23b1f9dce7ff81826b9 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 00:05:35 +0100 Subject: [PATCH 33/68] change --- .github/workflows/build-backend.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 2bc1da9e2e..7400576f9a 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -22,6 +22,10 @@ jobs: aws-region: eu-central-1 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 + - name: Test + if: always() + run: | + sleep - name: Run kaniko working-directory: ./backend run: | @@ -30,7 +34,3 @@ jobs: --context=./ \ --verbosity debug \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} - - name: Test - if: always() - run: | - sleep From c85b94269f3f12df0f7a4a39a5fd2798e1146c9e Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 00:07:42 +0100 Subject: [PATCH 34/68] change --- .github/workflows/build-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 7400576f9a..338c55c76d 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -25,7 +25,7 @@ jobs: - name: Test if: always() run: | - sleep + tail -f /dev/null - name: Run kaniko working-directory: ./backend run: | From ae9194b4032f10d49f044f50e1454f86eecbc58d Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 00:34:37 +0100 Subject: [PATCH 35/68] changes --- .github/workflows/build-backend.yml | 2 +- backend/Dockerfile | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 338c55c76d..54613dccd2 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -31,6 +31,6 @@ jobs: run: | /kaniko/executor \ --dockerfile=./Dockerfile \ - --context=./ \ + --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --verbosity debug \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} diff --git a/backend/Dockerfile b/backend/Dockerfile index 9a4c81780b..798adfa486 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,18 +14,16 @@ ENV DJANGO_SETTINGS_MODULE=pycon.settings.prod \ CELERY_RESULT_BACKEND=demo \ HASHID_DEFAULT_SECRET_SALT=demo -RUN apt-get update -y && apt-get install -y \ - # weasyprint - libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 \ - # postgres - libpq-dev - FROM base AS build-stage ARG FUNCTION_DIR RUN apt-get update -y && apt-get install -y \ - gcc git + gcc git \ + # weasyprint + libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 \ + # postgres + libpq-dev RUN mkdir -p ${FUNCTION_DIR} WORKDIR ${FUNCTION_DIR} @@ -68,6 +66,12 @@ ARG FUNCTION_DIR WORKDIR ${FUNCTION_DIR} +RUN apt-get update -y && apt-get install -y \ + # weasyprint + libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 \ + # postgres + libpq-dev + ENV LIBRARY_PATH=/lib:/usr/lib LD_LIBRARY_PATH=/lib:/usr/lib RUN apt-get update -y && apt-get install -y curl From 435fe7b21fee0179519d638ff78e0eb2cd4911bd Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 01:03:44 +0100 Subject: [PATCH 36/68] change --- .github/workflows/build-backend.yml | 8 -------- infrastructure/tools/github_runner_task.tf | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 54613dccd2..284aacfc35 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -10,10 +10,6 @@ jobs: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -22,10 +18,6 @@ jobs: aws-region: eu-central-1 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - - name: Test - if: always() - run: | - tail -f /dev/null - name: Run kaniko working-directory: ./backend run: | diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf index 8ef1839b4c..e880afa452 100644 --- a/infrastructure/tools/github_runner_task.tf +++ b/infrastructure/tools/github_runner_task.tf @@ -96,8 +96,8 @@ resource "aws_ecs_task_definition" "github_runner" { family = "github-runner" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" - cpu = 1024 - memory = 2048 + cpu = 8192 + memory = 16384 execution_role_arn = aws_iam_role.github_runner_execution_role.arn task_role_arn = aws_iam_role.github_runner_task_role.arn From e6c0dcf18e9514ef9793cb6693b4dacafd3228d3 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 01:05:31 +0100 Subject: [PATCH 37/68] change --- .github/workflows/build-backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 284aacfc35..cf7350be89 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -24,5 +24,6 @@ jobs: /kaniko/executor \ --dockerfile=./Dockerfile \ --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ + --context-sub-path=./backend \ --verbosity debug \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} From 4ba1ea2c1332c457e7e6d4796122e0e7dac206ea Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 01:07:12 +0100 Subject: [PATCH 38/68] change --- .github/workflows/build-backend.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index cf7350be89..6e802bc7c5 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -19,7 +19,6 @@ jobs: - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - name: Run kaniko - working-directory: ./backend run: | /kaniko/executor \ --dockerfile=./Dockerfile \ From 47a62344bc124b5283a59cf47bde439e91383b5e Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 01:24:13 +0100 Subject: [PATCH 39/68] change --- .github/workflows/build-backend.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 6e802bc7c5..cc3f519eae 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -10,6 +10,10 @@ jobs: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: From 7eaf443ec4dbe9bb157747a7136c0a3665034014 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 01:30:20 +0100 Subject: [PATCH 40/68] change --- .github/workflows/build-backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index cc3f519eae..bdccc736aa 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -22,6 +22,7 @@ jobs: aws-region: eu-central-1 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 + - run: tail -f /dev/null - name: Run kaniko run: | /kaniko/executor \ From 0251936a19998083153b55c97a1e479f35b7f61c Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 01:57:34 +0100 Subject: [PATCH 41/68] change --- .github/workflows/build-backend.yml | 1 + infrastructure/tools/images/kaniko/Dockerfile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index bdccc736aa..550d429fcd 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -30,4 +30,5 @@ jobs: --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --context-sub-path=./backend \ --verbosity debug \ + --cleanup \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile index 9444002e00..6d703c707b 100644 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ b/infrastructure/tools/images/kaniko/Dockerfile @@ -11,3 +11,6 @@ ENV DOCKER_CONFIG /kaniko/.docker/ ENV RUNNER_ALLOW_RUNASROOT 1 USER root RUN sed -i '/messagebus/d' /var/lib/dpkg/statoverride + +RUN mv /home/runner/ /kaniko/actions-runner/ +WORKDIR /kaniko/actions-runner/ From 7051a2e926eb91bdbc3def30facbf935e409e591 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 02:02:55 +0100 Subject: [PATCH 42/68] change --- .github/workflows/build-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 550d429fcd..5c9fa2f647 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -22,7 +22,7 @@ jobs: aws-region: eu-central-1 - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - - run: tail -f /dev/null + # - run: tail -f /dev/null - name: Run kaniko run: | /kaniko/executor \ From 39111b9c1abe007a1db5c94c840f4a4fa2634f6b Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 02:13:34 +0100 Subject: [PATCH 43/68] change --- .github/workflows/build-backend.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 5c9fa2f647..1e5237047e 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -29,6 +29,5 @@ jobs: --dockerfile=./Dockerfile \ --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --context-sub-path=./backend \ - --verbosity debug \ --cleanup \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} From 7857da2ba04d5ba2d4184d7a683eb2bddb5ab9f1 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 02:14:29 +0100 Subject: [PATCH 44/68] change --- .github/workflows/build-backend.yml | 2 +- backend/Dockerfile | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 1e5237047e..ad82f44b84 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -30,4 +30,4 @@ jobs: --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --context-sub-path=./backend \ --cleanup \ - --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-test diff --git a/backend/Dockerfile b/backend/Dockerfile index 798adfa486..d411e9abf5 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -91,4 +91,3 @@ RUN mkdir -p assets && .venv/bin/python manage.py collectstatic --noinput ENTRYPOINT ["/home/app/.venv/bin/gunicorn"] CMD [ "pycon.wsgi" ] -#test From eacca64b82ebf6f87a1887553b7978bcf9325460 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 02:17:10 +0100 Subject: [PATCH 45/68] change --- .github/workflows/build-backend.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index ad82f44b84..4e97187a24 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -29,5 +29,4 @@ jobs: --dockerfile=./Dockerfile \ --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --context-sub-path=./backend \ - --cleanup \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-test From 4a097dc277bf2f17ff05e8135ee1a3812277f930 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 02:31:26 +0100 Subject: [PATCH 46/68] change --- .github/workflows/build-backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 4e97187a24..2b08c9e444 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -29,4 +29,5 @@ jobs: --dockerfile=./Dockerfile \ --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --context-sub-path=./backend \ + --cache=true \ --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-test From d9a6ddc425949a82fa6e5d13116b537f95625871 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 03:27:17 +0100 Subject: [PATCH 47/68] change --- .github/workflows/build-backend.yml | 13 ----- infrastructure/global/.terraform.lock.hcl | 70 ++++++----------------- infrastructure/global/ecr_repos/main.tf | 44 +++++++++++++- infrastructure/global/main.tf | 2 +- 4 files changed, 61 insertions(+), 68 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 2b08c9e444..c44c106ae5 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -10,19 +10,6 @@ jobs: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@v2 - # - run: tail -f /dev/null - name: Run kaniko run: | /kaniko/executor \ diff --git a/infrastructure/global/.terraform.lock.hcl b/infrastructure/global/.terraform.lock.hcl index d553847820..a3fa96c6d9 100644 --- a/infrastructure/global/.terraform.lock.hcl +++ b/infrastructure/global/.terraform.lock.hcl @@ -1,61 +1,25 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/archive" { - version = "2.5.0" - hashes = [ - "h1:HXf8h8Z4JYEkBND/JiqC+CjluKqifKoDGrL1IsRo15M=", - "zh:3b5774d20e87058d6d67d9ad4ce3fc4a5f7ea7748d345fa6721e24a0cbb0a3d4", - "zh:3b94e706ac0f5151880ccc9e63d33c4113361f27e64224a942caa04a5a19cd44", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7d7201858fa9376029818c9d017b4b53a933cea75480306b1122663d1e8eea2b", - "zh:8c8c7537978adf12271fe143f93b3587bb5dbabf8202ff49d0e3955b7bddc24b", - "zh:a5942584665a2689e73f3a3c43296adeaeb7e8698631d157419aa931ff856907", - "zh:a63673abdba624d60c84b819184fe86422bdbdf6bc73f68d903a7191aed32c00", - "zh:bcd1586cc32b263265e09e78f56dba3a6b6b19f5371c099a9d7a1bfe0b0667cc", - "zh:cc9e70e186e4dcef60208b4a64b42e6813b197e21ea106a96bb4eb23b54c3e44", - "zh:d4c8a0f69412892507a2c9ec0e334bcc2812a54b81212420d4f2c96ef58f713a", - "zh:e91e6d90bbc15252310eca6400d4188b29260aab0539480a3fc7b45e4d19c446", - "zh:fc468449c0dbda56aae6cb924e4a67578d18504b5b06e8989783182c6b4a5f73", - ] -} - provider "registry.terraform.io/hashicorp/aws" { - version = "5.64.0" - constraints = "5.64.0" + version = "5.82.2" + constraints = "5.82.2" hashes = [ - "h1:Xasb457vfMG/1SGu6KSApCzAqUHMlsL028OQu3dZVv8=", - "zh:1d361f8062c68c9d5ac14b0aa8390709542129b8a9b258e61bbbabc706078b44", - "zh:39dcbf53e3896bdd77071384c8fad4a5862c222c73f3bcf356aca488101f22fd", - "zh:3fad63505f0c5b6f01cc9a6ef02b2226983b79424126a9caf6eb724f654299f4", - "zh:53a8b90d00829cc27e3171a13a8ff1404ee0ea018e73f31d3f916d246cc39613", - "zh:5734c25ef5a04b40f3c1ac5f817f11e42ee3328f74dbc141c0e64afbb0acc834", - "zh:66ea14dbd87f291ce4a877123363933d3ca4022f209f885807a6689c22c24e80", - "zh:68e79654ad0894a3d93134c3377748ace3058d5fad5ec09d1e9a8f8f9b8a47ea", - "zh:7b74259d0ceef0c49cea6bcd171df997b6bad141085bbadded15b440faeb0eee", - "zh:988ebfb5d115dc57070b5abf2e4200ad49cde535f27fd2ba5e34cf9ab336a57f", + "h1:RuPaHbllUB8a2TGTyc149wJfoh6zhIEjUvFYKR6iP2E=", + "zh:0262fc96012fb7e173e1b7beadd46dfc25b1dc7eaef95b90e936fc454724f1c8", + "zh:397413613d27f4f54d16efcbf4f0a43c059bd8d827fe34287522ae182a992f9b", + "zh:436c0c5d56e1da4f0a4c13129e12a0b519d12ab116aed52029b183f9806866f3", + "zh:4d942d173a2553d8d532a333a0482a090f4e82a2238acf135578f163b6e68470", + "zh:624aebc549bfbce06cc2ecfd8631932eb874ac7c10eb8466ce5b9a2fbdfdc724", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a0a2d4efe2835f0101a0a5024e044a3f28c00e10a8d87fce89c707ef6db75cea", - "zh:aecb3e4b9121771dee9cac7975bf5d0657b5f3e8b57788c455beaeb0f3c48d93", - "zh:d2d3393170b8ef761d3146f39f6788c4a3e876e6c5d4cedca4870c2680688ae6", - "zh:daba5a005c1baa4a5eefbfb86d43ccf880eb5b42e8136f0d932f55886d72bda0", - "zh:de16a6ff3baacdaf9609a0a89aa1913fc19cccaf5ee0fc1c49c5a075baa47c02", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + "zh:9e632dee2dfdf01b371cca7854b1ec63ceefa75790e619b0642b34d5514c6733", + "zh:a07567acb115b60a3df8f6048d12735b9b3bcf85ec92a62f77852e13d5a3c096", + "zh:ab7002df1a1be6432ac0eb1b9f6f0dd3db90973cd5b1b0b33d2dae54553dfbd7", + "zh:bc1ff65e2016b018b3e84db7249b2cd0433cb5c81dc81f9f6158f2197d6b9fde", + "zh:bcad84b1d767f87af6e1ba3dc97fdb8f2ad5de9224f192f1412b09aba798c0a8", + "zh:cf917dceaa0f9d55d9ff181b5dcc4d1e10af21b6671811b315ae2a6eda866a2a", + "zh:d8e90ecfb3216f3cc13ccde5a16da64307abb6e22453aed2ac3067bbf689313b", + "zh:d9054e0e40705df729682ad34c20db8695d57f182c65963abd151c6aba1ab0d3", + "zh:ecf3a4f3c57eb7e89f71b8559e2a71e4cdf94eea0118ec4f2cb37e4f4d71a069", ] } diff --git a/infrastructure/global/ecr_repos/main.tf b/infrastructure/global/ecr_repos/main.tf index f628aa0ca3..1578c8d6f7 100644 --- a/infrastructure/global/ecr_repos/main.tf +++ b/infrastructure/global/ecr_repos/main.tf @@ -1,11 +1,21 @@ locals { services = [ "pycon-backend", + "pycon-backend/cache", "pycon-frontend", - "pretix" + "pycon-frontend/cache", + "pretix", + "pretix/cache", ] + infrastructure_tools_account_id = [ + for account in data.aws_organizations_organization.organization.non_master_accounts : + account.id + if account.name == "Infrastructure Tools" + ][0] } +data "aws_organizations_organization" "organization" {} + resource "aws_ecr_repository" "service_repo" { for_each = toset(local.services) name = "pythonit/${each.key}" @@ -15,3 +25,35 @@ resource "aws_ecr_repository" "service_repo" { scan_on_push = false } } + +data "aws_iam_policy_document" "access_from_infrastructure_account" { + statement { + sid = "access from infrastructure account" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [local.infrastructure_tools_account_id] + } + + actions = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:DescribeRepositories", + "ecr:GetRepositoryPolicy", + "ecr:ListImages", + "ecr:BatchDeleteImage", + ] + } +} + +resource "aws_ecr_repository_policy" "access_from_infrastructure_account" { + for_each = toset(local.services) + repository = aws_ecr_repository.service_repo[each.key].name + policy = data.aws_iam_policy_document.access_from_infrastructure_account.json +} diff --git a/infrastructure/global/main.tf b/infrastructure/global/main.tf index 4e13042969..c94f8ef3b2 100644 --- a/infrastructure/global/main.tf +++ b/infrastructure/global/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.64.0" + version = "5.82.2" } } From 78a8d8aa0535becacc26fa295c9e09e506cfa246 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 03:38:16 +0100 Subject: [PATCH 48/68] change --- .github/workflows/build-backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index c44c106ae5..c2a98a855b 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -10,6 +10,7 @@ jobs: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} steps: + - run: aws sts get-caller-identity - name: Run kaniko run: | /kaniko/executor \ From 2e4b2aa49cc712c24363696bce65ef18cd3eb692 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 04:33:30 +0100 Subject: [PATCH 49/68] change --- .github/workflows/build-backend.yml | 6 +++++- infrastructure/tools/github_runner_task.tf | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index c2a98a855b..76c02b1d4b 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -10,7 +10,11 @@ jobs: name: Build runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} steps: - - run: aws sts get-caller-identity + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + # - run: tail -f /dev/null - name: Run kaniko run: | /kaniko/executor \ diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf index e880afa452..80952bfa31 100644 --- a/infrastructure/tools/github_runner_task.tf +++ b/infrastructure/tools/github_runner_task.tf @@ -41,6 +41,17 @@ resource "aws_iam_role_policy" "github_runner_execution_role_policy" { "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:DescribeRepositories", + "ecr:GetRepositoryPolicy", + "ecr:ListImages", + "ecr:BatchDeleteImage", ] Resource = "*" } From b79e1dc5c30e81feba5deeb56e4e4df941aef17c Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 04:38:28 +0100 Subject: [PATCH 50/68] change --- .github/workflows/build-backend.yml | 2 +- infrastructure/tools/lambdas/github_runner_webhook.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 76c02b1d4b..c57a38619e 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -14,7 +14,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-region: eu-central-1 - # - run: tail -f /dev/null + - run: tail -f /dev/null - name: Run kaniko run: | /kaniko/executor \ diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py index faa30200d1..bc0ff7f9ea 100644 --- a/infrastructure/tools/lambdas/github_runner_webhook.py +++ b/infrastructure/tools/lambdas/github_runner_webhook.py @@ -93,7 +93,6 @@ def handle_workflow_job(body, context): networkConfiguration=json.loads(NETWORK_CONFIGURATION), enableExecuteCommand=True, count=1, - launchType="FARGATE", overrides={ "containerOverrides": [ {"name": "runner", "command": [f"./run.sh --jitconfig {jit_config}"]} From 58c01ba9d80429ef0a7720b2a871c69c138e0b59 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 04:52:55 +0100 Subject: [PATCH 51/68] change --- infrastructure/tools/github_runner_task.tf | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf index 80952bfa31..e9e004dfc2 100644 --- a/infrastructure/tools/github_runner_task.tf +++ b/infrastructure/tools/github_runner_task.tf @@ -33,27 +33,6 @@ resource "aws_iam_role_policy" "github_runner_execution_role_policy" { aws_cloudwatch_log_group.github_runner.arn, "${aws_cloudwatch_log_group.github_runner.arn}*" ] - }, - { - Effect = "Allow" - Action = [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability", - "ecr:PutImage", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "ecr:DescribeRepositories", - "ecr:GetRepositoryPolicy", - "ecr:ListImages", - "ecr:BatchDeleteImage", - ] - Resource = "*" } ] }) @@ -92,6 +71,27 @@ resource "aws_iam_role_policy" "github_runner_task_role_policy" { "ssmmessages:*" ] Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:DescribeRepositories", + "ecr:GetRepositoryPolicy", + "ecr:ListImages", + "ecr:BatchDeleteImage", + ] + Resource = "*" } ] }) From 24f239289378bd7921042d4165f74c843d44a0bd Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 04:54:04 +0100 Subject: [PATCH 52/68] change --- .github/workflows/build-backend.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index c57a38619e..149bb0d70b 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -14,7 +14,6 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-region: eu-central-1 - - run: tail -f /dev/null - name: Run kaniko run: | /kaniko/executor \ From b26c967854d9741cca37796f2ded6516eec05566 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 12:30:02 +0100 Subject: [PATCH 53/68] change --- .github/workflows/build-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 149bb0d70b..684b28a276 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -21,4 +21,4 @@ jobs: --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ --context-sub-path=./backend \ --cache=true \ - --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-test + --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} From 69dd35e45c3a818f7918b548836d5d8770bcc2d1 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:03:31 +0100 Subject: [PATCH 54/68] Remove kaniko --- .github/workflows/build-backend.yml | 36 +++-- .github/workflows/build-frontend.yml | 2 +- .github/workflows/build-pretix.yml | 2 +- .tool-versions | 1 + infrastructure/tools/.tool-versions | 1 - infrastructure/tools/github_repo.tf | 3 - infrastructure/tools/github_runner_cluster.tf | 15 -- infrastructure/tools/github_runner_lambda.tf | 106 -------------- infrastructure/tools/github_runner_task.tf | 137 ------------------ infrastructure/tools/github_webhook.tf | 17 --- infrastructure/tools/images/kaniko/Dockerfile | 16 -- .../tools/lambdas/github_runner_webhook.py | 125 ---------------- 12 files changed, 30 insertions(+), 431 deletions(-) delete mode 100644 infrastructure/tools/.tool-versions delete mode 100644 infrastructure/tools/github_repo.tf delete mode 100644 infrastructure/tools/github_runner_cluster.tf delete mode 100644 infrastructure/tools/github_runner_lambda.tf delete mode 100644 infrastructure/tools/github_runner_task.tf delete mode 100644 infrastructure/tools/github_webhook.tf delete mode 100644 infrastructure/tools/images/kaniko/Dockerfile delete mode 100644 infrastructure/tools/lambdas/github_runner_webhook.py diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 684b28a276..4a2ddea91e 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -8,17 +8,35 @@ on: jobs: build: name: Build - runs-on: ${{ format('arm64-fargate-{0}{1}{2}', github.run_id, github.run_number, github.run_attempt) }} + runs-on: [self-hosted] steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} aws-region: eu-central-1 - - name: Run kaniko - run: | - /kaniko/executor \ - --dockerfile=./Dockerfile \ - --context=git://github.com/${{ github.repository }}#${{ github.ref }} \ - --context-sub-path=./backend \ - --cache=true \ - --destination=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + - name: Set up QEMU dependency + uses: docker/setup-qemu-action@v3 + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./backend + file: ./backend/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml index a521f9f4f6..c71008f76e 100644 --- a/.github/workflows/build-frontend.yml +++ b/.github/workflows/build-frontend.yml @@ -11,7 +11,7 @@ on: jobs: build: name: Build - runs-on: [self-hosted, arm64-fargate] + runs-on: [self-hosted] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/build-pretix.yml b/.github/workflows/build-pretix.yml index 677acd3c91..45e73c3b42 100644 --- a/.github/workflows/build-pretix.yml +++ b/.github/workflows/build-pretix.yml @@ -8,7 +8,7 @@ on: jobs: build: name: Build pretix - runs-on: [self-hosted, arm64-fargate] + runs-on: [self-hosted] steps: - uses: actions/checkout@v4 with: diff --git a/.tool-versions b/.tool-versions index 1a55cb8044..056a60c037 100644 --- a/.tool-versions +++ b/.tool-versions @@ -3,3 +3,4 @@ terraform 1.3.4 python 3.11.2 rust 1.61.0 pnpm 9 +opentofu 1.8.8 diff --git a/infrastructure/tools/.tool-versions b/infrastructure/tools/.tool-versions deleted file mode 100644 index 33dbae16aa..0000000000 --- a/infrastructure/tools/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -opentofu 1.8.8 diff --git a/infrastructure/tools/github_repo.tf b/infrastructure/tools/github_repo.tf deleted file mode 100644 index 6bedb2e402..0000000000 --- a/infrastructure/tools/github_repo.tf +++ /dev/null @@ -1,3 +0,0 @@ -data "github_repository" "pycon" { - full_name = "pythonitalia/pycon" -} diff --git a/infrastructure/tools/github_runner_cluster.tf b/infrastructure/tools/github_runner_cluster.tf deleted file mode 100644 index d7739d0646..0000000000 --- a/infrastructure/tools/github_runner_cluster.tf +++ /dev/null @@ -1,15 +0,0 @@ -resource "aws_ecs_cluster" "github_runners" { - name = "github-actions-runners" -} - -resource "aws_ecs_cluster_capacity_providers" "github_runners" { - cluster_name = aws_ecs_cluster.github_runners.name - - capacity_providers = ["FARGATE_SPOT"] - - default_capacity_provider_strategy { - base = 1 - weight = 100 - capacity_provider = "FARGATE_SPOT" - } -} diff --git a/infrastructure/tools/github_runner_lambda.tf b/infrastructure/tools/github_runner_lambda.tf deleted file mode 100644 index 2955f8ba6c..0000000000 --- a/infrastructure/tools/github_runner_lambda.tf +++ /dev/null @@ -1,106 +0,0 @@ -data "aws_iam_policy_document" "github_runner_webhook_assume_role" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["lambda.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - -resource "aws_iam_role" "github_runner_webhook_role" { - name = "github_runner_webhook_role" - assume_role_policy = data.aws_iam_policy_document.github_runner_webhook_assume_role.json -} - -data "aws_ssm_parameter" "github_token" { - name = "/github-runner/github-token" -} - -resource "aws_iam_role_policy" "github_runner_webhook_lambda_policy" { - name = "github_runner_webhook_lambda_policy" - role = aws_iam_role.github_runner_webhook_role.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - Resource = "*" - }, - { - Effect = "Allow" - Action = [ - "ssm:GetParameter" - ] - Resource = [ - data.aws_ssm_parameter.github_token.arn - ] - }, - { - Effect = "Allow" - Action = [ - "ecs:RunTask" - ] - Resource = [ - "${aws_ecs_task_definition.github_runner.arn}*", - ] - }, - { - Effect = "Allow" - Action = [ - "iam:PassRole" - ] - Resource = [ - aws_iam_role.github_runner_execution_role.arn, - aws_iam_role.github_runner_task_role.arn, - ] - } - ] - }) -} - -data "archive_file" "github_runner_webhook_artifact" { - type = "zip" - source_file = "${path.root}/lambdas/github_runner_webhook.py" - output_path = "${path.root}/.archive_files/github_runner_webhook.zip" -} - -resource "aws_lambda_function" "github_runner_webhook" { - function_name = "github_runner_webhook" - role = aws_iam_role.github_runner_webhook_role.arn - handler = "github_runner_webhook.handler" - runtime = "python3.13" - filename = data.archive_file.github_runner_webhook_artifact.output_path - source_code_hash = data.archive_file.github_runner_webhook_artifact.output_base64sha256 - timeout = 60 - - environment { - variables = { - WEBHOOK_SECRET = random_password.webhook_secret.result - GITHUB_TOKEN_SSM_NAME = data.aws_ssm_parameter.github_token.name - NETWORK_CONFIGURATION = jsonencode({ - "awsvpcConfiguration": { - "subnets": [aws_subnet.public["eu-central-1a"].id], - "securityGroups": [], - "assignPublicIp": "ENABLED" - } - }) - ECS_CLUSTER_NAME = aws_ecs_cluster.github_runners.name - ECS_TASK_DEFINITION = aws_ecs_task_definition.github_runner.arn - } - } -} - -resource "aws_lambda_function_url" "github_runner_webhook" { - function_name = aws_lambda_function.github_runner_webhook.function_name - authorization_type = "NONE" -} diff --git a/infrastructure/tools/github_runner_task.tf b/infrastructure/tools/github_runner_task.tf deleted file mode 100644 index e9e004dfc2..0000000000 --- a/infrastructure/tools/github_runner_task.tf +++ /dev/null @@ -1,137 +0,0 @@ -data "aws_iam_policy_document" "github_runner_execution_assume_role" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - -resource "aws_iam_role" "github_runner_execution_role" { - name = "github_runner_execution_role" - assume_role_policy = data.aws_iam_policy_document.github_runner_execution_assume_role.json -} - -resource "aws_iam_role_policy" "github_runner_execution_role_policy" { - name = "github_runner_execution_role_policy" - role = aws_iam_role.github_runner_execution_role.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - Resource = [ - aws_cloudwatch_log_group.github_runner.arn, - "${aws_cloudwatch_log_group.github_runner.arn}*" - ] - } - ] - }) -} - -data "aws_iam_policy_document" "github_runner_task_assume_role" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - - -resource "aws_iam_role" "github_runner_task_role" { - name = "github_runner_task_role" - assume_role_policy = data.aws_iam_policy_document.github_runner_task_assume_role.json -} - -resource "aws_iam_role_policy" "github_runner_task_role_policy" { - name = "github_runner_task_role_policy" - role = aws_iam_role.github_runner_task_role.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "ecs:ExecuteCommand", - "ssmmessages:*" - ] - Resource = "*" - }, - { - Effect = "Allow" - Action = [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability", - "ecr:PutImage", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "ecr:DescribeRepositories", - "ecr:GetRepositoryPolicy", - "ecr:ListImages", - "ecr:BatchDeleteImage", - ] - Resource = "*" - } - ] - }) -} - - -resource "aws_cloudwatch_log_group" "github_runner" { - name = "/github-runner/" - retention_in_days = 1 -} - -resource "aws_ecs_task_definition" "github_runner" { - family = "github-runner" - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - cpu = 8192 - memory = 16384 - execution_role_arn = aws_iam_role.github_runner_execution_role.arn - task_role_arn = aws_iam_role.github_runner_task_role.arn - - container_definitions = jsonencode([ - { - name = "runner" - image = "ghcr.io/pythonitalia/pycon/kaniko:latest" - essential = true - entrypoint = ["bash", "-c"] - portMappings = [] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.github_runner.name - "awslogs-region" = "eu-central-1" - "awslogs-stream-prefix" = "runner" - } - } - } - ]) - - runtime_platform { - operating_system_family = "LINUX" - cpu_architecture = "ARM64" - } -} diff --git a/infrastructure/tools/github_webhook.tf b/infrastructure/tools/github_webhook.tf deleted file mode 100644 index aa7d4c51c5..0000000000 --- a/infrastructure/tools/github_webhook.tf +++ /dev/null @@ -1,17 +0,0 @@ -resource "random_password" "webhook_secret" { - length = 64 - special = true - override_special = "!#$%&*()-_=+[]{}<>:?" -} - -resource "github_repository_webhook" "github_runner_notify" { - repository = data.github_repository.pycon.name - events = ["workflow_job"] - active = true - - configuration { - url = aws_lambda_function_url.github_runner_webhook.function_url - secret = random_password.webhook_secret.result - content_type = "json" - } -} diff --git a/infrastructure/tools/images/kaniko/Dockerfile b/infrastructure/tools/images/kaniko/Dockerfile deleted file mode 100644 index 6d703c707b..0000000000 --- a/infrastructure/tools/images/kaniko/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM gcr.io/kaniko-project/executor:debug AS kaniko - -FROM ghcr.io/actions/actions-runner:2.321.0 - -COPY --from=kaniko /kaniko/ /kaniko/ - -ENV SSL_CERT_DIR /kaniko/ssl/certs -ENV PATH $PATH:/usr/local/bin:/kaniko -ENV DOCKER_CONFIG /kaniko/.docker/ - -ENV RUNNER_ALLOW_RUNASROOT 1 -USER root -RUN sed -i '/messagebus/d' /var/lib/dpkg/statoverride - -RUN mv /home/runner/ /kaniko/actions-runner/ -WORKDIR /kaniko/actions-runner/ diff --git a/infrastructure/tools/lambdas/github_runner_webhook.py b/infrastructure/tools/lambdas/github_runner_webhook.py deleted file mode 100644 index bc0ff7f9ea..0000000000 --- a/infrastructure/tools/lambdas/github_runner_webhook.py +++ /dev/null @@ -1,125 +0,0 @@ -import boto3 -import json -import os -import hashlib -import hmac -from urllib import request - -WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"] -GITHUB_TOKEN_SSM_NAME = os.environ["GITHUB_TOKEN_SSM_NAME"] -NETWORK_CONFIGURATION = os.environ["NETWORK_CONFIGURATION"] -ECS_CLUSTER_NAME = os.environ["ECS_CLUSTER_NAME"] -ECS_TASK_DEFINITION = os.environ["ECS_TASK_DEFINITION"] - - -def handler(event, context): - body = event.get("body").encode("utf-8") - - if not verify_signature( - body, WEBHOOK_SECRET, event["headers"]["x-hub-signature-256"] - ): - return {"statusCode": 401, "body": ""} - - github_event = event["headers"]["x-github-event"] - body = json.loads(body) - - match github_event: - case "workflow_job": - handle_workflow_job(body, context) - case _: - ... - - return {"statusCode": 200, "body": ""} - - -def handle_workflow_job(body, context): - action = body["action"] - - if action != "queued": - return - - workflow_job = body["workflow_job"] - workflow_name = workflow_job["workflow_name"] - - if workflow_name != "Deploy": - return - - labels = workflow_job["labels"] - arm64_fargate_label = next( - (label for label in labels if "arm64-fargate-" in label), None - ) - if not arm64_fargate_label: - return - - unique_run_id = arm64_fargate_label.replace("arm64-fargate-", "") - - ssm_client = boto3.client("ssm") - github_token = ssm_client.get_parameter(Name=GITHUB_TOKEN_SSM_NAME)["Parameter"][ - "Value" - ] - - payload = { - "name": f"Runner for run #{unique_run_id}", - "runner_group_id": 3, - "labels": [arm64_fargate_label], - } - payload_encoded = json.dumps(payload).encode("utf-8") - - req = request.Request( - "https://api.github.com/orgs/pythonitalia/actions/runners/generate-jitconfig", - data=payload_encoded, - method="POST", - headers={ - "Authorization": f"Bearer {github_token}", - "Accept": "application/vnd.github.v3+json", - "X-GitHub-Api-Version": "2022-11-28", - }, - ) - - with request.urlopen(req) as response: - response_data = response.read().decode("utf-8") - print(response_data) - - jit_config = json.loads(response_data)["encoded_jit_config"] - - print("Handling workflow job - start?", jit_config) - print("Body:", body) - print("Context:", context) - - ecs_client = boto3.client("ecs") - ecs_client.run_task( - cluster=ECS_CLUSTER_NAME, - taskDefinition=ECS_TASK_DEFINITION, - networkConfiguration=json.loads(NETWORK_CONFIGURATION), - enableExecuteCommand=True, - count=1, - overrides={ - "containerOverrides": [ - {"name": "runner", "command": [f"./run.sh --jitconfig {jit_config}"]} - ] - }, - ) - - -def verify_signature(payload_body, secret_token, signature_header): - """Verify that the payload was sent from GitHub by validating SHA256. - - Raise and return 403 if not authorized. - - Args: - payload_body: original request body to verify (request.body()) - secret_token: GitHub app webhook token (WEBHOOK_SECRET) - signature_header: header received from GitHub (x-hub-signature-256) - """ - if not signature_header: - return False - - hash_object = hmac.new( - secret_token.encode("utf-8"), msg=payload_body, digestmod=hashlib.sha256 - ) - expected_signature = "sha256=" + hash_object.hexdigest() - - if not hmac.compare_digest(expected_signature, signature_header): - return False - - return True From 6f46dc85a65be9a1a9ee276603f1991ae5ed5671 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:04:46 +0100 Subject: [PATCH 55/68] change --- backend/Dockerfile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index d411e9abf5..32a237e27d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,17 +14,19 @@ ENV DJANGO_SETTINGS_MODULE=pycon.settings.prod \ CELERY_RESULT_BACKEND=demo \ HASHID_DEFAULT_SECRET_SALT=demo -FROM base AS build-stage - -ARG FUNCTION_DIR - RUN apt-get update -y && apt-get install -y \ - gcc git \ # weasyprint libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 \ # postgres libpq-dev +FROM base AS build-stage + +ARG FUNCTION_DIR + +RUN apt-get update -y && apt-get install -y \ + gcc git + RUN mkdir -p ${FUNCTION_DIR} WORKDIR ${FUNCTION_DIR} @@ -66,12 +68,6 @@ ARG FUNCTION_DIR WORKDIR ${FUNCTION_DIR} -RUN apt-get update -y && apt-get install -y \ - # weasyprint - libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 \ - # postgres - libpq-dev - ENV LIBRARY_PATH=/lib:/usr/lib LD_LIBRARY_PATH=/lib:/usr/lib RUN apt-get update -y && apt-get install -y curl From b1416797cff0ff9d7d2e130a201e8564ebf285c0 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:09:04 +0100 Subject: [PATCH 56/68] remove some access --- infrastructure/global/ecr_repos/main.tf | 40 ------------------------- 1 file changed, 40 deletions(-) diff --git a/infrastructure/global/ecr_repos/main.tf b/infrastructure/global/ecr_repos/main.tf index 1578c8d6f7..213beb8d83 100644 --- a/infrastructure/global/ecr_repos/main.tf +++ b/infrastructure/global/ecr_repos/main.tf @@ -2,19 +2,11 @@ locals { services = [ "pycon-backend", "pycon-backend/cache", - "pycon-frontend", - "pycon-frontend/cache", "pretix", "pretix/cache", ] - infrastructure_tools_account_id = [ - for account in data.aws_organizations_organization.organization.non_master_accounts : - account.id - if account.name == "Infrastructure Tools" - ][0] } -data "aws_organizations_organization" "organization" {} resource "aws_ecr_repository" "service_repo" { for_each = toset(local.services) @@ -25,35 +17,3 @@ resource "aws_ecr_repository" "service_repo" { scan_on_push = false } } - -data "aws_iam_policy_document" "access_from_infrastructure_account" { - statement { - sid = "access from infrastructure account" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [local.infrastructure_tools_account_id] - } - - actions = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability", - "ecr:PutImage", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "ecr:DescribeRepositories", - "ecr:GetRepositoryPolicy", - "ecr:ListImages", - "ecr:BatchDeleteImage", - ] - } -} - -resource "aws_ecr_repository_policy" "access_from_infrastructure_account" { - for_each = toset(local.services) - repository = aws_ecr_repository.service_repo[each.key].name - policy = data.aws_iam_policy_document.access_from_infrastructure_account.json -} From 7f382439cc4c14cba0ffd2c8cab8a082a160c7e2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:10:23 +0100 Subject: [PATCH 57/68] minor change --- infrastructure/global/ecr_repos/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/infrastructure/global/ecr_repos/main.tf b/infrastructure/global/ecr_repos/main.tf index 213beb8d83..d871ea7396 100644 --- a/infrastructure/global/ecr_repos/main.tf +++ b/infrastructure/global/ecr_repos/main.tf @@ -1,9 +1,7 @@ locals { services = [ "pycon-backend", - "pycon-backend/cache", "pretix", - "pretix/cache", ] } From acdfc5109f32512068843f58844721e9bd089901 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:12:39 +0100 Subject: [PATCH 58/68] separate workflow to build base pretix --- .github/workflows/build-base-pretix.yml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/build-base-pretix.yml diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml new file mode 100644 index 0000000000..e4cc6779ff --- /dev/null +++ b/.github/workflows/build-base-pretix.yml @@ -0,0 +1,37 @@ +on: + workflow_dispatch: + inputs: + version: + description: 'The version of pretix to build. Example: v2024.10.0' + required: true + type: string + +jobs: + build: + name: Build pretix + runs-on: [self-hosted] + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + with: + repository: pretix/pretix + ref: ${{ inputs.version }} + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Build pretix base + uses: docker/build-push-action@v6 + id: build-pretix-base + with: + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: ghcr.io/pythonitalia/pycon/arm-pretix:${{ inputs.version }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 From 440fa7a30b90ecd48989ddfeff2dc9c6e0c1bc2b Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:14:09 +0100 Subject: [PATCH 59/68] changes --- .github/workflows/build-base-pretix.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index e4cc6779ff..ca666544d7 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -1,3 +1,7 @@ +name: Build base pretix image + +concurrency: ${{ inputs.version }} + on: workflow_dispatch: inputs: From 5fc8ee243bbd04c9585dff6d08d1be397b2798be Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:14:17 +0100 Subject: [PATCH 60/68] change --- .github/workflows/build-base-pretix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index ca666544d7..ddb9423d3e 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -12,7 +12,7 @@ on: jobs: build: - name: Build pretix + name: Build runs-on: [self-hosted] permissions: contents: read From 77be4024756a0399e02656c5348f192426fc02ff Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:16:32 +0100 Subject: [PATCH 61/68] test --- .github/workflows/build-base-pretix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index ddb9423d3e..0c039ce310 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -3,6 +3,7 @@ name: Build base pretix image concurrency: ${{ inputs.version }} on: + pull_request: workflow_dispatch: inputs: version: From 63beb587765f1233587afc6b1664ef3956295f6c Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:17:28 +0100 Subject: [PATCH 62/68] remove --- .github/workflows/build-base-pretix.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index 0c039ce310..ddb9423d3e 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -3,7 +3,6 @@ name: Build base pretix image concurrency: ${{ inputs.version }} on: - pull_request: workflow_dispatch: inputs: version: From e3e6c05fa36a9fc90d17c30d40ccba46947b4495 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:19:55 +0100 Subject: [PATCH 63/68] change --- .github/workflows/build-base-pretix.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index ddb9423d3e..b37b9f5fa4 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -4,11 +4,6 @@ concurrency: ${{ inputs.version }} on: workflow_dispatch: - inputs: - version: - description: 'The version of pretix to build. Example: v2024.10.0' - required: true - type: string jobs: build: From 4a1dabf17eb268ed22a9bc3b93907c8f433e9b90 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 15:20:16 +0100 Subject: [PATCH 64/68] change --- .github/workflows/build-base-pretix.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index b37b9f5fa4..ddb9423d3e 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -4,6 +4,11 @@ concurrency: ${{ inputs.version }} on: workflow_dispatch: + inputs: + version: + description: 'The version of pretix to build. Example: v2024.10.0' + required: true + type: string jobs: build: From fe25799afbba481bdb3b2237ac41a8dda2dc5f86 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 16:10:11 +0100 Subject: [PATCH 65/68] Use separately build pretix base image --- .github/workflows/build-pretix.yml | 24 +----------------------- pretix/Dockerfile | 4 +--- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-pretix.yml b/.github/workflows/build-pretix.yml index 45e73c3b42..b8ed77d6ce 100644 --- a/.github/workflows/build-pretix.yml +++ b/.github/workflows/build-pretix.yml @@ -20,30 +20,11 @@ jobs: aws-access-key-id: ${{ secrets.aws_access_key_id }} aws-secret-access-key: ${{ secrets.aws_secret_access_key }} aws-region: eu-central-1 - - uses: actions/checkout@v4 - with: - repository: pretix/pretix - ref: v2024.10.0 - path: ./pretix-clone - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - name: Build pretix base - uses: docker/build-push-action@v6 - id: build-pretix-base - with: - context: ./pretix-clone - file: ./pretix-clone/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ inputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 - name: Build and push pretix uses: docker/build-push-action@v6 with: @@ -52,10 +33,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ inputs.githash }} + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ inputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 - build-args: | - PRETIX_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ inputs.githash }} diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 9a9c243439..302cf855c7 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,6 +1,4 @@ -# v2024.10.0 -ARG PRETIX_IMAGE -FROM ${PRETIX_IMAGE} +FROM ghcr.io/pythonitalia/pycon/arm-pretix:v2024.10.0 COPY ./settings.py /pretix/src/production_settings.py From 858eb42fa193af98aeb7da647abd45b82c0b366a Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 20:01:32 +0100 Subject: [PATCH 66/68] test --- frontend/src/global.css | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/global.css b/frontend/src/global.css index ed1d3b7377..eff6a6eed9 100644 --- a/frontend/src/global.css +++ b/frontend/src/global.css @@ -47,3 +47,4 @@ body:has(.brochure-page) { .pricing-page-table div { margin-top: -1px; } +/*test*/ From 44ab803b48fec57f3887d79ae3ee50f1adb6b449 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 5 Jan 2025 20:07:41 +0100 Subject: [PATCH 67/68] remove test changes --- backend/README.md | 1 - frontend/src/global.css | 1 - 2 files changed, 2 deletions(-) diff --git a/backend/README.md b/backend/README.md index 2f9518f566..e1edd09758 100644 --- a/backend/README.md +++ b/backend/README.md @@ -63,4 +63,3 @@ And for the admin go to: http://localhost:8000/admin You should be able to login with the user we create a few moments ago. -#testchange diff --git a/frontend/src/global.css b/frontend/src/global.css index eff6a6eed9..ed1d3b7377 100644 --- a/frontend/src/global.css +++ b/frontend/src/global.css @@ -47,4 +47,3 @@ body:has(.brochure-page) { .pricing-page-table div { margin-top: -1px; } -/*test*/ From a728db1a16d45092b30582415a4162f2071db3c4 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 12 Jan 2025 01:21:06 +0100 Subject: [PATCH 68/68] changs --- .github/workflows/build-backend.yml | 3 +-- .github/workflows/build-base-pretix.yml | 2 +- pretix/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 4a2ddea91e..066e0514ea 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -35,8 +35,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pycon-backend:arm-${{ inputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 diff --git a/.github/workflows/build-base-pretix.yml b/.github/workflows/build-base-pretix.yml index 0cd01282f2..d4b30095f8 100644 --- a/.github/workflows/build-base-pretix.yml +++ b/.github/workflows/build-base-pretix.yml @@ -39,7 +39,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false push: true - tags: ghcr.io/pythonitalia/pycon/arm-pretix:${{ inputs.version }} + tags: ghcr.io/pythonitalia/pycon/base-pretix:${{ inputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 302cf855c7..5a78ef7716 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/pythonitalia/pycon/arm-pretix:v2024.10.0 +FROM ghcr.io/pythonitalia/pycon/base-pretix:v2024.10.0 COPY ./settings.py /pretix/src/production_settings.py