From 9832d12468e1c450657749b92debf54fd8130b22 Mon Sep 17 00:00:00 2001 From: Jeffrey Date: Sun, 29 Mar 2026 10:44:28 -0700 Subject: [PATCH 1/9] CI fixes Clean up the CI: disable all auto-triggered CIs (#1190) CI switching to GHCR, 1st try bug fixes remove all AWS refs CI fixes More CI fixes --- .dockerignore | 9 +- .github/workflows/Dockerfile.build | 6 +- .github/workflows/build-coordinator.yml | 42 ++------ .github/workflows/build-packages.yml | 6 +- .github/workflows/ecr-pw-refresh.yml | 38 ------- .github/workflows/nightly.yml | 32 +----- .github/workflows/pr-workflow.yml | 31 +----- .github/workflows/release-build.yml | 50 +++------ .github/workflows/rw-build-container.yml | 52 +++------ .github/workflows/rw-run-integration-test.yml | 100 +++++------------- .github/workflows/rw-run-performance-test.yml | 65 ++++-------- .github/workflows/rw-run-unit-test.yml | 37 +------ docker/ansible/playbook.ci.yml | 1 + docker/ansible/roles/common/tasks/main.yml | 3 +- .../ansible/roles/custom-pg/defaults/main.yml | 2 +- docker/ansible/roles/custom-pg/tasks/main.yml | 59 +++++------ src/proxy/client_session.cc | 2 +- 17 files changed, 142 insertions(+), 393 deletions(-) delete mode 100644 .github/workflows/ecr-pw-refresh.yml diff --git a/.dockerignore b/.dockerignore index 8a40eede1..cf02a8f58 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,4 +16,11 @@ vcpkg_installed build_scripts venv releases -shared-lib \ No newline at end of file +shared-lib +.agent-system +.gemini +.work +.artifacts +.docs +.github +.githooks \ No newline at end of file diff --git a/.github/workflows/Dockerfile.build b/.github/workflows/Dockerfile.build index c4265e90a..5c0ccc4aa 100644 --- a/.github/workflows/Dockerfile.build +++ b/.github/workflows/Dockerfile.build @@ -1,5 +1,5 @@ # The Dockerfile for CI build & tests (for GitHub Actions) -ARG BASE_IMAGE=381492230138.dkr.ecr.us-east-1.amazonaws.com/devsupport:invalid +ARG BASE_IMAGE=ghcr.io/springtail-inc/devsupport:invalid FROM ${BASE_IMAGE} ARG CI_BUILD_TYPE @@ -26,9 +26,9 @@ RUN ln -s /home/dev/debug debug && \ RUN if [ "$SPRINGTAIL_INCLUDE_TIME_TRACES" = "false" ]; then unset SPRINGTAIL_INCLUDE_TIME_TRACES ; fi && \ if [ "$CI_BUILD_TYPE" = "debug" ]; then \ - cmake -B "debug" -S . -D'CMAKE_BUILD_TYPE=Debug' -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build"; \ + cmake -B "debug" -S . -D'CMAKE_BUILD_TYPE=Debug' -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build"; \ else \ - cmake -B "release" -S . -D'CMAKE_BUILD_TYPE=Release' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DNDEBUG=1 -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build"; \ + cmake -B "release" -S . -D'CMAKE_BUILD_TYPE=Release' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DNDEBUG=1 -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build"; \ fi && \ rm -rf /home/dev/external/vcpkg/buildtrees /home/dev/external/vcpkg/packages /home/dev/external/vcpkg/downloads diff --git a/.github/workflows/build-coordinator.yml b/.github/workflows/build-coordinator.yml index 7dec20a18..fa48e61ec 100644 --- a/.github/workflows/build-coordinator.yml +++ b/.github/workflows/build-coordinator.yml @@ -15,37 +15,21 @@ permissions: id-token: write contents: read checks: write + packages: write jobs: build: - runs-on: warp-ubuntu-latest-arm64-4x-spot + runs-on: ubuntu-22.04-arm timeout-minutes: 35 container: - image: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.build_type == 'release' && vars.RELEASE_BUILDER_IMAGE_VERSION || vars.BUILDER_IMAGE_VERSION }} + image: ghcr.io/springtail-inc/${{ inputs.build_type == 'release' && vars.RELEASE_BUILDER_IMAGE_VERSION || vars.BUILDER_IMAGE_VERSION }} credentials: - username: AWS - password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" - env: - PACKAGE_BUCKET_NAME: ${{ inputs.build_type == 'release' && 'prod-data-share.springtail.internal' || 'data-share.springtail.internal' }} - COORDINATOR_S3_PREFIX: stc + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout code uses: actions/checkout@v4 - - name: install-aws-cli-action - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2 # default - verbose: false # default - arch: arm64 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE_ARN }} - role-session-name: tempSession - aws-region: us-east-1 - - name: Build run: | git config --global --add safe.directory /__w/springtail/springtail @@ -66,14 +50,8 @@ jobs: echo "log_rotation_count: 10" >> stc/coordinator/config.yaml tar -czf stc.tar.gz stc - - name: Upload Coordinator - run: | - # generate a name with timestamp - TIMESTAMP=$(date +%s) - name="stc-${TIMESTAMP}.tar.gz" - # Add this name to an current file - echo $name > current - aws s3 cp stc.tar.gz s3://${PACKAGE_BUCKET_NAME}/${COORDINATOR_S3_PREFIX}/${name} - aws s3 cp current s3://${PACKAGE_BUCKET_NAME}/${COORDINATOR_S3_PREFIX}/current - rm stc.tar.gz - rm current + - name: Upload Coordinator Artifact + uses: actions/upload-artifact@v4 + with: + name: 'Coordinator Package' + path: stc.tar.gz diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index a2ea7b881..528972d1d 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -19,6 +19,7 @@ permissions: id-token: write contents: read checks: write + packages: write jobs: system-settings-check: @@ -32,9 +33,6 @@ jobs: with: num_procs: ${{ inputs.num_procs || 4 }} package_bucket_name: ${{ inputs.build_type == 'release' && 'prod-data-share.springtail.internal' || 'data-share.springtail.internal' }} - image_version_tag: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.build_type == 'release' && vars.RELEASE_BUILDER_IMAGE_VERSION || vars.BUILDER_IMAGE_VERSION }} + image_version_tag: ghcr.io/springtail-inc/${{ inputs.build_type == 'release' && vars.RELEASE_BUILDER_IMAGE_VERSION || vars.BUILDER_IMAGE_VERSION }} slack_webhook_url: "${{ vars.GENERIC_SLACK_WEBHOOK_URL }}" build_type: ${{ inputs.build_type || 'release' }} - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" diff --git a/.github/workflows/ecr-pw-refresh.yml b/.github/workflows/ecr-pw-refresh.yml deleted file mode 100644 index 6f37578e5..000000000 --- a/.github/workflows/ecr-pw-refresh.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This workflow refreshes on the 37th minutes every 4 hours. It is used to refresh the ECR password for the test workflow. -# It leverages the `gh` CLI to set the secrets for the dev support repo for testing purpose (DEV_SUPPORT_ECR_REPO_PW). -name: ECR PW Refresh (Dev Support) -on: - workflow_dispatch: - -permissions: - id-token: write - contents: read - -jobs: - refresh-ecr-pw: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install GitHub CLI - run: | - sudo apt update - sudo apt install -y gh - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE_ARN }} - role-session-name: tempSession - aws-region: us-east-1 - - - name: Login GH with Token - run: | - echo "Logging into GH with Token" - gh auth login --with-token <<< ${{ secrets.GH_TOKEN }} - - - name: Grab and Set ECR PW - run: | - DEV_SUPPORT_ECR_REPO_PW=$(aws ecr get-login-password --region us-east-1 --output text) - gh secret set DEV_SUPPORT_ECR_PW --body "$DEV_SUPPORT_ECR_REPO_PW" \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 15386fc52..095d8220e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,6 +6,7 @@ permissions: id-token: write contents: read checks: write + packages: write jobs: prepare: @@ -31,9 +32,6 @@ jobs: with: base_image_tag: ${{ vars.IMAGE_TAG_DEBUG }} build_type: "debug" - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" release-build-with-time-traces: uses: ./.github/workflows/rw-build-container.yml @@ -41,50 +39,35 @@ jobs: base_image_tag: ${{ vars.IMAGE_TAG_RELEASE }} build_type: "release" springtail_include_time_traces: true - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" unit-test: uses: ./.github/workflows/rw-run-unit-test.yml needs: [ debug-build, prepare ] with: image_tag: ${{ needs.debug-build.outputs.output_image_tag }} - s3_log_prefix: "nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}" build_type: "debug" reuse_code: true - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" multi-rounds-integration-test: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ debug-build, prepare ] with: image_tag: ${{ needs.debug-build.outputs.output_image_tag }} - s3_log_prefix: "nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}" config_to_run: "nightly" system_settings_override: '.github/workflows/nightly.overrides.json' iterations: "1" build_type: "debug" reuse_code: true timeout_minutes: 120 - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" performance-test: uses: ./.github/workflows/rw-run-performance-test.yml needs: [ release-build-with-time-traces, prepare ] with: image_tag: ${{ needs.release-build-with-time-traces.outputs.output_image_tag }} - s3_log_prefix: "nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}" system_settings_override: '.github/workflows/nightly.overrides.json' build_type: "release" reuse_code: true - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" slack-notification: uses: ./.github/workflows/rw-notify-slack.yml @@ -98,32 +81,27 @@ jobs: Nightly builds has completed for ${{ needs.prepare.outputs.date_prefix }} - ${{ needs.prepare.outputs.commit_sha }}. attachment: | { - color": "${{ needs.unit-test.result == 'success' && needs.multi-rounds-integration-test.result == 'success' && needs.performance-test.result == 'success' && 'good' || 'danger' }}", + "color": "${{ needs.unit-test.result == 'success' && needs.multi-rounds-integration-test.result == 'success' && needs.performance-test.result == 'success' && 'good' || 'danger' }}", "fields": [ { "title": "Unit Test", - "value": "${{ needs.unit-test.result }}; logs: <${{ needs.unit-test.outputs.test_log_s3_uri }}|here>", + "value": "${{ needs.unit-test.result }}; <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>", "short": false }, { "title": "Multi-rounds Integration Test", - "value": "${{ needs.multi-rounds-integration-test.result }}; last logs: <${{ needs.multi-rounds-integration-test.outputs.test_log_s3_uri }}|here>; last springtail logs: <${{ needs.multi-rounds-integration-test.outputs.springtail_log_s3_uri }}|here>", + "value": "${{ needs.multi-rounds-integration-test.result }}; <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>", "short": false }, { "title": "Performance Test", - "value": "${{ needs.performance-test.result }}; logs: <${{ needs.performance-test.outputs.test_log_s3_uri }}|here>", + "value": "${{ needs.performance-test.result }}; <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>", "short": false }, { "title": "Commit SHA", "value": "${{ needs.prepare.outputs.commit_sha }}", "short": false - }, - { - "title": "All Logs", - "value": "s3://devsupport.springtail.internal/nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}", - "short": false } ] } diff --git a/.github/workflows/pr-workflow.yml b/.github/workflows/pr-workflow.yml index 4e85fc614..f2376ae5a 100644 --- a/.github/workflows/pr-workflow.yml +++ b/.github/workflows/pr-workflow.yml @@ -8,6 +8,7 @@ permissions: id-token: write contents: read checks: write + packages: write jobs: @@ -26,9 +27,6 @@ jobs: build_type: "debug" num_procs: "6" push: true - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" unit-test: uses: ./.github/workflows/rw-run-unit-test.yml @@ -36,13 +34,9 @@ jobs: if: ${{ github.event.pull_request.draft == false }} with: image_tag: ${{ needs.build.outputs.output_image_tag }} - s3_log_prefix: "testlogs/${{ github.event.pull_request.number }}" build_type: "debug" reuse_code: true timeout_minutes: ${{ vars.UNIT_TEST_TIMEOUT_MINUTES || 20 }} - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" integration-test-p1: uses: ./.github/workflows/rw-run-integration-test.yml @@ -51,13 +45,9 @@ jobs: with: image_tag: ${{ needs.build.outputs.output_image_tag }} config_to_run: "github_ci_p1" - s3_log_prefix: "testlogs/${{ github.event.pull_request.number }}" build_type: "debug" reuse_code: true timeout_minutes: ${{ vars.INTEGRATION_TEST_TIMEOUT_MINUTES || 20 }} - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" integration-test-p2: uses: ./.github/workflows/rw-run-integration-test.yml @@ -66,13 +56,9 @@ jobs: with: image_tag: ${{ needs.build.outputs.output_image_tag }} config_to_run: "github_ci_p2" - s3_log_prefix: "testlogs/${{ github.event.pull_request.number }}" build_type: "debug" reuse_code: true timeout_minutes: ${{ vars.INTEGRATION_TEST_TIMEOUT_MINUTES || 20 }} - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" integration-test-p3: uses: ./.github/workflows/rw-run-integration-test.yml @@ -81,23 +67,16 @@ jobs: with: image_tag: ${{ needs.build.outputs.output_image_tag }} config_to_run: "github_ci_p3" - s3_log_prefix: "testlogs/${{ github.event.pull_request.number }}" build_type: "debug" reuse_code: true timeout_minutes: ${{ vars.INTEGRATION_TEST_TIMEOUT_MINUTES || 20 }} - secrets: - aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}" - aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}" integration-test-join: needs: [ build, unit-test, integration-test-p1, integration-test-p2, integration-test-p3 ] runs-on: ubuntu-latest if: ${{ always() && !cancelled() && github.event.pull_request.draft == false }} outputs: - it_test_log_s3_uris: ${{ steps.set_outputs.outputs.it_test_log_s3_uris }} - it_springtail_log_s3_uris: ${{ steps.set_outputs.outputs.it_springtail_log_s3_uris }} it_result: ${{ steps.set_outputs.outputs.it_result }} - ut_test_log_s3_uri: ${{ needs.unit-test.outputs.test_log_s3_uri }} ut_result: ${{ steps.set_outputs.outputs.ut_result }} build_result: ${{ steps.set_outputs.outputs.build_result }} env: @@ -107,9 +86,6 @@ jobs: - name: Set Outputs id: set_outputs run: | - echo "it_test_log_s3_uris=${{ needs.integration-test-p1.outputs.test_log_s3_uri }} ; ${{ needs.integration-test-p2.outputs.test_log_s3_uri }} ; ${{ needs.integration-test-p3.outputs.test_log_s3_uri }} ;" >> $GITHUB_OUTPUT - echo "it_springtail_log_s3_uris=${{ needs.integration-test-p1.outputs.springtail_log_s3_uri }} ; ${{ needs.integration-test-p2.outputs.springtail_log_s3_uri }} ; ${{ needs.integration-test-p3.outputs.springtail_log_s3_uri }} ; " >> $GITHUB_OUTPUT - echo "ut_test_log_s3_uri=${{ needs.unit-test.outputs.test_log_s3_uri }}" >> $GITHUB_OUTPUT echo "build_result=${{ needs.build.result }}" >> $GITHUB_OUTPUT it_result="unknown" @@ -182,15 +158,16 @@ jobs: }, { "title": "Unit Test - *${{ needs.integration-test-join.outputs.ut_result }}*", - "value": "test logs:\n${{ needs.integration-test-join.outputs.ut_test_log_s3_uri }}", + "value": "logs: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>", "short": false }, { "title": "Integration Test - *${{ needs.integration-test-join.outputs.it_result }}*", - "value": "test logs:\n${{ needs.integration-test-join.outputs.it_test_log_s3_uris }}\nspringtail logs:\n${{ needs.integration-test-join.outputs.it_springtail_log_s3_uris }}", + "value": "logs: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>", "short": false } ] } + diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 5ac4a8d01..17752bb5f 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -8,12 +8,12 @@ on: type: string required: true build_type: - description: 'The type of build to perform. This also determines the S3 bucket we upload to. For release build, we upload to the "release-builds.springtail.internal" bucket (in production account), for debug builds we upload to the "debug-builds.springtail.internal" bucket (in development account).' + description: 'The type of build to perform.' type: string package_bucket_name: - description: 'The name of the S3 bucket to store the build artifacts.' + description: 'Legacy field, no longer used.' type: string - required: true + required: false image_version_tag: description: 'The image version tag for the base builder to pull.' type: string @@ -22,29 +22,20 @@ on: type: string description: "The Slack webhook URL." required: true - secrets: - aws_role_arn: - description: "The ARN of the role to assume." - required: true - aws_ecr_docker_password: - description: "The password for the ECR docker registry." - required: true jobs: build: name: Build Springtail Packages - runs-on: warp-ubuntu-latest-arm64-4x-spot + runs-on: ubuntu-22.04-arm timeout-minutes: 30 container: image: ${{ inputs.image_version_tag }} credentials: - username: AWS - password: ${{ secrets.aws_ecr_docker_password }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} env: VCPKG_FORCE_SYSTEM_BINARIES: 1 ZIC: true - PACKAGE_BUCKET_NAME: ${{ inputs.package_bucket_name }} - PACKAGES_S3_PREFIX: packages NUM_PROCS: ${{ inputs.num_procs }} SLACK_WEBHOOK_URL: ${{ inputs.slack_webhook_url }} SLACK_CHANNEL: '#release' @@ -55,20 +46,6 @@ jobs: with: clean: true - - name: install-aws-cli-action - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2 # default - verbose: false # default - arch: arm64 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.aws_role_arn }} - role-session-name: tempSession - aws-region: us-east-1 - - name: Build Springtail Service run: | git config --global --add safe.directory /__w/springtail/springtail @@ -84,7 +61,7 @@ jobs: cd "$BUILD_TYPE" make install - - name: Package and Upload Build artifacts to S3 + - name: Package Build artifacts if: success() run: | git config --global --add safe.directory /__w/springtail/springtail @@ -96,8 +73,14 @@ jobs: } >> $GITHUB_ENV cd releases PKG_NAME="$(ls -t *.tgz | head -n1)" - aws s3 cp $PKG_NAME s3://${PACKAGE_BUCKET_NAME}/${PACKAGES_S3_PREFIX}/ - echo "PACKAGE_PATH=s3://${PACKAGE_BUCKET_NAME}/${PACKAGES_S3_PREFIX}/${PKG_NAME}" >> $GITHUB_ENV + echo "PACKAGE_NAME=${PKG_NAME}" >> $GITHUB_ENV + + - name: Upload Build Artifact + if: success() + uses: actions/upload-artifact@v4 + with: + name: 'Release Package' + path: ./releases/*.tgz - name: Notify Slack Success if: success() @@ -119,7 +102,7 @@ jobs: }, { "title": "Build Artifacts", - "value": "${{ env.PACKAGE_PATH }}", + "value": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>", "short": false }, { @@ -138,4 +121,3 @@ jobs: with: name: release-info path: install/INFO.txt - diff --git a/.github/workflows/rw-build-container.yml b/.github/workflows/rw-build-container.yml index 51b05320a..0208a3e3e 100644 --- a/.github/workflows/rw-build-container.yml +++ b/.github/workflows/rw-build-container.yml @@ -12,7 +12,7 @@ on: required: true default: "debug" push: - description: 'Whether to push the built image to ECR. If false, the image will only be loaded into the local Docker daemon. Otherwise it will be pushed to ECR but not loaded locally.' + description: 'Whether to push the built image to GHCR. If false, the image will only be loaded into the local Docker daemon. Otherwise it will be pushed to GHCR but not loaded locally.' type: boolean required: false default: true @@ -31,13 +31,6 @@ on: type: boolean required: false default: false - secrets: - aws_role_arn: - description: "The ARN of the role to assume." - required: true - aws_ecr_docker_password: - description: "The password for the ECR docker registry." - required: true outputs: output_image_tag: description: "The tag of the built image. Ready to use for the next steps." @@ -45,11 +38,11 @@ on: jobs: build-container: name: Build Springtail Container - runs-on: warp-ubuntu-latest-arm64-4x + runs-on: ubuntu-22.04-arm timeout-minutes: 20 env: CI_BUILD_TYPE: ${{ inputs.build_type }} - ECR_REPO_URI: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }} + REGISTRY_URI: ghcr.io/springtail-inc VCPKG_FORCE_SYSTEM_BINARIES: 1 ZIC: true USER: root @@ -64,13 +57,6 @@ jobs: with: clean: true - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.aws_role_arn }} - role-session-name: tempSession - aws-region: us-east-1 - - name: Prepare Build id: prepare-build run: | @@ -82,8 +68,8 @@ jobs: output_image_tag="tmp-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${r}" fi - # Logs in ECR - aws ecr get-login-password | docker login --username AWS --password-stdin ${ECR_REPO_URI} + # Log in to GHCR + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin echo "output_image_tag=$output_image_tag" >> $GITHUB_OUTPUT # Figure out the build args (line separated for later use) @@ -95,7 +81,7 @@ jobs: echo "springtail_include_time_traces=SPRINGTAIL_INCLUDE_TIME_TRACES=false" >> $GITHUB_OUTPUT fi - - name: Make ECR-safe tag from ref_name + - name: Make registry-safe tag from ref_name id: slug shell: bash run: | @@ -119,9 +105,8 @@ jobs: echo "tag=cache-$tag" >> "$GITHUB_OUTPUT" - name: Docker Build Push Action - uses: Warpbuilds/build-push-action@v6 + uses: docker/build-push-action@v6 with: - profile-name: "Springtail-CI" context: . push: ${{ inputs.push }} load: ${{ !inputs.push }} @@ -135,13 +120,13 @@ jobs: compression=zstd, # much faster than gzip compression-level=3, # good speed/size tradeoff force-compression=true - cache-from: type=registry,ref=381492230138.dkr.ecr.us-east-1.amazonaws.com/devsupport:cache-latest + cache-from: type=registry,ref=ghcr.io/springtail-inc/cache-latest cache-to: type=inline tags: | - ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ steps.prepare-build.outputs.output_image_tag }} - ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ steps.slug.outputs.tag }} + ghcr.io/springtail-inc/${{ steps.prepare-build.outputs.output_image_tag }} + ghcr.io/springtail-inc/${{ steps.slug.outputs.tag }} build-args: | - BASE_IMAGE=${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.base_image_tag }} + BASE_IMAGE=ghcr.io/springtail-inc/${{ inputs.base_image_tag }} CI_BUILD_TYPE=${{ inputs.build_type }} NCORES=${{ inputs.num_procs }} ${{ steps.prepare-build.outputs.springtail_include_time_traces }} @@ -149,16 +134,7 @@ jobs: - name: Clean up secrets if: always() run: | - docker logout ${ECR_REPO_URI} - rm -f $HOME/.aws $HOME/.ssh $HOME/.git-credentials + docker logout ghcr.io + rm -f $HOME/.ssh $HOME/.git-credentials + - # Reuse the Image Tag as the snapshot ID. This allows reusing the - # same snapshot in subsequent jobs. - - name: Snapshot the VM - uses: WarpBuilds/snapshot-save@v1 - if: ${{ !inputs.push }} - with: - alias: ${{ steps.prepare-build.outputs.output_image_tag }} - fail-on-error: true - wait-timeout-minutes: 30 - diff --git a/.github/workflows/rw-run-integration-test.yml b/.github/workflows/rw-run-integration-test.yml index 7f9d8933a..83296b17b 100644 --- a/.github/workflows/rw-run-integration-test.yml +++ b/.github/workflows/rw-run-integration-test.yml @@ -22,10 +22,6 @@ on: type: string required: false default: "1" - s3_log_prefix: - description: 'The S3 prefix for logs. This is the folder inside the S3 bucket where logs will be stored.' - type: string - required: true build_type: description: 'The type of build to base the tests on. "release" or "debug". The type of build need to match the base image type.' type: string @@ -55,20 +51,7 @@ on: type: boolean required: false default: false - secrets: - aws_role_arn: - description: "The ARN of the role to assume." - required: true - aws_ecr_docker_password: - description: "The password for the ECR docker registry." - required: true outputs: - test_log_s3_uri: - description: "The full S3 prefix where the test logs are stored." - value: ${{ jobs.test.outputs.test_log_s3_uri }} - springtail_log_s3_uri: - description: "The full S3 prefix where the springtail service logs are stored." - value: ${{ jobs.test.outputs.springtail_log_s3_uri }} skipped_tests: description: "The number of skipped tests during the integration tests." value: ${{ jobs.test.outputs.skipped_tests }} @@ -78,25 +61,21 @@ on: jobs: test: name: integration-test - runs-on: ${{ inputs.reuse_code && inputs.shapshot_alias != '' && format('warp-ubuntu-latest-arm64-4x;snapshot.key={0}', inputs['snapshot_alias']) || 'warp-ubuntu-latest-arm64-4x' }} + runs-on: ubuntu-22.04-arm timeout-minutes: ${{ inputs.timeout_minutes }} container: - image: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.image_tag }} + image: ghcr.io/springtail-inc/${{ inputs.image_tag }} credentials: - username: AWS - password: ${{ secrets.aws_ecr_docker_password }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: >- --init -v /var/run/docker.sock:/var/run/docker.sock outputs: - test_log_s3_uri: ${{ steps.run-test.outputs.test_log_s3_uri }} - springtail_log_s3_uri: ${{ steps.run-test.outputs.springtail_log_s3_uri }} skipped_tests: ${{ steps.run-test.outputs.skipped_tests }} env: CI_WORKSPACE: /__w/springtail/springtail CI_BUILD_TYPE: ${{ inputs.build_type }} - S3_LOGS_PREFIX_FULL: "s3://devsupport.springtail.internal/${{ inputs.s3_log_prefix }}" - ECR_REPO_URI: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }} IMAGE_TAG: ${{ inputs.image_tag }} SYSTEM_SETTINGS_OVERRIDE: ${{ inputs.system_settings_override }} VCPKG_FORCE_SYSTEM_BINARIES: 1 @@ -126,13 +105,6 @@ jobs: echo "Reusing existing code from /home/dev/springtail" rsync -a /home/dev/springtail/ . - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.aws_role_arn }} - role-session-name: tempSession - aws-region: us-east-1 - - name: Prepare Tests run: | git config --global --add safe.directory ${CI_WORKSPACE} @@ -252,23 +224,13 @@ jobs: _TIMESTAMP=$(date +%s) - # Test logs - log_file="integration-test-${_LOG_SHA}-${_LOG_FLAG}-${_TIMESTAMP}-${i}.log" - log_s3_key="${S3_LOGS_PREFIX_FULL}/${log_file}" - aws s3 cp tmplog $log_s3_key - echo "*** Integration Test Log S3 Key: $log_s3_key" + # Save test logs locally for artifact upload mv tmplog integration-test-${_LOG_SHA}-${_LOG_FLAG}-${_TIMESTAMP}-${i}.log - echo "test_log_s3_uri=${log_s3_key}" >> $GITHUB_OUTPUT # Springtail service logs - springtail_log_file="springtail-${_LOG_SHA}-${_LOG_FLAG}-${_TIMESTAMP}-${i}-${INTEGRATION_TEST_CONFIG}.tar.gz" output=$(python3 springtail.py --dump -f ../../system.json.test -b ../../${CI_BUILD_TYPE}) fname=$(echo $output | cut -f2 -d':' | xargs | head -1) - springtail_log_s3_key="${S3_LOGS_PREFIX_FULL}/${springtail_log_file}" - aws s3 cp $fname $springtail_log_s3_key - echo "*** Springtail Log S3 Key: $springtail_log_s3_key" mv $fname springtail-${_LOG_SHA}-${_LOG_FLAG}-${_TIMESTAMP}-${i}.tar.gz - echo "springtail_log_s3_uri=${springtail_log_s3_key}" >> $GITHUB_OUTPUT # If the current iteration of test failed, break out of the loop for now if [ "$_LOG_FLAG" = "failed" ]; then @@ -324,58 +286,46 @@ jobs: _TIMESTAMP=$(date +%s) - # Test log - log_file="proxy-regress-${GITHUB_SHORT_SHA}-${_TIMESTAMP}-${_LOG_FLAG}.log" - log_s3_key="${TEST_LOG_S3_FULL_PREFIX}/${log_file}" - aws s3 cp proxy_regress.log $log_s3_key - echo "*** Proxy Test Log S3 Key: $log_s3_key" + # Save proxy test log locally for artifact upload + if [ -f proxy_regress.log ]; then + mv proxy_regress.log "proxy-regress-${_TIMESTAMP}-${_LOG_FLAG}.log" + fi # Springtail service logs (proxy) - springtail_proxy_log_file="proxy-springtail-svc-${GITHUB_SHORT_SHA}-${_TIMESTAMP}.tar.gz" - # tar all logs under the Springtail log path - # only do the taring and uploading if there are something inside the log path - # checks if the log path is empty if [ "$(ls -A ${SPRINGTAIL_LOG_PATH})" ]; then - tar -czf $springtail_proxy_log_file ${SPRINGTAIL_LOG_PATH}/* - springtail_log_s3_key="${TEST_LOG_S3_FULL_PREFIX}/${springtail_proxy_log_file}" - aws s3 cp $springtail_proxy_log_file $springtail_log_s3_key - echo "*** Springtail Proxy Logs S3 Key: $springtail_log_s3_key" + tar -czf "proxy-springtail-svc-${_TIMESTAMP}.tar.gz" ${SPRINGTAIL_LOG_PATH}/* else - echo "*** No Springtail Proxy Logs to found inside ${SPRINGTAIL_LOG_PATH}" + echo "*** No Springtail Proxy Logs found inside ${SPRINGTAIL_LOG_PATH}" fi - # Find the file matching this name exactly: `regression.diff.pg.out` - # under the $TMPDIR directory (recursive) and upload it to S3 + # Find regression diff files and copy them to the workspace for artifact upload regression_pg_diff_file=$(find /tmp -name "regression.diff.pg.out") - # Only upload to S3 if the file exists if [ -f "$regression_pg_diff_file" ]; then - regression_pg_diff_s3_key="${TEST_LOG_S3_FULL_PREFIX}/regression_pg.diff.${GITHUB_SHORT_SHA}-${_TIMESTAMP}.out" - aws s3 cp $regression_pg_diff_file $regression_pg_diff_s3_key - echo "*** PG Regression Diff S3 Key: $regression_pg_diff_s3_key" + cp $regression_pg_diff_file "regression_pg.diff.${_TIMESTAMP}.out" fi - # Find the file matching this name exactly: `regression.diff.proxy.out` - # under the $TMPDIR directory (recursive) and upload it to S3 regression_proxy_diff_file=$(find /tmp -name "regression.diff.proxy.out") - # Only upload to S3 if the file exists if [ -f "$regression_proxy_diff_file" ]; then - regression_proxy_diff_s3_key="${TEST_LOG_S3_FULL_PREFIX}/regression_proxy.diff.${GITHUB_SHORT_SHA}-${_TIMESTAMP}.out" - aws s3 cp $regression_proxy_diff_file $regression_proxy_diff_s3_key - echo "*** Proxy Regression Diff S3 Key: $regression_proxy_diff_s3_key" + cp $regression_proxy_diff_file "regression_proxy.diff.${_TIMESTAMP}.out" fi + # Eventually fail the step if [ "$_LOG_FLAG" = "failed" ]; then - # Find the file matching this name exactly: `regression.result.proxy.out` - # under the $TMPDIR directory (recursive) and upload it to S3 regression_result_diff_file=$(find /tmp -name "regression_result.diff.out") - # Only upload to S3 if the file exists if [ -f "$regression_result_diff_file" ]; then - regression_result_diff_s3_key="${TEST_LOG_S3_FULL_PREFIX}/regression_result.diff.${GITHUB_SHORT_SHA}-${_TIMESTAMP}.out" - aws s3 cp $regression_result_diff_file $regression_result_diff_s3_key - echo "*** Proxy Regression Result Diff S3 Key: $regression_result_diff_s3_key" + cp $regression_result_diff_file "regression_result.diff.${_TIMESTAMP}.out" fi exit 1 fi + - uses: actions/upload-artifact@v4 + if: ${{ inputs.run_proxy_test && (success() || failure()) }} + with: + name: "Proxy Test Logs - ${{ inputs.config_to_run }}" + path: | + ./python/testing/proxy/proxy-regress-*.log + ./python/testing/proxy/proxy-springtail-svc-*.tar.gz + ./python/testing/proxy/regression_*.out + - name: Clean up Volume and Network if: always() run: | diff --git a/.github/workflows/rw-run-performance-test.yml b/.github/workflows/rw-run-performance-test.yml index e687f8252..72ec7bd16 100644 --- a/.github/workflows/rw-run-performance-test.yml +++ b/.github/workflows/rw-run-performance-test.yml @@ -12,10 +12,6 @@ on: type: string required: false default: '.github/workflows/ci-general.overrides.json' - s3_log_prefix: - description: 'The S3 prefix for logs. This is the folder inside the S3 bucket where logs will be stored.' - type: string - required: true build_type: description: 'The type of build to base the tests on. "release" or "debug". The type of build need to match the base image type.' type: string @@ -40,49 +36,31 @@ on: type: string required: false default: "4" - secrets: - aws_role_arn: - description: "The ARN of the role to assume." - required: true - aws_ecr_docker_password: - description: "The password for the ECR docker registry." - required: true - outputs: - test_log_s3_uri: - description: "The full S3 prefix where the test logs are stored." - value: ${{ jobs.test.outputs.test_log_s3_uri }} - springtail_log_s3_uri: - description: "The full S3 prefix where the springtail service logs are stored." - value: ${{ jobs.test.outputs.springtail_log_s3_uri }} workflow_dispatch: permissions: id-token: write contents: read checks: write + packages: write jobs: test: name: performance-test - runs-on: ${{ inputs.reuse_code && inputs.shapshot_alias != '' && format('warp-ubuntu-latest-arm64-4x;snapshot.key={0}', inputs['snapshot_alias']) || 'warp-ubuntu-latest-arm64-4x' }} + runs-on: ubuntu-22.04-arm timeout-minutes: ${{ inputs.timeout_minutes }} container: - image: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.image_tag }} + image: ghcr.io/springtail-inc/${{ inputs.image_tag }} credentials: - username: AWS - password: ${{ secrets.aws_ecr_docker_password }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: >- --init -v /var/run/docker.sock:/var/run/docker.sock - outputs: - test_log_s3_uri: ${{ steps.run-test.outputs.test_log_s3_uri }} - springtail_log_s3_uri: ${{ steps.run-test.outputs.springtail_log_s3_uri }} env: CI_WORKSPACE: /__w/springtail/springtail CI_BUILD_TYPE: ${{ inputs.build_type }} - S3_LOGS_PREFIX_FULL: "s3://devsupport.springtail.internal/${{ inputs.s3_log_prefix }}" - ECR_REPO_URI: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }} IMAGE_TAG: ${{ inputs.image_tag }} SYSTEM_SETTINGS_OVERRIDE: ${{ inputs.system_settings_override }} VCPKG_FORCE_SYSTEM_BINARIES: 1 @@ -112,13 +90,6 @@ jobs: echo "Reusing existing code from /home/dev/springtail" rsync -a /home/dev/springtail/ . - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.aws_role_arn }} - role-session-name: tempSession - aws-region: us-east-1 - - name: Prepare Tests run: | git config --global --add safe.directory ${CI_WORKSPACE} @@ -218,14 +189,10 @@ jobs: # Grab the short SHA _LOG_SHA=$(echo $GITHUB_LATEST_SHA | cut -c1-7) - # Upload performance test logs + # Save performance test logs locally for artifact upload _TIMESTAMP=$(date +%s) log_file="performance-test-${_LOG_SHA}-${_LOG_FLAG}-${_TIMESTAMP}.log" - log_s3_key="${S3_LOGS_PREFIX_FULL}/${log_file}" - aws s3 cp tmplog $log_s3_key - echo "*** Performance Test Log S3 Key: $log_s3_key" - rm -f tmplog - echo "test_log_s3_uri=$log_s3_key" >> $GITHUB_OUTPUT + mv tmplog $log_file # Go back to the testing folder to use the springtail.py to dump the logs cd ../testing @@ -234,11 +201,7 @@ jobs: springtail_log_file="springtail-performance-${_LOG_SHA}-${_LOG_FLAG}-${_TIMESTAMP}.tar.gz" output=$(python3 springtail.py --dump -f ../../system.json.test -b ../../${CI_BUILD_TYPE}}) log_dump_file_name=$(echo $output | cut -f2 -d':' | xargs | head -1) - springtail_log_s3_key="${S3_LOGS_PREFIX_FULL}/${springtail_log_file}" - aws s3 cp $log_dump_file_name $springtail_log_s3_key - echo "*** Springtail Log S3 Key: $springtail_log_s3_key" - rm -f $log_dump_file_name - echo "springtail_log_s3_uri=$springtail_log_s3_key" >> $GITHUB_OUTPUT + mv $log_dump_file_name $springtail_log_file if [ "$_LOG_FLAG" = "failed" ]; then echo "*** Performance test failed" @@ -246,6 +209,18 @@ jobs: fi echo "*** Finished running performance tests" + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: 'Performance Test Logs' + path: ./python/performance/performance-test-*.log + + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: 'Performance Springtail Logs' + path: ./python/testing/springtail-performance-*.tar.gz + - name: Clean up Volume and Network if: always() run: | diff --git a/.github/workflows/rw-run-unit-test.yml b/.github/workflows/rw-run-unit-test.yml index 3bfb31256..97c4b5fb2 100644 --- a/.github/workflows/rw-run-unit-test.yml +++ b/.github/workflows/rw-run-unit-test.yml @@ -13,10 +13,6 @@ on: type: string required: false default: '.github/workflows/ci-general.overrides.json' - s3_log_prefix: - description: 'The S3 prefix for logs. This is the folder inside the S3 bucket where logs will be stored.' - type: string - required: true build_type: description: 'The type of build to base the tests on. "release" or "debug". The type of build need to match the base image type.' type: string @@ -41,17 +37,7 @@ on: type: string required: false default: "" - secrets: - aws_role_arn: - description: "The ARN of the role to assume." - required: true - aws_ecr_docker_password: - description: "The password for the ECR docker registry." - required: true outputs: - test_log_s3_uri: - description: "The full S3 prefix where the test logs are stored." - value: ${{ jobs.test.outputs.test_log_s3_uri }} skipped_tests: description: "The number of skipped tests during the unit test run." value: ${{ jobs.test.outputs.skipped_tests }} @@ -60,24 +46,21 @@ on: jobs: test: name: unit-test - runs-on: ${{ inputs.reuse_code && inputs.shapshot_alias != '' && format('warp-ubuntu-latest-arm64-4x;snapshot.key={0}', inputs['snapshot_alias']) || 'warp-ubuntu-latest-arm64-4x' }} + runs-on: ubuntu-22.04-arm timeout-minutes: ${{ inputs.timeout_minutes }} container: - image: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.image_tag }} + image: ghcr.io/springtail-inc/${{ inputs.image_tag }} credentials: - username: AWS - password: ${{ secrets.aws_ecr_docker_password }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: >- --init -v /var/run/docker.sock:/var/run/docker.sock outputs: - test_log_s3_uri: ${{ steps.run-test.outputs.test_log_s3_uri }} skipped_tests: ${{ steps.run-test.outputs.skipped_tests }} env: CI_WORKSPACE: /__w/springtail/springtail CI_BUILD_TYPE: ${{ inputs.build_type }} - S3_LOGS_PREFIX_FULL: "s3://devsupport.springtail.internal/${{ inputs.s3_log_prefix }}" - ECR_REPO_URI: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }} IMAGE_TAG: ${{ inputs.image_tag }} SYSTEM_SETTINGS_OVERRIDE: ${{ inputs.system_settings_override }} VCPKG_FORCE_SYSTEM_BINARIES: 1 @@ -105,13 +88,6 @@ jobs: echo "Reusing existing code from /home/dev/springtail" rsync -a /home/dev/springtail/ . - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.aws_role_arn }} - role-session-name: tempSession - aws-region: us-east-1 - - name: Prepare Tests run: | git config --global --add safe.directory ${CI_WORKSPACE} @@ -227,11 +203,6 @@ jobs: tar zcvf $log_file tmplog rm tmplog - unit_test_log_s3key="${S3_LOGS_PREFIX_FULL}/$log_file" - aws s3 cp "$log_file" "$unit_test_log_s3key" - echo "*** Unit Test Log S3 Key: $unit_test_log_s3key" - echo "test_log_s3_uri=${unit_test_log_s3key}" >> $GITHUB_OUTPUT - # Fail eventually if [ "$_LOG_FLAG" = "failed" ]; then exit 1 diff --git a/docker/ansible/playbook.ci.yml b/docker/ansible/playbook.ci.yml index 9becfbcbc..b545b88aa 100644 --- a/docker/ansible/playbook.ci.yml +++ b/docker/ansible/playbook.ci.yml @@ -4,6 +4,7 @@ roles: - role: common install_additional_packages: false + install_aws_cli: false - role: python - role: custom-pg - role: postgres-config diff --git a/docker/ansible/roles/common/tasks/main.yml b/docker/ansible/roles/common/tasks/main.yml index 35cf83112..18ac59a26 100644 --- a/docker/ansible/roles/common/tasks/main.yml +++ b/docker/ansible/roles/common/tasks/main.yml @@ -88,4 +88,5 @@ curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "/tmp/awscliv2.zip" unzip -q /tmp/awscliv2.zip -d /tmp /tmp/aws/install --update - rm -f /tmp/awscliv2.zip \ No newline at end of file + rm -f /tmp/awscliv2.zip + when: install_aws_cli | default(true) diff --git a/docker/ansible/roles/custom-pg/defaults/main.yml b/docker/ansible/roles/custom-pg/defaults/main.yml index 1cb9e0daf..cc396eb1d 100644 --- a/docker/ansible/roles/custom-pg/defaults/main.yml +++ b/docker/ansible/roles/custom-pg/defaults/main.yml @@ -1,3 +1,3 @@ -custom_pg_package_url: "https://s3.us-east-1.amazonaws.com/public-share.springtail.io/postgresql-16_16.9-1_93cd1a8d-78c2-4979-8204-fa73ab5521a4.tar.gz" +custom_pg_package_url: "https://s3.us-east-1.amazonaws.com/public-share.springtail.io/postgres_apt.tgz" superuser_name: "springtail" superuser_password: "springtail" \ No newline at end of file diff --git a/docker/ansible/roles/custom-pg/tasks/main.yml b/docker/ansible/roles/custom-pg/tasks/main.yml index c0d07f837..912223240 100644 --- a/docker/ansible/roles/custom-pg/tasks/main.yml +++ b/docker/ansible/roles/custom-pg/tasks/main.yml @@ -20,19 +20,14 @@ path: /tmp/springtail-pg.tar.gz register: pg_tarball -- name: Download Custom PG build from Custom URL +- name: Download Custom PG .deb packages from URL ansible.builtin.get_url: url: "{{ custom_pg_package_url }}" dest: /tmp/springtail-pg.tar.gz mode: '0644' when: not pg_tarball.stat.exists -- name: Copy to remote host - ansible.builtin.copy: - src: /tmp/springtail-pg.tar.gz - dest: /tmp/springtail-pg.tar.gz - -- name: Install PG build dependencies +- name: Install PG runtime dependencies ansible.builtin.apt: name: - libreadline-dev @@ -45,38 +40,36 @@ - liblz4-dev - libzstd-dev - uuid-dev - - bison - - flex state: present -- name: Build and install Custom PG from source +- name: Install Custom PG from .deb packages ansible.builtin.shell: | set -e cd /tmp - tar xzf springtail-pg.tar.gz - cd postgresql-16.9 - ./configure \ - --prefix=/usr/lib/postgresql/16 \ - --bindir=/usr/lib/postgresql/16/bin \ - --datadir=/usr/share/postgresql/16 \ - --sysconfdir=/etc/postgresql-common \ - --libdir=/usr/lib/postgresql/16/lib \ - --includedir=/usr/include/postgresql/16 \ - --with-openssl \ - --with-libxml \ - --with-libxslt \ - --with-icu \ - --with-uuid=e2fs \ - --with-lz4 \ - --with-zstd \ - --with-systemd \ - --with-pgport=5432 - make -j$(nproc) world - make install-world - # Link system zoneinfo as PG timezone data (source tarball lacks timezone source files) - ln -sfn /usr/share/zoneinfo /usr/share/postgresql/16/timezone + mkdir -p postgres_apt + tar xzf springtail-pg.tar.gz -C postgres_apt + cd postgres_apt + + # Install packages in dependency order, skipping debug symbol (.ddeb) packages + dpkg -i libpq5_*.deb || apt-get install -f -y + dpkg -i libpq-dev_*.deb || apt-get install -f -y + dpkg -i libecpg6_*.deb libecpg-compat3_*.deb libpgtypes3_*.deb libecpg-dev_*.deb || apt-get install -f -y + dpkg -i postgresql-client-16_*.deb || apt-get install -f -y + dpkg -i postgresql-16_*.deb || apt-get install -f -y + dpkg -i postgresql-server-dev-16_*.deb || apt-get install -f -y + + # Optional PL language packages — install if present + dpkg -i postgresql-plpython3-16_*.deb 2>/dev/null || true + dpkg -i postgresql-plperl-16_*.deb 2>/dev/null || true + dpkg -i postgresql-pltcl-16_*.deb 2>/dev/null || true + dpkg -i postgresql-doc-16_*.deb 2>/dev/null || true + + # Fix any remaining dependency issues + apt-get install -f -y + + # Clean up cd /tmp - rm -rf postgresql-16.9 springtail-pg.tar.gz + rm -rf postgres_apt springtail-pg.tar.gz - name: Enable PG16 ansible.builtin.copy: diff --git a/src/proxy/client_session.cc b/src/proxy/client_session.cc index f6bcc2e25..8b9344992 100644 --- a/src/proxy/client_session.cc +++ b/src/proxy/client_session.cc @@ -780,7 +780,7 @@ namespace springtail::pg_proxy { // XXX debugging auto qs_deps = msg->qs_dependencies(); - for (const auto& qs : qs_deps) { + for (const auto& _ : qs_deps) { LOG_DEBUG(LOG_PROXY, LOG_LEVEL_DEBUG3, "[C:{}] Query dependency: {}", _id, qs->to_string()); } From 74bb0711c4b49beaee017396aa95585cc3ce77ea Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 16:15:42 -0700 Subject: [PATCH 2/9] fix: Github permissions --- .github/workflows/build-packages.yml | 3 +++ .github/workflows/nightly.yml | 18 ++++++++++++++++++ .github/workflows/pr-workflow.yml | 20 +++++++++++++++++++- .github/workflows/rw-build-container.yml | 16 ++++++++++++---- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 528972d1d..5aebda3d8 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -30,6 +30,9 @@ jobs: release-build: uses: ./.github/workflows/release-build.yml needs: [ system-settings-check ] + permissions: + contents: read + packages: write with: num_procs: ${{ inputs.num_procs || 4 }} package_bucket_name: ${{ inputs.build_type == 'release' && 'prod-data-share.springtail.internal' || 'data-share.springtail.internal' }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 095d8220e..c36adf4a4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,12 +29,18 @@ jobs: debug-build: uses: ./.github/workflows/rw-build-container.yml + permissions: + contents: read + packages: write with: base_image_tag: ${{ vars.IMAGE_TAG_DEBUG }} build_type: "debug" release-build-with-time-traces: uses: ./.github/workflows/rw-build-container.yml + permissions: + contents: read + packages: write with: base_image_tag: ${{ vars.IMAGE_TAG_RELEASE }} build_type: "release" @@ -43,6 +49,10 @@ jobs: unit-test: uses: ./.github/workflows/rw-run-unit-test.yml needs: [ debug-build, prepare ] + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.debug-build.outputs.output_image_tag }} build_type: "debug" @@ -51,6 +61,10 @@ jobs: multi-rounds-integration-test: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ debug-build, prepare ] + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.debug-build.outputs.output_image_tag }} config_to_run: "nightly" @@ -63,6 +77,10 @@ jobs: performance-test: uses: ./.github/workflows/rw-run-performance-test.yml needs: [ release-build-with-time-traces, prepare ] + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.release-build-with-time-traces.outputs.output_image_tag }} system_settings_override: '.github/workflows/nightly.overrides.json' diff --git a/.github/workflows/pr-workflow.yml b/.github/workflows/pr-workflow.yml index f2376ae5a..b26cfa2db 100644 --- a/.github/workflows/pr-workflow.yml +++ b/.github/workflows/pr-workflow.yml @@ -22,6 +22,9 @@ jobs: uses: ./.github/workflows/rw-build-container.yml needs: [ system-settings-check ] if: ${{ github.event.pull_request.draft == false }} + permissions: + contents: read + packages: write with: base_image_tag: ${{ vars.IMAGE_TAG_DEBUG }} build_type: "debug" @@ -32,6 +35,10 @@ jobs: uses: ./.github/workflows/rw-run-unit-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.build.outputs.output_image_tag }} build_type: "debug" @@ -42,6 +49,10 @@ jobs: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.build.outputs.output_image_tag }} config_to_run: "github_ci_p1" @@ -53,6 +64,10 @@ jobs: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.build.outputs.output_image_tag }} config_to_run: "github_ci_p2" @@ -64,6 +79,10 @@ jobs: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + permissions: + contents: read + packages: write + checks: write with: image_tag: ${{ needs.build.outputs.output_image_tag }} config_to_run: "github_ci_p3" @@ -170,4 +189,3 @@ jobs: } - diff --git a/.github/workflows/rw-build-container.yml b/.github/workflows/rw-build-container.yml index 0208a3e3e..f3693661e 100644 --- a/.github/workflows/rw-build-container.yml +++ b/.github/workflows/rw-build-container.yml @@ -35,6 +35,11 @@ on: output_image_tag: description: "The tag of the built image. Ready to use for the next steps." value: ${{ jobs.build-container.outputs.output_image_tag }} + +permissions: + contents: read + packages: write + jobs: build-container: name: Build Springtail Container @@ -57,6 +62,13 @@ jobs: with: clean: true + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Prepare Build id: prepare-build run: | @@ -67,9 +79,6 @@ jobs: r=$(head /dev/urandom | tr -dc a-f0-9 | head -c6) output_image_tag="tmp-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${r}" fi - - # Log in to GHCR - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin echo "output_image_tag=$output_image_tag" >> $GITHUB_OUTPUT # Figure out the build args (line separated for later use) @@ -137,4 +146,3 @@ jobs: docker logout ghcr.io rm -f $HOME/.ssh $HOME/.git-credentials - From ab14552b26a48fbeab5a8ce8dee63869940f9955 Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 16:23:15 -0700 Subject: [PATCH 3/9] fix: Github --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c36adf4a4..37cda573d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -78,6 +78,7 @@ jobs: uses: ./.github/workflows/rw-run-performance-test.yml needs: [ release-build-with-time-traces, prepare ] permissions: + id-token: write contents: read packages: write checks: write From 000f52503b69ad337f51cd3fd07b05866948b76d Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 16:32:37 -0700 Subject: [PATCH 4/9] fix: 3rd try --- .github/workflows/build-packages.yml | 2 ++ .github/workflows/nightly.yml | 6 ++++++ .github/workflows/pr-workflow.yml | 7 ++++++- .github/workflows/rw-build-container.yml | 14 +++++++++++--- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 5aebda3d8..d4a90fdd9 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -24,12 +24,14 @@ permissions: jobs: system-settings-check: uses: ./.github/workflows/system-settings-check.yml + secrets: inherit with: canonical_copy: 'system.json.settings' target_files: 'prod.system.settings.json' release-build: uses: ./.github/workflows/release-build.yml needs: [ system-settings-check ] + secrets: inherit permissions: contents: read packages: write diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 37cda573d..4a2c535c5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,6 +29,7 @@ jobs: debug-build: uses: ./.github/workflows/rw-build-container.yml + secrets: inherit permissions: contents: read packages: write @@ -38,6 +39,7 @@ jobs: release-build-with-time-traces: uses: ./.github/workflows/rw-build-container.yml + secrets: inherit permissions: contents: read packages: write @@ -49,6 +51,7 @@ jobs: unit-test: uses: ./.github/workflows/rw-run-unit-test.yml needs: [ debug-build, prepare ] + secrets: inherit permissions: contents: read packages: write @@ -61,6 +64,7 @@ jobs: multi-rounds-integration-test: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ debug-build, prepare ] + secrets: inherit permissions: contents: read packages: write @@ -77,6 +81,7 @@ jobs: performance-test: uses: ./.github/workflows/rw-run-performance-test.yml needs: [ release-build-with-time-traces, prepare ] + secrets: inherit permissions: id-token: write contents: read @@ -92,6 +97,7 @@ jobs: uses: ./.github/workflows/rw-notify-slack.yml needs: [ prepare, unit-test, multi-rounds-integration-test, performance-test ] if: needs.unit-test.result != 'skipped' && needs.multi-rounds-integration-test.result != 'skipped' && needs.performance-test.result != 'skipped' + secrets: inherit with: slack_webhook_url: ${{ vars.GENERIC_SLACK_WEBHOOK_URL }} # Need to create a new channel for PR notifications. diff --git a/.github/workflows/pr-workflow.yml b/.github/workflows/pr-workflow.yml index b26cfa2db..f18b45e3c 100644 --- a/.github/workflows/pr-workflow.yml +++ b/.github/workflows/pr-workflow.yml @@ -22,6 +22,7 @@ jobs: uses: ./.github/workflows/rw-build-container.yml needs: [ system-settings-check ] if: ${{ github.event.pull_request.draft == false }} + secrets: inherit permissions: contents: read packages: write @@ -35,6 +36,7 @@ jobs: uses: ./.github/workflows/rw-run-unit-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + secrets: inherit permissions: contents: read packages: write @@ -49,6 +51,7 @@ jobs: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + secrets: inherit permissions: contents: read packages: write @@ -64,6 +67,7 @@ jobs: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + secrets: inherit permissions: contents: read packages: write @@ -79,6 +83,7 @@ jobs: uses: ./.github/workflows/rw-run-integration-test.yml needs: [ build ] if: ${{ github.event.pull_request.draft == false }} + secrets: inherit permissions: contents: read packages: write @@ -154,6 +159,7 @@ jobs: needs: [ integration-test-join ] # Only if none of the tests is skipped if: ${{ always() && !cancelled() && github.event.pull_request.draft == false }} + secrets: inherit with: slack_webhook_url: ${{ vars.GENERIC_SLACK_WEBHOOK_URL }} # Need to create a new channel for PR notifications. @@ -188,4 +194,3 @@ jobs: ] } - diff --git a/.github/workflows/rw-build-container.yml b/.github/workflows/rw-build-container.yml index f3693661e..ab236755e 100644 --- a/.github/workflows/rw-build-container.yml +++ b/.github/workflows/rw-build-container.yml @@ -31,6 +31,13 @@ on: type: boolean required: false default: false + secrets: + GHCR_TOKEN: + description: 'Optional token with access to private GHCR images used as CI base images.' + required: false + GHCR_USERNAME: + description: 'Optional username to pair with GHCR_TOKEN for private GHCR auth.' + required: false outputs: output_image_tag: description: "The tag of the built image. Ready to use for the next steps." @@ -54,6 +61,8 @@ jobs: PYTHONUNBUFFERED: 1 BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} OUTPUT_IMAGE_TAG: ${{ inputs.output_image_tag }} + GHCR_USERNAME: ${{ secrets.GHCR_USERNAME != '' && secrets.GHCR_USERNAME || github.actor }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN != '' && secrets.GHCR_TOKEN || github.token }} outputs: output_image_tag: ${{ steps.prepare-build.outputs.output_image_tag }} steps: @@ -66,8 +75,8 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} + username: ${{ env.GHCR_USERNAME }} + password: ${{ env.GHCR_TOKEN }} - name: Prepare Build id: prepare-build @@ -145,4 +154,3 @@ jobs: run: | docker logout ghcr.io rm -f $HOME/.ssh $HOME/.git-credentials - From c594f277bd8739f9577f49a684596c54f83225c2 Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 16:51:08 -0700 Subject: [PATCH 5/9] fix: Added back the variable --- src/proxy/client_session.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proxy/client_session.cc b/src/proxy/client_session.cc index 8b9344992..f6bcc2e25 100644 --- a/src/proxy/client_session.cc +++ b/src/proxy/client_session.cc @@ -780,7 +780,7 @@ namespace springtail::pg_proxy { // XXX debugging auto qs_deps = msg->qs_dependencies(); - for (const auto& _ : qs_deps) { + for (const auto& qs : qs_deps) { LOG_DEBUG(LOG_PROXY, LOG_LEVEL_DEBUG3, "[C:{}] Query dependency: {}", _id, qs->to_string()); } From fce09197ed13af14e20b4316ef5f69b8ee5c4faf Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 17:15:35 -0700 Subject: [PATCH 6/9] fix: Extended build timeout --- .github/workflows/rw-build-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rw-build-container.yml b/.github/workflows/rw-build-container.yml index ab236755e..8364cd2e1 100644 --- a/.github/workflows/rw-build-container.yml +++ b/.github/workflows/rw-build-container.yml @@ -51,7 +51,7 @@ jobs: build-container: name: Build Springtail Container runs-on: ubuntu-22.04-arm - timeout-minutes: 20 + timeout-minutes: 30 env: CI_BUILD_TYPE: ${{ inputs.build_type }} REGISTRY_URI: ghcr.io/springtail-inc From dea2f76330cbe8eba2378babd33ca6c478614ac8 Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 17:48:20 -0700 Subject: [PATCH 7/9] fix: unused variable issue under release build --- src/proxy/client_session.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proxy/client_session.cc b/src/proxy/client_session.cc index f6bcc2e25..93f5cb3b3 100644 --- a/src/proxy/client_session.cc +++ b/src/proxy/client_session.cc @@ -782,6 +782,7 @@ namespace springtail::pg_proxy { auto qs_deps = msg->qs_dependencies(); for (const auto& qs : qs_deps) { LOG_DEBUG(LOG_PROXY, LOG_LEVEL_DEBUG3, "[C:{}] Query dependency: {}", _id, qs->to_string()); + (void)qs; // suppress unused variable warning } // add message to front of queue From df308a4ed25e3bfd636ad6b443cd36c1f4ee59d7 Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sat, 11 Apr 2026 17:51:30 -0700 Subject: [PATCH 8/9] fix: Explicitly calls Python3 --- .github/workflows/rw-run-integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rw-run-integration-test.yml b/.github/workflows/rw-run-integration-test.yml index 83296b17b..551aeac55 100644 --- a/.github/workflows/rw-run-integration-test.yml +++ b/.github/workflows/rw-run-integration-test.yml @@ -109,7 +109,7 @@ jobs: run: | git config --global --add safe.directory ${CI_WORKSPACE} # Override some settings in the config file - .github/workflows/scripts/merge_system_settings.py system.json.settings ${SYSTEM_SETTINGS_OVERRIDE} system.json.test + python3 .github/workflows/scripts/merge_system_settings.py system.json.settings ${SYSTEM_SETTINGS_OVERRIDE} system.json.test mkdir -p /opt/springtail/pids # Make sure we are using the correct config for Redis @@ -331,4 +331,4 @@ jobs: run: | docker rm -f pg16 || true docker volume rm ${DKR_VOLUME_NAME} || true - docker network rm ${DKR_NETWORK_NAME} || true \ No newline at end of file + docker network rm ${DKR_NETWORK_NAME} || true From 18da802c1f7c2197f0944d65840eef2ea16c576f Mon Sep 17 00:00:00 2001 From: Jeffrey Bian Date: Sun, 12 Apr 2026 09:39:22 -0700 Subject: [PATCH 9/9] fix: missing .github/workflows --- .github/workflows/rw-run-integration-test.yml | 20 ++++++++++++++++- .github/workflows/rw-run-performance-test.yml | 22 +++++++++++++++++-- .github/workflows/rw-run-unit-test.yml | 22 +++++++++++++++++-- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rw-run-integration-test.yml b/.github/workflows/rw-run-integration-test.yml index 551aeac55..602791201 100644 --- a/.github/workflows/rw-run-integration-test.yml +++ b/.github/workflows/rw-run-integration-test.yml @@ -99,6 +99,15 @@ jobs: with: clean: true + - name: Checkout workflow assets + uses: actions/checkout@v4 + if: inputs.reuse_code == true + with: + clean: true + path: .ci-workflow-assets + sparse-checkout: | + .github/workflows + - name: Reuse existing code if: inputs.reuse_code == true run: | @@ -108,8 +117,17 @@ jobs: - name: Prepare Tests run: | git config --global --add safe.directory ${CI_WORKSPACE} + workflow_repo_root="." + if [ -d .ci-workflow-assets/.github/workflows ]; then + workflow_repo_root=".ci-workflow-assets" + fi + merge_script="${workflow_repo_root}/.github/workflows/scripts/merge_system_settings.py" + override_file="${SYSTEM_SETTINGS_OVERRIDE}" + if [ ! -f "${override_file}" ] && [ -f "${workflow_repo_root}/${SYSTEM_SETTINGS_OVERRIDE}" ]; then + override_file="${workflow_repo_root}/${SYSTEM_SETTINGS_OVERRIDE}" + fi # Override some settings in the config file - python3 .github/workflows/scripts/merge_system_settings.py system.json.settings ${SYSTEM_SETTINGS_OVERRIDE} system.json.test + python3 "${merge_script}" system.json.settings "${override_file}" system.json.test mkdir -p /opt/springtail/pids # Make sure we are using the correct config for Redis diff --git a/.github/workflows/rw-run-performance-test.yml b/.github/workflows/rw-run-performance-test.yml index 72ec7bd16..3a3884360 100644 --- a/.github/workflows/rw-run-performance-test.yml +++ b/.github/workflows/rw-run-performance-test.yml @@ -84,6 +84,15 @@ jobs: with: clean: true + - name: Checkout workflow assets + uses: actions/checkout@v4 + if: inputs.reuse_code == true + with: + clean: true + path: .ci-workflow-assets + sparse-checkout: | + .github/workflows + - name: Reuse existing code if: inputs.reuse_code == true run: | @@ -93,8 +102,17 @@ jobs: - name: Prepare Tests run: | git config --global --add safe.directory ${CI_WORKSPACE} + workflow_repo_root="." + if [ -d .ci-workflow-assets/.github/workflows ]; then + workflow_repo_root=".ci-workflow-assets" + fi + merge_script="${workflow_repo_root}/.github/workflows/scripts/merge_system_settings.py" + override_file="${SYSTEM_SETTINGS_OVERRIDE}" + if [ ! -f "${override_file}" ] && [ -f "${workflow_repo_root}/${SYSTEM_SETTINGS_OVERRIDE}" ]; then + override_file="${workflow_repo_root}/${SYSTEM_SETTINGS_OVERRIDE}" + fi # Override some settings in the config file - .github/workflows/scripts/merge_system_settings.py system.json.settings ${SYSTEM_SETTINGS_OVERRIDE} system.json.test + python3 "${merge_script}" system.json.settings "${override_file}" system.json.test mkdir -p /opt/springtail/pids # Make sure we are using the correct config for Redis @@ -226,4 +244,4 @@ jobs: run: | docker rm -f pg16 || true docker volume rm ${DKR_VOLUME_NAME} || true - docker network rm ${DKR_NETWORK_NAME} || true \ No newline at end of file + docker network rm ${DKR_NETWORK_NAME} || true diff --git a/.github/workflows/rw-run-unit-test.yml b/.github/workflows/rw-run-unit-test.yml index 97c4b5fb2..05c5c6edc 100644 --- a/.github/workflows/rw-run-unit-test.yml +++ b/.github/workflows/rw-run-unit-test.yml @@ -82,6 +82,15 @@ jobs: with: clean: true + - name: Checkout workflow assets + uses: actions/checkout@v4 + if: inputs.reuse_code == true + with: + clean: true + path: .ci-workflow-assets + sparse-checkout: | + .github/workflows + - name: Reuse existing code if: inputs.reuse_code == true run: | @@ -91,8 +100,17 @@ jobs: - name: Prepare Tests run: | git config --global --add safe.directory ${CI_WORKSPACE} + workflow_repo_root="." + if [ -d .ci-workflow-assets/.github/workflows ]; then + workflow_repo_root=".ci-workflow-assets" + fi + merge_script="${workflow_repo_root}/.github/workflows/scripts/merge_system_settings.py" + override_file="${SYSTEM_SETTINGS_OVERRIDE}" + if [ ! -f "${override_file}" ] && [ -f "${workflow_repo_root}/${SYSTEM_SETTINGS_OVERRIDE}" ]; then + override_file="${workflow_repo_root}/${SYSTEM_SETTINGS_OVERRIDE}" + fi # Override some settings in the config file - .github/workflows/scripts/merge_system_settings.py system.json.settings ${SYSTEM_SETTINGS_OVERRIDE} system.json.test + python3 "${merge_script}" system.json.settings "${override_file}" system.json.test mkdir -p /opt/springtail/pids # Make sure we are using the correct config for Redis @@ -227,4 +245,4 @@ jobs: run: | docker rm -f pg16 || true docker volume rm ${DKR_VOLUME_NAME} || true - docker network rm ${DKR_NETWORK_NAME} || true \ No newline at end of file + docker network rm ${DKR_NETWORK_NAME} || true