diff --git a/.github/workflows/build-wheel-and-run-perf-regression-test.yml b/.github/workflows/build-wheel-and-run-perf-regression-test.yml new file mode 100644 index 0000000000..d434e84def --- /dev/null +++ b/.github/workflows/build-wheel-and-run-perf-regression-test.yml @@ -0,0 +1,29 @@ +permissions: + statuses: write + contents: read + packages: read + +on: + workflow_dispatch: + inputs: + test-definition: + required: true + description: Test definition + +jobs: + build-wheel-for-feature-branch: + uses: ./.github/workflows/build-wheels.yml + with: + # TODO: single source + platform-tag: manylinux_x86_64 + python-tags: '["cp310"]' + sha-to-build-and-test: ${{ github.sha }} + secrets: inherit + + run-qe-test: + needs: build-wheel-for-feature-branch + uses: ./.github/workflows/run-qe-test.yml + with: + get-wheel-from-jfrog: false + test-definition: ${{ inputs.test-definition }} + secrets: inherit diff --git a/.github/workflows/run-qe-test.yml b/.github/workflows/run-qe-test.yml new file mode 100644 index 0000000000..e55b6e4ce7 --- /dev/null +++ b/.github/workflows/run-qe-test.yml @@ -0,0 +1,357 @@ +run-name: "Run QE test (python-client-version=${{ inputs.python-client-version-in-jfrog-to-test }}, feature=${{ inputs.feature }}, num_records=${{ inputs.num_records }}, insert_mode=${{ inputs.insert_mode }})" + +permissions: + contents: read + id-token: write + +on: + workflow_dispatch: + inputs: + python-client-version-in-jfrog-to-test: + type: string + required: true + description: Python client version + test-definition: + type: string + required: true + description: "Test definition without the branch (i.e remove '@')" + test-code-revision: + type: string + required: true + description: "Test code revision" + feature: + required: true + description: "Client feature to test" + num_records: + type: number + default: 1000000 + description: Number of records to use in workload + insert_mode: + required: true + type: choice + description: Insert mode means don't check for outliers. + options: + - insert + - outlier + workflow_call: + inputs: + # workflow_call hack + is-workflow-call: + type: boolean + default: true + required: false + get-wheel-from-jfrog: + type: boolean + default: true + required: false + # Optional since calling workflow can also pass in their own wheel built from a feature branch + python-client-version-in-jfrog-to-test: + type: string + required: false + description: Python client version + test-code-revision: + type: string + required: true + description: "Test code revision" + test-definition: + type: string + required: true + description: "Test definition without the branch (i.e remove '@')" + +# Prevent another job from overriding the Docker image +concurrency: + group: ${{ github.workflow }} + +jobs: + # For this job, I used Andrew Kim's workflow in qe-docker repo as a reference + # https://github.com/citrusleaf/qe-docker/commit/d72f2d3fda70e199e24f4f670811d6e33e7f92be + add-python-client-to-qe-test-image-for-performance-regression-tests: + env: + WHEEL_ARCH: x86_64 + outputs: + new-test-image-digest: ${{ steps.build.outputs.digest }} + runs-on: ubuntu-24.04-arm + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@df199fb7be9f65074067a9eb93f12bb4c5547cf2 # v2.13.3 + with: + egress-policy: audit + - name: Get Github action to download wheel from JFrog + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + path: aerospike-client-python + + - name: Get Github actions workflow to build Docker image + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + repository: citrusleaf/qe-docker + path: qe-docker + ref: cicd-client-performance-regression-tests-requirements-txt + token: ${{ secrets.CLIENT_BOT_PAT }} + + - uses: ./aerospike-client-python/.github/actions/get-artifact-for-stage-tests + with: + # Calling workflow only builds wheel from source + get_from_jfrog: ${{ inputs.is-workflow-call == false && true || inputs.get-wheel-from-jfrog }} + jfrog_build_version: ${{ inputs.python-client-version-in-jfrog-to-test }} + dist_type_to_get: 'wheel' + wheel_python_version: '3.10' + wheel_os: manylinux + wheel_cpu_arch: ${{ env.WHEEL_ARCH }} + JFROG_PLATFORM_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JFROG_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + JFROG_REPO_NAME: ${{ vars.JFROG_REPO_NAME }} + + - name: Add wheel to staging folder for building Docker image + run: mv *linux*${{ env.WHEEL_ARCH }}*.whl qe-docker/test/python/_files/ + + - name: Log in to QE's Docker registry to download base image and publish new image + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ secrets.QE_DOCKER_REGISTRY_URL }} + username: ${{ secrets.QE_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.QE_DOCKER_REGISTRY_PASSWORD }} + + # No way to set our own tag + # - run: ./bin/build -b -p -m test/python/3.11-mavenAnsible-client.Dockerfile + + - name: Build and push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + id: build + with: + push: true + # TODO: set branch in tag + file: ./qe-docker/test/python/${{ env.TEST_IMAGE_TAG }}.Dockerfile + # Although our client perf test definitions state the test images where the tag doesn't have the arm- prefix + # Bob seems to look for the same tag from the test definition, but prefixed with arm- + tags: ${{ secrets.QE_DOCKER_REGISTRY_URL }}/test/python:arm-${{ env.TEST_IMAGE_TAG }} + context: ./qe-docker/test/python/_files + env: + TEST_IMAGE_TAG: 3.11-mavenAnsible-client + + register-test-definition-with-custom-env-var-parameters: + needs: add-python-client-to-qe-test-image-for-performance-regression-tests + runs-on: ubuntu-24.04 + env: + REGISTER_FILE_PATH_IN_REPO: perf_regression/register-client-tests.yaml + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@df199fb7be9f65074067a9eb93f12bb4c5547cf2 # v2.13.3 + with: + egress-policy: audit + # TODO: composite action for this would be helpful + # TODO: need special testctl not in jfrog yet + - name: Get test-register + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + repository: citrusleaf/qe-tools + token: ${{ secrets.CLIENT_BOT_PAT }} + path: qe-tools + sparse-checkout: | + bin/test-register + sparse-checkout-cone-mode: false + + - name: Add test-register to PATH + run: echo "$(realpath qe-tools/bin)" >> $GITHUB_PATH + + - uses: jfrog/setup-jfrog-cli@5b06f730cc5a6f55d78b30753f8583454b08c0aa # v4.8.1 + env: + JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + + - run: jf rt download --fail-no-op qe-go-dev-local/testctl.amd64.linux qe-tools/bin/testctl + - run: chmod u+x ./testctl + working-directory: qe-tools/bin + + - name: Add AWS credentials + run: | + mkdir -p ~/.aws + cd ~/.aws + sections=("default" "qe") + for section in ${sections[@]}; + do + cat <<-EOF >> credentials + [$section] + aws_access_key_id = ${{ secrets.QE_TEST_ENQUEUE_AWS_ACCESS_KEY_ID }} + aws_secret_access_key = ${{ secrets.QE_TEST_ENQUEUE_AWS_SECRET_ACCESS_KEY }} + region = us-west-1 + EOF + done + shell: bash + + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + # Entire repo must be cloned because the test definition includes this repo + repository: citrusleaf/aerospike-tests-python + token: ${{ secrets.CLIENT_BOT_PAT }} + path: aerospike-tests-python + ref: ${{ inputs.test-code-revision }} + fetch-depth: 0 + + - run: cp ${{ env.REGISTER_FILE_PATH_IN_REPO }}.template ${{ env.REGISTER_FILE_PATH_IN_REPO }} + working-directory: aerospike-tests-python/ + + - name: Environment variable overrides + run: | + yq eval ".perf_regression/client_centered.environment.MODE = \"${{ inputs.insert_mode }}\"" -i ${{ env.REGISTER_FILE_PATH_IN_REPO }} + yq eval ".perf_regression/client_centered.environment.NUM_RECORDS = \"${{ inputs.num_records }}\"" -i ${{ env.REGISTER_FILE_PATH_IN_REPO }} + yq eval ".perf_regression/client_centered.environment.FEATURE = \"${{ inputs.feature }}\"" -i ${{ env.REGISTER_FILE_PATH_IN_REPO }} + yq eval ".perf_regression/client_centered.environment.TEST_IMAGE_DIGEST = \"${{ needs.add-python-client-to-qe-test-image-for-performance-regression-tests.outputs.new-test-image-digest }}\"" -i ${{ env.REGISTER_FILE_PATH_IN_REPO }} + cat ${{ env.REGISTER_FILE_PATH_IN_REPO }} + working-directory: aerospike-tests-python + + - uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 + with: + oauth-client-id: ${{ vars.TAILSCALE_OIDC_CLIENT_ID }} + audience: ${{ vars.TAILSCALE_OIDC_AUDIENCE }} + tags: tag:aerospike-oidc + args: --exit-node=${{ vars.TAILSCALE_EXIT_NODE}} + version: 1.92.5 + + - run: test-register --config staging ./${{ env.REGISTER_FILE_PATH_IN_REPO }} + working-directory: aerospike-tests-python + + run-qe-test: + env: + TEST_CONFIG_FILE_NAME: client_perf.yaml + TESTCTL_BINARY_NAME: testctl + runs-on: ubuntu-24.04 + needs: [ + register-test-definition-with-custom-env-var-parameters + ] + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@df199fb7be9f65074067a9eb93f12bb4c5547cf2 # v2.13.3 + with: + egress-policy: audit + # - name: Setup Rclone + # uses: AnimMouse/setup-rclone@0d99fa3878a334d3e307c1a8372ad55550fdaea7 + # with: + # rclone_config: ${{ secrets.RCLONE_CONFIG }} + + # - name: Download test-enqueue and testctl + # run: 'rclone --drive-root-folder-id ${{ secrets.TEST_ENQUEUE_PERF_GOOGLE_DRIVE_FOLDER_ID }} copy remote:/ ~/bin' + + # - name: Add test-enqueue and testctl to PATH + # run: | + # ls -l ~/bin + # echo "$(realpath ~/bin)" >> $GITHUB_PATH + + - name: Get test-enqueue + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + repository: citrusleaf/qe-tools + token: ${{ secrets.CLIENT_BOT_PAT }} + path: qe-tools + sparse-checkout: | + bin/test-enqueue + sparse-checkout-cone-mode: false + + - name: Add test-enqueue to PATH + run: echo "$(realpath qe-tools/bin)" >> $GITHUB_PATH + + - uses: jfrog/setup-jfrog-cli@5b06f730cc5a6f55d78b30753f8583454b08c0aa # v4.8.1 + env: + JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} + + - run: jf rt download qe-go-dev-local/testctl.amd64.linux qe-tools/bin/testctl + - run: chmod u+x ./testctl + working-directory: qe-tools/bin + + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + path: aerospike-client-python + sparse-checkout: | + test/${{ env.TEST_CONFIG_FILE_NAME }} + sparse-checkout-cone-mode: false + + - name: Add AWS credentials + run: | + mkdir -p ~/.aws + cd ~/.aws + sections=("default" "qe") + for section in ${sections[@]}; + do + cat <<-EOF >> credentials + [$section] + aws_access_key_id = ${{ secrets.QE_TEST_ENQUEUE_AWS_ACCESS_KEY_ID }} + aws_secret_access_key = ${{ secrets.QE_TEST_ENQUEUE_AWS_SECRET_ACCESS_KEY }} + region = us-west-1 + EOF + done + shell: bash + + - run: testctl --version + + # test-enqueue hides testctl's output, so this can be helpful for debugging + # - run: testctl --config staging run 901b1178-0159-452d-9b36-64d455e568f2 --deployment perf_client + working-directory: aerospike-client-python/test/ + + - name: Allows us to get the exact test run id that was created by test-enqueue. + run: echo TESTCTL_USER=$(uuidgen) >> $GITHUB_ENV + + - name: Set the test code revision to run. + run: yq -i '.definitions.std.[0] = "${{ inputs.test-code-revision }}@${{ inputs.test-definition }}"' ${{ env.TEST_CONFIG_FILE_NAME }} + working-directory: aerospike-client-python/test/ + + - name: Tailscale VPN + uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 + with: + oauth-client-id: ${{ vars.TAILSCALE_OIDC_CLIENT_ID }} + audience: ${{ vars.TAILSCALE_OIDC_AUDIENCE }} + tags: tag:aerospike-oidc + args: --exit-node=${{ vars.TAILSCALE_EXIT_NODE}} + version: 1.92.5 + + - name: Enqueue test run and fail fast if unable to enqueue + # if: ${{ false }} + run: | + test-enqueue --config staging --deployment perf_client ${{ env.TEST_CONFIG_FILE_NAME }} | tee test-enqueue-output.log + enqueue_count=$(grep "COUNT:" test-enqueue-output.log | sed 's/COUNT\://' | xargs) + if [[ "$enqueue_count" != "1" ]]; then + echo "We expected 1 test run to be enqueued, but $enqueue_count was actually enqueued." + exit 1 + fi + working-directory: aerospike-client-python/test/ + + # - name: Print test logs in real time + # run: sshpass -p ${{ secrets.PERF_PASSWORD }} ssh ${{ secrets.PERF_USERNAME }}@${{ secrets.PERF_HOST }} docker logs -f test_0 + + - name: Wait for test run to finish + id: poll-run-id + run: | + while true; do + # Color messes up the run_id's string + # Use xargs to trim whitespace from testctl's output + run_id=$(${{ env.TESTCTL_BINARY_NAME }} --config staging ps --no-color --user ${{ env.TESTCTL_USER }} -n 1 -status completed --columns RUN_ID | sed '1d' | xargs) + if [[ -z "$run_id" ]]; then + echo "Test run has not finished yet..." + sleep 2 + else + echo "Test run $run_id has finished." + break + fi + done + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + + # Downloading builds doesn't work here, so we disable it + - run: ${{ env.TESTCTL_BINARY_NAME }} --config staging download --skip-build-download ${{ steps.poll-run-id.outputs.run_id }} + + - name: Print test run logs + run: | + cd ${{ steps.poll-run-id.outputs.run_id }}*/work/test_0 + echo "stdout:" + cat stdout + echo "stderr:" + cat stderr + + - name: Set Github job status + run: | + run_status=$(${{ env.TESTCTL_BINARY_NAME }} --config staging ps --no-color --user ${{ env.TESTCTL_USER }} -n 1 -status completed --columns STATUS | sed '1d' | xargs) + echo $run_status + if [[ "$run_status" == "Failure" ]]; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/stage-tests.yml b/.github/workflows/stage-tests.yml index 1c76ea938a..8c6a5e1905 100644 --- a/.github/workflows/stage-tests.yml +++ b/.github/workflows/stage-tests.yml @@ -5,6 +5,9 @@ name: Stage tests # The purpose is to test that our artifacts work on the Linux distros / OS versions that the client supports # and QE doesn't have enough disk space for more Linux distros, so we have some tests here in Github Actions +permissions: + contents: read + on: workflow_call: inputs: diff --git a/test/client_perf.yaml b/test/client_perf.yaml new file mode 100644 index 0000000000..9ee1398cc5 --- /dev/null +++ b/test/client_perf.yaml @@ -0,0 +1,15 @@ +build: + repo: citrusleaf/aerospike-server + ref: + - 8.0.0.8 + + script: enterprise + target: + - x86-ubuntu-22.04 + +scenarios: + std: + - perfasd +definitions: + std: + - CLIENT-3541-add-client-centered-performance-regression-tests@perf_regression/client_centered