From b30074228188424c1685217b27b9852074cfa176 Mon Sep 17 00:00:00 2001 From: dkaruga-piwikpro Date: Mon, 2 Mar 2026 15:19:17 +0100 Subject: [PATCH 1/3] MID-6955: Add s5cmd actions for S3 upload and download --- allure/history_fast/README.md | 31 +++++ allure/history_fast/action.yaml | 116 ++++++++++++++++++ s3/s5cmd/README.md | 53 ++++++++ s3/s5cmd/download/action.yaml | 78 ++++++++++++ s3/s5cmd/upload/action.yaml | 87 +++++++++++++ .../upload/create-index-html-if-not-exists.py | 44 +++++++ 6 files changed, 409 insertions(+) create mode 100644 allure/history_fast/README.md create mode 100644 allure/history_fast/action.yaml create mode 100644 s3/s5cmd/README.md create mode 100644 s3/s5cmd/download/action.yaml create mode 100644 s3/s5cmd/upload/action.yaml create mode 100644 s3/s5cmd/upload/create-index-html-if-not-exists.py diff --git a/allure/history_fast/README.md b/allure/history_fast/README.md new file mode 100644 index 0000000..877f51b --- /dev/null +++ b/allure/history_fast/README.md @@ -0,0 +1,31 @@ +# Allure history action +Same as `PiwikPRO/actions/allure/history` but uses faster `s5cmd`. + +## โš–๏ธ Why is this a separate action? +This as a separate action rather than replacing our existing workflow because previous implementation was unstable due to proxy timeouts. +We need to verify that this current implementation does not have any issues before making it the default. + +--- + +## ๐Ÿ›  Usage +```yaml +- name: Generate S3 paths + uses: PiwikPRO/actions/allure/s3_path@master + with: + environment: ${{ inputs.environment }} # usually itโ€™s just inputs.environment or matrix.environment + team: 'qa-team' # required field. cia/mit etc. + matrix_block: ${{ matrix.testblock }} # optional field. Use if the matrix strategy is used. + retention: '30days' # optional field. Default value is 30days + +- name: Report generating + uses: PiwikPRO/actions/allure/history_fast@master + with: + aws-access-key-id: ${{ secrets.ARTIFACTORY_S3_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.ARTIFACTORY_S3_SECRET_ACCESS_KEY }} + aws-http-proxy: ${{ secrets.FORWARD_PROXY_HTTP }} + aws-https-proxy: ${{ secrets.FORWARD_PROXY_HTTPS }} + environment: # usually itโ€™s just inputs.environment or matrix.environment + team: 'qa-team' # required field. cia/mit etc. + enable-history: 'true' # optional field. Default value is false. + retention: '60days' # optional field. Default value is 30days +``` \ No newline at end of file diff --git a/allure/history_fast/action.yaml b/allure/history_fast/action.yaml new file mode 100644 index 0000000..a653855 --- /dev/null +++ b/allure/history_fast/action.yaml @@ -0,0 +1,116 @@ +name: 'Report generating' +description: 'All required steps after tests' +inputs: + aws-access-key-id: + required: true + description: AWS access key id + aws-secret-access-key: + required: true + description: AWS secret access key + aws-http-proxy: + required: true + description: AWS http proxy + aws-https-proxy: + required: true + description: AWS https proxy + environment: + required: true + description: Environment name + enable-history: + required: true + default: 'false' + description: Enable history + retention: + required: false + default: '30days' + description: Test report storage folder + team: + required: true + description: Team name + path_to_artifacts: + required: false + default: 'artifacts/' + description: 'Path to parent artifacts directory containing allure/ folder' + matrix_block: + required: false + description: 'Optional field if someone uses a matrix in the repository' + + +runs: + using: "composite" + steps: + + - name: Set environment variables + shell: bash + run: | + MATRIX_BLOCK="${{ inputs.matrix_block }}" + MATRIX_BLOCK="${MATRIX_BLOCK// /_}" + MATRIX_SUFFIX="${MATRIX_BLOCK:+/${MATRIX_BLOCK}}" + + echo "S3_BUCKET=piwikpro-artifactory" >> $GITHUB_ENV + echo "S3_HISTORY_PATH=${{ inputs.retention }}/${{ inputs.team }}/tests/history/${{ github.workflow_ref }}/${{ inputs.environment }}${MATRIX_SUFFIX}" >> $GITHUB_ENV + + - name: Prepare history directory + if: ${{ inputs.enable-history == 'true' && !cancelled() }} + shell: bash + run: | + ARTIFACTS_PARENT="${{ inputs.path_to_artifacts }}" + ARTIFACTS_PARENT="${ARTIFACTS_PARENT%/}" + + TARGET_USER=${SUDO_USER:-$(id -un)} + TARGET_GROUP=$(id -gn "$TARGET_USER") + ALLURE_REPORT_HISTORY_DIR="${ARTIFACTS_PARENT}/allure/history" + sudo mkdir -p "${ALLURE_REPORT_HISTORY_DIR}" + sudo chown -R "$TARGET_USER:$TARGET_GROUP" "${ARTIFACTS_PARENT}/allure" + + - name: Download Allure history from S3 before generating report + if: ${{ inputs.enable-history == 'true' && !cancelled() }} + uses: PiwikPRO/actions/s3/s5cmd/download@master + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-bucket: ${{ env.S3_BUCKET }} + aws-region: eu-central-1 + aws-http-proxy: ${{ inputs.aws-http-proxy }} + aws-https-proxy: ${{ inputs.aws-https-proxy }} + src-path: ${{ env.S3_HISTORY_PATH }}/ + dst-path: ${{ inputs.path_to_artifacts }}allure/history/ + ignore-missing: 'true' + + - name: Generate allure report + uses: PiwikPRO/actions/allure/report@master + with: + path_to_artifacts: ${{ inputs.path_to_artifacts }} + + - name: Upload HTML report to S3 + uses: PiwikPRO/actions/s3/s5cmd/upload@master + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-http-proxy: ${{ inputs.aws-http-proxy }} + aws-https-proxy: ${{ inputs.aws-https-proxy }} + aws-bucket: ${{ env.S3_BUCKET }} + aws-region: eu-central-1 + src-path: ${{ inputs.path_to_artifacts }} + dst-path: ${{ env.S3_PATH }} + + - name: Upload new Allure history to S3 + if: ${{ inputs.enable-history == 'true' && !cancelled() }} + uses: PiwikPRO/actions/s3/s5cmd/upload@master + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-bucket: ${{ env.S3_BUCKET }} + aws-region: eu-central-1 + aws-http-proxy: ${{ inputs.aws-http-proxy }} + aws-https-proxy: ${{ inputs.aws-https-proxy }} + src-path: ${{ inputs.path_to_artifacts }}allure-report/history/ + dst-path: ${{ env.S3_HISTORY_PATH }}/ + echo-destination-index-html: 'false' + + - name: Generate summary + shell: bash + run: | + echo "[Allure Report](${{ env.ALLURE_REPORT_URL }})" >> $GITHUB_STEP_SUMMARY + echo "Branch: ${{ github.head_ref || github.ref_name }}" >> $GITHUB_STEP_SUMMARY + echo "ALLURE_REPORT_URL=${{ env.ALLURE_REPORT_URL }}" >> $GITHUB_ENV \ No newline at end of file diff --git a/s3/s5cmd/README.md b/s3/s5cmd/README.md new file mode 100644 index 0000000..1dace36 --- /dev/null +++ b/s3/s5cmd/README.md @@ -0,0 +1,53 @@ +# S3 Fast Transfer Actions +This GitHub Actions provides a high-performance alternative for uploading and downloading objects to/from Amazon S3 using s5cmd. + +## ๐Ÿš€ Why use `s5cmd`? +While the standard AWS CLI is the reliable "go-to" for most S3 operations, it often becomes a bottleneck when dealing with a large volume of small files. +**s5cmd** is a faster, parallelized alternative written in Go. +The performance gains are significant when uploading our standard Allure report: +* **AWS CLI:** ~7 minutes +* **`s5cmd`:** ~1 minute + +## โš–๏ธ Why is this a separate action? +This as a separate action rather than replacing our existing AWS CLI-based workflow for several reasons: +- **Stability:** A previous implementation using `s5cmd` was unstable due to proxy timeouts. This version needs to be monitored under various network conditions. +- **Maturity:** We need to verify that this current implementation does not have any unforeseen drawbacks before making it the default. +- **Rollback Safety:** Keeping it separate allows teams to revert to the proven AWS CLI solution instantly if issues arise. + +--- + +## ๐Ÿ›  Usage +The behavior and inputs are designed to be a drop-in replacement for the standard `PiwikPRO/actions/s3/upload`. + +### Upload +```yaml +- name: Upload to S3 + uses: PiwikPRO/actions/s3/s5cmd/upload@master + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-http-proxy: ${{ inputs.aws-http-proxy }} + aws-https-proxy: ${{ inputs.aws-https-proxy }} + aws-bucket: piwikpro-artifactory + aws-region: eu-central-1 + src-path: artifacts/ + dst-path: ${{ github.repository }}/@${{ github.ref_name }}/artifacts/ + echo-destination-index-html: true +``` + +### Download +The download action includes an additional `ignore-missing` flag, which prevents the step from failing if no files are found at the source path. +```yaml +- name: Download from S3 + uses: PiwikPRO/actions/s3/s5cmd/download@master + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-http-proxy: ${{ inputs.aws-http-proxy }} + aws-https-proxy: ${{ inputs.aws-https-proxy }} + aws-bucket: piwikpro-artifactory + aws-region: eu-central-1 + src-path: ${{ github.repository }}/@${{ github.ref_name }}/artifacts/ + dst-path: dest-path/ + ignore-missing: true +``` \ No newline at end of file diff --git a/s3/s5cmd/download/action.yaml b/s3/s5cmd/download/action.yaml new file mode 100644 index 0000000..f4d9423 --- /dev/null +++ b/s3/s5cmd/download/action.yaml @@ -0,0 +1,78 @@ +name: 'Download from S3' +description: 'Download recursively from s3.' +inputs: + src-path: + required: false + description: Path to source dir in S3. Leading/trailing slashes will be normalized. + default: "${{ github.repository }}/@${{ github.ref_name }}/artifacts" + dst-path: + required: false + description: Path to local destination dir. Path must be relative to `github.workspace`. Leading/trailing slashes will be normalized. + default: 'artifacts' + aws-access-key-id: + required: true + description: AWS Access Key ID + aws-secret-access-key: + required: true + description: AWS Secret Access Key + aws-bucket: + required: true + description: AWS Bucket + aws-region: + required: false + description: AWS Region + default: eu-central-1 + ignore-missing: + required: false + description: Do not fail if no objects are found at the source path. + default: 'false' + aws-http-proxy: + required: false + description: URI for aws-cli HTTP proxy + aws-https-proxy: + required: false + description: URI for aws-cli HTTPS proxy +runs: + using: 'composite' + steps: + - name: Install s5cmd + shell: bash + run: | + S5CMD_VERSION="2.3.0" + curl -fsSL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/s5cmd_${S5CMD_VERSION}_Linux-64bit.tar.gz" -o /tmp/s5cmd.tar.gz + sudo tar xzf /tmp/s5cmd.tar.gz -C /usr/local/bin s5cmd + + - name: Normalize paths + shell: bash + run: | + SOURCE_PATH="${{ inputs.src-path }}" + DESTINATION_PATH="${{ inputs.dst-path }}" + + # Remove leading slashes + SOURCE_PATH="${SOURCE_PATH#/}" + DESTINATION_PATH="${DESTINATION_PATH#/}" + + # Ensure trailing slashes + DESTINATION_PATH="${DESTINATION_PATH%/}/" + SOURCE_PATH="${SOURCE_PATH%/}/" + + echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV + echo "DESTINATION_PATH=$DESTINATION_PATH" >> $GITHUB_ENV + + - name: Download from s3 + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} + AWS_DEFAULT_REGION: ${{ inputs.aws-region }} + HTTP_PROXY: ${{ inputs.aws-http-proxy }} + HTTPS_PROXY: ${{ inputs.aws-https-proxy }} + run: >- + s5cmd + --log error + --numworkers 50 + --retry-count 30 + cp + "s3://${{ inputs.aws-bucket }}/${{ env.SOURCE_PATH }}*" + "${{ github.workspace }}/${{ env.DESTINATION_PATH }}" + || [[ "${{ inputs.ignore-missing }}" == "true" ]] || exit 1 diff --git a/s3/s5cmd/upload/action.yaml b/s3/s5cmd/upload/action.yaml new file mode 100644 index 0000000..ef94270 --- /dev/null +++ b/s3/s5cmd/upload/action.yaml @@ -0,0 +1,87 @@ +name: 'Upload to S3' +description: 'Upload recursively to s3.' +inputs: + src-path: + required: false + description: Path to dir that may be uploaded. Path must be relative to `github.workspace`. Leading/trailing slashes will be normalized. + default: 'artifacts' + dst-path: + required: false + description: Path to destination dir. Leading/trailing slashes will be normalized. + default: "${{ github.repository }}/@${{ github.ref_name }}/artifacts" + aws-access-key-id: + required: true + description: AWS Access Key ID + aws-secret-access-key: + required: true + description: AWS Secret Access Key + aws-bucket: + required: true + description: AWS Bucket + aws-region: + required: false + description: AWS Region + default: eu-central-1 + echo-destination-index-html: + required: false + description: Print url to index.html in copied dir. + default: 'true' + aws-http-proxy: + required: false + description: URI for aws-cli HTTP proxy + aws-https-proxy: + required: false + description: URI for aws-cli HTTPS proxy +runs: + using: 'composite' + steps: + - name: Install s5cmd + shell: bash + run: | + S5CMD_VERSION="2.3.0" + curl -fsSL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/s5cmd_${S5CMD_VERSION}_Linux-64bit.tar.gz" -o /tmp/s5cmd.tar.gz + sudo tar xzf /tmp/s5cmd.tar.gz -C /usr/local/bin s5cmd + + - name: Normalize paths + shell: bash + run: | + SOURCE_PATH="${{ inputs.src-path }}" + DESTINATION_PATH="${{ inputs.dst-path }}" + + # Remove leading slashes + SOURCE_PATH="${SOURCE_PATH#/}" + DESTINATION_PATH="${DESTINATION_PATH#/}" + + # Ensure trailing slashes + DESTINATION_PATH="${DESTINATION_PATH%/}/" + SOURCE_PATH="${SOURCE_PATH%/}/" + + echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV + echo "DESTINATION_PATH=$DESTINATION_PATH" >> $GITHUB_ENV + + - name: Generate index.html if it doesn't exist + if: ${{ inputs.echo-destination-index-html == 'true' }} + shell: bash + run: python ${{ github.action_path }}/create-index-html-if-not-exists.py ${{ github.workspace }}/${{ env.SOURCE_PATH }} + + - name: Copy to s3 + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} + AWS_DEFAULT_REGION: ${{ inputs.aws-region }} + HTTP_PROXY: ${{ inputs.aws-http-proxy }} + HTTPS_PROXY: ${{ inputs.aws-https-proxy }} + run: >- + s5cmd + --log error + --numworkers 50 + --retry-count 30 + cp + "${{ github.workspace }}/${{ env.SOURCE_PATH }}" + "s3://${{ inputs.aws-bucket }}/${{ env.DESTINATION_PATH }}" + + - name: Report URL + if: ${{ inputs.echo-destination-index-html == 'true' }} + shell: bash + run: echo https://${{ inputs.aws-bucket }}.s3.amazonaws.com/${{ env.DESTINATION_PATH }}index.html diff --git a/s3/s5cmd/upload/create-index-html-if-not-exists.py b/s3/s5cmd/upload/create-index-html-if-not-exists.py new file mode 100644 index 0000000..dedd3fe --- /dev/null +++ b/s3/s5cmd/upload/create-index-html-if-not-exists.py @@ -0,0 +1,44 @@ +import os +import sys +import glob + + +assert ( + len(sys.argv) == 2 +), "Pass a single argument, that is a path to directory where index.html should be created" + +directory_to_create_index_of = sys.argv[1] + +if not os.path.exists(directory_to_create_index_of): + print("The requested directory does not exists - skipping generation of index.html") + sys.exit(0) + +index_html_path = os.path.join(directory_to_create_index_of, "index.html") + +if os.path.isfile(index_html_path): + sys.exit(0) + +files_to_be_included_in_the_list = set() +for file in glob.glob(directory_to_create_index_of + "/**/**", recursive=True): + if os.path.isdir(file): + continue + files_to_be_included_in_the_list.add( + os.path.relpath(os.path.abspath(file), os.path.abspath(directory_to_create_index_of)) + ) + +index_html_content = ( + "

Index of artifacts:

" + "
" + "" +) + +with open(index_html_path, "w") as f: + f.write(index_html_content) From 474a9cba1b2de74248a68f9679bb7dd5404ae20b Mon Sep 17 00:00:00 2001 From: dkaruga-piwikpro Date: Mon, 2 Mar 2026 15:34:41 +0100 Subject: [PATCH 2/3] MID-6955: Add test metrics extraction to action --- allure/history_fast/action.yaml | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/allure/history_fast/action.yaml b/allure/history_fast/action.yaml index a653855..56f8493 100644 --- a/allure/history_fast/action.yaml +++ b/allure/history_fast/action.yaml @@ -35,6 +35,25 @@ inputs: required: false description: 'Optional field if someone uses a matrix in the repository' +outputs: + total: + description: 'Total number of tests' + value: ${{ steps.allure_metrics.outputs.total }} + passed: + description: 'Number of passed tests' + value: ${{ steps.allure_metrics.outputs.passed }} + failed: + description: 'Number of failed tests' + value: ${{ steps.allure_metrics.outputs.failed }} + broken: + description: 'Number of broken tests' + value: ${{ steps.allure_metrics.outputs.broken }} + skipped: + description: 'Number of skipped tests' + value: ${{ steps.allure_metrics.outputs.skipped }} + unknown: + description: 'Number of unknown status tests' + value: ${{ steps.allure_metrics.outputs.unknown }} runs: using: "composite" @@ -82,6 +101,30 @@ runs: with: path_to_artifacts: ${{ inputs.path_to_artifacts }} + - name: Extract Allure metrics + id: allure_metrics + shell: bash + run: | + ARTIFACTS_PARENT="${{ inputs.path_to_artifacts }}" + ARTIFACTS_PARENT="${ARTIFACTS_PARENT%/}" + SUMMARY_JSON="${ARTIFACTS_PARENT}/allure-report/widgets/summary.json" + if [ -f "$SUMMARY_JSON" ]; then + TOTAL=$(jq -r '.statistic.total // 0' "$SUMMARY_JSON") + PASSED=$(jq -r '.statistic.passed // 0' "$SUMMARY_JSON") + FAILED=$(jq -r '.statistic.failed // 0' "$SUMMARY_JSON") + BROKEN=$(jq -r '.statistic.broken // 0' "$SUMMARY_JSON") + SKIPPED=$(jq -r '.statistic.skipped // 0' "$SUMMARY_JSON") + UNKNOWN=$(jq -r '.statistic.unknown // 0' "$SUMMARY_JSON") + else + TOTAL=0; PASSED=0; FAILED=0; BROKEN=0; SKIPPED=0; UNKNOWN=0 + fi + echo "total=${TOTAL}" >> $GITHUB_OUTPUT + echo "passed=${PASSED}" >> $GITHUB_OUTPUT + echo "failed=${FAILED}" >> $GITHUB_OUTPUT + echo "broken=${BROKEN}" >> $GITHUB_OUTPUT + echo "skipped=${SKIPPED}" >> $GITHUB_OUTPUT + echo "unknown=${UNKNOWN}" >> $GITHUB_OUTPUT + - name: Upload HTML report to S3 uses: PiwikPRO/actions/s3/s5cmd/upload@master with: @@ -113,4 +156,5 @@ runs: run: | echo "[Allure Report](${{ env.ALLURE_REPORT_URL }})" >> $GITHUB_STEP_SUMMARY echo "Branch: ${{ github.head_ref || github.ref_name }}" >> $GITHUB_STEP_SUMMARY + echo "Tests: total ${{ steps.allure_metrics.outputs.total }} | passed ${{ steps.allure_metrics.outputs.passed }} | failed ${{ steps.allure_metrics.outputs.failed }} | broken ${{ steps.allure_metrics.outputs.broken }} | skipped ${{ steps.allure_metrics.outputs.skipped }}" >> $GITHUB_STEP_SUMMARY echo "ALLURE_REPORT_URL=${{ env.ALLURE_REPORT_URL }}" >> $GITHUB_ENV \ No newline at end of file From 650e82f7459249b34d193514857e9fd3b4860e39 Mon Sep 17 00:00:00 2001 From: dkaruga-piwikpro Date: Mon, 2 Mar 2026 15:37:50 +0100 Subject: [PATCH 3/3] MID-6955: Fix missing newline in action.yaml file --- allure/history_fast/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/allure/history_fast/action.yaml b/allure/history_fast/action.yaml index 56f8493..3f81935 100644 --- a/allure/history_fast/action.yaml +++ b/allure/history_fast/action.yaml @@ -157,4 +157,4 @@ runs: echo "[Allure Report](${{ env.ALLURE_REPORT_URL }})" >> $GITHUB_STEP_SUMMARY echo "Branch: ${{ github.head_ref || github.ref_name }}" >> $GITHUB_STEP_SUMMARY echo "Tests: total ${{ steps.allure_metrics.outputs.total }} | passed ${{ steps.allure_metrics.outputs.passed }} | failed ${{ steps.allure_metrics.outputs.failed }} | broken ${{ steps.allure_metrics.outputs.broken }} | skipped ${{ steps.allure_metrics.outputs.skipped }}" >> $GITHUB_STEP_SUMMARY - echo "ALLURE_REPORT_URL=${{ env.ALLURE_REPORT_URL }}" >> $GITHUB_ENV \ No newline at end of file + echo "ALLURE_REPORT_URL=${{ env.ALLURE_REPORT_URL }}" >> $GITHUB_ENV