MID-6955: S3 actions using s5cmd

allure/history_fast/README.md

@@ -0,0 +1,31 @@
+# Allure history action
+Same as `PiwikPRO/actions/allure/history` but uses faster `s5cmd`.
+
+## ⚖️ Why is this a separate action?
+This as a separate action rather than replacing our existing workflow because previous implementation was unstable due to proxy timeouts.
+We need to verify that this current implementation does not have any issues before making it the default.
+
+---
+
+## 🛠 Usage
+```yaml
+- name: Generate S3 paths
+  uses: PiwikPRO/actions/allure/s3_path@master
+  with:
+    environment: ${{ inputs.environment }} # usually it’s just inputs.environment or matrix.environment
+    team: 'qa-team'  # required field. cia/mit etc.
+    matrix_block: ${{ matrix.testblock }} # optional field. Use if the matrix strategy is used.
+    retention: '30days'  # optional field. Default value is 30days
+
+- name: Report generating
+  uses: PiwikPRO/actions/allure/history_fast@master
+  with:
+    aws-access-key-id: ${{ secrets.ARTIFACTORY_S3_ACCESS_KEY_ID }}
+    aws-secret-access-key: ${{ secrets.ARTIFACTORY_S3_SECRET_ACCESS_KEY }}
+    aws-http-proxy: ${{ secrets.FORWARD_PROXY_HTTP }}
+    aws-https-proxy: ${{ secrets.FORWARD_PROXY_HTTPS }}
+    environment:  # usually it’s just inputs.environment or matrix.environment
+    team: 'qa-team' # required field. cia/mit etc.
+    enable-history: 'true'  # optional field. Default value is false. 
+    retention: '60days' # optional field. Default value is 30days
+```

allure/history_fast/action.yaml

@@ -0,0 +1,160 @@
+name: 'Report generating'
+description: 'All required steps after tests'
+inputs:
+  aws-access-key-id:
+    required: true
+    description: AWS access key id
+  aws-secret-access-key:
+    required: true
+    description: AWS secret access key
+  aws-http-proxy:
+    required: true
+    description: AWS http proxy
+  aws-https-proxy:
+    required: true
+    description: AWS https proxy
+  environment:
+    required: true
+    description: Environment name
+  enable-history:
+    required: true
+    default: 'false'
+    description: Enable history
+  retention:
+    required: false
+    default: '30days'
+    description: Test report storage folder
+  team:
+    required: true
+    description: Team name
+  path_to_artifacts:
+    required: false
+    default: 'artifacts/'
+    description: 'Path to parent artifacts directory containing allure/ folder'
+  matrix_block:
+    required: false
+    description: 'Optional field if someone uses a matrix in the repository'
+
+outputs:
+  total:
+    description: 'Total number of tests'
+    value: ${{ steps.allure_metrics.outputs.total }}
+  passed:
+    description: 'Number of passed tests'
+    value: ${{ steps.allure_metrics.outputs.passed }}
+  failed:
+    description: 'Number of failed tests'
+    value: ${{ steps.allure_metrics.outputs.failed }}
+  broken:
+    description: 'Number of broken tests'
+    value: ${{ steps.allure_metrics.outputs.broken }}
+  skipped:
+    description: 'Number of skipped tests'
+    value: ${{ steps.allure_metrics.outputs.skipped }}
+  unknown:
+    description: 'Number of unknown status tests'
+    value: ${{ steps.allure_metrics.outputs.unknown }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Set environment variables
+      shell: bash
+      run: |
+        MATRIX_BLOCK="${{ inputs.matrix_block }}"
+        MATRIX_BLOCK="${MATRIX_BLOCK// /_}"
+        MATRIX_SUFFIX="${MATRIX_BLOCK:+/${MATRIX_BLOCK}}"
+
+        echo "S3_BUCKET=piwikpro-artifactory" >> $GITHUB_ENV
+        echo "S3_HISTORY_PATH=${{ inputs.retention }}/${{ inputs.team }}/tests/history/${{ github.workflow_ref }}/${{ inputs.environment }}${MATRIX_SUFFIX}" >> $GITHUB_ENV
+
+    - name: Prepare history directory
+      if: ${{ inputs.enable-history == 'true' && !cancelled() }}
+      shell: bash
+      run: |
+        ARTIFACTS_PARENT="${{ inputs.path_to_artifacts }}"
+        ARTIFACTS_PARENT="${ARTIFACTS_PARENT%/}"
+
+        TARGET_USER=${SUDO_USER:-$(id -un)}
+        TARGET_GROUP=$(id -gn "$TARGET_USER")
+        ALLURE_REPORT_HISTORY_DIR="${ARTIFACTS_PARENT}/allure/history"
+        sudo mkdir -p "${ALLURE_REPORT_HISTORY_DIR}"
+        sudo chown -R "$TARGET_USER:$TARGET_GROUP" "${ARTIFACTS_PARENT}/allure"
+
+    - name: Download Allure history from S3 before generating report
+      if: ${{ inputs.enable-history == 'true' && !cancelled() }}
+      uses: PiwikPRO/actions/s3/s5cmd/download@master
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-bucket: ${{ env.S3_BUCKET }}
+        aws-region: eu-central-1
+        aws-http-proxy: ${{ inputs.aws-http-proxy }}
+        aws-https-proxy: ${{ inputs.aws-https-proxy }}
+        src-path: ${{ env.S3_HISTORY_PATH }}/
+        dst-path: ${{ inputs.path_to_artifacts }}allure/history/
+        ignore-missing: 'true'
+
+    - name: Generate allure report
+      uses: PiwikPRO/actions/allure/report@master
+      with:
+        path_to_artifacts: ${{ inputs.path_to_artifacts }}
+
+    - name: Extract Allure metrics
+      id: allure_metrics
+      shell: bash
+      run: |
+        ARTIFACTS_PARENT="${{ inputs.path_to_artifacts }}"
+        ARTIFACTS_PARENT="${ARTIFACTS_PARENT%/}"
+        SUMMARY_JSON="${ARTIFACTS_PARENT}/allure-report/widgets/summary.json"
+        if [ -f "$SUMMARY_JSON" ]; then
+          TOTAL=$(jq -r '.statistic.total // 0' "$SUMMARY_JSON")
+          PASSED=$(jq -r '.statistic.passed // 0' "$SUMMARY_JSON")
+          FAILED=$(jq -r '.statistic.failed // 0' "$SUMMARY_JSON")
+          BROKEN=$(jq -r '.statistic.broken // 0' "$SUMMARY_JSON")
+          SKIPPED=$(jq -r '.statistic.skipped // 0' "$SUMMARY_JSON")
+          UNKNOWN=$(jq -r '.statistic.unknown // 0' "$SUMMARY_JSON")
+        else
+          TOTAL=0; PASSED=0; FAILED=0; BROKEN=0; SKIPPED=0; UNKNOWN=0
+        fi
+        echo "total=${TOTAL}" >> $GITHUB_OUTPUT
+        echo "passed=${PASSED}" >> $GITHUB_OUTPUT
+        echo "failed=${FAILED}" >> $GITHUB_OUTPUT
+        echo "broken=${BROKEN}" >> $GITHUB_OUTPUT
+        echo "skipped=${SKIPPED}" >> $GITHUB_OUTPUT
+        echo "unknown=${UNKNOWN}" >> $GITHUB_OUTPUT
+
+    - name: Upload HTML report to S3
+      uses: PiwikPRO/actions/s3/s5cmd/upload@master
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-http-proxy: ${{ inputs.aws-http-proxy }}
+        aws-https-proxy: ${{ inputs.aws-https-proxy }}
+        aws-bucket: ${{ env.S3_BUCKET }}
+        aws-region: eu-central-1
+        src-path: ${{ inputs.path_to_artifacts }}
+        dst-path: ${{ env.S3_PATH }}
+
+    - name: Upload new Allure history to S3
+      if: ${{ inputs.enable-history == 'true' && !cancelled() }}
+      uses: PiwikPRO/actions/s3/s5cmd/upload@master
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-bucket: ${{ env.S3_BUCKET }}
+        aws-region: eu-central-1
+        aws-http-proxy: ${{ inputs.aws-http-proxy }}
+        aws-https-proxy: ${{ inputs.aws-https-proxy }}
+        src-path: ${{ inputs.path_to_artifacts }}allure-report/history/
+        dst-path: ${{ env.S3_HISTORY_PATH }}/
+        echo-destination-index-html: 'false'
+
+    - name: Generate summary
+      shell: bash
+      run: |
+        echo "[Allure Report](${{ env.ALLURE_REPORT_URL }})" >> $GITHUB_STEP_SUMMARY
+        echo "Branch: ${{ github.head_ref || github.ref_name }}" >> $GITHUB_STEP_SUMMARY
+        echo "Tests: total ${{ steps.allure_metrics.outputs.total }} | passed ${{ steps.allure_metrics.outputs.passed }} | failed ${{ steps.allure_metrics.outputs.failed }} | broken ${{ steps.allure_metrics.outputs.broken }} | skipped ${{ steps.allure_metrics.outputs.skipped }}" >> $GITHUB_STEP_SUMMARY
+        echo "ALLURE_REPORT_URL=${{ env.ALLURE_REPORT_URL }}" >> $GITHUB_ENV

s3/s5cmd/README.md

@@ -0,0 +1,53 @@
+# S3 Fast Transfer Actions
+This GitHub Actions provides a high-performance alternative for uploading and downloading objects to/from Amazon S3 using s5cmd.
+
+## 🚀 Why use `s5cmd`?
+While the standard AWS CLI is the reliable "go-to" for most S3 operations, it often becomes a bottleneck when dealing with a large volume of small files.
+**s5cmd** is a faster, parallelized alternative written in Go. 
+The performance gains are significant when uploading our standard Allure report:
+* **AWS CLI:** ~7 minutes
+* **`s5cmd`:** ~1 minute
+
+## ⚖️ Why is this a separate action?
+This as a separate action rather than replacing our existing AWS CLI-based workflow for several reasons:
+- **Stability:** A previous implementation using `s5cmd` was unstable due to proxy timeouts. This version needs to be monitored under various network conditions.
+- **Maturity:** We need to verify that this current implementation does not have any unforeseen drawbacks before making it the default.
+- **Rollback Safety:** Keeping it separate allows teams to revert to the proven AWS CLI solution instantly if issues arise.
+
+---
+
+## 🛠 Usage
+The behavior and inputs are designed to be a drop-in replacement for the standard `PiwikPRO/actions/s3/upload`.
+
+### Upload
+```yaml
+- name: Upload to S3
+  uses: PiwikPRO/actions/s3/s5cmd/upload@master
+  with:
+    aws-access-key-id: ${{ inputs.aws-access-key-id }}
+    aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+    aws-http-proxy: ${{ inputs.aws-http-proxy }}
+    aws-https-proxy: ${{ inputs.aws-https-proxy }}
+    aws-bucket: piwikpro-artifactory
+    aws-region: eu-central-1
+    src-path: artifacts/
+    dst-path: ${{ github.repository }}/@${{ github.ref_name }}/artifacts/
+    echo-destination-index-html: true
+```
+
+### Download
+The download action includes an additional `ignore-missing` flag, which prevents the step from failing if no files are found at the source path.
+```yaml
+- name: Download from S3
+  uses: PiwikPRO/actions/s3/s5cmd/download@master
+  with:
+    aws-access-key-id: ${{ inputs.aws-access-key-id }}
+    aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+    aws-http-proxy: ${{ inputs.aws-http-proxy }}
+    aws-https-proxy: ${{ inputs.aws-https-proxy }}
+    aws-bucket: piwikpro-artifactory
+    aws-region: eu-central-1
+    src-path: ${{ github.repository }}/@${{ github.ref_name }}/artifacts/
+    dst-path: dest-path/
+    ignore-missing: true
+```

s3/s5cmd/download/action.yaml

@@ -0,0 +1,78 @@
+name: 'Download from S3'
+description: 'Download recursively from s3.'
+inputs:
+  src-path:
+    required: false
+    description: Path to source dir in S3. Leading/trailing slashes will be normalized.
+    default: "${{ github.repository }}/@${{ github.ref_name }}/artifacts"
+  dst-path:
+    required: false
+    description: Path to local destination dir. Path must be relative to `github.workspace`. Leading/trailing slashes will be normalized.
+    default: 'artifacts'
+  aws-access-key-id:
+    required: true
+    description: AWS Access Key ID
+  aws-secret-access-key:
+    required: true
+    description: AWS Secret Access Key
+  aws-bucket:
+    required: true
+    description: AWS Bucket
+  aws-region:
+    required: false
+    description: AWS Region
+    default: eu-central-1
+  ignore-missing:
+    required: false
+    description: Do not fail if no objects are found at the source path.
+    default: 'false'
+  aws-http-proxy:
+    required: false
+    description: URI for aws-cli HTTP proxy
+  aws-https-proxy:
+    required: false
+    description: URI for aws-cli HTTPS proxy
+runs:
+  using: 'composite'
+  steps:
+    - name: Install s5cmd
+      shell: bash
+      run: |
+        S5CMD_VERSION="2.3.0"
+        curl -fsSL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/s5cmd_${S5CMD_VERSION}_Linux-64bit.tar.gz" -o /tmp/s5cmd.tar.gz
+        sudo tar xzf /tmp/s5cmd.tar.gz -C /usr/local/bin s5cmd
+
+    - name: Normalize paths
+      shell: bash
+      run: |
+        SOURCE_PATH="${{ inputs.src-path }}"
+        DESTINATION_PATH="${{ inputs.dst-path }}"
+
+        # Remove leading slashes
+        SOURCE_PATH="${SOURCE_PATH#/}"
+        DESTINATION_PATH="${DESTINATION_PATH#/}"
+
+        # Ensure trailing slashes
+        DESTINATION_PATH="${DESTINATION_PATH%/}/"
+        SOURCE_PATH="${SOURCE_PATH%/}/"
+
+        echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV
+        echo "DESTINATION_PATH=$DESTINATION_PATH" >> $GITHUB_ENV
+
+    - name: Download from s3
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+        HTTP_PROXY: ${{ inputs.aws-http-proxy }}
+        HTTPS_PROXY: ${{ inputs.aws-https-proxy }}
+      run: >-
+        s5cmd
+        --log error
+        --numworkers 50
+        --retry-count 30
+        cp
+        "s3://${{ inputs.aws-bucket }}/${{ env.SOURCE_PATH }}*"
+        "${{ github.workspace }}/${{ env.DESTINATION_PATH }}"
+        || [[ "${{ inputs.ignore-missing }}" == "true" ]] || exit 1