action.yaml

name: 'Benchmarkoor'
description: 'Run Ethereum execution client benchmarks using benchmarkoor'
author: 'ethpandaops'
branding:
  icon: 'activity'
  color: 'gray-dark'

inputs:
  github-token:
    description: 'GitHub token for API access'
    required: true

  image:
    description: 'Docker image for benchmarkoor (e.g., ghcr.io/ethpandaops/benchmarkoor:master). If not provided, will build locally.'
    required: false
    default: ''

  git-ref:
    description: 'Git branch or commit hash to build from. Only used when image is not provided. Defaults to master.'
    required: false
    default: ''

  upload-artifacts:
    description: 'Whether to upload run results as GitHub artifacts'
    required: false
    default: 'false'

  run-config-urls:
    description: 'Comma-separated URLs for config files to download and pass via --config (merged in order)'
    required: false
    default: ''

  run-config:
    description: 'Raw YAML config content to pass via --config (appended after URL config)'
    required: false
    default: ''

  run-args:
    description: 'Extra flags passed to benchmarkoor run command'
    required: false
    default: ''

  tmp-dir:
    description: 'Temporary directory path used for benchmarkoor data (overlayfs) and cache'
    required: false
    default: '/tmp'

  docker-env:
    description: 'Newline-separated environment variables to pass into the Docker container (e.g. AWS_ACCESS_KEY_ID, MY_VAR=value)'
    required: false
    default: ''

runs:
  using: 'composite'
  steps:
    - name: Verify and install dependencies
      uses: ethpandaops/benchmarkoor/.github/actions/dependencies@0ff45ea539387c901ac4a7845e2c6e6aeb9d8f25

    - name: Build Docker image locally
      if: inputs.image == ''
      shell: bash
      run: |
        GIT_REF="${{ inputs.git-ref }}"
        if [ -z "$GIT_REF" ]; then
          GIT_REF="master"
        fi

        echo "Cloning https://github.com/ethpandaops/benchmarkoor.git at ref $GIT_REF"
        git clone https://github.com/ethpandaops/benchmarkoor.git benchmarkoor-build
        cd benchmarkoor-build
        git checkout "$GIT_REF"

        echo "Building Docker image..."
        docker build -t benchmarkoor:local .
        cd ..

        rm -rf benchmarkoor-build

    - name: Determine which image to use
      id: determine-image
      shell: bash
      run: |
        if [ -n "${{ inputs.image }}" ]; then
          IMAGE="${{ inputs.image }}"
          echo "Using provided Docker image: ${IMAGE}"
        else
          IMAGE="benchmarkoor:local"
          echo "Using locally built image: ${IMAGE}"
        fi
        echo "image=${IMAGE}" >> $GITHUB_OUTPUT

        if [ -z "$IMAGE" ]; then
          echo "ERROR: Docker image not specified"
          exit 1
        fi

        if [ "$IMAGE" != "benchmarkoor:local" ]; then
          echo "Pulling Docker image: ${IMAGE}"
          docker pull "${IMAGE}" || {
            echo "Failed to pull Docker image: ${IMAGE}"
            exit 1
          }
        fi

    - name: Get Job ID from GH API
      id: get-job-id
      shell: bash
      env:
        GH_TOKEN: ${{ inputs.github-token }}
      run: |
        jobs=$(gh api -F per_page=100 -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs)
        job_id=$(echo "$jobs" | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}" and .status=="in_progress") | .id')
        echo "job_id=${job_id}" >> "$GITHUB_OUTPUT"

    - name: Run Benchmarkoor
      id: run-benchmarkoor
      shell: bash
      env:
        INPUT_RUN_CONFIG: ${{ inputs.run-config }}
        INPUT_RUN_CONFIG_URLS: ${{ inputs.run-config-urls }}
        INPUT_DOCKER_ENV: ${{ inputs.docker-env }}
      run: |
        # Remove any leftover containers from a previous run
        docker rm -f benchmarkoor-run 2>/dev/null || true
        docker rm -f benchmarkoor-md 2>/dev/null || true

        mkdir -p ${{ runner.temp }}/results

        IMAGE="${{ steps.determine-image.outputs.image }}"
        echo "Running benchmarkoor with Docker image: ${IMAGE}"

        # Download URL configs if provided (comma-separated)
        if [ -n "$INPUT_RUN_CONFIG_URLS" ]; then
          IFS=',' read -ra CONFIG_URLS <<< "$INPUT_RUN_CONFIG_URLS"
          for i in "${!CONFIG_URLS[@]}"; do
            URL=$(echo "${CONFIG_URLS[$i]}" | xargs)  # trim whitespace
            echo "Downloading config from ${URL}"
            curl -fsSL "${URL}" -o "${{ runner.temp }}/benchmarkoor-config-url-${i}.yaml"
          done
        fi

        # Write inline config if provided
        if [ -n "$INPUT_RUN_CONFIG" ]; then
          echo "$INPUT_RUN_CONFIG" > "${{ runner.temp }}/benchmarkoor-config-inline.yaml"
        fi

        # Extract unique filesystem mount points from datadir source_dir values
        DATADIR_MOUNT_POINTS=""
        CONFIG_FILES=()
        for cfg in "${{ runner.temp }}"/benchmarkoor-config-url-*.yaml; do
          [ -f "$cfg" ] && CONFIG_FILES+=("$cfg")
        done
        if [ -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml" ]; then
          CONFIG_FILES+=("${{ runner.temp }}/benchmarkoor-config-inline.yaml")
        fi

        if [ ${#CONFIG_FILES[@]} -gt 0 ]; then
          ALL_SOURCE_DIRS=""
          for cfg in "${CONFIG_FILES[@]}"; do
            # Extract source_dir from runner.client.datadirs map values and runner.instances[].datadir
            DIRS=$(yq '(
              .runner.client.datadirs[].source_dir,
              .runner.instances[].datadir.source_dir
            )' "$cfg" 2>/dev/null | grep -v '^null$' || true)
            if [ -n "$DIRS" ]; then
              ALL_SOURCE_DIRS="${ALL_SOURCE_DIRS}${DIRS}"$'\n'
            fi
          done

          # Resolve each source_dir to its filesystem mountpoint, deduplicate
          if [ -n "$ALL_SOURCE_DIRS" ]; then
            DATADIR_MOUNT_POINTS=$(echo "$ALL_SOURCE_DIRS" | while IFS= read -r dir; do
              dir=$(echo "$dir" | xargs)
              [ -z "$dir" ] && continue
              if [ -d "$dir" ]; then
                mp=$(findmnt -n -o TARGET --target "$dir" 2>/dev/null || true)
                if [ -n "$mp" ] && [[ "$mp" == /* ]]; then
                  echo "$mp"
                else
                  echo "Warning: could not determine mount point for '$dir', skipping" >&2
                fi
              else
                echo "Warning: datadir source_dir '$dir' does not exist, skipping" >&2
              fi
            done | sort -u)
          fi
        fi

        if [ -n "$DATADIR_MOUNT_POINTS" ]; then
          echo "Detected datadir filesystem mount points for rshared propagation:"
          echo "$DATADIR_MOUNT_POINTS"
        fi

        # Build docker run command
        DOCKER_CMD="docker run --network=host --rm --name=benchmarkoor-run"
        DOCKER_CMD="$DOCKER_CMD -v /var/run/docker.sock:/var/run/docker.sock"
        DOCKER_CMD="$DOCKER_CMD -v /run/podman/podman.sock:/run/podman/podman.sock"
        DOCKER_CMD="$DOCKER_CMD -v /var/lib/containers:/var/lib/containers:rshared"
        DOCKER_CMD="$DOCKER_CMD -v ${{ runner.temp }}/results:/app/results"
        DOCKER_CMD="$DOCKER_CMD -v ${{ runner.temp }}:/app/configs"
        DOCKER_CMD="$DOCKER_CMD --mount type=bind,source=${{ inputs.tmp-dir }},target=${{ inputs.tmp-dir }},bind-propagation=rshared"
        DOCKER_CMD="$DOCKER_CMD --privileged --pid=host"
        DOCKER_CMD="$DOCKER_CMD -v /proc/sys/vm/drop_caches:/host_drop_caches"
        DOCKER_CMD="$DOCKER_CMD -v /sys/devices/system/cpu:/host_sys_cpu"
        DOCKER_CMD="$DOCKER_CMD -v /sys/fs/cgroup:/sys/fs/cgroup:ro"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_GITHUB_TOKEN=${{ inputs.github-token }}"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_DROP_CACHES_PATH=/host_drop_caches"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_CPU_SYSFS_PATH=/host_sys_cpu"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_BENCHMARK_RESULTS_DIR=/app/results"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_BENCHMARK_RESULTS_OWNER=$(id -u):$(id -g)"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_DATADIR=${{ inputs.tmp-dir }}"
        DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_CACHEDIR=${{ inputs.tmp-dir }}/benchmarkoor-cache"

        # Forward user-specified environment variables into the container.
        if [ -n "$INPUT_DOCKER_ENV" ]; then
          while IFS= read -r env_entry; do
            env_entry=$(echo "$env_entry" | xargs)  # trim whitespace
            [ -z "$env_entry" ] && continue
            DOCKER_CMD="$DOCKER_CMD -e ${env_entry}"
          done <<< "$INPUT_DOCKER_ENV"
        fi

        # Add rshared mounts for datadir filesystem mount points (ZFS/overlayfs)
        if [ -n "$DATADIR_MOUNT_POINTS" ]; then
          while IFS= read -r mp; do
            [ -z "$mp" ] && continue
            DOCKER_CMD="$DOCKER_CMD --mount type=bind,source=${mp},target=${mp},bind-propagation=rshared"
          done <<< "$DATADIR_MOUNT_POINTS"
        fi

        DOCKER_CMD="$DOCKER_CMD ${IMAGE}"
        DOCKER_CMD="$DOCKER_CMD run"

        # Add config files in order: URL configs, inline config, override config (last wins)
        for cfg in "${{ runner.temp }}"/benchmarkoor-config-url-*.yaml; do
          [ -f "$cfg" ] && DOCKER_CMD="$DOCKER_CMD --config=/app/configs/$(basename "$cfg")"
        done

        if [ -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml" ]; then
          DOCKER_CMD="$DOCKER_CMD --config=/app/configs/benchmarkoor-config-inline.yaml"
        fi

        # Add GitHub Actions context as metadata labels
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.run_id=${{ github.run_id }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.run_number=${{ github.run_number }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.job=${{ github.job }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.job_id=${{ steps.get-job-id.outputs.job_id }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.repository=${{ github.repository }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.workflow=\"${{ github.workflow }}\""
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.sha=${{ github.sha }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.actor=${{ github.actor }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.event_name=${{ github.event_name }}"
        DOCKER_CMD="$DOCKER_CMD --metadata.label=github.ref=${{ github.ref }}"

        # Append extra run args if provided
        if [ -n "${{ inputs.run-args }}" ]; then
          DOCKER_CMD="$DOCKER_CMD ${{ inputs.run-args }}"
        fi

        # Execute the command
        echo "Running: $DOCKER_CMD"
        set +e
        eval $DOCKER_CMD &
        wait $!
        DOCKER_EXIT_CODE=$?
        set -e

        # Find run directories
        RUN_DIRS=""
        if [ -d "${{ runner.temp }}/results/runs" ]; then
          RUN_DIRS=$(find "${{ runner.temp }}/results/runs" -mindepth 1 -maxdepth 1 -type d | sort | tr '\n' ' ')
        fi

        echo "run-dirs=${RUN_DIRS}" >> $GITHUB_OUTPUT

        if [ $DOCKER_EXIT_CODE -ne 0 ]; then
          echo "Benchmarkoor failed with exit code $DOCKER_EXIT_CODE"
          exit $DOCKER_EXIT_CODE
        fi

        echo "Benchmarkoor completed successfully"

    - name: Generate markdown summaries
      if: always() && steps.run-benchmarkoor.outputs.run-dirs != ''
      shell: bash
      run: |
        IMAGE="${{ steps.determine-image.outputs.image }}"

        for RUN_DIR_HOST in ${{ steps.run-benchmarkoor.outputs.run-dirs }}; do
          RUN_DIR_NAME=$(basename "$RUN_DIR_HOST")
          CONTAINER_RUN_DIR="/app/results/runs/${RUN_DIR_NAME}"

          echo "Generating summary for ${RUN_DIR_NAME}"

          docker run --rm --name=benchmarkoor-md \
            -v "${{ runner.temp }}/results:/app/results" \
            "${IMAGE}" \
            generate-markdown-summary \
            --run-dir="${CONTAINER_RUN_DIR}" \
            --output="${CONTAINER_RUN_DIR}/summary.md"

          if [ -f "${RUN_DIR_HOST}/summary.md" ]; then
            cat "${RUN_DIR_HOST}/summary.md" >> "$GITHUB_STEP_SUMMARY"
            echo "" >> "$GITHUB_STEP_SUMMARY"
          fi
        done

    - name: Create results archive
      if: always() && inputs.upload-artifacts == 'true'
      shell: bash
      run: |
        if [ -d "${{ runner.temp }}/results" ]; then
          tar -czf "${{ runner.temp }}/benchmarkoor-results.tar.gz" -C "${{ runner.temp }}" results
        fi

    - name: Upload artifacts
      uses: actions/upload-artifact@v4
      if: always() && inputs.upload-artifacts == 'true'
      with:
        name: benchmarkoor-${{ github.run_id }}
        path: ${{ runner.temp }}/benchmarkoor-results.tar.gz

    - name: Cleanup
      shell: bash
      if: always()
      run: |
        echo "Cleaning up"
        rm -rf ${{ runner.temp }}/results
        rm -f ${{ runner.temp }}/benchmarkoor-config-url-*.yaml
        rm -f ${{ runner.temp }}/benchmarkoor-config-inline.yaml
        rm -f ${{ runner.temp }}/benchmarkoor-results.tar.gz
        docker rm -f benchmarkoor-run 2>/dev/null || true
        docker rm -f benchmarkoor-md 2>/dev/null || true