Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 69 additions & 18 deletions .github/workflows/pxf-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,19 @@ env:
JAVA_HOME: "/usr/lib/jvm/java-11-openjdk"
GO_VERSION: "1.21"
GPHOME: "/usr/local/cloudberry-db"
CLOUDBERRY_VERSION: "main"
PXF_HOME: "/usr/local/pxf"

jobs:
# Stage 1: Build artifacts (runs in parallel)
build-cloudberry-deb:
name: Build Cloudberry DEB Package
name: Build Cloudberry DEB (${{ matrix.cloudberry_branch }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cloudberry_branch:
- main
- REL_2_STABLE
container:
image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest
options: --user root
Expand All @@ -35,7 +40,7 @@ jobs:
uses: actions/checkout@v4
with:
repository: apache/cloudberry
ref: ${{ env.CLOUDBERRY_VERSION }}
ref: ${{ matrix.cloudberry_branch }}
path: workspace/cloudberry
submodules: true

Expand All @@ -47,7 +52,10 @@ jobs:
- name: Build Cloudberry DEB
run: |
export WORKSPACE=$PWD/workspace
export CLOUDBERRY_VERSION=99.0.0
# Sanitize branch name for Debian version (replace _ with -)
BRANCH_NAME="${{ matrix.cloudberry_branch }}"
CLEAN_BRANCH=$(echo "$BRANCH_NAME" | tr '_' '-')
export CLOUDBERRY_VERSION=99.0.0-${CLEAN_BRANCH}
export CLOUDBERRY_BUILD=1
bash cloudberry-pxf/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh

Expand All @@ -59,14 +67,14 @@ jobs:
- name: Upload DEB artifact
uses: actions/upload-artifact@v4
with:
name: cloudberry-deb
name: cloudberry-deb-${{ matrix.cloudberry_branch }}
path: workspace/cloudberry-deb/*.deb
retention-days: 7

- name: Upload Cloudberry source artifact
uses: actions/upload-artifact@v4
with:
name: cloudberry-source
name: cloudberry-source-${{ matrix.cloudberry_branch }}
path: workspace/cloudberry-source.tar.gz
retention-days: 7

Expand Down Expand Up @@ -94,12 +102,15 @@ jobs:

# Stage 2: Parallel test jobs using matrix strategy
pxf-test:
name: Test PXF - ${{ matrix.test_group }}
name: Test PXF - ${{ matrix.test_group }} (${{ matrix.cloudberry_branch }})
needs: [build-cloudberry-deb, build-docker-images]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cloudberry_branch:
- main
- REL_2_STABLE
test_group:
- cli
- external-table
Expand Down Expand Up @@ -143,13 +154,13 @@ jobs:
- name: Download Cloudberry DEB
uses: actions/download-artifact@v4
with:
name: cloudberry-deb
name: cloudberry-deb-${{ matrix.cloudberry_branch }}
path: /tmp

- name: Download Cloudberry source
uses: actions/download-artifact@v4
with:
name: cloudberry-source
name: cloudberry-source-${{ matrix.cloudberry_branch }}
path: /tmp

- name: Download singlecluster image
Expand Down Expand Up @@ -248,7 +259,7 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results-${{ matrix.test_group }}
name: test-results-${{ matrix.cloudberry_branch }}-${{ matrix.test_group }}
path: artifacts/**
if-no-files-found: ignore
retention-days: 7
Expand Down Expand Up @@ -291,19 +302,40 @@ jobs:
GROUPS_FAILED=0
FAILED_GROUP_NAMES=""

# Branch-specific counters
declare -A BRANCH_PASSED
declare -A BRANCH_FAILED

# Collect all test stats
declare -A GROUP_STATS

for dir in all-artifacts/test-results-*; do
if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then
# Extract branch and group from directory name: test-results-<branch>-<group>
dirname=$(basename "$dir")
# Try to extract branch from test_stats.json if available, fallback to directory parsing
branch=$(cat "$dir/test_stats.json" | grep -oP '"branch":\s*"\K[^"]+' || echo "")
if [ -z "$branch" ]; then
# Parse from directory name: test-results-main-cli or test-results-REL_2_STABLE-cli
if [[ "$dirname" == *"-main-"* ]]; then
branch="main"
elif [[ "$dirname" == *"-REL_2_STABLE-"* ]]; then
branch="REL_2_STABLE"
else
branch="unknown"
fi
fi

group=$(cat "$dir/test_stats.json" | grep -oP '"group":\s*"\K[^"]+' || basename "$dir" | sed 's/test-results-//')
result=$(cat "$dir/test_stats.json" | grep -oP '"result":\s*"\K[^"]+' || echo "unknown")
total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+' || echo "0")
passed=$(cat "$dir/test_stats.json" | grep -oP '"passed":\s*\K\d+' || echo "0")
failed=$(cat "$dir/test_stats.json" | grep -oP '"failed":\s*\K\d+' || echo "0")
skipped=$(cat "$dir/test_stats.json" | grep -oP '"skipped":\s*\K\d+' || echo "0")

GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped"
# Use branch-group as key
key="${branch}/${group}"
GROUP_STATS[$key]="$result,$total,$passed,$failed,$skipped,$branch,$group"

OVERALL_TOTAL=$((OVERALL_TOTAL + total))
OVERALL_PASSED=$((OVERALL_PASSED + passed))
Expand All @@ -312,9 +344,11 @@ jobs:

if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
GROUPS_PASSED=$((GROUPS_PASSED + 1))
BRANCH_PASSED[$branch]=$((${BRANCH_PASSED[$branch]:-0} + 1))
else
GROUPS_FAILED=$((GROUPS_FAILED + 1))
FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} "
BRANCH_FAILED[$branch]=$((${BRANCH_FAILED[$branch]:-0} + 1))
FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${key} "
fi
fi
done
Expand All @@ -334,20 +368,36 @@ jobs:
echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY

# Summary by branch
echo "### Summary by Cloudberry Branch" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Branch | Passed Groups | Failed Groups |" >> $GITHUB_STEP_SUMMARY
echo "|--------|-------------:|-------------:|" >> $GITHUB_STEP_SUMMARY
for branch in main REL_2_STABLE; do
p=${BRANCH_PASSED[$branch]:-0}
f=${BRANCH_FAILED[$branch]:-0}
if [ $f -eq 0 ]; then
echo "| $branch | ✅ $p | $f |" >> $GITHUB_STEP_SUMMARY
else
echo "| $branch | $p | ❌ $f |" >> $GITHUB_STEP_SUMMARY
fi
done
echo "" >> $GITHUB_STEP_SUMMARY

# Detailed table
echo "### Test Results by Group" >> $GITHUB_STEP_SUMMARY
echo "### Test Results by Branch and Group" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Test Group | Status | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY
echo "|------------|--------|-------:|-------:|--------:|------:|" >> $GITHUB_STEP_SUMMARY
echo "| Branch | Test Group | Status | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY
echo "|--------|------------|--------|-------:|-------:|--------:|------:|" >> $GITHUB_STEP_SUMMARY

for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do
IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}"
for key in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do
IFS=',' read -r result total passed failed skipped branch group <<< "${GROUP_STATS[$key]}"
if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
status="✅ PASS"
else
status="❌ FAIL"
fi
echo "| $group | $status | $passed | $failed | $skipped | $total |" >> $GITHUB_STEP_SUMMARY
echo "| $branch | $group | $status | $passed | $failed | $skipped | $total |" >> $GITHUB_STEP_SUMMARY
done

echo "" >> $GITHUB_STEP_SUMMARY
Expand All @@ -357,3 +407,4 @@ jobs:
echo "::error::${GROUPS_FAILED} test group(s) failed: ${FAILED_GROUP_NAMES}"
exit 1
fi

Loading