Skip to content
cloudberry-backup-ci

CI: refactor CI workflow for cloudberry-backup #6

Sign in to view logs

CI: refactor CI workflow for cloudberry-backup

CI: refactor CI workflow for cloudberry-backup #6

Workflow file for this run

.github/workflows/cloudberry-backup-ci.yml at 6a8bc2d
# --------------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed
# with this work for additional information regarding copyright
# ownership. The ASF licenses this file to You under the Apache
# License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the
# License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# --------------------------------------------------------------------
# GitHub Actions Workflow: Cloudberry-backup CI
# --------------------------------------------------------------------
# Description:
#
# Builds Apache Cloudberry from source, packages the installation,
# and runs Cloudberry-backup tests against a demo Cloudberry cluster.
#
# Workflow Overview:
# 1. Build Cloudberry from source and upload the installation as an artifact.
# 2. For each test target (unit, integration, end_to_end), restore the
# Cloudberry installation, create a demo cluster, and run tests.
#
# Notes:
# - Each test job runs in an isolated environment and creates its own demo cluster.
# - Artifacts are used to avoid rebuilding Cloudberry for every test target.
# --------------------------------------------------------------------
name: cloudberry-backup-ci
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
types: [ opened, synchronize, reopened, edited, ready_for_review ]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
permissions:
contents: read
env:
CLOUDBERRY_REPO: apache/cloudberry
CLOUDBERRY_REF: main
CLOUDBERRY_DIR: cloudberry
CLOUDBERRY_BACKUP_DIR: cloudberry-backup
jobs:
build_cloudberry:
name: Build Cloudberry From Source
runs-on: ubuntu-22.04
timeout-minutes: 180
container:
image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
options: >-
--user root
-h cdw
-v /usr/share:/host_usr_share
-v /usr/local:/host_usr_local
-v /opt:/host_opt
steps:
- name: Free Disk Space
run: |
set -euo pipefail
echo "=== Disk space before cleanup ==="
df -h /
rm -rf /host_opt/hostedtoolcache || true
rm -rf /host_usr_local/lib/android || true
rm -rf /host_usr_share/dotnet || true
rm -rf /host_opt/ghc || true
rm -rf /host_usr_local/.ghcup || true
rm -rf /host_usr_share/swift || true
rm -rf /host_usr_local/share/powershell || true
rm -rf /host_usr_local/share/chromium || true
rm -rf /host_usr_share/miniconda || true
rm -rf /host_opt/az || true
rm -rf /host_usr_share/sbt || true
echo "=== Disk space after cleanup ==="
df -h /
- name: Checkout Cloudberry Source
uses: actions/checkout@v4
with:
repository: ${{ env.CLOUDBERRY_REPO }}
ref: ${{ env.CLOUDBERRY_REF }}
fetch-depth: 1
submodules: true
path: ${{ env.CLOUDBERRY_DIR }}
- name: Cloudberry Environment Initialization
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
if ! su - gpadmin -c "/tmp/init_system.sh"; then
echo "::error::Container initialization failed"
exit 1
fi
mkdir -p "${SRC_DIR}/build-logs/details"
chown -R gpadmin:gpadmin .
chmod -R 755 .
chmod 777 "${SRC_DIR}/build-logs"
df -h /
rm -rf /__t/*
df -h /
- name: Configure Cloudberry
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"
if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
echo "::error::Configure script failed"
exit 1
fi
- name: Build Cloudberry
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"
if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then
echo "::error::Build script failed"
exit 1
fi
- name: Package Cloudberry Source
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
tar -C "${GITHUB_WORKSPACE}" -czf cloudberry-src.tgz "${CLOUDBERRY_DIR}"
- name: Package Cloudberry Installation
run: |
set -euo pipefail
tar -C /usr/local -czf cloudberry-db.tgz cloudberry-db
- name: Upload Cloudberry Installation
uses: actions/upload-artifact@v4
with:
name: cloudberry-db-install
path: cloudberry-db.tgz
if-no-files-found: error
retention-days: 7
- name: Upload Cloudberry Source
uses: actions/upload-artifact@v4
with:
name: cloudberry-source
path: cloudberry-src.tgz
if-no-files-found: error
retention-days: 7
test_cloudberry_backup:
name: Cloudberry-backup Tests (${{ matrix.test_target }})
needs: [build_cloudberry]
runs-on: ubuntu-22.04
timeout-minutes: 180
container:
image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
options: >-
--user root
-h cdw
-v /usr/share:/host_usr_share
-v /usr/local:/host_usr_local
-v /opt:/host_opt
strategy:
fail-fast: false
matrix:
test_target: [unit, integration, end_to_end]
steps:
- name: Free Disk Space
run: |
set -euo pipefail
echo "=== Disk space before cleanup ==="
df -h /
rm -rf /host_opt/hostedtoolcache || true
rm -rf /host_usr_local/lib/android || true
rm -rf /host_usr_share/dotnet || true
rm -rf /host_opt/ghc || true
rm -rf /host_usr_local/.ghcup || true
rm -rf /host_usr_share/swift || true
rm -rf /host_usr_local/share/powershell || true
rm -rf /host_usr_local/share/chromium || true
rm -rf /host_usr_share/miniconda || true
rm -rf /host_opt/az || true
rm -rf /host_usr_share/sbt || true
echo "=== Disk space after cleanup ==="
df -h /
- name: Checkout Cloudberry-backup
uses: actions/checkout@v4
with:
fetch-depth: 0
path: ${{ env.CLOUDBERRY_BACKUP_DIR }}
- name: Download Cloudberry Installation
uses: actions/download-artifact@v4
with:
name: cloudberry-db-install
- name: Download Cloudberry Source
uses: actions/download-artifact@v4
with:
name: cloudberry-source
- name: Restore Cloudberry Installation
run: |
set -euo pipefail
tar -C /usr/local -xzf cloudberry-db.tgz
- name: Restore Cloudberry Source
run: |
set -euo pipefail
tar -C "${GITHUB_WORKSPACE}" -xzf cloudberry-src.tgz
- name: Cloudberry Environment Initialization
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
if ! su - gpadmin -c "/tmp/init_system.sh"; then
echo "::error::Container initialization failed"
exit 1
fi
mkdir -p "${SRC_DIR}/build-logs/details"
chown -R gpadmin:gpadmin .
chmod -R 755 .
chmod 777 "${SRC_DIR}/build-logs"
df -h /
rm -rf /__t/*
df -h /
- name: Create Cloudberry Demo Cluster
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"
if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then
echo "::error::Demo cluster creation failed"
exit 1
fi
- name: Cloudberry-backup Tests
env:
CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }}
CLOUDBERRY_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
TEST_TARGET: ${{ matrix.test_target }}
run: |
set -euo pipefail
TEST_LOG_ROOT="${GITHUB_WORKSPACE}/test-logs/${TEST_TARGET}"
mkdir -p "${TEST_LOG_ROOT}"
chown -R gpadmin:gpadmin "${TEST_LOG_ROOT}"
cat <<'SCRIPT' > /tmp/run_cloudberry_backup_tests.sh
#!/bin/bash
set -euo pipefail
export GOPATH=/home/gpadmin/go
export PATH=/usr/local/go/bin:${GOPATH}/bin:${PATH}
source /usr/local/cloudberry-db/cloudberry-env.sh
source ${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh
pushd ${CLOUDBERRY_BACKUP_SRC}
make depend
dummy_dir=$(find ${CLOUDBERRY_SRC} -name dummy_seclabel -type d | head -n 1 || true)
if [ -n "${dummy_dir}" ]; then
pushd "${dummy_dir}"
make install
popd
gpconfig -c shared_preload_libraries -v dummy_seclabel
gpstop -ra
gpconfig -s shared_preload_libraries | grep dummy_seclabel
else
echo "dummy_seclabel not found, skipping preload setup"
fi
psql postgres -c 'DROP TABLESPACE IF EXISTS test_tablespace'
set +e
case "${TEST_TARGET}" in
unit)
make unit 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-unit.log"
;;
integration)
sudo dnf install glibc-langpack-de -y
make integration 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-integration.log"
;;
end_to_end)
make end_to_end 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-end_to_end.log"
;;
*)
echo "unknown test target: ${TEST_TARGET}"
exit 2
;;
esac
test_status=${PIPESTATUS[0]}
set -e
popd
if [ -n "${MASTER_DATA_DIRECTORY:-}" ] && [ -d "${MASTER_DATA_DIRECTORY}/log" ]; then
cp -a "${MASTER_DATA_DIRECTORY}/log" "${TEST_LOG_ROOT}/gpdb-log" || true
fi
if [ -d "${CLOUDBERRY_SRC}/build-logs" ]; then
cp -a "${CLOUDBERRY_SRC}/build-logs" "${TEST_LOG_ROOT}/cloudberry-build-logs" || true
fi
exit ${test_status}
SCRIPT
chmod +x /tmp/run_cloudberry_backup_tests.sh
set +e
su - gpadmin -c "TEST_LOG_ROOT=${TEST_LOG_ROOT} CLOUDBERRY_BACKUP_SRC=${CLOUDBERRY_BACKUP_SRC} CLOUDBERRY_SRC=${CLOUDBERRY_SRC} TEST_TARGET=${TEST_TARGET} /tmp/run_cloudberry_backup_tests.sh"
status=$?
set -e
{
echo "## Cloudberry-backup Test Summary"
echo "- Target: ${TEST_TARGET}"
if [ ${status} -eq 0 ]; then
echo "- Result: PASS"
else
echo "- Result: FAIL"
fi
echo "- Logs: ${TEST_LOG_ROOT}"
} >> "$GITHUB_STEP_SUMMARY"
exit ${status}
- name: Upload Test Logs (On Failure)
if: failure()
uses: actions/upload-artifact@v4
with:
name: cloudberry-backup-logs-${{ matrix.test_target }}
path: test-logs/${{ matrix.test_target }}
if-no-files-found: warn
retention-days: 7
test_s3_local:
name: Cloudberry-backup Tests (test-s3-local)
needs: [build_cloudberry]
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout Cloudberry-backup
uses: actions/checkout@v4
with:
fetch-depth: 0
path: ${{ env.CLOUDBERRY_BACKUP_DIR }}
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
- name: Run test-s3-local
env:
CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }}
run: |
set -euo pipefail
export GOPATH="${GITHUB_WORKSPACE}/go"
export PATH="${GOPATH}/bin:/usr/local/go/bin:${PATH}"
pushd "${CLOUDBERRY_BACKUP_SRC}"
make depend
make build
${PWD}/plugins/generate_minio_config.sh
mkdir -p /tmp/minio/gpbackup-s3-test
docker run -d --name s3-minio --memory="2g" -p 9000:9000 -p 9001:9001 -v /tmp/minio:/data/minio quay.io/minio/minio server /data/minio --console-address ":9001"
sleep 2
${PWD}/plugins/plugin_test.sh "${GOPATH}/bin/gpbackup_s3_plugin" /tmp/minio_config.yaml 2>&1 | tee "${GITHUB_WORKSPACE}/test-s3-local.log"
docker stop s3-minio
docker rm s3-minio
popd
- name: Upload test-s3-local logs (On Failure)
if: failure()
uses: actions/upload-artifact@v4
with:
name: cloudberry-backup-logs-test-s3-local
path: test-s3-local.log
if-no-files-found: warn
retention-days: 7
test_regression:
name: Cloudberry-backup Tests (regression)
needs: [build_cloudberry]
runs-on: ubuntu-22.04
timeout-minutes: 120
container:
image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
options: >-
--user root
-h cdw
-v /usr/share:/host_usr_share
-v /usr/local:/host_usr_local
-v /opt:/host_opt
steps:
- name: Free Disk Space
run: |
set -euo pipefail
echo "=== Disk space before cleanup ==="
df -h /
rm -rf /host_opt/hostedtoolcache || true
rm -rf /host_usr_local/lib/android || true
rm -rf /host_usr_share/dotnet || true
rm -rf /host_opt/ghc || true
rm -rf /host_usr_local/.ghcup || true
rm -rf /host_usr_share/swift || true
rm -rf /host_usr_local/share/powershell || true
rm -rf /host_usr_local/share/chromium || true
rm -rf /host_usr_share/miniconda || true
rm -rf /host_opt/az || true
rm -rf /host_usr_share/sbt || true
echo "=== Disk space after cleanup ==="
df -h /
- name: Checkout Cloudberry-backup
uses: actions/checkout@v4
with:
fetch-depth: 0
path: ${{ env.CLOUDBERRY_BACKUP_DIR }}
- name: Download Cloudberry Installation
uses: actions/download-artifact@v4
with:
name: cloudberry-db-install
- name: Download Cloudberry Source
uses: actions/download-artifact@v4
with:
name: cloudberry-source
- name: Restore Cloudberry Installation
run: |
set -euo pipefail
tar -C /usr/local -xzf cloudberry-db.tgz
- name: Restore Cloudberry Source
run: |
set -euo pipefail
tar -C "${GITHUB_WORKSPACE}" -xzf cloudberry-src.tgz
- name: Cloudberry Environment Initialization
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
if ! su - gpadmin -c "/tmp/init_system.sh"; then
echo "::error::Container initialization failed"
exit 1
fi
mkdir -p "${SRC_DIR}/build-logs/details"
chown -R gpadmin:gpadmin .
chmod -R 755 .
chmod 777 "${SRC_DIR}/build-logs"
df -h /
rm -rf /__t/*
df -h /
- name: Create Cloudberry Demo Cluster
env:
SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"
if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then
echo "::error::Demo cluster creation failed"
exit 1
fi
- name: Cloudberry-backup Regression Test
env:
CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }}
CLOUDBERRY_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }}
run: |
set -euo pipefail
TEST_LOG_ROOT="${GITHUB_WORKSPACE}/test-logs/regression"
mkdir -p "${TEST_LOG_ROOT}"
chown -R gpadmin:gpadmin "${TEST_LOG_ROOT}"
cat <<'SCRIPT' > /tmp/run_cloudberry_backup_regression.sh
#!/bin/bash
set -euo pipefail
export GOPATH=/home/gpadmin/go
export PATH=/usr/local/go/bin:${GOPATH}/bin:${PATH}
source /usr/local/cloudberry-db/cloudberry-env.sh
source ${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh
pushd ${CLOUDBERRY_BACKUP_SRC}
make depend
make build
make install
popd
pushd ${CLOUDBERRY_SRC}/src/test/regress
./pg_regress --dbname=regression --host=localhost --port=7000 --init-file=init_file --schedule=./minimal_schedule || true
cat regression.diffs 2>/dev/null || true
popd
psql -d postgres -c 'DROP TABLESPACE IF EXISTS test_tablespace'
pg_dump regression -f /tmp/regression_schema_before.sql --schema-only
backup_dir=/tmp/regression_backup
rm -rf "${backup_dir}"
mkdir -p "${backup_dir}"
gpbackup --dbname regression --backup-dir "${backup_dir}" --metadata-only
find "${backup_dir}" -name "*_metadata.sql" | sed -nr 's#.*gpseg-[0-9]/backups/.*/([0-9]{14})/.*#\\1#p' > /tmp/regression_timestamp
read timestamp < /tmp/regression_timestamp
echo "backup timestamp: ${timestamp}"
psql -d postgres -c 'DROP DATABASE IF EXISTS regression'
set +e
gprestore --create-db --timestamp ${timestamp} --backup-dir "${backup_dir}" --with-globals --on-error-continue
set -e
pg_dump regression -f /tmp/regression_schema_after.sql --schema-only
set +e
diff -u /tmp/regression_schema_before.sql /tmp/regression_schema_after.sql > /tmp/regression_schema.diff
diff_status=$?
set -e
cp -a /tmp/regression_schema_before.sql "${TEST_LOG_ROOT}/regression_schema_before.sql" || true
cp -a /tmp/regression_schema_after.sql "${TEST_LOG_ROOT}/regression_schema_after.sql" || true
cp -a /tmp/regression_schema.diff "${TEST_LOG_ROOT}/regression_schema.diff" || true
if [ -n "${MASTER_DATA_DIRECTORY:-}" ] && [ -d "${MASTER_DATA_DIRECTORY}/log" ]; then
cp -a "${MASTER_DATA_DIRECTORY}/log" "${TEST_LOG_ROOT}/gpdb-log" || true
fi
if [ -d "${CLOUDBERRY_SRC}/build-logs" ]; then
cp -a "${CLOUDBERRY_SRC}/build-logs" "${TEST_LOG_ROOT}/cloudberry-build-logs" || true
fi
if [ ${diff_status} -ne 0 ]; then
echo "regression schema diff found"
exit ${diff_status}
fi
SCRIPT
chmod +x /tmp/run_cloudberry_backup_regression.sh
set +e
su - gpadmin -c "TEST_LOG_ROOT=${TEST_LOG_ROOT} CLOUDBERRY_BACKUP_SRC=${CLOUDBERRY_BACKUP_SRC} CLOUDBERRY_SRC=${CLOUDBERRY_SRC} /tmp/run_cloudberry_backup_regression.sh"
status=$?
set -e
{
echo "## Cloudberry-backup Test Summary"
echo "- Target: regression"
if [ ${status} -eq 0 ]; then
echo "- Result: PASS"
else
echo "- Result: FAIL"
fi
echo "- Logs: ${TEST_LOG_ROOT}"
} >> "$GITHUB_STEP_SUMMARY"
exit ${status}
- name: Upload Regression Logs (On Failure)
if: failure()
uses: actions/upload-artifact@v4
with:
name: cloudberry-backup-logs-regression
path: test-logs/regression
if-no-files-found: warn
retention-days: 7