diff --git a/.github/workflows/centos-image-automation.yml b/.github/workflows/centos-image-automation.yml new file mode 100644 index 00000000..5225cfb6 --- /dev/null +++ b/.github/workflows/centos-image-automation.yml @@ -0,0 +1,174 @@ +name: CentOS Image Automation + +on: + schedule: + # Run every 15 days at 2:00 AM UTC + - cron: "0 2 */15 * *" + workflow_dispatch: + inputs: + environment: + description: "Target environment for image deployment" + required: true + type: choice + options: + - production + - staging + default: "staging" + centos_version: + description: "CentOS Stream version (e.g., 9, 10)" + required: false + default: "10" + image_size: + description: "Image disk size in GB" + required: false + default: "120" + +env: + WORK_DIR: /tmp/centos-images + +jobs: + centos-image-automation: + runs-on: [self-hosted, Linux, ppc64le] + timeout-minutes: 240 # 4 hours max + strategy: + matrix: + centos_version: [9, 10] + fail-fast: false # Continue with other versions even if one fails + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up environment + run: | + mkdir -p ${{ env.WORK_DIR }} + + # Ensure required tools are available + command -v curl >/dev/null 2>&1 || { echo "curl is required but not installed"; exit 1; } + command -v wget >/dev/null 2>&1 || { echo "wget is required but not installed"; exit 1; } + command -v jq >/dev/null 2>&1 || { echo "jq is required but not installed"; exit 1; } + + - name: Verify pvsadm installation + run: | + if ! command -v pvsadm &> /dev/null; then + echo "ERROR: pvsadm is not installed on this runner" + echo "Please install pvsadm on the self-hosted runner" + exit 1 + fi + pvsadm version + + - name: Copy scripts and template to work directory + run: | + echo "Copying scripts to work directory..." + cp hack/scripts/download-and-convert-centos.sh ${{ env.WORK_DIR }}/ + cp hack/scripts/upload-to-cos.sh ${{ env.WORK_DIR }}/ + cp hack/scripts/import-to-powervs.sh ${{ env.WORK_DIR }}/ + cp hack/scripts/image-prep.template.static ${{ env.WORK_DIR }}/ + + # Make scripts executable + chmod +x ${{ env.WORK_DIR }}/*.sh + + - name: Download and convert CentOS ${{ matrix.centos_version }} image + env: + CENTOS_VERSION: ${{ github.event.inputs.centos_version || matrix.centos_version }} + IMAGE_SIZE: ${{ github.event.inputs.image_size || '120' }} + IMAGE_PREP_TEMPLATE: ${{ env.WORK_DIR }}/image-prep.template.static + run: | + echo "Starting conversion for CentOS Stream ${{ env.CENTOS_VERSION }}..." + cd ${{ env.WORK_DIR }} + + if ! ./download-and-convert-centos.sh; then + echo "ERROR: Conversion failed" + exit 1 + fi + + echo "Conversion completed successfully" + + - name: Set environment-specific variables + id: set-env + run: | + ENVIRONMENT="${{ github.event.inputs.environment || 'production' }}" + echo "environment=${ENVIRONMENT}" >> $GITHUB_OUTPUT + + if [ "$ENVIRONMENT" = "staging" ]; then + echo "powervs_instance_id=${{ secrets.POWERVS_STAGING_INSTANCE_ID }}" >> $GITHUB_OUTPUT + echo "cos_bucket_name=${{ secrets.COS_STAGING_BUCKET_NAME }}" >> $GITHUB_OUTPUT + else + echo "powervs_instance_id=${{ secrets.POWERVS_PROD_INSTANCE_ID }}" >> $GITHUB_OUTPUT + echo "cos_bucket_name=${{ secrets.COS_PROD_BUCKET_NAME }}" >> $GITHUB_OUTPUT + fi + + - name: Upload OVA to IBM Cloud Object Storage + env: + IBM_API_KEY: ${{ secrets.IBM_API_KEY }} + COS_BUCKET_NAME: ${{ steps.set-env.outputs.cos_bucket_name }} + COS_INSTANCE_NAME: ${{ secrets.COS_INSTANCE_NAME }} + COS_REGION: ${{ secrets.COS_REGION }} + run: | + echo "Uploading to bucket: ${{ env.COS_BUCKET_NAME }}" + cd ${{ env.WORK_DIR }} + + if ! ./upload-to-cos.sh; then + echo "ERROR: Upload failed" + exit 1 + fi + + echo "Upload completed successfully" + + - name: Import image to PowerVS workspace (${{ steps.set-env.outputs.environment }}) + env: + IBM_API_KEY: ${{ secrets.IBM_API_KEY }} + POWERVS_INSTANCE_ID: ${{ steps.set-env.outputs.powervs_instance_id }} + COS_BUCKET_NAME: ${{ steps.set-env.outputs.cos_bucket_name }} + COS_REGION: ${{ secrets.COS_REGION }} + COS_HMAC_ACCESS_KEY: ${{ secrets.COS_HMAC_ACCESS_KEY }} + COS_HMAC_SECRET_KEY: ${{ secrets.COS_HMAC_SECRET_KEY }} + run: | + echo "Deploying to ${{ steps.set-env.outputs.environment }} environment" + echo "PowerVS Instance ID: ${{ env.POWERVS_INSTANCE_ID }}" + echo "COS Bucket: ${{ env.COS_BUCKET_NAME }}" + + cd ${{ env.WORK_DIR }} + + if ! ./import-to-powervs.sh; then + echo "ERROR: Import failed" + exit 1 + fi + + echo "Import completed successfully" + + - name: Cleanup after successful completion + if: success() + run: | + echo "All steps completed successfully - cleaning up..." + rm -rf ${{ env.WORK_DIR }}/* || echo "Cleanup warning (non-critical)" + + # Clean up pvsadm temp directories and loop devices + sudo find /tmp -name "qcow2ova*" -type d -exec rm -rf {} + 2>/dev/null || true + sudo losetup -D 2>/dev/null || true + + echo "Cleanup completed" + + - name: Cleanup on failure + if: failure() + run: | + echo "Workflow failed - performing partial cleanup..." + + # Keep qcow2 and OVA files for debugging, but clean up temp files + sudo find /tmp -name "qcow2ova*" -type d -exec rm -rf {} + 2>/dev/null || true + sudo losetup -D 2>/dev/null || true + + echo "Partial cleanup completed. Check ${{ env.WORK_DIR }} for debugging." + + # Notification step can be added here if needed + # Options: Email, Slack, or other notification services + + - name: Upload workflow artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: centos-${{ matrix.centos_version }}-automation-${{ github.run_number }} + path: | + ${{ env.WORK_DIR }}/*.log + ${{ env.WORK_DIR }}/*.json + retention-days: 30 diff --git a/hack/scripts/cleanup-resources.sh b/hack/scripts/cleanup-resources.sh new file mode 100755 index 00000000..1e0bce13 --- /dev/null +++ b/hack/scripts/cleanup-resources.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# Script to cleanup temporary files and resources + +set -o errexit +set -o nounset +set -o pipefail + +# Configuration +WORK_DIR="${WORK_DIR:-/tmp/centos-images}" +KEEP_METADATA="${KEEP_METADATA:-true}" +KEEP_LOGS="${KEEP_LOGS:-true}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Display disk usage before cleanup +show_disk_usage_before() { + log_info "Disk usage before cleanup:" + if [ -d "$WORK_DIR" ]; then + du -sh "$WORK_DIR" 2>/dev/null || log_warn "Could not calculate disk usage" + log_info "Files in work directory:" + ls -lh "$WORK_DIR" 2>/dev/null || log_warn "Could not list files" + else + log_info "Work directory does not exist: $WORK_DIR" + fi +} + +# Cleanup qcow2 files +cleanup_qcow2_files() { + log_info "Cleaning up qcow2 files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.qcow2; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + ((count++)) + fi + done + + # Also remove checksum files + for file in "$WORK_DIR"/*.SHA256SUM; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + ((count++)) + fi + done + fi + + log_info "Removed $count qcow2 and checksum files" +} + +# Cleanup OVA files +cleanup_ova_files() { + log_info "Cleaning up OVA files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.ova.gz "$WORK_DIR"/*.ova; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + ((count++)) + fi + done + fi + + log_info "Removed $count OVA files" +} + +# Cleanup temporary files +cleanup_temp_files() { + log_info "Cleaning up temporary files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + # Remove temp directories created by pvsadm + for dir in "$WORK_DIR"/tmp* "$WORK_DIR"/pvsadm-*; do + if [ -d "$dir" ]; then + log_info "Removing directory: $(basename "$dir")" + rm -rf "$dir" + ((count++)) + fi + done + + # Remove other temporary files + for file in "$WORK_DIR"/*.tmp "$WORK_DIR"/*.temp; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + ((count++)) + fi + done + fi + + log_info "Removed $count temporary files/directories" +} + +# Cleanup logs (optional) +cleanup_logs() { + if [ "$KEEP_LOGS" == "false" ]; then + log_info "Cleaning up log files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.log; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + ((count++)) + fi + done + fi + + log_info "Removed $count log files" + else + log_info "Keeping log files (KEEP_LOGS=true)" + fi +} + +# Cleanup metadata (optional) +cleanup_metadata() { + if [ "$KEEP_METADATA" == "false" ]; then + log_info "Cleaning up metadata files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.json; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + ((count++)) + fi + done + fi + + log_info "Removed $count metadata files" + else + log_info "Keeping metadata files (KEEP_METADATA=true)" + fi +} + +# Archive important files before cleanup +archive_important_files() { + log_info "Archiving important files..." + + if [ -d "$WORK_DIR" ]; then + local archive_dir="${WORK_DIR}/archive" + mkdir -p "$archive_dir" + + # Archive metadata + for file in "$WORK_DIR"/*.json; do + if [ -f "$file" ]; then + cp "$file" "$archive_dir/" 2>/dev/null || true + fi + done + + # Archive logs + for file in "$WORK_DIR"/*.log; do + if [ -f "$file" ]; then + cp "$file" "$archive_dir/" 2>/dev/null || true + fi + done + + # Create tarball + if [ -d "$archive_dir" ] && [ "$(ls -A "$archive_dir")" ]; then + local timestamp=$(date +%Y%m%d-%H%M%S) + local archive_file="${WORK_DIR}/centos-image-automation-${timestamp}.tar.gz" + + tar -czf "$archive_file" -C "$WORK_DIR" archive/ + log_info "Created archive: $archive_file" + + # Remove archive directory + rm -rf "$archive_dir" + fi + fi +} + +# Display disk usage after cleanup +show_disk_usage_after() { + log_info "Disk usage after cleanup:" + if [ -d "$WORK_DIR" ]; then + du -sh "$WORK_DIR" 2>/dev/null || log_warn "Could not calculate disk usage" + + if [ "$(ls -A "$WORK_DIR" 2>/dev/null)" ]; then + log_info "Remaining files:" + ls -lh "$WORK_DIR" 2>/dev/null || true + else + log_info "Work directory is empty" + fi + else + log_info "Work directory does not exist: $WORK_DIR" + fi +} + +# Remove work directory completely (optional) +remove_work_dir() { + if [ "${REMOVE_WORK_DIR:-false}" == "true" ]; then + log_warn "Removing entire work directory: $WORK_DIR" + rm -rf "$WORK_DIR" + log_info "Work directory removed" + fi +} + +# Main execution +main() { + log_info "Starting cleanup process" + log_info "Work directory: $WORK_DIR" + log_info "Keep metadata: $KEEP_METADATA" + log_info "Keep logs: $KEEP_LOGS" + + # Show disk usage before + show_disk_usage_before + + echo "" + + # Archive important files first + archive_important_files + + # Cleanup in order + cleanup_qcow2_files + cleanup_ova_files + cleanup_temp_files + cleanup_logs + cleanup_metadata + + echo "" + + # Show disk usage after + show_disk_usage_after + + # Optionally remove entire work directory + remove_work_dir + + log_info "==========================================" + log_info "Cleanup completed successfully!" + log_info "==========================================" +} + +# Run main function +main "$@" + + diff --git a/hack/scripts/download-and-convert-centos.sh b/hack/scripts/download-and-convert-centos.sh new file mode 100755 index 00000000..eed384f8 --- /dev/null +++ b/hack/scripts/download-and-convert-centos.sh @@ -0,0 +1,454 @@ +#!/usr/bin/env bash +# Script to download latest CentOS Stream image and convert to OVA using pvsadm +# This script runs directly on a PowerVM with ppc64le architecture + +set -o errexit +set -o nounset +set -o pipefail + +# Configuration +CENTOS_VERSION="${CENTOS_VERSION:-10}" +WORK_DIR="${WORK_DIR:-/tmp/centos-images}" +IMAGE_PREP_TEMPLATE="${IMAGE_PREP_TEMPLATE:-./image-prep.template.static}" +IMAGE_SIZE="${IMAGE_SIZE:-120}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Check if running on ppc64le +check_architecture() { + local arch=$(uname -m) + if [[ "$arch" != "ppc64le" ]]; then + log_error "This script must run on ppc64le architecture. Current: $arch" + exit 1 + fi + log_info "Architecture check passed: $arch" +} + +# Check required tools +check_requirements() { + local missing_tools=() + local need_install=() + + # Check which tools are missing + for tool in curl jq; do + if ! command -v $tool &> /dev/null; then + missing_tools+=($tool) + need_install+=($tool) + fi + done + + # Check wget or curl (at least one needed) + if ! command -v wget &> /dev/null && ! command -v curl &> /dev/null; then + missing_tools+=(wget) + need_install+=(wget) + fi + + # Check qemu-img (required by pvsadm) + if ! command -v qemu-img &> /dev/null; then + missing_tools+=(qemu-img) + need_install+=(qemu-img) + fi + + # Check pvsadm separately + if ! command -v pvsadm &> /dev/null; then + missing_tools+=(pvsadm) + fi + + if [ ${#missing_tools[@]} -ne 0 ]; then + log_warn "Missing tools: ${missing_tools[*]}" + log_info "Installing missing tools..." + + # Install system packages first + if [ ${#need_install[@]} -ne 0 ]; then + log_info "Installing system packages: ${need_install[*]}" + if command -v dnf &> /dev/null; then + sudo dnf install -y "${need_install[@]}" || log_warn "Some packages failed to install" + elif command -v yum &> /dev/null; then + sudo yum install -y "${need_install[@]}" || log_warn "Some packages failed to install" + else + log_error "No package manager found (yum/dnf). Please install: ${need_install[*]}" + exit 1 + fi + fi + + # Install pvsadm if missing + if [[ " ${missing_tools[*]} " =~ " pvsadm " ]]; then + install_pvsadm + fi + fi + + log_info "All required tools are available" +} + +# Install pvsadm +install_pvsadm() { + log_info "Installing pvsadm..." + local PVSADM_VERSION="v0.1.13" + local ARCH="ppc64le" + local DOWNLOAD_URL="https://github.com/ppc64le-cloud/pvsadm/releases/download/${PVSADM_VERSION}/pvsadm-linux-${ARCH}.tar.gz" + local TAR_FILE="pvsadm-linux-${ARCH}.tar.gz" + + # Try wget first, fall back to curl + if command -v wget &> /dev/null; then + wget -q "$DOWNLOAD_URL" -O "$TAR_FILE" + elif command -v curl &> /dev/null; then + curl -sL "$DOWNLOAD_URL" -o "$TAR_FILE" + else + log_error "Neither wget nor curl is available. Please install one of them." + exit 1 + fi + + tar -xzf "$TAR_FILE" + sudo mv pvsadm /usr/local/bin/ 2>/dev/null || mv pvsadm /usr/local/bin/ + rm -f "$TAR_FILE" + + pvsadm version + log_info "pvsadm installed successfully" +} + +# Check disk space +check_disk_space() { + local required_space_gb=100 + local available_space_gb=$(df -BG "$WORK_DIR" | awk 'NR==2 {print $4}' | sed 's/G//') + + if [ "$available_space_gb" -lt "$required_space_gb" ]; then + log_error "Insufficient disk space. Required: ${required_space_gb}GB, Available: ${available_space_gb}GB" + exit 1 + fi + + log_info "Disk space check passed: ${available_space_gb}GB available" +} + +# Get latest CentOS Stream image URL +get_latest_image_url() { + log_info "Detecting latest CentOS Stream ${CENTOS_VERSION} image..." + + local base_url="https://cloud.centos.org/centos/${CENTOS_VERSION}-stream/ppc64le/images" + + # Get the latest image filename + local latest_image=$(curl -s "$base_url/" | \ + grep -oP 'CentOS-Stream-GenericCloud-'"${CENTOS_VERSION}"'-[0-9]{8}\.[0-9]+\.ppc64le\.qcow2' | \ + sort -V | tail -1) + + if [ -z "$latest_image" ]; then + log_error "Could not detect latest CentOS Stream ${CENTOS_VERSION} image" + exit 1 + fi + + echo "${base_url}/${latest_image}" +} + +# Download CentOS image +download_image() { + local image_url="$1" + local image_name=$(basename "$image_url") + local image_path="${WORK_DIR}/${image_name}" + + # Extract date from image name to check for existing OVA + local date_part=$(echo "$image_name" | grep -oP '\d{8}' | head -1) + local ova_pattern="centos-${CENTOS_VERSION}-stream-${date_part}*.ova.gz" + + # Check if OVA already exists - if so, skip download entirely + local existing_ova=$(find "$WORK_DIR" -name "$ova_pattern" -type f 2>/dev/null | head -1) + if [ -n "$existing_ova" ] && [ -f "$existing_ova" ]; then + log_info "==========================================" + log_info "OVA already exists: $(basename "$existing_ova")" + log_info "Skipping download and conversion" + log_info "To force re-download, delete: $existing_ova" + log_info "==========================================" + # Return a dummy path since we won't use it + echo "$image_path" + return 0 + fi + + # Check if qcow2 image already exists and is valid + if [ -f "$image_path" ]; then + local file_size=$(stat -f%z "$image_path" 2>/dev/null || stat -c%s "$image_path" 2>/dev/null) + if [ "$file_size" -gt 100000000 ]; then # > 100MB, likely valid + log_info "==========================================" + log_info "Image already exists: $image_name" + log_info "Size: $(du -h "$image_path" | cut -f1)" + log_info "Skipping download to save time" + log_info "To force re-download, delete: $image_path" + log_info "==========================================" + echo "$image_path" + return 0 + else + log_warn "Existing file is too small, re-downloading..." + rm -f "$image_path" + fi + fi + + log_info "Downloading image: $image_name" + log_info "URL: $image_url" + + # Download with progress - try wget first, fall back to curl + if command -v wget &> /dev/null; then + log_info "Using wget for download..." + wget -c -O "$image_path" "$image_url" 2>&1 | \ + grep --line-buffered -oP '\d+%' | \ + awk '{if ($1 ~ /^(20|40|60|80|100)%$/) print "[INFO] Download progress: " $1}' >&2 + elif command -v curl &> /dev/null; then + log_info "Using curl for download (this may take 5-10 minutes)..." + curl -L -C - -o "$image_path" "$image_url" 2>&1 | \ + grep --line-buffered -oP '\d+\.\d+' | \ + awk '{p=int($1); if (p>=20 && p%20==0 && p!=last) {print "[INFO] Download progress: " p "%"; last=p}}' >&2 + else + log_error "Neither wget nor curl is available" + exit 1 + fi + + # Check if download succeeded + if [ ! -f "$image_path" ]; then + log_error "Download failed: $image_path does not exist" + exit 1 + fi + + log_info "Download completed: $(du -h "$image_path" | cut -f1)" + + # Download checksum if available + local checksum_url="${image_url}.SHA256SUM" + local checksum_exists=false + + if command -v wget &> /dev/null; then + wget -q --spider "$checksum_url" 2>&1 && checksum_exists=true + elif command -v curl &> /dev/null; then + curl -s -I "$checksum_url" 2>&1 | grep -q "200 OK" && checksum_exists=true + fi + + if [ "$checksum_exists" = true ]; then + log_info "Downloading checksum..." + if command -v wget &> /dev/null; then + wget -q -O "${image_path}.SHA256SUM" "$checksum_url" >&2 + else + curl -sL -o "${image_path}.SHA256SUM" "$checksum_url" >&2 + fi + + # Verify checksum + log_info "Verifying checksum..." + cd "$WORK_DIR" + if sha256sum -c "${image_name}.SHA256SUM" 2>&1 | grep -q "OK"; then + log_info "Checksum verification passed" + else + log_warn "Checksum verification failed or not available" + fi + cd - >/dev/null + else + log_warn "Checksum file not available, skipping verification" + fi + + echo "$image_path" +} + +# Convert qcow2 to OVA using pvsadm +convert_to_ova() { + local qcow2_path="$1" + local image_name=$(basename "$qcow2_path" .qcow2) + + # Extract date from image name (e.g., CentOS-Stream-GenericCloud-10-20251215.0.ppc64le.qcow2) + local date_part=$(echo "$image_name" | grep -oP '\d{8}' | head -1) + local ova_name="centos-${CENTOS_VERSION}-stream-${date_part}" + + # Check if OVA already exists + local existing_ova=$(find "$WORK_DIR" -name "${ova_name}*.ova.gz" -type f 2>/dev/null | head -1) + if [ -n "$existing_ova" ] && [ -f "$existing_ova" ]; then + log_info "==========================================" + log_info "OVA already exists: $(basename "$existing_ova")" + log_info "Skipping conversion to save time" + log_info "To force reconversion, delete: $existing_ova" + log_info "==========================================" + echo "$existing_ova" + return 0 + fi + + log_info "Converting image to OVA format..." + log_info "Image name: $ova_name" + log_info "Source: $qcow2_path" + + # Check if prep template exists + if [ ! -f "$IMAGE_PREP_TEMPLATE" ]; then + log_error "Image prep template not found: $IMAGE_PREP_TEMPLATE" + exit 1 + fi + + # Run pvsadm conversion + cd "$WORK_DIR" + + log_info "Running pvsadm qcow2ova..." + log_info "This may take 30-60 minutes..." + + # Run pvsadm and capture output - use PIPESTATUS to catch errors in pipeline + set +e # Temporarily disable errexit to handle error properly + pvsadm image qcow2ova \ + --image-name "$ova_name" \ + --image-url "$qcow2_path" \ + --image-dist centos \ + --prep-template "$IMAGE_PREP_TEMPLATE" \ + --skip-os-password \ + --image-size "$IMAGE_SIZE" \ + 2>&1 | tee "${WORK_DIR}/conversion.log" >&2 + + local pvsadm_exit_code=${PIPESTATUS[0]} + set -e # Re-enable errexit + + if [ $pvsadm_exit_code -ne 0 ]; then + log_error "==========================================" + log_error "pvsadm conversion FAILED with exit code: $pvsadm_exit_code" + log_error "==========================================" + + # Show last 30 lines of log for debugging + if [ -f "${WORK_DIR}/conversion.log" ]; then + log_error "Last 30 lines of conversion log:" + tail -30 "${WORK_DIR}/conversion.log" >&2 + fi + + log_error "Common issues:" + log_error " - Loop device exhaustion: Run 'sudo losetup -D' and 'rm -rf /tmp/qcow2ova*'" + log_error " - Disk space: Check 'df -h /tmp'" + log_error " - Permissions: Ensure you can run 'sudo losetup'" + + exit 1 + fi + + log_info "pvsadm conversion completed successfully" + + # Find the generated OVA file + local ova_file=$(find "$WORK_DIR" -name "${ova_name}*.ova.gz" -type f 2>/dev/null | head -1) + + if [ -z "$ova_file" ] || [ ! -f "$ova_file" ]; then + log_error "OVA file not found after conversion" + log_error "Expected pattern: ${ova_name}*.ova.gz" + log_error "Files in work directory:" + ls -lh "$WORK_DIR" >&2 + exit 1 + fi + + log_info "Conversion completed successfully" + log_info "OVA file: $ova_file" + + # Clean up pvsadm temporary directories immediately after conversion + log_info "Cleaning up pvsadm temporary directories..." + sleep 2 # Wait for pvsadm to fully release file handles + find /tmp -maxdepth 1 -type d -name "qcow2ova*" ! -newer "$ova_file" -exec rm -rf {} \; 2>/dev/null || true + + # Clean up qcow2 file if OVA exists (saves ~1-2GB) + if [ -f "$ova_file" ] && [ -f "$qcow2_path" ]; then + log_info "Removing qcow2 file to save space (OVA already created)..." + rm -f "$qcow2_path" + fi + + # Save metadata + cat > "${WORK_DIR}/image-metadata.json" </dev/null | grep -q '[p]vsadm'; then + ps aux | grep '[p]vsadm' | awk '{print $2}' | while read pid; do + if [ -n "$pid" ]; then + # Check if process is older than 4 hours + start_time=$(ps -p $pid -o etimes= 2>/dev/null | tr -d ' ') + if [ -n "$start_time" ] && [ "$start_time" -gt 14400 ]; then + log_warn "Killing old pvsadm process: $pid (running for ${start_time}s)" + kill -9 $pid 2>/dev/null || true + fi + fi + done + fi + + # Remove ALL old pvsadm temp directories (not just 1 day old) + log_info "Removing all old qcow2ova temp directories..." + local temp_dirs=$(find /tmp -maxdepth 1 -type d -name "qcow2ova*" 2>/dev/null | wc -l) + if [ "$temp_dirs" -gt 0 ]; then + log_info "Found $temp_dirs old qcow2ova directories to clean" + find /tmp -maxdepth 1 -type d -name "qcow2ova*" -exec rm -rf {} \; 2>/dev/null || true + fi + + # Clean up any loop devices that might be stuck + log_info "Cleaning up loop devices..." + if command -v losetup &> /dev/null; then + sudo losetup -D 2>/dev/null || losetup -D 2>/dev/null || true + local available_loops=$(losetup -f 2>&1 | grep -c "^/dev/loop" || echo "unknown") + log_info "Loop devices cleaned up (available: $available_loops)" + fi + + # Wait a moment for file handles to be released + sleep 2 +} + +# Main execution +main() { + log_info "Starting CentOS Stream ${CENTOS_VERSION} image download and conversion" + log_info "Work directory: $WORK_DIR" + + # Create work directory + mkdir -p "$WORK_DIR" + cd "$WORK_DIR" + + # Cleanup old pvsadm artifacts first + cleanup_old_pvsadm + + # Pre-flight checks + check_architecture + check_requirements + check_disk_space + + # Get latest image URL + local image_url=$(get_latest_image_url) + log_info "Latest image URL: $image_url" + + # Download image + local qcow2_path=$(download_image "$image_url") + log_info "Downloaded to: $qcow2_path" + + # Convert to OVA + local ova_file=$(convert_to_ova "$qcow2_path") + + log_info "==========================================" + log_info "Process completed successfully!" + log_info "OVA file: $ova_file" + log_info "Metadata: ${WORK_DIR}/image-metadata.json" + log_info "==========================================" + + # Display file sizes (only show files that exist) + log_info "File sizes:" + if [ -f "$qcow2_path" ]; then + ls -lh "$qcow2_path" 2>/dev/null || true + fi + ls -lh "$ova_file" 2>/dev/null || true +} + +# Run main function +main "$@" + + diff --git a/hack/scripts/import-to-powervs.sh b/hack/scripts/import-to-powervs.sh new file mode 100755 index 00000000..788947ed --- /dev/null +++ b/hack/scripts/import-to-powervs.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# Script to import OVA image from COS to PowerVS workspace + +set -o errexit +set -o nounset +set -o pipefail + +# Configuration +WORK_DIR="${WORK_DIR:-/tmp/centos-images}" +IBM_API_KEY="${IBM_API_KEY:?IBM_API_KEY environment variable is required}" # pragma: allowlist secret +POWERVS_INSTANCE_ID="${POWERVS_INSTANCE_ID:-}" +POWERVS_INSTANCE_NAME="${POWERVS_INSTANCE_NAME:-}" +POWERVS_WORKSPACE_CRN="${POWERVS_WORKSPACE_CRN:-}" +COS_BUCKET_NAME="${COS_BUCKET_NAME:?COS_BUCKET_NAME environment variable is required}" +COS_REGION="${COS_REGION:-us-south}" +COS_HMAC_ACCESS_KEY="${COS_HMAC_ACCESS_KEY:-}" +COS_HMAC_SECRET_KEY="${COS_HMAC_SECRET_KEY:-}" +IMPORT_TIMEOUT="${IMPORT_TIMEOUT:-3600}" # 1 hour timeout + +# Check that at least one PowerVS identifier is provided +if [ -z "$POWERVS_INSTANCE_ID" ] && [ -z "$POWERVS_INSTANCE_NAME" ] && [ -z "$POWERVS_WORKSPACE_CRN" ]; then + log_error "One of POWERVS_INSTANCE_ID, POWERVS_INSTANCE_NAME, or POWERVS_WORKSPACE_CRN is required" + exit 1 +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check required tools +check_requirements() { + local missing_tools=() + + for tool in pvsadm jq; do + if ! command -v $tool &> /dev/null; then + missing_tools+=($tool) + fi + done + + if [ ${#missing_tools[@]} -ne 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + exit 1 + fi + + log_info "All required tools are available" +} + +# Get image metadata +get_image_metadata() { + local metadata_file="${WORK_DIR}/image-metadata.json" + + if [ ! -f "$metadata_file" ]; then + log_error "Metadata file not found: $metadata_file" + exit 1 + fi + + cat "$metadata_file" +} + +# Get PowerVS instance identifier (ID or name) +get_pvs_instance() { + # Priority: POWERVS_INSTANCE_ID > POWERVS_INSTANCE_NAME > extract from CRN + if [ -n "$POWERVS_INSTANCE_ID" ]; then + echo "$POWERVS_INSTANCE_ID" + return 0 + elif [ -n "$POWERVS_INSTANCE_NAME" ]; then + echo "$POWERVS_INSTANCE_NAME" + return 0 + elif [ -n "$POWERVS_WORKSPACE_CRN" ]; then + # CRN format: crn:v1:bluemix:public:power-iaas:region:a/account:workspace-id:: + # Extract the workspace-id part (second to last segment) + local instance_id=$(echo "$POWERVS_WORKSPACE_CRN" | awk -F: '{print $(NF-1)}') + if [ -n "$instance_id" ]; then + echo "$instance_id" + return 0 + fi + fi + + log_error "Could not determine PowerVS instance ID or name" + return 1 +} + +# Check if image already exists in PowerVS +check_image_exists() { + local image_name="$1" + local pvs_instance="$2" + local get_flag="$3" + + log_info "Checking if image already exists in PowerVS..." + + export IBMCLOUD_API_KEY="$IBM_API_KEY" + + # List images and check if our image exists + local existing_image=$(pvsadm get images $get_flag "$pvs_instance" --json 2>/dev/null | \ + jq -r --arg name "$image_name" '.[] | select(.name == $name) | .name' | head -1) + + if [ -n "$existing_image" ]; then + log_info "Image already exists in PowerVS: $existing_image" + return 0 + else + log_info "Image not found in PowerVS, will proceed with import" + return 1 + fi +} + +# Import image to PowerVS +import_image() { + local image_name="$1" + local cos_object_name="$2" + + log_info "Importing image to PowerVS workspace" + log_info "Image name: $image_name" + log_info "COS object: $cos_object_name" + + # Set IBM Cloud API key + export IBMCLOUD_API_KEY="$IBM_API_KEY" + + # Get PowerVS instance identifier + local pvs_instance=$(get_pvs_instance) + if [ $? -ne 0 ]; then + log_error "Failed to get PowerVS instance identifier" + exit 1 + fi + + # Determine if it's an ID or name + local import_flag="" + local get_flag="" + if [ -n "$POWERVS_INSTANCE_ID" ] || [ -n "$POWERVS_WORKSPACE_CRN" ]; then + import_flag="--pvs-instance-id" + get_flag="--pvs-instance-id" + log_info "Using PowerVS Instance ID: $pvs_instance" + else + import_flag="--pvs-instance-name" + get_flag="--pvs-instance-name" + log_info "Using PowerVS Instance Name: $pvs_instance" + fi + + # Check if image already exists + if check_image_exists "$image_name" "$pvs_instance" "$get_flag"; then + log_info "==========================================" + log_info "Image already exists in PowerVS workspace" + log_info "Skipping import to avoid duplicate" + log_info "To force re-import, delete the existing image first" + log_info "==========================================" + return 0 + fi + + # Import using pvsadm + log_info "Starting import with pvsadm..." + + local import_output="${WORK_DIR}/import-output.json" + + # Build import command + local import_cmd="pvsadm image import $import_flag \"$pvs_instance\" --bucket \"$COS_BUCKET_NAME\" --bucket-region \"$COS_REGION\" --object \"$cos_object_name\" --pvs-image-name \"$image_name\"" + + # Add HMAC credentials if provided + if [ -n "$COS_HMAC_ACCESS_KEY" ] && [ -n "$COS_HMAC_SECRET_KEY" ]; then + import_cmd="$import_cmd --accesskey \"$COS_HMAC_ACCESS_KEY\" --secretkey \"$COS_HMAC_SECRET_KEY\"" + log_info "Using HMAC credentials for COS access" + else + log_info "No HMAC credentials provided, pvsadm will auto-generate service credentials" + fi + + import_cmd="$import_cmd --watch" + + log_info "Running import command..." + eval "$import_cmd" 2>&1 | tee "${WORK_DIR}/import.log" + + local import_status=$? + + if [ $import_status -eq 0 ]; then + log_info "Import completed successfully" + else + log_error "Import failed with status: $import_status" + exit 1 + fi +} + +# Get imported image ID +get_image_id() { + local image_name="$1" + + log_info "Retrieving image ID from PowerVS workspace..." + + export IBMCLOUD_API_KEY="$IBM_API_KEY" + + # Extract PowerVS instance ID from CRN + local pvs_instance_id=$(extract_pvs_instance "$POWERVS_WORKSPACE_CRN") + + # List images and find the one we just imported + local image_id=$(pvsadm get images --pvs-instance-id "$pvs_instance_id" --json 2>/dev/null | \ + jq -r --arg name "$image_name" '.[] | select(.name == $name) | .imageID' | head -1) + + if [ -z "$image_id" ] || [ "$image_id" == "null" ]; then + log_warn "Could not retrieve image ID automatically" + return 1 + fi + + log_info "Image ID: $image_id" + echo "$image_id" +} + +# Update metadata with PowerVS information +update_metadata() { + local image_id="$1" + local metadata_file="${WORK_DIR}/image-metadata.json" + local temp_file="${metadata_file}.tmp" + + jq --arg workspace "$POWERVS_WORKSPACE_CRN" \ + --arg image_id "$image_id" \ + '. + { + powervs_workspace_crn: $workspace, + powervs_image_id: $image_id, + import_date: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) + }' "$metadata_file" > "$temp_file" + + mv "$temp_file" "$metadata_file" + + log_info "Metadata updated with PowerVS information" +} + +# Main execution +main() { + log_info "Starting image import to PowerVS workspace" + log_info "Work directory: $WORK_DIR" + + # Check requirements + check_requirements + + # Get metadata + local metadata=$(get_image_metadata) + local image_name=$(echo "$metadata" | jq -r '.image_name') + local cos_object_name=$(echo "$metadata" | jq -r '.cos_object_name') + + if [ -z "$image_name" ] || [ "$image_name" == "null" ]; then + log_error "Image name not found in metadata" + exit 1 + fi + + if [ -z "$cos_object_name" ] || [ "$cos_object_name" == "null" ]; then + log_error "COS object name not found in metadata" + exit 1 + fi + + log_info "Image name: $image_name" + log_info "COS object: $cos_object_name" + + # Import image + import_image "$image_name" "$cos_object_name" + + # Get image ID + local image_id="" + if image_id=$(get_image_id "$image_name"); then + # Update metadata + update_metadata "$image_id" + else + log_warn "Could not retrieve image ID, but import may have succeeded" + fi + + log_info "==========================================" + log_info "Import completed successfully!" + log_info "Workspace: $POWERVS_WORKSPACE_CRN" + log_info "Image name: $image_name" + if [ -n "$image_id" ]; then + log_info "Image ID: $image_id" + fi + log_info "==========================================" + + # Display next steps + log_info "" + log_info "Next steps:" + log_info "1. Verify the image in PowerVS console" + log_info "2. Test deploy a VM with the new image" + log_info "3. Update Catalog CRD with new image ID" +} + +# Run main function +main "$@" + + diff --git a/hack/scripts/send-notification.sh b/hack/scripts/send-notification.sh new file mode 100755 index 00000000..ced61041 --- /dev/null +++ b/hack/scripts/send-notification.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +# Script to send Slack notifications about the automation status + +set -o nounset +set -o pipefail + +# Configuration +WORK_DIR="${WORK_DIR:-/tmp/centos-images}" +SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL:-}" +SLACK_BOT_TOKEN="${SLACK_BOT_TOKEN:-}" +SLACK_CHANNEL_ID="${SLACK_CHANNEL_ID:-}" +JOB_STATUS="${JOB_STATUS:-unknown}" +ENVIRONMENT="${ENVIRONMENT:-production}" +GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-}" +GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if Slack is configured (webhook or bot token + channel) +check_slack_config() { + if [ -n "$SLACK_WEBHOOK_URL" ]; then + log_info "Using Slack Webhook URL" + return 0 + elif [ -n "$SLACK_BOT_TOKEN" ] && [ -n "$SLACK_CHANNEL_ID" ]; then + log_info "Using Slack Bot Token with Channel ID: $SLACK_CHANNEL_ID" + return 0 + else + log_warn "Slack not configured. Set either:" + log_warn " - SLACK_WEBHOOK_URL, or" + log_warn " - SLACK_BOT_TOKEN + SLACK_CHANNEL_ID" + return 1 + fi +} + +# Get image metadata +get_metadata() { + local metadata_file="${WORK_DIR}/image-metadata.json" + + if [ -f "$metadata_file" ]; then + cat "$metadata_file" + else + echo "{}" + fi +} + +# Build success message +build_success_message() { + local metadata=$(get_metadata) + local image_name=$(echo "$metadata" | jq -r '.image_name // "N/A"') + local centos_version=$(echo "$metadata" | jq -r '.centos_version // "N/A"') + local powervs_image_id=$(echo "$metadata" | jq -r '.powervs_image_id // "N/A"') + local cos_bucket=$(echo "$metadata" | jq -r '.cos_bucket // "N/A"') + local cos_object=$(echo "$metadata" | jq -r '.cos_object_name // "N/A"') + + local workflow_url="" + if [ -n "$GITHUB_RUN_ID" ] && [ -n "$GITHUB_REPOSITORY" ]; then + workflow_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + fi + + # Determine environment emoji + local env_emoji="🚀" + if [ "$ENVIRONMENT" = "staging" ]; then + env_emoji="🧪" + fi + + cat <" + } + } +EOF + fi + + cat <" + } + } +EOF + fi + + cat < /dev/null; then + missing_tools+=($tool) + fi + done + + if [ ${#missing_tools[@]} -ne 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + exit 1 + fi + + log_info "All required tools are available" +} + +# Get OVA file from metadata +get_ova_file() { + local metadata_file="${WORK_DIR}/image-metadata.json" + + if [ ! -f "$metadata_file" ]; then + log_error "Metadata file not found: $metadata_file" + exit 1 + fi + + local ova_file=$(jq -r '.ova_file' "$metadata_file") + + if [ -z "$ova_file" ] || [ "$ova_file" == "null" ]; then + log_error "OVA file path not found in metadata" + exit 1 + fi + + if [ ! -f "$ova_file" ]; then + log_error "OVA file does not exist: $ova_file" + exit 1 + fi + + echo "$ova_file" +} + +# Note: pvsadm doesn't have a direct command to list COS objects +# We'll handle the duplicate error gracefully during upload + +# Upload OVA to COS using pvsadm +upload_to_cos() { + local ova_file="$1" + local ova_filename=$(basename "$ova_file") + + log_info "Uploading OVA to IBM Cloud Object Storage" + log_info "File: $ova_filename" + log_info "Bucket: $COS_BUCKET_NAME" + log_info "Region: $COS_REGION" + + # Set IBM Cloud API key + export IBMCLOUD_API_KEY="$IBM_API_KEY" + + # Upload using pvsadm (will handle duplicate error gracefully) + log_info "Starting upload with pvsadm..." + + # Capture output and check for duplicate error + local upload_output="${WORK_DIR}/upload_attempt.log" + + if pvsadm image upload \ + --bucket "$COS_BUCKET_NAME" \ + --bucket-region "$COS_REGION" \ + --cos-instance-name "$COS_INSTANCE_NAME" \ + -f "$ova_file" \ + 2>&1 | tee "$upload_output" | awk ' + /[0-9]+%/ { + match($0, /([0-9]+)%/, arr) + percent = arr[1] + if (percent >= 20 && percent % 20 == 0 && percent != last) { + print "[INFO] Upload progress: " percent "%" + last = percent + } + next + } + { print } + ' | tee "${WORK_DIR}/upload.log"; then + log_info "Upload completed successfully" + else + # Check if error is due to file already existing + if grep -q "object already exists" "$upload_output"; then + log_warn "==========================================" + log_warn "File already exists in COS bucket" + log_warn "This is expected on retry - continuing..." + log_warn "==========================================" + # Not a fatal error, continue + else + log_error "Upload failed with unexpected error" + cat "$upload_output" >&2 + exit 1 + fi + fi + + # Update metadata with COS information + local metadata_file="${WORK_DIR}/image-metadata.json" + local temp_file="${metadata_file}.tmp" + + jq --arg bucket "$COS_BUCKET_NAME" \ + --arg region "$COS_REGION" \ + --arg object "$ova_filename" \ + '. + { + cos_bucket: $bucket, + cos_region: $region, + cos_object_name: $object, + upload_date: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) + }' "$metadata_file" > "$temp_file" + + mv "$temp_file" "$metadata_file" + + log_info "Metadata updated with COS information" +} + +# Verify upload +verify_upload() { + local ova_filename="$1" + + log_info "Verifying upload..." + + export IBMCLOUD_API_KEY="$IBM_API_KEY" + + # List objects in bucket to verify + if pvsadm get cos-objects --bucket "$COS_BUCKET_NAME" --bucket-region "$COS_REGION" 2>&1 | grep -q "$ova_filename"; then + log_info "Upload verification successful: $ova_filename found in bucket" + return 0 + else + log_warn "Could not verify upload in bucket listing" + return 1 + fi +} + +# Main execution +main() { + log_info "Starting OVA upload to IBM Cloud Object Storage" + log_info "Work directory: $WORK_DIR" + + # Check requirements + check_requirements + + # Get OVA file path + local ova_file=$(get_ova_file) + log_info "OVA file: $ova_file" + + # Get file size + local file_size=$(du -h "$ova_file" | cut -f1) + log_info "File size: $file_size" + + # Upload to COS + upload_to_cos "$ova_file" + + # Verify upload + local ova_filename=$(basename "$ova_file") + verify_upload "$ova_filename" || log_warn "Verification skipped or failed" + + log_info "==========================================" + log_info "Upload completed successfully!" + log_info "Bucket: $COS_BUCKET_NAME" + log_info "Region: $COS_REGION" + log_info "Object: $ova_filename" + log_info "==========================================" +} + +# Run main function +main "$@" + +