diff --git a/catalog/concurrency-utils.sh b/catalog/concurrency-utils.sh new file mode 100755 index 000000000..2ec71f671 --- /dev/null +++ b/catalog/concurrency-utils.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Copyright AppsCode Inc. and Contributors +# +# Licensed under the AppsCode Community License 1.0.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +MAX_WORKERS=${MAX_WORKERS:-30} +MAX_RETRIES=${MAX_RETRIES:-10} +TIMEOUT=${TIMEOUT:-300} +FAIL_LOG=$(mktemp) +declare -a ASYNC_PIDS=() + +FIFO=$(mktemp) +rm -f "$FIFO" +mkfifo "$FIFO" +exec 3<>"$FIFO" +rm -f "$FIFO" + +for ((i = 0; i < MAX_WORKERS; i++)); do + echo >&3 +done + +cleanup() { + trap - EXIT INT TERM + local pid + exec 3>&- 2>/dev/null || true + for pid in "${ASYNC_PIDS[@]:-}"; do + kill "$pid" 2>/dev/null || true + done + for pid in $(jobs -pr); do + kill "$pid" 2>/dev/null || true + done + rm -f "$FAIL_LOG" +} +trap cleanup EXIT INT TERM + +# Usage: run_async command [args...] +run_async() { + local -a cmd=("$@") + local desc="${cmd[*]}" + + read -r -u 3 + + ( + trap 'echo >&3' EXIT + + local out exit_code attempt=1 + while [[ $attempt -le $MAX_RETRIES ]]; do + out=$(timeout "$TIMEOUT" "${cmd[@]}" 2>&1) && { + printf '\033[32m✓ %s\033[0m\n' "$desc" >&2 + exit 0 + } + exit_code=$? + ((attempt++)) + [[ $attempt -gt $MAX_RETRIES ]] && break + printf '\033[38;5;208m⚠ %s (attempt %d/%d, exit %d)\n%s\033[0m\n' "$desc" "$attempt" "$MAX_RETRIES" "$exit_code" "$out" >&2 + sleep $((attempt * 2 + RANDOM % (3 + attempt * 2))) + done + printf '\033[31m✗ %s (failed, exit %d)\n%s\033[0m\n' "$desc" "$exit_code" "$out" >&2 + { + flock 200 + printf '%s\nexit=%d\n%s\n---\n' "$desc" "$exit_code" "$out" >&200 + } 200>>"$FAIL_LOG" + exit 1 + ) & + ASYNC_PIDS+=("$!") +} + +# Wait for all jobs, print summary, return 1 if any failed +wait_all() { + local pid + for pid in "${ASYNC_PIDS[@]}"; do + wait "$pid" || true + done + ASYNC_PIDS=() + + echo "---" >&2 + if [[ -s "$FAIL_LOG" ]]; then + printf '\033[31mFailed tasks:\033[0m\n' >&2 + cat "$FAIL_LOG" >&2 + return 1 + fi + printf '\033[32mAll tasks completed successfully\033[0m\n' >&2 +} diff --git a/catalog/copy-images.sh b/catalog/copy-images.sh index fcb79879c..2eb1d168c 100755 --- a/catalog/copy-images.sh +++ b/catalog/copy-images.sh @@ -14,9 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/concurrency-utils.sh" + +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi @@ -33,7 +36,7 @@ curl -sL "https://github.com/google/go-containerregistry/releases/latest/downloa tar -zxvf /tmp/go-containerregistry.tar.gz -C /tmp/ mv /tmp/crane . -CMD="./crane" +CMD="run_async ./crane" $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace-installer:v2026.3.30 $IMAGE_REGISTRY/appscode-charts/ace-installer:v2026.3.30 $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace:v2026.3.30 $IMAGE_REGISTRY/appscode-charts/ace:v2026.3.30 @@ -301,3 +304,5 @@ $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/va $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/virtual-secrets-server:v2026.2.27 $IMAGE_REGISTRY/appscode-charts/virtual-secrets-server:v2026.2.27 $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager-gateway:v2026.1.15 $IMAGE_REGISTRY/appscode-charts/voyager-gateway:v2026.1.15 $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager:v2026.3.23 $IMAGE_REGISTRY/appscode-charts/voyager:v2026.3.23 + +wait_all diff --git a/catalog/export-images.sh b/catalog/export-images.sh index 21e8c7cc6..bd8f37f55 100755 --- a/catalog/export-images.sh +++ b/catalog/export-images.sh @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/concurrency-utils.sh" mkdir -p images @@ -30,7 +33,8 @@ curl -sL "https://github.com/google/go-containerregistry/releases/latest/downloa tar -zxvf /tmp/go-containerregistry.tar.gz -C /tmp/ mv /tmp/crane images -CMD="./images/crane" +CMD="run_async ./images/crane" +EXPECTED_COUNT=$(grep -cE '^\$CMD pull ' "${BASH_SOURCE[0]}") $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace-installer:v2026.3.30 images/appscode-charts-ace-installer-v2026.3.30.tar $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace:v2026.3.30 images/appscode-charts-ace-v2026.3.30.tar @@ -299,4 +303,15 @@ $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager-gateway:v2026.1.15 images/appscode-charts-voyager-gateway-v2026.1.15.tar $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager:v2026.3.23 images/appscode-charts-voyager-v2026.3.23.tar +if ! wait_all; then + exit 1 +fi + +actual=$(find images -name '*.tar' | wc -l) +if [[ "$actual" -ne "$EXPECTED_COUNT" ]]; then + printf '\033[31mValidation failed: expected %d files, got %d\033[0m\n' "$EXPECTED_COUNT" "$actual" >&2 + exit 1 +fi +printf '\033[32mValidation passed: %d files\033[0m\n' "$actual" >&2 + tar -czvf images.tar.gz images diff --git a/catalog/import-images.sh b/catalog/import-images.sh index cd830de62..988cfacf8 100755 --- a/catalog/import-images.sh +++ b/catalog/import-images.sh @@ -14,17 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/concurrency-utils.sh" + +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi TARBALL=${1:-} -tar -zxvf $TARBALL +if [ -n "$TARBALL" ]; then + if [ ! -f "$TARBALL" ]; then + echo "Error: Tarball '$TARBALL' does not exist." + exit 1 + fi + echo "Extracting $TARBALL..." + tar -zxvf "$TARBALL" +else + echo "Error: No tarball provided." + echo "Usage: $0 [images.tar.gz]" + exit 1 +fi -CMD="./crane" +CMD="run_async ./images/crane" $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-ace-installer-v2026.3.30.tar $IMAGE_REGISTRY/appscode-charts/ace-installer:v2026.3.30 $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-ace-v2026.3.30.tar $IMAGE_REGISTRY/appscode-charts/ace:v2026.3.30 @@ -292,3 +306,5 @@ $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-v $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-virtual-secrets-server-v2026.2.27.tar $IMAGE_REGISTRY/appscode-charts/virtual-secrets-server:v2026.2.27 $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-voyager-gateway-v2026.1.15.tar $IMAGE_REGISTRY/appscode-charts/voyager-gateway:v2026.1.15 $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-voyager-v2026.3.23.tar $IMAGE_REGISTRY/appscode-charts/voyager:v2026.3.23 + +wait_all diff --git a/catalog/import-into-k3s.sh b/catalog/import-into-k3s.sh index c6542483d..458969c3c 100755 --- a/catalog/import-into-k3s.sh +++ b/catalog/import-into-k3s.sh @@ -14,15 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euxo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi TARBALL=${1:-} -tar -zxvf $TARBALL +if [ -n "$TARBALL" ]; then + if [ ! -f "$TARBALL" ]; then + echo "Error: Tarball '$TARBALL' does not exist." + exit 1 + fi + echo "Extracting $TARBALL..." + tar -zxvf "$TARBALL" +else + echo "Error: No tarball provided." + echo "Usage: $0 [images.tar.gz]" + exit 1 +fi k3s ctr images import images/appscode-charts-ace-installer-v2026.3.30.tar k3s ctr images import images/appscode-charts-ace-v2026.3.30.tar diff --git a/catalog/sync-gcp-mp-images.sh b/catalog/sync-gcp-mp-images.sh index 5aa22ca5e..f73f0fd63 100755 --- a/catalog/sync-gcp-mp-images.sh +++ b/catalog/sync-gcp-mp-images.sh @@ -14,13 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euxo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi -if [ -z "${TAG}" ]; then +if [ -z "${TAG:-}" ]; then echo "TAG is not set" exit 1 fi