From 21d43708d214f96220787107a3db33495d397c2e Mon Sep 17 00:00:00 2001 From: sami Date: Fri, 13 Mar 2026 14:58:25 +0600 Subject: [PATCH 1/3] Initialize token based bash workerpool Signed-off-by: sami --- catalog/concurrency_utils.sh | 96 ++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100755 catalog/concurrency_utils.sh diff --git a/catalog/concurrency_utils.sh b/catalog/concurrency_utils.sh new file mode 100755 index 000000000..2ec71f671 --- /dev/null +++ b/catalog/concurrency_utils.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Copyright AppsCode Inc. and Contributors +# +# Licensed under the AppsCode Community License 1.0.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +MAX_WORKERS=${MAX_WORKERS:-30} +MAX_RETRIES=${MAX_RETRIES:-10} +TIMEOUT=${TIMEOUT:-300} +FAIL_LOG=$(mktemp) +declare -a ASYNC_PIDS=() + +FIFO=$(mktemp) +rm -f "$FIFO" +mkfifo "$FIFO" +exec 3<>"$FIFO" +rm -f "$FIFO" + +for ((i = 0; i < MAX_WORKERS; i++)); do + echo >&3 +done + +cleanup() { + trap - EXIT INT TERM + local pid + exec 3>&- 2>/dev/null || true + for pid in "${ASYNC_PIDS[@]:-}"; do + kill "$pid" 2>/dev/null || true + done + for pid in $(jobs -pr); do + kill "$pid" 2>/dev/null || true + done + rm -f "$FAIL_LOG" +} +trap cleanup EXIT INT TERM + +# Usage: run_async command [args...] +run_async() { + local -a cmd=("$@") + local desc="${cmd[*]}" + + read -r -u 3 + + ( + trap 'echo >&3' EXIT + + local out exit_code attempt=1 + while [[ $attempt -le $MAX_RETRIES ]]; do + out=$(timeout "$TIMEOUT" "${cmd[@]}" 2>&1) && { + printf '\033[32m✓ %s\033[0m\n' "$desc" >&2 + exit 0 + } + exit_code=$? + ((attempt++)) + [[ $attempt -gt $MAX_RETRIES ]] && break + printf '\033[38;5;208m⚠ %s (attempt %d/%d, exit %d)\n%s\033[0m\n' "$desc" "$attempt" "$MAX_RETRIES" "$exit_code" "$out" >&2 + sleep $((attempt * 2 + RANDOM % (3 + attempt * 2))) + done + printf '\033[31m✗ %s (failed, exit %d)\n%s\033[0m\n' "$desc" "$exit_code" "$out" >&2 + { + flock 200 + printf '%s\nexit=%d\n%s\n---\n' "$desc" "$exit_code" "$out" >&200 + } 200>>"$FAIL_LOG" + exit 1 + ) & + ASYNC_PIDS+=("$!") +} + +# Wait for all jobs, print summary, return 1 if any failed +wait_all() { + local pid + for pid in "${ASYNC_PIDS[@]}"; do + wait "$pid" || true + done + ASYNC_PIDS=() + + echo "---" >&2 + if [[ -s "$FAIL_LOG" ]]; then + printf '\033[31mFailed tasks:\033[0m\n' >&2 + cat "$FAIL_LOG" >&2 + return 1 + fi + printf '\033[32mAll tasks completed successfully\033[0m\n' >&2 +} From 05b7400178f81f0265061ca460453e9452e85e37 Mon Sep 17 00:00:00 2001 From: sami Date: Fri, 13 Mar 2026 16:11:21 +0600 Subject: [PATCH 2/3] Concurrently export/import charts/images Signed-off-by: sami --- ...currency_utils.sh => concurrency-utils.sh} | 0 catalog/copy-images.sh | 11 +++++--- catalog/export-images.sh | 19 ++++++++++++-- catalog/import-images.sh | 26 ++++++++++++++++--- catalog/import-into-k3s.sh | 19 +++++++++++--- catalog/sync-gcp-mp-images.sh | 6 ++--- 6 files changed, 66 insertions(+), 15 deletions(-) rename catalog/{concurrency_utils.sh => concurrency-utils.sh} (100%) diff --git a/catalog/concurrency_utils.sh b/catalog/concurrency-utils.sh similarity index 100% rename from catalog/concurrency_utils.sh rename to catalog/concurrency-utils.sh diff --git a/catalog/copy-images.sh b/catalog/copy-images.sh index fcb79879c..2eb1d168c 100755 --- a/catalog/copy-images.sh +++ b/catalog/copy-images.sh @@ -14,9 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/concurrency-utils.sh" + +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi @@ -33,7 +36,7 @@ curl -sL "https://github.com/google/go-containerregistry/releases/latest/downloa tar -zxvf /tmp/go-containerregistry.tar.gz -C /tmp/ mv /tmp/crane . -CMD="./crane" +CMD="run_async ./crane" $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace-installer:v2026.3.30 $IMAGE_REGISTRY/appscode-charts/ace-installer:v2026.3.30 $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace:v2026.3.30 $IMAGE_REGISTRY/appscode-charts/ace:v2026.3.30 @@ -301,3 +304,5 @@ $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/va $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/virtual-secrets-server:v2026.2.27 $IMAGE_REGISTRY/appscode-charts/virtual-secrets-server:v2026.2.27 $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager-gateway:v2026.1.15 $IMAGE_REGISTRY/appscode-charts/voyager-gateway:v2026.1.15 $CMD cp --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager:v2026.3.23 $IMAGE_REGISTRY/appscode-charts/voyager:v2026.3.23 + +wait_all diff --git a/catalog/export-images.sh b/catalog/export-images.sh index 21e8c7cc6..bd8f37f55 100755 --- a/catalog/export-images.sh +++ b/catalog/export-images.sh @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/concurrency-utils.sh" mkdir -p images @@ -30,7 +33,8 @@ curl -sL "https://github.com/google/go-containerregistry/releases/latest/downloa tar -zxvf /tmp/go-containerregistry.tar.gz -C /tmp/ mv /tmp/crane images -CMD="./images/crane" +CMD="run_async ./images/crane" +EXPECTED_COUNT=$(grep -cE '^\$CMD pull ' "${BASH_SOURCE[0]}") $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace-installer:v2026.3.30 images/appscode-charts-ace-installer-v2026.3.30.tar $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ace:v2026.3.30 images/appscode-charts-ace-v2026.3.30.tar @@ -299,4 +303,15 @@ $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/ $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager-gateway:v2026.1.15 images/appscode-charts-voyager-gateway-v2026.1.15.tar $CMD pull --allow-nondistributable-artifacts --insecure ghcr.io/appscode-charts/voyager:v2026.3.23 images/appscode-charts-voyager-v2026.3.23.tar +if ! wait_all; then + exit 1 +fi + +actual=$(find images -name '*.tar' | wc -l) +if [[ "$actual" -ne "$EXPECTED_COUNT" ]]; then + printf '\033[31mValidation failed: expected %d files, got %d\033[0m\n' "$EXPECTED_COUNT" "$actual" >&2 + exit 1 +fi +printf '\033[32mValidation passed: %d files\033[0m\n' "$actual" >&2 + tar -czvf images.tar.gz images diff --git a/catalog/import-images.sh b/catalog/import-images.sh index cd830de62..fe9e1b507 100755 --- a/catalog/import-images.sh +++ b/catalog/import-images.sh @@ -14,17 +14,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/concurrency-utils.sh" + +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi TARBALL=${1:-} -tar -zxvf $TARBALL +if [ -n "$TARBALL" ]; then + if [ ! -f "$TARBALL" ]; then + echo "Error: Tarball '$TARBALL' does not exist." + exit 1 + fi + echo "Extracting $TARBALL..." + tar -zxvf "$TARBALL" +elif [ -d "images" ] && [ -f "images/crane" ] && ls images/*.tar >/dev/null 2>&1; then + echo "Found existing images directory with tarballs and crane binary. Skipping extraction..." +else + echo "Usage: $0 [images.tar.gz]" + echo "Error: No tarball provided and valid images directory not found." + exit 1 +fi -CMD="./crane" +CMD="run_async ./images/crane" $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-ace-installer-v2026.3.30.tar $IMAGE_REGISTRY/appscode-charts/ace-installer:v2026.3.30 $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-ace-v2026.3.30.tar $IMAGE_REGISTRY/appscode-charts/ace:v2026.3.30 @@ -292,3 +308,5 @@ $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-v $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-virtual-secrets-server-v2026.2.27.tar $IMAGE_REGISTRY/appscode-charts/virtual-secrets-server:v2026.2.27 $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-voyager-gateway-v2026.1.15.tar $IMAGE_REGISTRY/appscode-charts/voyager-gateway:v2026.1.15 $CMD push --allow-nondistributable-artifacts --insecure images/appscode-charts-voyager-v2026.3.23.tar $IMAGE_REGISTRY/appscode-charts/voyager:v2026.3.23 + +wait_all diff --git a/catalog/import-into-k3s.sh b/catalog/import-into-k3s.sh index c6542483d..cdeb11796 100755 --- a/catalog/import-into-k3s.sh +++ b/catalog/import-into-k3s.sh @@ -14,15 +14,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euxo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi TARBALL=${1:-} -tar -zxvf $TARBALL +if [ -n "$TARBALL" ]; then + if [ ! -f "$TARBALL" ]; then + echo "Error: Tarball '$TARBALL' does not exist." + exit 1 + fi + echo "Extracting $TARBALL..." + tar -zxvf "$TARBALL" +elif [ -d "images" ] && ls images/*.tar >/dev/null 2>&1; then + echo "Found existing images directory with tarballs. Skipping extraction..." +else + echo "Usage: $0 [images.tar.gz]" + echo "Error: No tarball provided and valid images directory not found." + exit 1 +fi k3s ctr images import images/appscode-charts-ace-installer-v2026.3.30.tar k3s ctr images import images/appscode-charts-ace-v2026.3.30.tar diff --git a/catalog/sync-gcp-mp-images.sh b/catalog/sync-gcp-mp-images.sh index 5aa22ca5e..f73f0fd63 100755 --- a/catalog/sync-gcp-mp-images.sh +++ b/catalog/sync-gcp-mp-images.sh @@ -14,13 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x +set -euxo pipefail -if [ -z "${IMAGE_REGISTRY}" ]; then +if [ -z "${IMAGE_REGISTRY:-}" ]; then echo "IMAGE_REGISTRY is not set" exit 1 fi -if [ -z "${TAG}" ]; then +if [ -z "${TAG:-}" ]; then echo "TAG is not set" exit 1 fi From e60efba096306cb9422e5978d2a03f72cdf8282e Mon Sep 17 00:00:00 2001 From: sami Date: Tue, 31 Mar 2026 11:29:38 +0600 Subject: [PATCH 3/3] Always force tarball extraction Signed-off-by: sami --- catalog/import-images.sh | 4 +--- catalog/import-into-k3s.sh | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/catalog/import-images.sh b/catalog/import-images.sh index fe9e1b507..988cfacf8 100755 --- a/catalog/import-images.sh +++ b/catalog/import-images.sh @@ -32,11 +32,9 @@ if [ -n "$TARBALL" ]; then fi echo "Extracting $TARBALL..." tar -zxvf "$TARBALL" -elif [ -d "images" ] && [ -f "images/crane" ] && ls images/*.tar >/dev/null 2>&1; then - echo "Found existing images directory with tarballs and crane binary. Skipping extraction..." else + echo "Error: No tarball provided." echo "Usage: $0 [images.tar.gz]" - echo "Error: No tarball provided and valid images directory not found." exit 1 fi diff --git a/catalog/import-into-k3s.sh b/catalog/import-into-k3s.sh index cdeb11796..458969c3c 100755 --- a/catalog/import-into-k3s.sh +++ b/catalog/import-into-k3s.sh @@ -29,11 +29,9 @@ if [ -n "$TARBALL" ]; then fi echo "Extracting $TARBALL..." tar -zxvf "$TARBALL" -elif [ -d "images" ] && ls images/*.tar >/dev/null 2>&1; then - echo "Found existing images directory with tarballs. Skipping extraction..." else + echo "Error: No tarball provided." echo "Usage: $0 [images.tar.gz]" - echo "Error: No tarball provided and valid images directory not found." exit 1 fi