⚡ Bolt: Parallelize repository downloads in CI

.github/workflows/build.yml

@@ -40,8 +40,8 @@ jobs:
           NEW_LOG=""
           download_file() {
             local url=$1 output=$2 mandatory=$3
-            echo "Downloading file: $output from $url" && curl -L -H "Authorization: token $GITHUB_TOKEN" -o "$output" "$url"
-            if [ $? -ne 0 ]; then
+            echo "Downloading file: $output from $url"
+            if ! curl -sS -L -H "Authorization: token $GITHUB_TOKEN" -o "$output" "$url"; then
               echo "Failed to download $output from $url"
               if [ "$mandatory" = "true" ]; then
                 echo "should_continue=false" >> $GITHUB_OUTPUT
@@ -53,6 +53,7 @@ jobs:
 
           extract_archive() {
             local file=$1 target=$2 archive_pattern=$3 additional_files_json=$4 
+            local temp_dir
             mkdir -p "$target" && temp_dir=$(mktemp -d) 
 
             case "$file" in 
@@ -122,28 +123,17 @@ jobs:
             [ -d "$temp_dir" ] && { rm -rf "$temp_dir" && echo "Deleted temporary directory: $temp_dir"; }
           }
 
-          check_for_updates() {
-            local repo=$1 response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$repo/releases/latest") || { echo "Error: Failed to fetch latest release information for $repo."; return 1; }
-            local latest_release=$(echo "$response" | jq -r ".tag_name // empty") || { echo "Error: Unable to find the latest release for $repo."; return 1; }
-
-            echo "Checking for updates for $repo"
-            local current_version=$(echo "$RELEASE_VERSIONS" | jq -r ".repos[\"$repo\"].version // 0")
-
-            if [[ "$current_version" != "$latest_release" ]]; then
-              RELEASE_VERSIONS=$(echo "$RELEASE_VERSIONS" | jq ".repos[\"$repo\"] = { \"version\": \"$latest_release\" }")
-              if [[ "$current_version" == "0" ]]; then
-                NEW_LOG="${NEW_LOG}- **$repo**: $latest_release\n"
-                echo "NEW_LOG=${NEW_LOG}" >> $GITHUB_ENV
-                echo "New Repo: $repo (version $latest_release)"
-              else
-                CHANGES_LOG="${CHANGES_LOG}- **$repo**: ${current_version} to $latest_release\n"
-                echo "CHANGES_LOG=$CHANGES_LOG" >> $GITHUB_ENV
-                echo "Updates found for $repo: current version $current_version, latest version $latest_release"
-              fi
-              UPDATES_COUNT=$((UPDATES_COUNT + 1))
+          fetch_release_info() {
+            local repo=$1
+            local safe_repo=${repo//\//_}
+            local response
+            response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$repo/releases/latest")
+            local tag=$(echo "$response" | jq -r ".tag_name // empty")
+            if [ -n "$tag" ]; then
+              echo "$repo|$tag" > "release_info_${safe_repo}.txt"
+            else
+               echo "Error: Unable to find the latest release for $repo."
             fi
-
-            echo "$RELEASE_VERSIONS" > release_versions.json
           }
 
           handle_downloads() {
@@ -206,18 +196,49 @@ jobs:
               fi
           }
 
-          echo "Checking and updating releases..."
+          echo "Checking for updates..."
+          rm -f release_info_*.txt
           for repo in $(echo "$DOWNLOAD_FILES" | jq -r '.repos | keys[]'); do
-            check_for_updates "$repo"
+            fetch_release_info "$repo" &
           done
+          wait
+
+          cat release_info_*.txt > release_info.txt 2>/dev/null || true
+
+          if [ -s release_info.txt ]; then
+            while IFS='|' read -r repo latest_release; do
+              local current_version=$(echo "$RELEASE_VERSIONS" | jq -r ".repos[\"$repo\"].version // 0")
+
+              if [[ "$current_version" != "$latest_release" ]]; then
+                RELEASE_VERSIONS=$(echo "$RELEASE_VERSIONS" | jq ".repos[\"$repo\"] = { \"version\": \"$latest_release\" }")
+                if [[ "$current_version" == "0" ]]; then
+                  NEW_LOG="${NEW_LOG}- **$repo**: $latest_release\n"
+                  echo "NEW_LOG=${NEW_LOG}" >> $GITHUB_ENV
+                  echo "New Repo: $repo (version $latest_release)"
+                else
+                  CHANGES_LOG="${CHANGES_LOG}- **$repo**: ${current_version} to $latest_release\n"
+                  echo "CHANGES_LOG=$CHANGES_LOG" >> $GITHUB_ENV
+                  echo "Updates found for $repo: current version $current_version, latest version $latest_release"
+                fi
+                UPDATES_COUNT=$((UPDATES_COUNT + 1))
+              fi
+            done < release_info.txt
+
+            echo "$RELEASE_VERSIONS" > release_versions.json
+          fi
+
+          rm -f release_info_*.txt release_info.txt
 
           UPDATES_COUNT=${UPDATES_COUNT:-0}
           echo "DEBUG: UPDATES_COUNT is $UPDATES_COUNT"
           if [ "$UPDATES_COUNT" -ge "$MIN_UPDATES" ] || [ "$EVENT_NAME" = "workflow_dispatch" ]; then
             echo "should_continue=true" >> $GITHUB_OUTPUT
             echo "$RELEASE_VERSIONS" > release_versions.json
             echo "Downloading files for updated repositories..." && mkdir -p downloads
-            for repo in $(echo "$RELEASE_VERSIONS" | jq -r '.repos | keys[]'); do handle_downloads "$repo"; done
+            for repo in $(echo "$RELEASE_VERSIONS" | jq -r '.repos | keys[]'); do
+              handle_downloads "$repo" &
+            done
+            wait
             echo "Final contents of release/:"
             find release/ -type f | sort
             echo "NEW_LOG<<EOF" >> $GITHUB_ENV
@@ -386,4 +407,3 @@ jobs:
           repository: ${{ github.repository }}
           retain_days: 7
           keep_minimum_runs: 7
-

.jules/bolt.md

@@ -0,0 +1,7 @@
+## 2024-10-24 - Parallel Bash Workflows in CI
+**Learning:** Sequential processing of independent network/IO tasks in Bash scripts within CI workflows is a major performance bottleneck. Bash's '&' background operator and 'wait' command allow for easy parallelization without complex dependencies.
+**Action:** Identify loops iterating over independent items (like multiple file downloads or repo processing) and parallelize them using `&` and `wait`. Ensure local variables are used in functions to prevent state pollution between parallel subshells.
+
+## 2024-10-24 - Bash set -e and Error Handling
+**Learning:** GitHub Actions defaults to `set -e`. Code like `command; if [ $? -ne 0 ]; ...` is dangerous because `command` triggers immediate exit on failure before reaching the `if`.
+**Action:** Use `if ! command; then ... fi` to properly handle errors in scripts that might run with `set -e`. This is also crucial for ensuring cleanup or signaling logic runs even when commands fail.

build_version.txt

@@ -1 +1 @@
-28
+29

release_versions.json

@@ -1,7 +1,7 @@
 {
   "repos": {
     "Atmosphere-NX/Atmosphere": {
-      "version": "1.10.1"
+      "version": "1.10.2"
     },
     "averne/Fizeau": {
       "version": "v2.8.2"
@@ -19,7 +19,7 @@
       "version": "v1.05"
     },
     "CTCaer/hekate": {
-      "version": "v6.4.2"
+      "version": "v6.5.0"
     },
     "DarkMatterCore/nxdumptool": {
       "version": "v1.1.15"
@@ -67,7 +67,7 @@
       "version": "2.1.2"
     },
     "ndeadly/MissionControl": {
-      "version": "v0.14.1"
+      "version": "v0.14.2"
     },
     "nedex/QuickNTP": {
       "version": "1.6.0"
@@ -76,7 +76,7 @@
       "version": "2.3.0"
     },
     "ppkantorski/Ultrahand-Overlay": {
-      "version": "v2.2.6"
+      "version": "v2.2.7"
     },
     "proferabg/EdiZon-Overlay": {
       "version": "v1.0.14"