Rolling Rebuilds #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rolling Rebuilds | |
| on: | |
| schedule: | |
| - cron: '0 2 * * 1/3' # At 2:00am on Monday and Thursday | |
| workflow_dispatch: | |
| inputs: | |
| package_filter: | |
| description: 'Package name filter (regex)' | |
| type: string | |
| default: '' | |
| force: | |
| description: 'Force rebuild even if no changes detected' | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: rolling-rebuilds | |
| cancel-in-progress: false | |
| jobs: | |
| find-rolling-packages: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| packages: ${{ steps.find.outputs.packages }} | |
| has_packages: ${{ steps.find.outputs.has_packages }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Find rolling packages | |
| id: find | |
| run: | | |
| PACKAGES="[]" | |
| # Find recipes without top-level pkgver/version (these are rolling packages) | |
| for recipe in $(find binaries packages -name "*.yaml" 2>/dev/null); do | |
| # Skip disabled packages | |
| if grep -qE '^_disabled:\s*(true|yes)' "$recipe"; then | |
| continue | |
| fi | |
| # Check if package has a hardcoded version (pkgver or version at top level) | |
| if grep -qE '^(pkgver|version):' "$recipe"; then | |
| continue | |
| fi | |
| # Must have x_exec.pkgver to be useful | |
| if ! grep -qE '^[[:space:]]+pkgver:' "$recipe"; then | |
| continue | |
| fi | |
| pkg_family=$(echo "$recipe" | cut -d'/' -f2) | |
| recipe_name=$(basename "$recipe" .yaml) | |
| # Apply filter if provided | |
| if [ -n "${{ inputs.package_filter }}" ]; then | |
| if ! echo "$pkg_family" | grep -qE "${{ inputs.package_filter }}"; then | |
| continue | |
| fi | |
| fi | |
| PACKAGES=$(echo "$PACKAGES" | jq --arg path "$recipe" --arg family "$pkg_family" --arg name "$recipe_name" \ | |
| '. + [{"path": $path, "family": $family, "name": $name}]') | |
| done | |
| PKG_COUNT=$(echo "$PACKAGES" | jq 'length') | |
| echo "::notice::Found ${PKG_COUNT} rolling packages" | |
| echo "packages=$(echo "$PACKAGES" | jq -c .)" >> $GITHUB_OUTPUT | |
| if [ "$PKG_COUNT" -gt 0 ]; then | |
| echo "has_packages=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_packages=false" >> $GITHUB_OUTPUT | |
| fi | |
| check-for-updates: | |
| needs: find-rolling-packages | |
| if: needs.find-rolling-packages.outputs.has_packages == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| to_rebuild: ${{ steps.check.outputs.to_rebuild }} | |
| has_updates: ${{ steps.check.outputs.has_updates }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Install skopeo | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y skopeo | |
| - name: Check for source updates | |
| id: check | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| GHCR_OWNER: ${{ github.repository_owner }} | |
| run: | | |
| TO_REBUILD="[]" | |
| PACKAGES='${{ needs.find-rolling-packages.outputs.packages }}' | |
| # Function to extract pkgver script from x_exec block | |
| extract_pkgver_script() { | |
| local file="$1" | |
| awk ' | |
| /^x_exec:/ { in_xexec=1; next } | |
| in_xexec && /^[a-z_]+:/ && !/^[[:space:]]/ { in_xexec=0 } | |
| in_xexec && /^[[:space:]]+pkgver:[[:space:]]*\|/ { in_pkgver=1; next } | |
| in_pkgver && /^[[:space:]]+[a-z_]+:/ { in_pkgver=0 } | |
| in_pkgver { gsub(/^[[:space:]]{4}/, ""); print } | |
| ' "$file" | |
| } | |
| # Function to run pkgver script with timeout | |
| run_pkgver() { | |
| local script="$1" | |
| timeout 120s bash -c "$script" 2>/dev/null | tr -d '\n\r' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | head -c 100 | |
| } | |
| # Sanitize version for OCI tag comparison | |
| # OCI tags only allow: [a-zA-Z0-9_.-], other chars become - | |
| sanitize_version() { | |
| local version="$1" | |
| echo "$version" | sed 's/[^a-zA-Z0-9_.-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' | |
| } | |
| # Function to extract pkg name from recipe | |
| # Priority: first provides entry > pkg field | |
| get_pkg_name() { | |
| local file="$1" | |
| # Try to get first provides entry | |
| local provides_entry | |
| provides_entry=$(awk ' | |
| /^provides:/ { in_provides=1; next } | |
| in_provides && /^[a-z_]+:/ && !/^[[:space:]]/ { exit } | |
| in_provides && /^[[:space:]]*-/ { | |
| gsub(/^[[:space:]]*-[[:space:]]*/, "") | |
| gsub(/^"/, ""); gsub(/"$/, "") | |
| # Extract base name before any separator (=>, ==, :) | |
| split($0, parts, /[=:>]/) | |
| print parts[1] | |
| exit | |
| } | |
| ' "$file") | |
| if [ -n "$provides_entry" ]; then | |
| echo "$provides_entry" | |
| else | |
| # Fallback to pkg field | |
| grep -E "^pkg:" "$file" | head -1 | sed 's/^pkg:[[:space:]]*//; s/^"//; s/"$//' | |
| fi | |
| } | |
| # Function to get remote_pkgver (for upstream comparison) | |
| get_remote_pkgver() { | |
| local file="$1" | |
| # Try to get remote_pkgver first, fall back to pkgver | |
| local remote_ver | |
| remote_ver=$(grep -E "^remote_pkgver:" "$file" | head -1 | sed 's/^remote_pkgver:[[:space:]]*//; s/^"//; s/"$//') | |
| if [ -z "$remote_ver" ]; then | |
| remote_ver=$(grep -E "^pkgver:" "$file" | head -1 | sed 's/^pkgver:[[:space:]]*//; s/^"//; s/"$//') | |
| fi | |
| echo "$remote_ver" | |
| } | |
| # Function to extract ghcr_pkg from recipe (custom GHCR path) | |
| get_ghcr_pkg() { | |
| local file="$1" | |
| grep -E "^ghcr_pkg:" "$file" | head -1 | sed 's/^ghcr_pkg:[[:space:]]*//; s/^"//; s/"$//' | |
| } | |
| # Function to get current version from GHCR | |
| # GHCR path: {owner}/{ghcr_pkg}/{provides} | |
| get_ghcr_version() { | |
| local ghcr_pkg="$1" | |
| local provides_entry="$2" | |
| local ghcr_path="ghcr.io/${GHCR_OWNER}/${ghcr_pkg}/${provides_entry}" | |
| echo "Checking GHCR: $ghcr_path" >&2 | |
| # List tags and extract version from tag name | |
| # Tags are formatted as {version}-{arch} (e.g., v1.2.3-x86_64-linux) | |
| local tags_json | |
| tags_json=$(skopeo list-tags "docker://${ghcr_path}" 2>/dev/null) || { | |
| echo "" | |
| return | |
| } | |
| # Get first tag matching x86_64-linux and extract version | |
| local tag version | |
| tag=$(echo "$tags_json" | jq -r '.Tags[]? | select(endswith("-x86_64-linux"))' | head -1) | |
| # If no x86_64, try aarch64 | |
| if [ -z "$tag" ]; then | |
| tag=$(echo "$tags_json" | jq -r '.Tags[]? | select(endswith("-aarch64-linux"))' | head -1) | |
| version=$(echo "$tag" | sed 's/-aarch64-linux$//') | |
| else | |
| version=$(echo "$tag" | sed 's/-x86_64-linux$//') | |
| fi | |
| echo "$version" | |
| } | |
| echo "$PACKAGES" | jq -c '.[]' | while IFS= read -r pkg; do | |
| path=$(echo "$pkg" | jq -r '.path') | |
| family=$(echo "$pkg" | jq -r '.family') | |
| name=$(echo "$pkg" | jq -r '.name') | |
| # Extract pkg_name from recipe | |
| pkg_name=$(get_pkg_name "$path") | |
| if [ -z "$pkg_name" ]; then | |
| pkg_name="$family" # fallback to family name | |
| fi | |
| # Check for custom ghcr_pkg (required for GHCR version check) | |
| ghcr_pkg=$(get_ghcr_pkg "$path") | |
| echo "::group::Checking $family/$name (pkg: $pkg_name)" | |
| # Skip if no ghcr_pkg - we can't determine the GHCR path | |
| if [ -z "$ghcr_pkg" ]; then | |
| echo "No ghcr_pkg set - cannot check GHCR version, skipping" | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| NEEDS_REBUILD=false | |
| NEW_VERSION="" | |
| CURRENT_VERSION="" | |
| # Get remote_pkgver from recipe for comparison | |
| REMOTE_PKGVER=$(get_remote_pkgver "$path") | |
| # Try to extract and run pkgver script | |
| PKGVER_SCRIPT=$(extract_pkgver_script "$path") | |
| if [ -n "$PKGVER_SCRIPT" ]; then | |
| echo "Found pkgver script, executing..." | |
| NEW_VERSION=$(run_pkgver "$PKGVER_SCRIPT") | |
| if [ -n "$NEW_VERSION" ]; then | |
| echo "New version from pkgver: $NEW_VERSION" | |
| echo "Current remote_pkgver in recipe: $REMOTE_PKGVER" | |
| # Sanitize version for comparison (OCI tags have restricted charset) | |
| SANITIZED_VERSION=$(sanitize_version "$NEW_VERSION") | |
| SANITIZED_REMOTE=$(sanitize_version "$REMOTE_PKGVER") | |
| echo "Sanitized new version: $SANITIZED_VERSION" | |
| echo "Sanitized remote version: $SANITIZED_REMOTE" | |
| # Compare new version with remote_pkgver (not pkgver) | |
| if [ "$SANITIZED_VERSION" != "$SANITIZED_REMOTE" ]; then | |
| echo "Version changed: $SANITIZED_REMOTE -> $SANITIZED_VERSION" | |
| NEEDS_REBUILD=true | |
| else | |
| echo "Version unchanged (matches remote_pkgver)" | |
| fi | |
| else | |
| echo "pkgver script returned empty result" | |
| fi | |
| else | |
| echo "No pkgver script found, skipping version check" | |
| fi | |
| # Force rebuild if requested | |
| if [ "${{ inputs.force }}" == "true" ]; then | |
| NEEDS_REBUILD=true | |
| fi | |
| if [ "$NEEDS_REBUILD" == "true" ]; then | |
| echo "Marking for rebuild" | |
| # Write to temp file to persist across subshell | |
| echo "$path|$NEW_VERSION" >> /tmp/to_rebuild.txt | |
| else | |
| echo "No rebuild needed" | |
| fi | |
| echo "::endgroup::" | |
| done | |
| # Build TO_REBUILD from temp file | |
| if [ -f /tmp/to_rebuild.txt ]; then | |
| while IFS='|' read -r path version; do | |
| TO_REBUILD=$(echo "$TO_REBUILD" | jq --arg path "$path" --arg version "$version" \ | |
| '. + [{"path": $path, "new_version": $version}]') | |
| done < /tmp/to_rebuild.txt | |
| fi | |
| # Write final output | |
| REBUILD_COUNT=$(echo "$TO_REBUILD" | jq 'length') | |
| echo "::notice::${REBUILD_COUNT} packages need rebuilding" | |
| echo "to_rebuild=$(echo "$TO_REBUILD" | jq -c .)" >> $GITHUB_OUTPUT | |
| if [ "$REBUILD_COUNT" -gt 0 ]; then | |
| echo "has_updates=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_updates=false" >> $GITHUB_OUTPUT | |
| fi | |
| rebuild: | |
| needs: [find-rolling-packages, check-for-updates] | |
| if: needs.check-for-updates.outputs.has_updates == 'true' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| package: ${{ fromJson(needs.check-for-updates.outputs.to_rebuild) }} | |
| uses: ./.github/workflows/matrix_builds.yaml | |
| with: | |
| host: "ALL" | |
| sbuild-url: "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}/${{ matrix.package.path }}" | |
| ghcr-url: ${{ format('ghcr.io/{0}', github.repository_owner) }} | |
| pkg-family: ${{ github.event.repository.name }} | |
| rebuild: true | |
| logs: true | |
| secrets: inherit | |
| summary: | |
| needs: [find-rolling-packages, check-for-updates, rebuild] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate summary | |
| run: | | |
| echo "## 🔄 Rolling Rebuild Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| FOUND=$(echo '${{ needs.find-rolling-packages.outputs.packages }}' | jq 'length') | |
| echo "| Rolling packages found | ${FOUND} |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ needs.check-for-updates.outputs.has_updates }}" == "true" ]; then | |
| REBUILT=$(echo '${{ needs.check-for-updates.outputs.to_rebuild }}' | jq 'length') | |
| echo "| Packages rebuilt | ${REBUILT} |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| Packages rebuilt | 0 |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Build result:** ${{ needs.rebuild.result || 'skipped' }}" >> $GITHUB_STEP_SUMMARY |