Add S3/CloudFront mirror fallback to installers

.github/workflows/ci.yml

@@ -93,6 +93,11 @@ jobs:
         shell: bash
         run: sh scripts/test-installer-messages.sh
 
+      - name: Unit test installer fallback logic
+        if: runner.os == 'Linux'
+        shell: bash
+        run: sh scripts/test-installer-fallback.sh
+
       - name: Check PowerShell installer syntax
         if: runner.os == 'Windows'
         shell: pwsh
@@ -148,48 +153,30 @@ jobs:
             (cd "$download_dir" && shasum -a 256 "$archive" > checksums.txt)
           fi
 
+          # Fixed-port fixture server (matches the S3 fallback steps below).
+          # Avoids the ephemeral-port + port-file write race that intermittently
+          # hung before writing its port on macOS runners.
           server_log="$fixture_root/http-server.log"
-          server_port_file="$fixture_root/http-server.port"
-          FIXTURE_ROOT="$fixture_root" SERVER_PORT_FILE="$server_port_file" python3 - <<'PY' >"$server_log" 2>&1 &
-          import functools
-          import http.server
-          import os
-          import pathlib
-
-          root = os.environ["FIXTURE_ROOT"]
-          port_file = pathlib.Path(os.environ["SERVER_PORT_FILE"])
-          handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=root)
-
-          with http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) as httpd:
-              port_file.write_text(str(httpd.server_port), encoding="utf-8")
-              httpd.serve_forever()
-          PY
+          server_url="http://127.0.0.1:18080"
+          archive_url="${server_url}/download/v${version}/${archive}"
+          python3 -m http.server 18080 --directory "$fixture_root" >"$server_log" 2>&1 &
           server_pid=$!
           trap 'kill "$server_pid" 2>/dev/null || true' EXIT
 
           server_ready=0
-          server_url=""
-          archive_url=""
-          for attempt in $(seq 1 40); do
+          for attempt in $(seq 1 120); do
             if ! kill -0 "$server_pid" 2>/dev/null; then
               echo "Fixture HTTP server exited before serving the test archive." >&2
               sed 's/^/  /' "$server_log" >&2 || true
               exit 1
             fi
-            if [ -s "$server_port_file" ]; then
-              server_url="http://127.0.0.1:$(cat "$server_port_file")"
-              archive_url="${server_url}/download/v${version}/${archive}"
-            fi
-            if [ -n "$archive_url" ] && curl -fsS --head --connect-timeout 2 "$archive_url" >/dev/null 2>&1; then
+            if curl -fsS --head --connect-timeout 2 "$archive_url" >/dev/null 2>&1; then
               server_ready=1
               break
             fi
             sleep 0.25
           done
           if [ "$server_ready" != "1" ]; then
-            if [ -z "$archive_url" ]; then
-              archive_url="download/v${version}/${archive}"
-            fi
             echo "Fixture HTTP server did not serve $archive_url." >&2
             sed 's/^/  /' "$server_log" >&2 || true
             exit 1
@@ -200,7 +187,13 @@ jobs:
           # restriction (test only).
           export INSTALLER_CURL_PROTO_OPTS="--proto =http,https"
 
+          # Pin the source to github so this test only exercises the fixture
+          # (RELEASES_DOWNLOAD_BASE_URL) path; auto mode would fast-fail a
+          # transient fixture hiccup and divert to the real dl.agora.io mirror.
+          # github mode also uses the retry profile, restoring pre-fallback
+          # resilience.
           VERSION="$version" \
+          AGORA_INSTALL_SOURCE=github \
           RELEASES_DOWNLOAD_BASE_URL="${server_url}/download" \
           RELEASES_PAGE_URL="$server_url" \
           INSTALL_DIR="$install_dir" \
@@ -210,6 +203,7 @@ jobs:
 
           printf '%064d  %s\n' 0 "$archive" > "$download_dir/checksums.txt"
           if VERSION="$version" \
+            AGORA_INSTALL_SOURCE=github \
             RELEASES_DOWNLOAD_BASE_URL="${server_url}/download" \
             RELEASES_PAGE_URL="$server_url" \
             INSTALL_DIR="$bad_install_dir" \
@@ -218,6 +212,78 @@ jobs:
             exit 1
           fi
 
+      - name: Smoke test installer S3 fallback (Unix)
+        if: runner.os != 'Windows'
+        shell: bash
+        run: |
+          set -euo pipefail
+          version='0.0.0-ci'
+          arch="$(uname -m)"; case "$arch" in x86_64|amd64) arch=amd64;; aarch64|arm64) arch=arm64;; esac
+          os="$(uname -s | tr '[:upper:]' '[:lower:]')"
+          archive="agora-cli_v${version}_${os}_${arch}.tar.gz"
+
+          root=".tmp/fallback-fixture"
+          rel="${root}/releases/v${version}"
+          install_dir="$PWD/.tmp/fallback-bin"
+          rm -rf "$root" "$install_dir"
+          mkdir -p "$rel" "$install_dir"
+
+          cp dist/agora "$root/agora"
+          tar -C "$root" -czf "${rel}/${archive}" agora
+          if command -v sha256sum >/dev/null 2>&1; then
+            ( cd "$rel" && sha256sum "$archive" > checksums.txt )
+          else
+            ( cd "$rel" && shasum -a 256 "$archive" > checksums.txt )
+          fi
+          printf '{"tag_name":"v%s","version":"%s"}' "$version" "$version" > "${root}/latest.json"
+
+          server_log="${root}/http.log"
+          python3 -m http.server 18082 --directory "$root" >"$server_log" 2>&1 &
+          server_pid=$!
+          trap 'kill "$server_pid" 2>/dev/null || true' EXIT
+          server_ready=0
+          for _ in $(seq 1 120); do
+            if ! kill -0 "$server_pid" 2>/dev/null; then
+              echo "Fixture HTTP server exited before becoming ready." >&2
+              sed 's/^/  /' "$server_log" >&2 || true
+              exit 1
+            fi
+            if curl -fsS --head --connect-timeout 2 "http://127.0.0.1:18082/latest.json" >/dev/null 2>&1; then
+              server_ready=1; break
+            fi
+            sleep 0.25
+          done
+          if [ "$server_ready" != "1" ]; then
+            echo "Fixture HTTP server did not become ready." >&2
+            sed 's/^/  /' "$server_log" >&2 || true
+            exit 1
+          fi
+
+          export INSTALLER_CURL_PROTO_OPTS="--proto =http,https"
+          GITHUB_API_URL="http://127.0.0.1:1/api" \
+          RELEASES_DOWNLOAD_BASE_URL="http://127.0.0.1:1/download" \
+          S3_DOWNLOAD_BASE_URL="http://127.0.0.1:18082/releases" \
+          S3_LATEST_URL="http://127.0.0.1:18082/latest.json" \
+          RELEASES_PAGE_URL="http://127.0.0.1:18082" \
+          INSTALL_DIR="$install_dir" \
+          sh ./install.sh --force
+
+          "$install_dir/agora" --help >/dev/null
+          echo "Fallback install succeeded via mock S3."
+
+          printf '%064d  %s\n' 0 "$archive" > "${rel}/checksums.txt"
+          if GITHUB_API_URL="http://127.0.0.1:1/api" \
+             RELEASES_DOWNLOAD_BASE_URL="http://127.0.0.1:1/download" \
+             S3_DOWNLOAD_BASE_URL="http://127.0.0.1:18082/releases" \
+             S3_LATEST_URL="http://127.0.0.1:18082/latest.json" \
+             RELEASES_PAGE_URL="http://127.0.0.1:18082" \
+             INSTALL_DIR="$PWD/.tmp/fallback-bin-bad" \
+             sh ./install.sh --force; then
+            echo "Expected checksum verification to fail on the mirrored archive." >&2
+            exit 1
+          fi
+          echo "Checksum verification correctly rejected the tampered mirror archive."
+
       - name: Smoke test PowerShell installer
         if: runner.os == 'Windows'
         shell: pwsh
@@ -247,7 +313,7 @@ jobs:
           $server = Start-Process -FilePath python -ArgumentList '-m', 'http.server', '18081', '--directory', $fixtureRoot -RedirectStandardOutput $serverOutLog -RedirectStandardError $serverErrLog -PassThru
           $archiveUrl = "http://127.0.0.1:18081/download/v$version/$archive"
           $serverReady = $false
-          for ($attempt = 1; $attempt -le 20; $attempt++) {
+          for ($attempt = 1; $attempt -le 60; $attempt++) {
             if ($server.HasExited) {
               if (Test-Path -LiteralPath $serverOutLog) { Get-Content -Path $serverOutLog | ForEach-Object { Write-Host $_ } }
               if (Test-Path -LiteralPath $serverErrLog) { Get-Content -Path $serverErrLog | ForEach-Object { Write-Host $_ } }
@@ -271,6 +337,9 @@ jobs:
 
           try {
             $env:VERSION = $version
+            # Pin source to github so this test only exercises the fixture path
+            # (auto mode could divert a transient hiccup to the real mirror).
+            $env:AGORA_INSTALL_SOURCE = 'github'
             $env:RELEASES_DOWNLOAD_BASE_URL = 'http://127.0.0.1:18081/download'
             $env:RELEASES_PAGE_URL = 'http://127.0.0.1:18081'
 
@@ -297,5 +366,64 @@ jobs:
             $global:LASTEXITCODE = 0
           } finally {
             Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue
-            Remove-Item Env:VERSION, Env:RELEASES_DOWNLOAD_BASE_URL, Env:RELEASES_PAGE_URL -ErrorAction SilentlyContinue
+            Remove-Item Env:VERSION, Env:AGORA_INSTALL_SOURCE, Env:RELEASES_DOWNLOAD_BASE_URL, Env:RELEASES_PAGE_URL -ErrorAction SilentlyContinue
+          }
+
+      - name: Smoke test PowerShell installer S3 fallback
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $version = '0.0.0-ci'
+          $arch = switch ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant()) {
+            'x64' { 'amd64' } 'arm64' { 'arm64' } default { throw "Unsupported CI arch" }
+          }
+          $root = Join-Path $PWD '.tmp/fallback-fixture'
+          $rel = Join-Path $root "releases/v$version"
+          $installDir = Join-Path $PWD '.tmp/fallback-bin'
+          $archive = "agora-cli_v$version" + "_windows_${arch}.zip"
+          Remove-Item -Recurse -Force $root, $installDir -ErrorAction SilentlyContinue
+          New-Item -ItemType Directory -Force -Path $rel, $installDir | Out-Null
+
+          Compress-Archive -Path (Join-Path $PWD 'dist/agora.exe') -DestinationPath (Join-Path $rel $archive) -Force
+          $hash = (Get-FileHash -Path (Join-Path $rel $archive) -Algorithm SHA256).Hash.ToLowerInvariant()
+          Set-Content -Path (Join-Path $rel 'checksums.txt') -Value "$hash  $archive"
+          Set-Content -NoNewline -Path (Join-Path $root 'latest.json') -Value ('{"tag_name":"v' + $version + '","version":"' + $version + '"}')
+
+          $server = Start-Process -FilePath python -ArgumentList '-m', 'http.server', '18083', '--directory', $root -PassThru
+          try {
+            $ready = $false
+            for ($i = 0; $i -lt 120; $i++) {
+              if ($server.HasExited) { throw 'Fixture HTTP server exited before becoming ready.' }
+              try { if ((Invoke-WebRequest -Uri 'http://127.0.0.1:18083/latest.json' -UseBasicParsing).StatusCode -eq 200) { $ready = $true; break } } catch { Start-Sleep -Milliseconds 250 }
+            }
+            if (-not $ready) { throw 'fixture server not ready at http://127.0.0.1:18083/latest.json' }
+
+            $env:GITHUB_API_URL = 'http://127.0.0.1:1/api'
+            $env:RELEASES_DOWNLOAD_BASE_URL = 'http://127.0.0.1:1/download'
+            $env:S3_DOWNLOAD_BASE_URL = 'http://127.0.0.1:18083/releases'
+            $env:S3_LATEST_URL = 'http://127.0.0.1:18083/latest.json'
+            $env:RELEASES_PAGE_URL = 'http://127.0.0.1:18083'
+
+            & ./install.ps1 -InstallDir $installDir
+            if ($LASTEXITCODE -ne 0) { throw "install.ps1 fallback failed: $LASTEXITCODE" }
+            & (Join-Path $installDir 'agora.exe') --help *> $null
+            Write-Host 'Fallback install succeeded via mock S3.'
+
+            # Checksum verification must still fire on S3-served bytes: corrupt
+            # the mirror's checksums.txt and assert install.ps1 refuses.
+            Set-Content -Path (Join-Path $rel 'checksums.txt') -Value ('0' * 64 + "  $archive")
+            $badInstallDir = Join-Path $PWD '.tmp/fallback-bin-bad'
+            $previousNativePreference = $PSNativeCommandUseErrorActionPreference
+            $PSNativeCommandUseErrorActionPreference = $false
+            pwsh -NoProfile -ExecutionPolicy Bypass -File ./install.ps1 -InstallDir $badInstallDir
+            $badExitCode = $LASTEXITCODE
+            $PSNativeCommandUseErrorActionPreference = $previousNativePreference
+            if ($badExitCode -eq 0) {
+              throw 'Expected checksum verification to fail on the mirrored archive.'
+            }
+            $global:LASTEXITCODE = 0
+            Write-Host 'Checksum verification correctly rejected the tampered mirror archive.'
+          } finally {
+            Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue
+            Remove-Item Env:GITHUB_API_URL, Env:RELEASES_DOWNLOAD_BASE_URL, Env:S3_DOWNLOAD_BASE_URL, Env:S3_LATEST_URL, Env:RELEASES_PAGE_URL -ErrorAction SilentlyContinue
           }

.github/workflows/release.yml

@@ -366,3 +366,136 @@ jobs:
             sleep "$delay"
             delay=$((delay * 2))
           done
+
+  mirror-to-s3:
+    name: Mirror release to S3
+    needs: [goreleaser]
+    if: github.event_name == 'push' && needs.goreleaser.result == 'success'
+    runs-on: ubuntu-latest
+    env:
+      AWS_REGION: us-east-1
+      S3_BUCKET: dl-agora-io
+      S3_PREFIX: cli
+      CF_DISTRIBUTION_ID: E2U1WWAZBG33XY
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Resolve tag/version
+        id: ver
+        shell: bash
+        run: |
+          set -euo pipefail
+          tag="${GITHUB_REF_NAME}"
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+          echo "version=${tag#v}" >> "$GITHUB_OUTPUT"
+
+      - name: Download release artifacts
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          mkdir -p dist
+          gh release download "${{ steps.ver.outputs.tag }}" \
+            --pattern "agora-cli_*_linux_*.tar.gz" \
+            --pattern "agora-cli_*_darwin_*.tar.gz" \
+            --pattern "agora-cli_*_windows_*.zip" \
+            --pattern "checksums.txt" \
+            --dir dist
+          # Sigstore bundle is best-effort; its absence must not block mirroring.
+          gh release download "${{ steps.ver.outputs.tag }}" \
+            --pattern "checksums.txt.sigstore.json" \
+            --dir dist || echo "::warning::checksums.txt.sigstore.json not present in release; skipping."
+          echo "Downloaded:"; ls -la dist/
+
+      - name: Verify SHA-256 checksums before upload
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [ ! -f dist/checksums.txt ]; then
+            echo "::error::checksums.txt missing; refusing to mirror unverified binaries."
+            exit 1
+          fi
+          cd dist
+          while IFS= read -r line; do
+            [ -z "$line" ] && continue
+            expected="$(echo "$line" | awk '{print $1}')"
+            file="$(echo "$line" | awk '{print $2}')"
+            case "$file" in
+              agora-cli_*_linux_*.tar.gz|agora-cli_*_darwin_*.tar.gz|agora-cli_*_windows_*.zip) ;;
+              *) continue ;;
+            esac
+            [ -f "$file" ] || { echo "::error::missing $file"; exit 1; }
+            actual="$(sha256sum "$file" | awk '{print $1}')"
+            [ "$expected" = "$actual" ] || { echo "::error::checksum mismatch for $file"; exit 1; }
+            echo "  ok: $file"
+          done < checksums.txt
+          echo "All archives verified."
+
+      - name: Determine prerelease flag
+        id: pre
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          is_pre="$(gh release view "${{ steps.ver.outputs.tag }}" --json isPrerelease --jq '.isPrerelease')"
+          echo "is_prerelease=${is_pre}" >> "$GITHUB_OUTPUT"
+          echo "Prerelease: ${is_pre}"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Upload versioned artifacts (immutable)
+        shell: bash
+        run: |
+          set -euo pipefail
+          dest="s3://${S3_BUCKET}/${S3_PREFIX}/releases/${{ steps.ver.outputs.tag }}/"
+          aws s3 cp dist/ "$dest" --recursive \
+            --exclude "*" \
+            --include "agora-cli_*_linux_*.tar.gz" \
+            --include "agora-cli_*_darwin_*.tar.gz" \
+            --include "agora-cli_*_windows_*.zip" \
+            --include "checksums.txt" \
+            --include "checksums.txt.sigstore.json" \
+            --cache-control "public, max-age=31536000, immutable"
+          echo "Uploaded versioned artifacts to $dest"
+
+      - name: Upload installer scripts (mutable)
+        shell: bash
+        run: |
+          set -euo pipefail
+          base="s3://${S3_BUCKET}/${S3_PREFIX}"
+          aws s3 cp install.sh  "${base}/install.sh"  --cache-control "public, max-age=300" --content-type "text/x-shellscript"
+          aws s3 cp install.ps1 "${base}/install.ps1" --cache-control "public, max-age=300" --content-type "text/plain"
+          echo "Uploaded installer scripts."
+
+      - name: Update latest.json pointer (stable releases only)
+        if: steps.pre.outputs.is_prerelease == 'false'
+        shell: bash
+        run: |
+          set -euo pipefail
+          v="${{ steps.ver.outputs.version }}"
+          printf '{"tag_name":"v%s","version":"%s"}' "$v" "$v" > latest.json
+          cat latest.json; echo
+          aws s3 cp latest.json "s3://${S3_BUCKET}/${S3_PREFIX}/latest.json" \
+            --cache-control "public, max-age=300" --content-type "application/json"
+          echo "Updated latest.json -> v${v}"
+
+      - name: Invalidate CloudFront for mutable paths
+        shell: bash
+        run: |
+          set -euo pipefail
+          paths="/${S3_PREFIX}/install.sh /${S3_PREFIX}/install.ps1"
+          if [ "${{ steps.pre.outputs.is_prerelease }}" = "false" ]; then
+            paths="${paths} /${S3_PREFIX}/latest.json"
+          fi
+          # shellcheck disable=SC2086
+          aws cloudfront create-invalidation \
+            --distribution-id "${CF_DISTRIBUTION_ID}" \
+            --paths $paths
+          echo "Invalidated: ${paths}"