From e3b4bfc02f1c78b07e70f2fbf709f2daf6cee7c3 Mon Sep 17 00:00:00 2001 From: Aaron Gable Date: Wed, 17 Jun 2026 17:15:08 -0700 Subject: [PATCH] Overhaul our github actions for better dep updates & testing --- .github/dependabot.yml | 49 ++++-- .github/workflows/boulder-ci.yml | 25 +-- .github/workflows/check-iana-registries.yml | 2 +- .github/workflows/codeql.yml | 10 +- .github/workflows/dep-tests.yml | 73 +++++++++ .github/workflows/issue-for-sre-handoff.yml | 5 +- .../merged-to-main-or-release-branch.yml | 11 +- .github/workflows/release.yml | 1 + .github/workflows/try-release.yml | 8 +- .github/workflows/zizmor.yml | 4 +- tools/update-container-versions.sh | 148 ++++++++++++++++++ 11 files changed, 283 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/dep-tests.yml create mode 100755 tools/update-container-versions.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ed1c2b799eb..c00d5c8fd5c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,37 +2,60 @@ version: 2 updates: - package-ecosystem: "gomod" + # Update a select few Go dependencies (mostly those we maintain, and ones + # that are important to keep up-to-date like zlint and psl) regularly. + # Group the updates so that one package can't starve out the others. + # No cooldown because of the very limited set of deps we're updating. + # Note that this also disables security updates for all other go deps: + # that's okay because we run govulncheck (which is more granular than + # dependabot) regularly and can handle those security updates manually. directory: "/" + allow: + - dependency-name: "github.com/go-jose/*" + - dependency-name: "github.com/letsencrypt/*" + - dependency-name: "github.com/weppos/publicsuffix-go" + - dependency-name: "github.com/zmap/*" groups: - aws: - patterns: - - "github.com/aws/*" - otel: + go: patterns: - - "go.opentelemetry.io/*" - open-pull-requests-limit: 1 + - "*" schedule: interval: "weekly" day: "wednesday" - cooldown: - default-days: 30 + open-pull-requests-limit: 1 + - package-ecosystem: "github-actions" + # Keep all github actions up to date on a monthly basis, but group the + # updates because they're generally easy to review. directory: "/" + groups: + gha: + patterns: + - "*" schedule: interval: monthly open-pull-requests-limit: 1 cooldown: default-days: 7 + - package-ecosystem: "docker-compose" + # Keep a select few Docker images (those which are only used in our CI, + # and don't have to match production image versions) up to date. Group the + # updates to prevent review fatigue and to ensure pkimetal doesn't get + # starved. directory: "/" + allow: + - dependency-name: "ghcr.io/pkimetal/pkimetal" + - dependency-name: "jaegertracing/all-in-one" + - dependency-name: "minio/minio" + - dependency-name: "minio/mc" + groups: + docker-compose: + patterns: + - "*" schedule: interval: "weekly" day: "wednesday" open-pull-requests-limit: 1 cooldown: default-days: 7 - allow: - - dependency-name: "ghcr.io/pkimetal/pkimetal" - - dependency-name: "jaegertracing/all-in-one" - - dependency-name: "minio/minio" - - dependency-name: "minio/mc" diff --git a/.github/workflows/boulder-ci.yml b/.github/workflows/boulder-ci.yml index b22fa6586bb..88382424ae6 100644 --- a/.github/workflows/boulder-ci.yml +++ b/.github/workflows/boulder-ci.yml @@ -8,7 +8,7 @@ on: push: branches: - main - - release-branch-* + - v0. pull_request: branches: - '**' @@ -109,29 +109,6 @@ jobs: name: container-logs path: ${{ github.workspace }}/containers.log - govulncheck: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - - steps: - # Checks out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Setup Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - # When Go produces a security release, we want govulncheck to run - # against the most recently released Go version. - check-latest: true - go-version: "stable" - cache: false - - - name: Run govulncheck - run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... - vendorcheck: runs-on: ubuntu-24.04 strategy: diff --git a/.github/workflows/check-iana-registries.yml b/.github/workflows/check-iana-registries.yml index 998a7562419..7e9bc5bbb03 100644 --- a/.github/workflows/check-iana-registries.yml +++ b/.github/workflows/check-iana-registries.yml @@ -1,7 +1,7 @@ name: Check for IANA special-purpose address registry updates on: - schedule: + schedule: # daily around 16:20 UTC - cron: "20 16 * * *" workflow_dispatch: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 41cbe01b27d..4872b413a9d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,11 +1,13 @@ name: "Code Scanning - Action" on: - pull_request: - branches: [ release-branch-*, main] push: - branches: [ release-branch-*, main] - + branches: + - main + - v0.* + pull_request: + branches: + - '**' jobs: CodeQL-Build: diff --git a/.github/workflows/dep-tests.yml b/.github/workflows/dep-tests.yml new file mode 100644 index 00000000000..2af8f3f7341 --- /dev/null +++ b/.github/workflows/dep-tests.yml @@ -0,0 +1,73 @@ +# Workflow to regularly check boulder's dependencies for vulnerabilities +# and updates. + +name: Dependency checks + +on: + schedule: # daily around 16:20 UTC + - cron: "20 16 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + # We want to run with the latest version of Go, just like we'll be + # running with the latest version of all of our dependencies. + go-version: "stable" + + - name: Update Go dependencies + run: go get -u -t ./... + + - name: Install crane + # This is the sole external dependency of update-container-versions.sh + run: go install github.com/google/go-containerregistry/cmd/crane@latest + + - name: Update Docker dependencies + run: ./tools/update-container-versions.sh + + - name: Build Docker images + # We do this locally, rather than pulling BOULDER_TOOLS_TAG like the + # CI job does, to also get the latest versions of everything in the + # DOCKERFILE. + run: docker compose build + + - name: Run tests + run: ./tn.sh -lui + + govulncheck: + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + # When Go produces a security release, we want govulncheck to run + # against the most recently released Go version. + check-latest: true + go-version: "stable" + cache: false + + - name: Run govulncheck + run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... diff --git a/.github/workflows/issue-for-sre-handoff.yml b/.github/workflows/issue-for-sre-handoff.yml index 241a2086fd1..f376013f1f3 100644 --- a/.github/workflows/issue-for-sre-handoff.yml +++ b/.github/workflows/issue-for-sre-handoff.yml @@ -7,9 +7,8 @@ on: - 'test/config-next/*.json' - 'test/config-next/*.yaml' - 'test/config-next/*.yml' - - 'sa/db-users/*.sql' - - 'sa/db-next/**/*.sql' - - 'sa/db/**/*.sql' + - 'sa/db/*.sql' + - 'sa/vtschema/*/.json' jobs: check-changes: diff --git a/.github/workflows/merged-to-main-or-release-branch.yml b/.github/workflows/merged-to-main-or-release-branch.yml index dad5e927581..77e3a4d15eb 100644 --- a/.github/workflows/merged-to-main-or-release-branch.yml +++ b/.github/workflows/merged-to-main-or-release-branch.yml @@ -2,17 +2,20 @@ # be used by tag protection rules to ensure that tags may only be pushed if # their corresponding commit was first pushed to one of those branches. name: Merged to main (or hotfix) -permissions: - contents: read + on: push: branches: - main - - release-branch-* + - v0.* + +permissions: + contents: read + jobs: merged-to-main: name: Merged to main (or hotfix) - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9e6d4c68d7d..54e377d8bdb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ # Keep the GO_VERSION matrix and the container-building steps in sync with # try-release.yml. name: Build release + on: push: tags: diff --git a/.github/workflows/try-release.yml b/.github/workflows/try-release.yml index f2728502e3f..f1836fcd4d2 100644 --- a/.github/workflows/try-release.yml +++ b/.github/workflows/try-release.yml @@ -4,11 +4,15 @@ # Keep the GO_VERSION matrix and the container-building steps in sync with # release.yml. name: Try release + on: push: - branches: [main] + branches: + - main + - v0. pull_request: - branches: [main] + branches: + - '**' workflow_dispatch: permissions: diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 2d981a0dcc2..bcac8be065f 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -4,7 +4,7 @@ on: push: branches: - main - - release-branch-* + - v0.* pull_request: branches: - '**' @@ -14,7 +14,7 @@ permissions: {} jobs: zizmor: - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest env: ZIZMOR_IMAGE: ghcr.io/zizmorcore/zizmor:1.25.2@sha256:14ea7f5cc7c67933394a35b5a38a277397818d232602635edb2010b313afb110 diff --git a/tools/update-container-versions.sh b/tools/update-container-versions.sh new file mode 100755 index 00000000000..01a31d6b0b6 --- /dev/null +++ b/tools/update-container-versions.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# +# bump-compose-images.sh — locally bump pinned image tags in a compose file to +# their newest compatible version so you can run your tests against them. +# Edits the file in place (no backup). Not a PR bot. +# +# Borrows Renovate's techniques: structure-aware extraction that honours +# `build:` and skips ${templated}/*aliased images; docker versioning that +# rejects commit-hash tags, strips a leading `v`, and only treats tags as +# updatable when release-length AND suffix match (so `-alpine` stays `-alpine` +# and `1.50` stays two-component). It never crosses a major version. Tag +# schemes docker versioning can't parse (e.g. MinIO RELEASE.* date stamps) +# fall back to shape-matching. +# +# Deps: crane (github.com/google/go-containerregistry), awk, sort, coreutils. +# Usage: ./bump-compose-images.sh [path/to/docker-compose.yml] +# +set -euo pipefail + +COMPOSE="${1:-docker-compose.yml}" +[ -f "$COMPOSE" ] || { echo "no such file: $COMPOSE" >&2; exit 1; } +command -v crane >/dev/null 2>&1 || { echo "need crane on PATH" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# 1. Extract eligible images: "\t\t" +# structure-aware; skips services with build:, and ${}/*/& image values. +# --------------------------------------------------------------------------- +read -r -d '' EXTRACT_AWK <<'AWK' || true +function indent(s, i){ i=match(s,/[^ ]/); return (i?i-1:length(s)) } +function eligible(r){ + if (r=="" || r ~ /[[:space:]]/) return 0 + if (r ~ /[$]/ || r ~ /^[*&]/) return 0 + return 1 +} +function name(s, t){ t=s; sub(/^[[:space:]]+/,"",t); sub(/:[[:space:]]*$/,"",t); return t } +function flush(){ + if (svc!="" && img_line>0 && !has_build && eligible(img_ref)) + print img_line "\t" name(svc) "\t" img_ref + svc=""; img_line=0; img_ref=""; has_build=0 +} +BEGIN{ svc_indent=-1; svc_parent=-1; expect_child=0 } +{ + line=$0 + if (line ~ /^[[:space:]]*#/ || line ~ /^[[:space:]]*$/) next + ind=indent(line) + if (svc_indent<0 && line ~ /^[[:space:]]*services:[[:space:]]*$/){ svc_parent=ind; expect_child=1; next } + if (expect_child && ind>svc_parent){ svc_indent=ind; expect_child=0 } + if (svc_indent<0) next + if (ind==svc_indent && line ~ /:[[:space:]]*$/){ flush(); svc=line; next } + if (svc=="") next + if (ind<=svc_indent){ flush(); next } + if (line ~ /^[[:space:]]*build:/) has_build=1 + if (img_line==0 && match(line,/^[[:space:]]*image:[[:space:]]*/)){ + img_ref=substr(line,RLENGTH+1); sub(/[[:space:]]+$/,"",img_ref); img_line=NR + } +} +END{ flush() } +AWK + +# --------------------------------------------------------------------------- +# 2. Selector: print updatable candidate tags for CUR (stdin = candidate tags) +# Renovate docker versioning, with shape-matching fallback. +# --------------------------------------------------------------------------- +read -r -d '' SELECT_AWK <<'AWK' || true +function pdock(v, o, s,n,parts,i,suf,prefix,rel,pre,tmp){ + delete o; o["ok"]=0 + if (v ~ /^[a-f0-9]{7,40}$/ && v !~ /^[0-9]+$/) return 0 + s=v; sub(/^v/,"",s) + n=split(s,parts,"-"); prefix=parts[1]; suf="" + for(i=2;i<=n;i++) suf=suf (i>2?"-":"") parts[i] + if (match(prefix,/^[0-9]+(\.[0-9]+)*/)){ + rel=substr(prefix,1,RLENGTH); pre=substr(prefix,RLENGTH+1) + if (pre ~ /^[A-Za-z0-9_]*$/ && rel!=""){ + o["ok"]=1; o["len"]=split(rel,tmp,"."); o["maj"]=tmp[1]; o["rel"]=rel; o["suf"]=suf; o["pre"]=pre; return 1 + } + } + return 0 +} +function shape(v, t){ t=v; gsub(/[0-9]+/,"#",t); return t } +BEGIN{ curok=pdock(CUR,C) } +{ + cand=$0; if (cand=="") next + if (curok){ + if (pdock(cand,D) && D["suf"]==C["suf"] && D["len"]==C["len"] && D["maj"]==C["maj"] && (C["pre"]!="" || D["pre"]=="")) print cand + } else if (shape(cand)==shape(CUR)) print cand +} +AWK + +# split an image ref into REPO / TAG / DIGEST (handles registry:port + @sha256) +parse_ref(){ + local ref="$1" namever last + DIGEST="" + if [[ "$ref" == *@* ]]; then DIGEST="${ref##*@}"; namever="${ref%@*}"; else namever="$ref"; fi + last="${namever##*/}" + if [[ "$last" == *:* ]]; then REPO="${namever%:*}"; TAG="${namever##*:}"; else REPO="$namever"; TAG=""; fi +} + +# choose the newest updatable tag for REPO:TAG, or empty +newest_tag(){ + local repo="$1" tag="$2" tags + tags=$(crane ls "$repo" 2>/dev/null) || return 0 + printf '%s\n' "$tags" | awk -v CUR="$tag" "$SELECT_AWK" | sort -V | tail -1 +} + +# --------------------------------------------------------------------------- +# 3. Decide updates, then rewrite the file by line number (Renovate LineMapper) +# --------------------------------------------------------------------------- +declare -A CACHE # ref -> new ref +mapfile=$(mktemp); report=$(mktemp); changed=0 + +while IFS=$'\t' read -r lineno svc ref; do + if [[ -n "${CACHE[$ref]+x}" ]]; then newref="${CACHE[$ref]}"; else + parse_ref "$ref" + newref="$ref" + if [[ -n "$TAG" && "$TAG" == *[0-9]* ]]; then + newtag=$(newest_tag "$REPO" "$TAG") + if [[ -n "$newtag" && "$newtag" != "$TAG" ]]; then + newref="$REPO:$newtag" + if [[ -n "$DIGEST" ]]; then + newdig=$(crane digest "$REPO:$newtag" 2>/dev/null || true) + [[ -n "$newdig" ]] && newref="$newref@$newdig" + fi + fi + fi + CACHE[$ref]="$newref" + fi + if [[ "$newref" != "$ref" ]]; then + printf '%s\t%s\n' "$lineno" "$newref" >> "$mapfile" + printf ' %-16s %s -> %s\n' "$svc" "$ref" "$newref" >> "$report" + changed=$((changed+1)) + fi +done < <(awk "$EXTRACT_AWK" "$COMPOSE") + +if [[ "$changed" -eq 0 ]]; then + rm -f "$mapfile" "$report"; echo "nothing to update"; exit 0 +fi + +tmp=$(mktemp) +awk -v MAP="$mapfile" ' + BEGIN{ while((getline l < MAP)>0){ split(l,a,"\t"); NEW[a[1]]=a[2] } } + { if ((FNR in NEW) && match($0,/^[[:space:]]*image:[[:space:]]*/)) + print substr($0,1,RLENGTH) NEW[FNR] + else print } +' "$COMPOSE" > "$tmp" + +cat "$tmp" > "$COMPOSE"; rm -f "$tmp" +cat "$report"; rm -f "$mapfile" "$report" +echo "updated $COMPOSE ($changed image(s))"