diff --git a/.github/scripts/security_sync.py b/.github/scripts/security_sync.py new file mode 100644 index 0000000..f90fce0 --- /dev/null +++ b/.github/scripts/security_sync.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +try: + from packaging.version import Version +except ImportError: + sys.exit("packaging package required: pip install packaging") + + +def run(command, cwd=None): + subprocess.run(command, cwd=cwd, check=True) + + +def capture(command, cwd=None): + return subprocess.run( + command, cwd=cwd, check=True, text=True, capture_output=True + ).stdout + + +def normalize_package(name): + return name.strip().lower() + + +def alert_section(scope): + # Dependabot scope: "runtime" -> [packages], "development" -> [dev-packages] + return "[dev-packages]" if scope == "development" else "[packages]" + + +def update_pipfile(pipfile_path, package, patched, section): + """Patch direct dep version, or append to the proper section. + Returns True if the file was modified, False otherwise.""" + content = pipfile_path.read_text() + pattern = re.compile( + rf'^(\s*"?{re.escape(package)}"?\s*=\s*)"[^"]*"\s*$', + re.IGNORECASE | re.MULTILINE, + ) + if pattern.search(content): + new_content = pattern.sub(rf'\g<1>">={patched}"', content) + if new_content != content: + pipfile_path.write_text(new_content) + return True + return False + + # Not a direct dep: append to the requested section so pipenv lock pulls it. + section_re = re.compile(rf'^{re.escape(section)}\s*$', re.MULTILINE) + m = section_re.search(content) + if not m: + # Section missing entirely; append at end. + addition = f'\n{section}\n{package} = ">={patched}"\n' + pipfile_path.write_text(content + addition) + else: + insert_at = m.end() + new_content = ( + content[:insert_at] + + f'\n{package} = ">={patched}"' + + content[insert_at:] + ) + pipfile_path.write_text(new_content) + return True + + +def lock_version(lockfile_path, package): + if not lockfile_path.exists(): + return None + data = json.loads(lockfile_path.read_text()) + for group in ("default", "develop"): + info = data.get(group, {}).get(package) + if info and "version" in info: + return info["version"].lstrip("=") + return None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--alerts", required=True) + parser.add_argument("--citus-root", required=True) + parser.add_argument("--summary-out", required=True, + help="Path to write per-alert outcome JSON") + args = parser.parse_args() + + citus_root = Path(args.citus_root) + + alerts = json.loads(Path(args.alerts).read_text()) + + # Deduplicate by package, keep highest patched version per package. + targets = {} + for alert in alerts: + package = normalize_package(alert["dependency"]["package"]["name"]) + patched = (alert.get("security_vulnerability", {}) + .get("first_patched_version") or {}).get("identifier") + scope = alert.get("dependency", {}).get("scope", "runtime") + if not patched: + continue + prev = targets.get(package) + if prev is None or Version(patched) > Version(prev["patched"]): + targets[package] = {"patched": patched, "scope": scope} + + if not targets: + Path(args.summary_out).write_text(json.dumps({"addressed": [], "details": []}, indent=2)) + sys.exit("No actionable alerts (none had a first_patched_version); aborting before opening PRs.") + + pipfile_paths = [ + citus_root / "src/test/regress/Pipfile", + citus_root / ".devcontainer/src/test/regress/Pipfile", + ] + lock_paths = [p.with_name("Pipfile.lock") for p in pipfile_paths] + + # Snapshot pre-state for each package. + pre_versions = {pkg: [lock_version(lp, pkg) for lp in lock_paths] + for pkg in targets} + + for pkg, info in targets.items(): + section = alert_section(info["scope"]) + for pf in pipfile_paths: + update_pipfile(pf, pkg, info["patched"], section) + + for pf in pipfile_paths: + run(["pipenv", "lock"], cwd=pf.parent) + + # Evaluate post-state and classify each target. + summary = [] + addressed = [] + for pkg, info in targets.items(): + post = [lock_version(lp, pkg) for lp in lock_paths] + statuses = [] + for before, after in zip(pre_versions[pkg], post): + if after is None: + statuses.append("absent") + elif Version(after) >= Version(info["patched"]): + if before is None or Version(before) < Version(info["patched"]): + statuses.append("applied") + else: + statuses.append("already-satisfied") + else: + statuses.append("not-fixed") + if any(s == "applied" for s in statuses) and \ + all(s in ("applied", "already-satisfied") for s in statuses): + overall = "addressed" + elif all(s == "already-satisfied" for s in statuses): + overall = "already-satisfied" + else: + overall = "failed" + summary.append({ + "package": pkg, "patched": info["patched"], + "scope": info["scope"], "statuses": statuses, "overall": overall, + }) + if overall == "addressed": + addressed.append(pkg) + + Path(args.summary_out).write_text(json.dumps({ + "addressed": addressed, + "details": summary, + }, indent=2)) + + for s in summary: + print(f"{s['overall']:20s} {s['package']} -> {s['patched']} ({s['scope']})") + + if not addressed: + sys.exit("No alerts were addressed by this run; aborting before opening PRs.") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/dependency-security-post-merge.yml b/.github/workflows/dependency-security-post-merge.yml new file mode 100644 index 0000000..3e75100 --- /dev/null +++ b/.github/workflows/dependency-security-post-merge.yml @@ -0,0 +1,62 @@ +name: dependency-security-post-merge + +on: + pull_request: + types: [closed] + branches: [master] + +permissions: + contents: write + pull-requests: write + +jobs: + notify-citus: + if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'automation/dependency-security-sync' + runs-on: ubuntu-latest + steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Checkout citus sync branch + uses: actions/checkout@v4 + with: + repository: citusdata/citus + ref: automation/dependency-security-sync + token: ${{ steps.app-token.outputs.token }} + path: citus + + - name: Update citus image_suffix with merged the-process SHA + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + run: | + cd citus + short_sha=$(echo "$MERGE_SHA" | cut -c1-7) + postfix="-v${short_sha}" + + citus_pr=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$citus_pr" ]; then + echo "No open citus sync PR found; skipping." + exit 0 + fi + + sed -i -E "s|(image_suffix:\s*)\"-v[0-9a-f]+\"|\1\"${postfix}\"|" .github/workflows/build_and_test.yml + + if git diff --quiet -- .github/workflows/build_and_test.yml; then + echo "No build_and_test.yml image_suffix change needed." + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}. image_suffix already matches ${postfix}." + exit 0 + fi + + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git add .github/workflows/build_and_test.yml + git commit -m "Update image_suffix from merged the-process commit" + git push origin automation/dependency-security-sync + + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}; updated build_and_test.yml image_suffix to ${postfix}." diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml new file mode 100644 index 0000000..86afeae --- /dev/null +++ b/.github/workflows/dependency-security-sync.yml @@ -0,0 +1,203 @@ +name: dependency-security-sync + +on: + schedule: + - cron: '0 2 * * 0' + workflow_dispatch: + +concurrency: + group: dependency-security-sync + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Checkout the-process + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + + - name: Checkout citus + uses: actions/checkout@v4 + with: + repository: citusdata/citus + path: citus + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pipenv and gh auth + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + python -m pip install --upgrade pip pipenv + gh auth setup-git + + - name: Fetch open Dependabot alerts from citus + id: alerts + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + gh api -H "Accept: application/vnd.github+json" \ + "/repos/citusdata/citus/dependabot/alerts?state=open&per_page=100" > alerts.json + count=$(jq 'length' alerts.json) + echo "count=$count" >> "$GITHUB_OUTPUT" + if [ "$count" -eq 0 ]; then + echo "No open alerts found. Exiting." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Run cross-repo sync + if: steps.alerts.outputs.count != '0' + id: sync + run: | + python -m pip install --quiet packaging + python .github/scripts/security_sync.py \ + --alerts alerts.json \ + --citus-root "$PWD/citus" \ + --summary-out sync-summary.json + addressed=$(jq -r '.addressed | join(" ")' sync-summary.json) + echo "addressed=$addressed" >> "$GITHUB_OUTPUT" + echo "## Sync summary" >> "$GITHUB_STEP_SUMMARY" + jq -r '.details[] | "- \(.overall): \(.package) -> \(.patched) (\(.scope))"' \ + sync-summary.json >> "$GITHUB_STEP_SUMMARY" + + - name: Create or update citus PR + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + cd citus + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git checkout -B automation/dependency-security-sync + git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock || true + if git diff --cached --quiet; then + echo "No citus dependency changes to commit." + else + git commit -m "Automate Dependabot alert security dependency sync" + git push -f origin automation/dependency-security-sync + fi + + pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$pr_number" ]; then + gh pr create \ + --repo citusdata/citus \ + --base main \ + --head automation/dependency-security-sync \ + --title "Automated security dependency sync from Dependabot alerts" \ + --body $'Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow.' \ + --label dependencies \ + --draft + pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number') + fi + echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV" + + - name: Regenerate the-process requirements + if: steps.alerts.outputs.count != '0' && steps.sync.outputs.addressed != '' + run: | + cd citus/src/test/regress + base=$(pipenv requirements) + dev=$(pipenv requirements --dev) + cd "$GITHUB_WORKSPACE" + ref="citusdata/citus#${CITUS_PR}" + base_header=$'# generated from Citus\'s Pipfile.lock (in src/test/regress) as of '"${ref}"$'\n# using `pipenv requirements > requirements.txt`, so as to avoid the\n# need for pipenv/pyenv in this image\n\n' + dev_header=$'# generated from Citus\'s Pipfile.lock (in src/test/regress) as of '"${ref}"$'\n# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n# need for pipenv/pyenv in this image\n\n' + for f in \ + circleci/images/citusupgradetester/files/etc/requirements.txt \ + circleci/images/failtester/files/etc/requirements.txt \ + circleci/images/pgupgradetester/files/etc/requirements.txt; do + printf '%s%s\n' "$base_header" "$base" > "$f" + done + printf '%s%s\n' "$dev_header" "$dev" > circleci/images/stylechecker/files/etc/requirements.txt + + - name: Create or update the-process PR + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git checkout -B automation/dependency-security-sync + git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt || true + if git diff --cached --quiet; then + echo "No the-process dependency changes to commit." + else + git commit -m "Automate security requirements sync from citus alerts" + git push -f origin automation/dependency-security-sync + fi + + pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$pr_number" ]; then + gh pr create \ + --repo citusdata/the-process \ + --base master \ + --head automation/dependency-security-sync \ + --title "Automated requirements sync for Dependabot security alerts" \ + --body $'Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow.' \ + --label dependencies + pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number') + fi + echo "THE_PROCESS_PR=$pr_number" >> "$GITHUB_ENV" + + - name: Close addressed Dependabot PRs in citus + if: steps.sync.outputs.addressed != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ADDRESSED: ${{ steps.sync.outputs.addressed }} + run: | + for pkg in $ADDRESSED; do + for n in $(gh pr list --repo citusdata/citus --state open \ + --search "author:app/dependabot in:title Bump $pkg" \ + --json number --jq '.[].number'); do + gh pr close "$n" --repo citusdata/citus \ + --comment "Closing in favor of consolidated automated security sync PR #${CITUS_PR} (addresses $pkg)." + done + done + gh pr comment "$CITUS_PR" --repo citusdata/citus \ + --body "Supersedes Dependabot PRs for: $ADDRESSED." + + - name: Close addressed Dependabot PRs in the-process + if: steps.sync.outputs.addressed != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ADDRESSED: ${{ steps.sync.outputs.addressed }} + run: | + for pkg in $ADDRESSED; do + for n in $(gh pr list --repo citusdata/the-process --state open \ + --search "author:app/dependabot in:title Bump $pkg" \ + --json number --jq '.[].number'); do + gh pr close "$n" --repo citusdata/the-process \ + --comment "Closing in favor of consolidated automated requirements sync PR #${THE_PROCESS_PR} (addresses $pkg)." + done + done + gh pr comment "$THE_PROCESS_PR" --repo citusdata/the-process \ + --body "Supersedes Dependabot PRs for: $ADDRESSED." + + - name: Summary + if: steps.alerts.outputs.count != '0' + run: | + { + echo "## Dependency security sync" + echo "- Alerts processed: ${{ steps.alerts.outputs.count }}" + echo "- the-process PR: #${THE_PROCESS_PR}" + echo "- citus PR: #${CITUS_PR}" + } >> "$GITHUB_STEP_SUMMARY"