From 76c2e1a74112970b596e4555b72a9a5eafc8d218 Mon Sep 17 00:00:00 2001 From: ihalatci Date: Tue, 3 Mar 2026 18:44:51 +0000 Subject: [PATCH 1/9] Add weekly cross-repo dependency security sync automation --- .github/dependabot.yml | 37 ++++ .github/scripts/security_sync.py | 102 +++++++++++ .../dependency-security-post-merge.yml | 39 +++++ .../workflows/dependency-security-sync.yml | 164 ++++++++++++++++++ 4 files changed, 342 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/scripts/security_sync.py create mode 100644 .github/workflows/dependency-security-post-merge.yml create mode 100644 .github/workflows/dependency-security-sync.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..531f37d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,37 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/circleci/images/citusupgradetester/files/etc" + schedule: + interval: weekly + day: sunday + time: "02:00" + timezone: "UTC" + open-pull-requests-limit: 0 + + - package-ecosystem: pip + directory: "/circleci/images/failtester/files/etc" + schedule: + interval: weekly + day: sunday + time: "02:00" + timezone: "UTC" + open-pull-requests-limit: 0 + + - package-ecosystem: pip + directory: "/circleci/images/pgupgradetester/files/etc" + schedule: + interval: weekly + day: sunday + time: "02:00" + timezone: "UTC" + open-pull-requests-limit: 0 + + - package-ecosystem: pip + directory: "/circleci/images/stylechecker/files/etc" + schedule: + interval: weekly + day: sunday + time: "02:00" + timezone: "UTC" + open-pull-requests-limit: 0 diff --git a/.github/scripts/security_sync.py b/.github/scripts/security_sync.py new file mode 100644 index 0000000..a6f48b8 --- /dev/null +++ b/.github/scripts/security_sync.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +import subprocess +from pathlib import Path + + +def run(command, cwd=None): + subprocess.run(command, cwd=cwd, check=True) + + +def capture(command, cwd=None): + result = subprocess.run(command, cwd=cwd, check=True, text=True, capture_output=True) + return result.stdout + + +def normalize_package(name): + return name.strip().lower() + + +def update_pipfile(pipfile_path, patched_versions): + content = pipfile_path.read_text() + original_content = content + + for package in ("cryptography", "werkzeug"): + patched = patched_versions.get(package) + if not patched: + continue + + pattern = re.compile( + rf'^(\s*"?{re.escape(package)}"?\s*=\s*)"[^"]*"\s*$', + re.IGNORECASE | re.MULTILINE, + ) + + def replacement(match): + return f'{match.group(1)}"=={patched}"' + + content = pattern.sub(replacement, content) + + if content != original_content: + pipfile_path.write_text(content) + + +def write_requirements(the_process_root, base_requirements, dev_requirements, citus_sha): + base_header = ( + f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n" + "# using `pipenv requirements > requirements.txt`, so as to avoid the\n" + "# need for pipenv/pyenv in this image\n\n" + ) + dev_header = ( + f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n" + "# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n" + "# need for pipenv/pyenv in this image\n\n" + ) + + base_targets = [ + the_process_root / "circleci/images/citusupgradetester/files/etc/requirements.txt", + the_process_root / "circleci/images/failtester/files/etc/requirements.txt", + the_process_root / "circleci/images/pgupgradetester/files/etc/requirements.txt", + ] + for target in base_targets: + target.write_text(base_header + base_requirements) + + dev_target = the_process_root / "circleci/images/stylechecker/files/etc/requirements.txt" + dev_target.write_text(dev_header + dev_requirements) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--alerts", required=True) + parser.add_argument("--citus-root", required=True) + parser.add_argument("--the-process-root", required=True) + args = parser.parse_args() + + citus_root = Path(args.citus_root) + the_process_root = Path(args.the_process_root) + + alerts = json.loads(Path(args.alerts).read_text()) + patched_versions = {} + for alert in alerts: + package = normalize_package(alert["dependency"]["package"]["name"]) + patched = (alert.get("security_vulnerability", {}).get("first_patched_version") or {}).get("identifier") + if patched: + patched_versions[package] = patched + + update_pipfile(citus_root / "src/test/regress/Pipfile", patched_versions) + update_pipfile(citus_root / ".devcontainer/src/test/regress/Pipfile", patched_versions) + + run(["pipenv", "lock"], cwd=citus_root / "src/test/regress") + run(["pipenv", "lock"], cwd=citus_root / ".devcontainer/src/test/regress") + + base_requirements = capture(["pipenv", "requirements"], cwd=citus_root / "src/test/regress") + dev_requirements = capture(["pipenv", "requirements", "--dev"], cwd=citus_root / "src/test/regress") + + citus_sha = capture(["git", "rev-parse", "--short", "HEAD"], cwd=citus_root).strip() + write_requirements(the_process_root, base_requirements, dev_requirements, f"citusdata/citus@{citus_sha}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/dependency-security-post-merge.yml b/.github/workflows/dependency-security-post-merge.yml new file mode 100644 index 0000000..f036a43 --- /dev/null +++ b/.github/workflows/dependency-security-post-merge.yml @@ -0,0 +1,39 @@ +name: dependency-security-post-merge + +on: + pull_request: + types: [closed] + branches: [master] + +permissions: + contents: read + pull-requests: write + +jobs: + notify-citus: + if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'automation/dependency-security-sync' + runs-on: ubuntu-latest + steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.PACKAGING_APP_ID }} + private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} + owner: citusdata + + - name: Comment on citus PR with image tag postfix + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + run: | + short_sha=$(echo "$MERGE_SHA" | cut -c1-7) + postfix="-v${short_sha}" + + citus_pr=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$citus_pr" ]; then + echo "No open citus sync PR found; skipping." + exit 0 + fi + + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}. Image tag postfix placeholder: ${postfix}. TODO: replace placeholder source when final postfix publication source is confirmed." diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml new file mode 100644 index 0000000..d5319b0 --- /dev/null +++ b/.github/workflows/dependency-security-sync.yml @@ -0,0 +1,164 @@ +name: dependency-security-sync + +on: + schedule: + - cron: '0 2 * * 0' + workflow_dispatch: + +concurrency: + group: dependency-security-sync + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.PACKAGING_APP_ID }} + private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} + owner: citusdata + + - name: Checkout the-process + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + + - name: Checkout citus + uses: actions/checkout@v4 + with: + repository: citusdata/citus + path: citus + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pipenv and gh auth + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + python -m pip install --upgrade pip pipenv + gh auth setup-git + + - name: Fetch open Dependabot alerts from citus + id: alerts + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + gh api -H "Accept: application/vnd.github+json" \ + "/repos/citusdata/citus/dependabot/alerts?state=open&per_page=100" > alerts.json + count=$(jq 'length' alerts.json) + echo "count=$count" >> "$GITHUB_OUTPUT" + if [ "$count" -eq 0 ]; then + echo "No open alerts found. Exiting." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Run cross-repo sync + if: steps.alerts.outputs.count != '0' + run: | + python .github/scripts/security_sync.py \ + --alerts alerts.json \ + --citus-root "$PWD/citus" \ + --the-process-root "$PWD" + + - name: Create or update citus PR + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + cd citus + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git checkout -B automation/dependency-security-sync + git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock .github/dependabot.yml || true + if git diff --cached --quiet; then + echo "No citus dependency changes to commit." + else + git commit -m "Automate Dependabot alert security dependency sync" + git push -f origin automation/dependency-security-sync + fi + + pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$pr_number" ]; then + gh pr create \ + --repo citusdata/citus \ + --base main \ + --head automation/dependency-security-sync \ + --title "Automated security dependency sync from Dependabot alerts" \ + --body "Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow." \ + --label dependencies + pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number') + fi + echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV" + + - name: Create or update the-process PR + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git checkout -B automation/dependency-security-sync + git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt .github/dependabot.yml || true + if git diff --cached --quiet; then + echo "No the-process dependency changes to commit." + else + git commit -m "Automate security requirements sync from citus alerts" + git push -f origin automation/dependency-security-sync + fi + + pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$pr_number" ]; then + gh pr create \ + --repo citusdata/the-process \ + --base master \ + --head automation/dependency-security-sync \ + --title "Automated requirements sync for Dependabot security alerts" \ + --body "Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow." \ + --label dependencies + pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number') + fi + echo "THE_PROCESS_PR=$pr_number" >> "$GITHUB_ENV" + + - name: Close superseded Dependabot PRs in citus + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + cd citus + for n in $(gh pr list --repo citusdata/citus --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do + gh pr close "$n" --repo citusdata/citus --comment "Closing in favor of consolidated automated security sync PR #${CITUS_PR}." + done + gh pr comment "$CITUS_PR" --repo citusdata/citus --body "Supersedes open Dependabot PRs with this consolidated automated security sync." + + - name: Close superseded Dependabot PRs in the-process + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + for n in $(gh pr list --repo citusdata/the-process --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do + gh pr close "$n" --repo citusdata/the-process --comment "Closing in favor of consolidated automated security sync PR #${THE_PROCESS_PR}." + done + gh pr comment "$THE_PROCESS_PR" --repo citusdata/the-process --body "Supersedes open Dependabot PRs with this consolidated automated requirements sync." + + - name: Summary + if: steps.alerts.outputs.count != '0' + run: | + { + echo "## Dependency security sync" + echo "- Alerts processed: ${{ steps.alerts.outputs.count }}" + echo "- the-process PR: #${THE_PROCESS_PR}" + echo "- citus PR: #${CITUS_PR}" + } >> "$GITHUB_STEP_SUMMARY" From b6ffb10586c29fffc4333168e175b2d066d41944 Mon Sep 17 00:00:00 2001 From: ihalatci Date: Tue, 3 Mar 2026 18:53:34 +0000 Subject: [PATCH 2/9] Generalize alert sync and wire image_suffix callback --- .github/dependabot.yml | 37 ------------------- .github/scripts/security_sync.py | 3 +- .../dependency-security-post-merge.yml | 29 +++++++++++++-- .../workflows/dependency-security-sync.yml | 4 +- 4 files changed, 29 insertions(+), 44 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 531f37d..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 -updates: - - package-ecosystem: pip - directory: "/circleci/images/citusupgradetester/files/etc" - schedule: - interval: weekly - day: sunday - time: "02:00" - timezone: "UTC" - open-pull-requests-limit: 0 - - - package-ecosystem: pip - directory: "/circleci/images/failtester/files/etc" - schedule: - interval: weekly - day: sunday - time: "02:00" - timezone: "UTC" - open-pull-requests-limit: 0 - - - package-ecosystem: pip - directory: "/circleci/images/pgupgradetester/files/etc" - schedule: - interval: weekly - day: sunday - time: "02:00" - timezone: "UTC" - open-pull-requests-limit: 0 - - - package-ecosystem: pip - directory: "/circleci/images/stylechecker/files/etc" - schedule: - interval: weekly - day: sunday - time: "02:00" - timezone: "UTC" - open-pull-requests-limit: 0 diff --git a/.github/scripts/security_sync.py b/.github/scripts/security_sync.py index a6f48b8..3f0cbbb 100644 --- a/.github/scripts/security_sync.py +++ b/.github/scripts/security_sync.py @@ -24,8 +24,7 @@ def update_pipfile(pipfile_path, patched_versions): content = pipfile_path.read_text() original_content = content - for package in ("cryptography", "werkzeug"): - patched = patched_versions.get(package) + for package, patched in patched_versions.items(): if not patched: continue diff --git a/.github/workflows/dependency-security-post-merge.yml b/.github/workflows/dependency-security-post-merge.yml index f036a43..58ac3d1 100644 --- a/.github/workflows/dependency-security-post-merge.yml +++ b/.github/workflows/dependency-security-post-merge.yml @@ -6,7 +6,7 @@ on: branches: [master] permissions: - contents: read + contents: write pull-requests: write jobs: @@ -22,11 +22,20 @@ jobs: private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} owner: citusdata - - name: Comment on citus PR with image tag postfix + - name: Checkout citus sync branch + uses: actions/checkout@v4 + with: + repository: citusdata/citus + ref: automation/dependency-security-sync + token: ${{ steps.app-token.outputs.token }} + path: citus + + - name: Update citus image_suffix with merged the-process SHA env: GH_TOKEN: ${{ steps.app-token.outputs.token }} MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} run: | + cd citus short_sha=$(echo "$MERGE_SHA" | cut -c1-7) postfix="-v${short_sha}" @@ -36,4 +45,18 @@ jobs: exit 0 fi - gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}. Image tag postfix placeholder: ${postfix}. TODO: replace placeholder source when final postfix publication source is confirmed." + sed -i -E "s|(image_suffix:\s*)\"-v[0-9a-f]+\"|\1\"${postfix}\"|" .github/workflows/build_and_test.yml + + if git diff --quiet -- .github/workflows/build_and_test.yml; then + echo "No build_and_test.yml image_suffix change needed." + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}. image_suffix already matches ${postfix}." + exit 0 + fi + + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git add .github/workflows/build_and_test.yml + git commit -m "Update image_suffix from merged the-process commit" + git push origin automation/dependency-security-sync + + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}; updated build_and_test.yml image_suffix to ${postfix}." diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml index d5319b0..9f429ba 100644 --- a/.github/workflows/dependency-security-sync.yml +++ b/.github/workflows/dependency-security-sync.yml @@ -82,7 +82,7 @@ jobs: git config user.name "packagingApp[bot]" git config user.email "packagingApp[bot]@users.noreply.github.com" git checkout -B automation/dependency-security-sync - git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock .github/dependabot.yml || true + git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock || true if git diff --cached --quiet; then echo "No citus dependency changes to commit." else @@ -111,7 +111,7 @@ jobs: git config user.name "packagingApp[bot]" git config user.email "packagingApp[bot]@users.noreply.github.com" git checkout -B automation/dependency-security-sync - git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt .github/dependabot.yml || true + git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt || true if git diff --cached --quiet; then echo "No the-process dependency changes to commit." else From fb4e1b84c5c352080b5ce0e77a059ff5d880b516 Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 11:55:04 +0000 Subject: [PATCH 3/9] Trigger CI re-run on automation branch From 6e921085936da7bd1aabf85f04390cc5ddb33f1d Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 13:01:31 +0000 Subject: [PATCH 4/9] Use $'...' for newline in automated PR bodies --- .github/workflows/dependency-security-sync.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml index 9f429ba..4ee75df 100644 --- a/.github/workflows/dependency-security-sync.yml +++ b/.github/workflows/dependency-security-sync.yml @@ -97,7 +97,7 @@ jobs: --base main \ --head automation/dependency-security-sync \ --title "Automated security dependency sync from Dependabot alerts" \ - --body "Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow." \ + --body $'Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow.' \ --label dependencies pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number') fi @@ -126,7 +126,7 @@ jobs: --base master \ --head automation/dependency-security-sync \ --title "Automated requirements sync for Dependabot security alerts" \ - --body "Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow." \ + --body $'Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow.' \ --label dependencies pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number') fi From cfd1fd225aebb0b7b1a15580a82c9405ab892e24 Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 13:13:53 +0000 Subject: [PATCH 5/9] Use GH_APP_ID / GH_APP_KEY org secrets for App auth --- .github/workflows/dependency-security-post-merge.yml | 4 ++-- .github/workflows/dependency-security-sync.yml | 4 ++-- .vscode/settings.json | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.github/workflows/dependency-security-post-merge.yml b/.github/workflows/dependency-security-post-merge.yml index 58ac3d1..3e75100 100644 --- a/.github/workflows/dependency-security-post-merge.yml +++ b/.github/workflows/dependency-security-post-merge.yml @@ -18,8 +18,8 @@ jobs: id: app-token uses: actions/create-github-app-token@v1 with: - app-id: ${{ secrets.PACKAGING_APP_ID }} - private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - name: Checkout citus sync branch diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml index 4ee75df..9c85011 100644 --- a/.github/workflows/dependency-security-sync.yml +++ b/.github/workflows/dependency-security-sync.yml @@ -22,8 +22,8 @@ jobs: id: app-token uses: actions/create-github-app-token@v1 with: - app-id: ${{ secrets.PACKAGING_APP_ID }} - private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - name: Checkout the-process diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file From d1a8d353b33a6bcd8b535c01c4cc91535753695f Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 13:14:02 +0000 Subject: [PATCH 6/9] Remove accidentally committed empty .vscode/settings.json --- .vscode/settings.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 9e26dfe..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From f6e75481f678ce08ad80116dcde1e3c023297b18 Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 13:51:53 +0000 Subject: [PATCH 7/9] security_sync: handle transitive deps, fail-fast, narrow PR closures - update_pipfile() appends to [packages]/[dev-packages] when alert is for a transitive dep, so pipenv lock can pull the patched version. - Use >= constraint instead of == to avoid over-pinning. - Verify per-alert outcome by reading both Pipfile.lock files post-lock, classify as addressed / already-satisfied / failed. - Write summary JSON consumed by the workflow; print one line per alert. - Exit non-zero (no PRs opened) when no alerts are actually addressed. - Workflow: pass --summary-out, surface details in run summary, expose addressed-package list as step output, only close Dependabot PRs whose title matches a package we addressed (instead of closing all of them). --- .github/scripts/security_sync.py | 162 ++++++++++++++---- .../workflows/dependency-security-sync.yml | 45 +++-- 2 files changed, 163 insertions(+), 44 deletions(-) diff --git a/.github/scripts/security_sync.py b/.github/scripts/security_sync.py index 3f0cbbb..67f0b9b 100644 --- a/.github/scripts/security_sync.py +++ b/.github/scripts/security_sync.py @@ -4,42 +4,76 @@ import json import re import subprocess +import sys from pathlib import Path +try: + from packaging.version import Version +except ImportError: + sys.exit("packaging package required: pip install packaging") + def run(command, cwd=None): subprocess.run(command, cwd=cwd, check=True) def capture(command, cwd=None): - result = subprocess.run(command, cwd=cwd, check=True, text=True, capture_output=True) - return result.stdout + return subprocess.run( + command, cwd=cwd, check=True, text=True, capture_output=True + ).stdout def normalize_package(name): return name.strip().lower() -def update_pipfile(pipfile_path, patched_versions): - content = pipfile_path.read_text() - original_content = content +def alert_section(scope): + # Dependabot scope: "runtime" -> [packages], "development" -> [dev-packages] + return "[dev-packages]" if scope == "development" else "[packages]" - for package, patched in patched_versions.items(): - if not patched: - continue - pattern = re.compile( - rf'^(\s*"?{re.escape(package)}"?\s*=\s*)"[^"]*"\s*$', - re.IGNORECASE | re.MULTILINE, +def update_pipfile(pipfile_path, package, patched, section): + """Patch direct dep version, or append to the proper section. + Returns True if the file was modified, False otherwise.""" + content = pipfile_path.read_text() + pattern = re.compile( + rf'^(\s*"?{re.escape(package)}"?\s*=\s*)"[^"]*"\s*$', + re.IGNORECASE | re.MULTILINE, + ) + if pattern.search(content): + new_content = pattern.sub(rf'\g<1>">={patched}"', content) + if new_content != content: + pipfile_path.write_text(new_content) + return True + return False + + # Not a direct dep: append to the requested section so pipenv lock pulls it. + section_re = re.compile(rf'^{re.escape(section)}\s*$', re.MULTILINE) + m = section_re.search(content) + if not m: + # Section missing entirely; append at end. + addition = f'\n{section}\n{package} = ">={patched}"\n' + pipfile_path.write_text(content + addition) + else: + insert_at = m.end() + new_content = ( + content[:insert_at] + + f'\n{package} = ">={patched}"' + + content[insert_at:] ) + pipfile_path.write_text(new_content) + return True - def replacement(match): - return f'{match.group(1)}"=={patched}"' - content = pattern.sub(replacement, content) - - if content != original_content: - pipfile_path.write_text(content) +def lock_version(lockfile_path, package): + if not lockfile_path.exists(): + return None + data = json.loads(lockfile_path.read_text()) + for group in ("default", "develop"): + info = data.get(group, {}).get(package) + if info and "version" in info: + return info["version"].lstrip("=") + return None def write_requirements(the_process_root, base_requirements, dev_requirements, citus_sha): @@ -53,7 +87,6 @@ def write_requirements(the_process_root, base_requirements, dev_requirements, ci "# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n" "# need for pipenv/pyenv in this image\n\n" ) - base_targets = [ the_process_root / "circleci/images/citusupgradetester/files/etc/requirements.txt", the_process_root / "circleci/images/failtester/files/etc/requirements.txt", @@ -61,7 +94,6 @@ def write_requirements(the_process_root, base_requirements, dev_requirements, ci ] for target in base_targets: target.write_text(base_header + base_requirements) - dev_target = the_process_root / "circleci/images/stylechecker/files/etc/requirements.txt" dev_target.write_text(dev_header + dev_requirements) @@ -71,30 +103,96 @@ def main(): parser.add_argument("--alerts", required=True) parser.add_argument("--citus-root", required=True) parser.add_argument("--the-process-root", required=True) + parser.add_argument("--summary-out", required=True, + help="Path to write per-alert outcome JSON") args = parser.parse_args() citus_root = Path(args.citus_root) the_process_root = Path(args.the_process_root) alerts = json.loads(Path(args.alerts).read_text()) - patched_versions = {} + + # Deduplicate by package, keep highest patched version per package. + targets = {} for alert in alerts: package = normalize_package(alert["dependency"]["package"]["name"]) - patched = (alert.get("security_vulnerability", {}).get("first_patched_version") or {}).get("identifier") - if patched: - patched_versions[package] = patched - - update_pipfile(citus_root / "src/test/regress/Pipfile", patched_versions) - update_pipfile(citus_root / ".devcontainer/src/test/regress/Pipfile", patched_versions) - - run(["pipenv", "lock"], cwd=citus_root / "src/test/regress") - run(["pipenv", "lock"], cwd=citus_root / ".devcontainer/src/test/regress") + patched = (alert.get("security_vulnerability", {}) + .get("first_patched_version") or {}).get("identifier") + scope = alert.get("dependency", {}).get("scope", "runtime") + if not patched: + continue + prev = targets.get(package) + if prev is None or Version(patched) > Version(prev["patched"]): + targets[package] = {"patched": patched, "scope": scope} - base_requirements = capture(["pipenv", "requirements"], cwd=citus_root / "src/test/regress") - dev_requirements = capture(["pipenv", "requirements", "--dev"], cwd=citus_root / "src/test/regress") + if not targets: + Path(args.summary_out).write_text(json.dumps({"addressed": [], "details": []}, indent=2)) + sys.exit("No actionable alerts (none had a first_patched_version); aborting before opening PRs.") + pipfile_paths = [ + citus_root / "src/test/regress/Pipfile", + citus_root / ".devcontainer/src/test/regress/Pipfile", + ] + lock_paths = [p.with_name("Pipfile.lock") for p in pipfile_paths] + + # Snapshot pre-state for each package. + pre_versions = {pkg: [lock_version(lp, pkg) for lp in lock_paths] + for pkg in targets} + + for pkg, info in targets.items(): + section = alert_section(info["scope"]) + for pf in pipfile_paths: + update_pipfile(pf, pkg, info["patched"], section) + + for pf in pipfile_paths: + run(["pipenv", "lock"], cwd=pf.parent) + + # Evaluate post-state and classify each target. + summary = [] + addressed = [] + for pkg, info in targets.items(): + post = [lock_version(lp, pkg) for lp in lock_paths] + statuses = [] + for before, after in zip(pre_versions[pkg], post): + if after is None: + statuses.append("absent") + elif Version(after) >= Version(info["patched"]): + if before is None or Version(before) < Version(info["patched"]): + statuses.append("applied") + else: + statuses.append("already-satisfied") + else: + statuses.append("not-fixed") + if any(s == "applied" for s in statuses) and \ + all(s in ("applied", "already-satisfied") for s in statuses): + overall = "addressed" + elif all(s == "already-satisfied" for s in statuses): + overall = "already-satisfied" + else: + overall = "failed" + summary.append({ + "package": pkg, "patched": info["patched"], + "scope": info["scope"], "statuses": statuses, "overall": overall, + }) + if overall == "addressed": + addressed.append(pkg) + + Path(args.summary_out).write_text(json.dumps({ + "addressed": addressed, + "details": summary, + }, indent=2)) + + for s in summary: + print(f"{s['overall']:20s} {s['package']} -> {s['patched']} ({s['scope']})") + + if not addressed: + sys.exit("No alerts were addressed by this run; aborting before opening PRs.") + + base_requirements = capture(["pipenv", "requirements"], cwd=pipfile_paths[0].parent) + dev_requirements = capture(["pipenv", "requirements", "--dev"], cwd=pipfile_paths[0].parent) citus_sha = capture(["git", "rev-parse", "--short", "HEAD"], cwd=citus_root).strip() - write_requirements(the_process_root, base_requirements, dev_requirements, f"citusdata/citus@{citus_sha}") + write_requirements(the_process_root, base_requirements, dev_requirements, + f"citusdata/citus@{citus_sha}") if __name__ == "__main__": diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml index 9c85011..73eedca 100644 --- a/.github/workflows/dependency-security-sync.yml +++ b/.github/workflows/dependency-security-sync.yml @@ -67,11 +67,19 @@ jobs: - name: Run cross-repo sync if: steps.alerts.outputs.count != '0' + id: sync run: | + python -m pip install --quiet packaging python .github/scripts/security_sync.py \ --alerts alerts.json \ --citus-root "$PWD/citus" \ - --the-process-root "$PWD" + --the-process-root "$PWD" \ + --summary-out sync-summary.json + addressed=$(jq -r '.addressed | join(" ")' sync-summary.json) + echo "addressed=$addressed" >> "$GITHUB_OUTPUT" + echo "## Sync summary" >> "$GITHUB_STEP_SUMMARY" + jq -r '.details[] | "- \(.overall): \(.package) -> \(.patched) (\(.scope))"' \ + sync-summary.json >> "$GITHUB_STEP_SUMMARY" - name: Create or update citus PR if: steps.alerts.outputs.count != '0' @@ -132,26 +140,39 @@ jobs: fi echo "THE_PROCESS_PR=$pr_number" >> "$GITHUB_ENV" - - name: Close superseded Dependabot PRs in citus - if: steps.alerts.outputs.count != '0' + - name: Close addressed Dependabot PRs in citus + if: steps.sync.outputs.addressed != '' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} + ADDRESSED: ${{ steps.sync.outputs.addressed }} run: | - cd citus - for n in $(gh pr list --repo citusdata/citus --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do - gh pr close "$n" --repo citusdata/citus --comment "Closing in favor of consolidated automated security sync PR #${CITUS_PR}." + for pkg in $ADDRESSED; do + for n in $(gh pr list --repo citusdata/citus --state open \ + --search "author:app/dependabot in:title Bump $pkg" \ + --json number --jq '.[].number'); do + gh pr close "$n" --repo citusdata/citus \ + --comment "Closing in favor of consolidated automated security sync PR #${CITUS_PR} (addresses $pkg)." + done done - gh pr comment "$CITUS_PR" --repo citusdata/citus --body "Supersedes open Dependabot PRs with this consolidated automated security sync." + gh pr comment "$CITUS_PR" --repo citusdata/citus \ + --body "Supersedes Dependabot PRs for: $ADDRESSED." - - name: Close superseded Dependabot PRs in the-process - if: steps.alerts.outputs.count != '0' + - name: Close addressed Dependabot PRs in the-process + if: steps.sync.outputs.addressed != '' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} + ADDRESSED: ${{ steps.sync.outputs.addressed }} run: | - for n in $(gh pr list --repo citusdata/the-process --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do - gh pr close "$n" --repo citusdata/the-process --comment "Closing in favor of consolidated automated security sync PR #${THE_PROCESS_PR}." + for pkg in $ADDRESSED; do + for n in $(gh pr list --repo citusdata/the-process --state open \ + --search "author:app/dependabot in:title Bump $pkg" \ + --json number --jq '.[].number'); do + gh pr close "$n" --repo citusdata/the-process \ + --comment "Closing in favor of consolidated automated requirements sync PR #${THE_PROCESS_PR} (addresses $pkg)." + done done - gh pr comment "$THE_PROCESS_PR" --repo citusdata/the-process --body "Supersedes open Dependabot PRs with this consolidated automated requirements sync." + gh pr comment "$THE_PROCESS_PR" --repo citusdata/the-process \ + --body "Supersedes Dependabot PRs for: $ADDRESSED." - name: Summary if: steps.alerts.outputs.count != '0' From f94d8cb0e3fb988637913aed817eede50aaea91b Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 14:45:58 +0000 Subject: [PATCH 8/9] Create citus dependency sync PR as draft --- .github/workflows/dependency-security-sync.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml index 73eedca..0b8a5c9 100644 --- a/.github/workflows/dependency-security-sync.yml +++ b/.github/workflows/dependency-security-sync.yml @@ -106,7 +106,8 @@ jobs: --head automation/dependency-security-sync \ --title "Automated security dependency sync from Dependabot alerts" \ --body $'Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow.' \ - --label dependencies + --label dependencies \ + --draft pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number') fi echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV" From d95c1bee2f0435fa3dba1787aea4c3a6b2ce4540 Mon Sep 17 00:00:00 2001 From: ihalatci Date: Wed, 29 Apr 2026 14:50:50 +0000 Subject: [PATCH 9/9] Reference citus PR number in regenerated requirements headers Defer requirements regeneration from the Python script to a workflow step that runs after the citus PR is created, so the header can reference `citusdata/citus#` (matching the existing convention) rather than a transient SHA. --- .github/scripts/security_sync.py | 30 ------------------- .../workflows/dependency-security-sync.yml | 19 +++++++++++- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/.github/scripts/security_sync.py b/.github/scripts/security_sync.py index 67f0b9b..f90fce0 100644 --- a/.github/scripts/security_sync.py +++ b/.github/scripts/security_sync.py @@ -76,39 +76,15 @@ def lock_version(lockfile_path, package): return None -def write_requirements(the_process_root, base_requirements, dev_requirements, citus_sha): - base_header = ( - f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n" - "# using `pipenv requirements > requirements.txt`, so as to avoid the\n" - "# need for pipenv/pyenv in this image\n\n" - ) - dev_header = ( - f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n" - "# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n" - "# need for pipenv/pyenv in this image\n\n" - ) - base_targets = [ - the_process_root / "circleci/images/citusupgradetester/files/etc/requirements.txt", - the_process_root / "circleci/images/failtester/files/etc/requirements.txt", - the_process_root / "circleci/images/pgupgradetester/files/etc/requirements.txt", - ] - for target in base_targets: - target.write_text(base_header + base_requirements) - dev_target = the_process_root / "circleci/images/stylechecker/files/etc/requirements.txt" - dev_target.write_text(dev_header + dev_requirements) - - def main(): parser = argparse.ArgumentParser() parser.add_argument("--alerts", required=True) parser.add_argument("--citus-root", required=True) - parser.add_argument("--the-process-root", required=True) parser.add_argument("--summary-out", required=True, help="Path to write per-alert outcome JSON") args = parser.parse_args() citus_root = Path(args.citus_root) - the_process_root = Path(args.the_process_root) alerts = json.loads(Path(args.alerts).read_text()) @@ -188,12 +164,6 @@ def main(): if not addressed: sys.exit("No alerts were addressed by this run; aborting before opening PRs.") - base_requirements = capture(["pipenv", "requirements"], cwd=pipfile_paths[0].parent) - dev_requirements = capture(["pipenv", "requirements", "--dev"], cwd=pipfile_paths[0].parent) - citus_sha = capture(["git", "rev-parse", "--short", "HEAD"], cwd=citus_root).strip() - write_requirements(the_process_root, base_requirements, dev_requirements, - f"citusdata/citus@{citus_sha}") - if __name__ == "__main__": main() diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml index 0b8a5c9..86afeae 100644 --- a/.github/workflows/dependency-security-sync.yml +++ b/.github/workflows/dependency-security-sync.yml @@ -73,7 +73,6 @@ jobs: python .github/scripts/security_sync.py \ --alerts alerts.json \ --citus-root "$PWD/citus" \ - --the-process-root "$PWD" \ --summary-out sync-summary.json addressed=$(jq -r '.addressed | join(" ")' sync-summary.json) echo "addressed=$addressed" >> "$GITHUB_OUTPUT" @@ -112,6 +111,24 @@ jobs: fi echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV" + - name: Regenerate the-process requirements + if: steps.alerts.outputs.count != '0' && steps.sync.outputs.addressed != '' + run: | + cd citus/src/test/regress + base=$(pipenv requirements) + dev=$(pipenv requirements --dev) + cd "$GITHUB_WORKSPACE" + ref="citusdata/citus#${CITUS_PR}" + base_header=$'# generated from Citus\'s Pipfile.lock (in src/test/regress) as of '"${ref}"$'\n# using `pipenv requirements > requirements.txt`, so as to avoid the\n# need for pipenv/pyenv in this image\n\n' + dev_header=$'# generated from Citus\'s Pipfile.lock (in src/test/regress) as of '"${ref}"$'\n# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n# need for pipenv/pyenv in this image\n\n' + for f in \ + circleci/images/citusupgradetester/files/etc/requirements.txt \ + circleci/images/failtester/files/etc/requirements.txt \ + circleci/images/pgupgradetester/files/etc/requirements.txt; do + printf '%s%s\n' "$base_header" "$base" > "$f" + done + printf '%s%s\n' "$dev_header" "$dev" > circleci/images/stylechecker/files/etc/requirements.txt + - name: Create or update the-process PR if: steps.alerts.outputs.count != '0' env: