Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions .github/scripts/check_quality.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
"""Quality ratchet: compare current ruff violations against a committed baseline.

Exit codes:
0 — no regressions (violations per file ≤ baseline)
1 — regressions detected (at least one file has more violations than baseline)

Usage:
python .github/scripts/check_quality.py [--baseline .quality-baseline.json]
"""

from __future__ import annotations

import argparse
import collections
import datetime
import json
import subprocess
import sys
from pathlib import Path


def run_ruff(repo_root: Path) -> dict[str, int]:
"""Run ruff and return per-file violation counts (relative paths)."""
try:
result = subprocess.run(
["ruff", "check", ".", "--output-format=json"],
cwd=repo_root,
capture_output=True,
text=True,
)
except FileNotFoundError:
print(
"❌ ruff is not installed or not on PATH. Install it with: pip install ruff",
file=sys.stderr,
)
sys.exit(1)

# ruff exits 1 when violations found; that is expected
try:
data = json.loads(result.stdout)
except json.JSONDecodeError:
print("❌ Failed to parse ruff JSON output.", file=sys.stderr)
if result.stdout.strip():
print(result.stdout, file=sys.stderr)
if result.stderr.strip():
print(result.stderr, file=sys.stderr)
sys.exit(1)

counts: dict[str, int] = collections.Counter()
repo_root_resolved = repo_root.resolve()
for item in data:
filename = item.get("filename", "")
p = Path(filename)
if p.is_absolute():
try:
rel = p.resolve().relative_to(repo_root_resolved).as_posix()
except ValueError:
rel = p.as_posix()
else:
rel = p.as_posix()
counts[rel] += 1

return dict(counts)
Comment thread
Copilot marked this conversation as resolved.


def load_baseline(path: Path) -> dict[str, int]:
"""Load the committed baseline file."""
if not path.exists():
print(f"❌ Baseline file not found: {path}", file=sys.stderr)
print(
" Run: python .github/scripts/check_quality.py --update-baseline",
file=sys.stderr,
)
sys.exit(1)

try:
data = json.loads(path.read_text())
except json.JSONDecodeError as e:
print(f"❌ Failed to parse baseline JSON: {path} ({e})", file=sys.stderr)
sys.exit(1)

per_file = data.get("per_file", {})
if not isinstance(per_file, dict):
print(f"❌ Baseline file has invalid 'per_file' shape: {path}", file=sys.stderr)
sys.exit(1)

return per_file


def compare(
current: dict[str, int], baseline: dict[str, int]
) -> tuple[list[str], list[str]]:
"""Return (regressions, improvements).

A regression is any file whose current count exceeds its baseline count.
A new file with violations (absent from baseline) is also a regression.
"""
all_files = set(current) | set(baseline)
regressions: list[str] = []
improvements: list[str] = []

for f in sorted(all_files):
curr = current.get(f, 0)
base = baseline.get(f, 0)
if curr > base:
regressions.append(f" ❌ {f} ({base} → {curr}, +{curr - base})")
elif curr < base:
improvements.append(f" ✅ {f} ({base} → {curr}, -{base - curr})")

return regressions, improvements


def update_baseline(repo_root: Path, baseline_path: Path) -> None:
"""Regenerate the baseline from the current ruff output and write it."""
result = subprocess.run(["ruff", "--version"], capture_output=True, text=True)
ruff_version = result.stdout.strip().replace("ruff ", "")

current = run_ruff(repo_root)
data = {
"generated_at": datetime.datetime.now(datetime.UTC).isoformat(
timespec="seconds"
),
"ruff_version": ruff_version,
"total_violations": sum(current.values()),
"per_file": dict(sorted(current.items())),
}
baseline_path.write_text(json.dumps(data, indent=2) + "\n")
print(
f"✅ Baseline updated: {sum(current.values())} violations across {len(current)} files"
)
print(f" Written to {baseline_path}")


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--baseline",
default=".quality-baseline.json",
help="Path to baseline JSON file (default: .quality-baseline.json)",
)
parser.add_argument(
"--update-baseline",
action="store_true",
help="Regenerate the baseline from current state and exit 0",
)
args = parser.parse_args()

repo_root = Path(__file__).resolve().parents[2]
baseline_path = repo_root / args.baseline

if args.update_baseline:
update_baseline(repo_root, baseline_path)
return

baseline = load_baseline(baseline_path)
current = run_ruff(repo_root)

total_baseline = sum(baseline.values())
total_current = sum(current.values())
regressions, improvements = compare(current, baseline)

print("=" * 60)
print(" Quality Ratchet Report")
print("=" * 60)
print(f" Baseline violations : {total_baseline}")
print(f" Current violations : {total_current}")
delta = total_current - total_baseline
delta_str = f"+{delta}" if delta > 0 else str(delta)
print(f" Delta : {delta_str}")
print()

if improvements:
print(f"Improvements ({len(improvements)} file(s)):")
print("\n".join(improvements))
print()

if regressions:
print(f"Regressions ({len(regressions)} file(s)) — BLOCKING:")
print("\n".join(regressions))
print()
print("Fix the regressions above, or run:")
print(" ruff check <file> --fix")
print()
print("❌ Quality ratchet FAILED — this PR introduces new violations.")
sys.exit(1)

print("✅ Quality ratchet PASSED — no new violations introduced.")


if __name__ == "__main__":
main()
4 changes: 2 additions & 2 deletions .github/scripts/generate_rollback_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"""

import os
from datetime import datetime, timezone
from datetime import UTC, datetime


def main():
Expand All @@ -17,7 +17,7 @@ def main():
author = os.environ.get("AUTHOR", "unknown")
files_changed = os.environ.get("FILES_CHANGED", "").split()

timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
timestamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")

path = "docs/ROLLBACK.md"
os.makedirs("docs", exist_ok=True)
Expand Down
94 changes: 94 additions & 0 deletions .github/workflows/quality-ratchet.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
name: Quality Ratchet

on:
pull_request:
types: [opened, synchronize, ready_for_review]
paths:
- '**.py'
- 'pyproject.toml'
- '.quality-baseline.json'

permissions:
contents: read
pull-requests: write

env:
RUFF_VERSION: '0.15.20'

jobs:
ratchet:
name: ruff ratchet (no new violations)
runs-on: ubuntu-latest
timeout-minutes: 10

steps:
- name: Checkout PR head
uses: actions/checkout@v5

- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.11'
cache: 'pip'

- name: Install ruff
run: pip install "ruff==${{ env.RUFF_VERSION }}"

- name: Run quality ratchet
id: ratchet
run: python .github/scripts/check_quality.py
continue-on-error: true

- name: Comment on PR
if: github.event_name == 'pull_request'
uses: actions/github-script@v8
with:
Comment on lines +42 to +45
script: |
const outcome = '${{ steps.ratchet.outcome }}';
const passed = outcome === 'success';

const header = passed
? '## ✅ Quality Ratchet — PASSED'
: '## ❌ Quality Ratchet — FAILED';

const body = passed
? `${header}\n\nNo new ruff violations introduced by this PR. 🎉`
: `${header}

This PR introduces new ruff violations compared to the baseline.

Run \`python .github/scripts/check_quality.py\` locally to see the details, then fix the regressions or run \`ruff check <file> --fix\` to auto-fix them.

> **Tip:** You can improve the baseline by fixing existing violations in the files you touch, then updating it with:
> \`python .github/scripts/check_quality.py --update-baseline\``;

const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});

const existing = comments.find(c =>
c.user.login === 'github-actions[bot]' &&
c.body.includes('Quality Ratchet')
);

if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body: body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: body,
});
}

- name: Fail if ratchet failed
if: steps.ratchet.outcome == 'failure'
run: exit 1
14 changes: 14 additions & 0 deletions .quality-baseline.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"generated_at": "2026-07-04T06:36:49+00:00",
"ruff_version": "0.15.20",
"total_violations": 24,
"per_file": {
".github/scripts/generate_rollback_manifest.py": 1,
"autopilot/staleness_engine.py": 5,
"tests/test_autonomous_agents.py": 6,
"tests/unit/test_dependency_agent.py": 1,
"tests/unit/test_security_scan_agent.py": 2,
"tests/unit/test_staleness_engine.py": 3,
"tests/unit/test_triage_agent.py": 6
}
}
10 changes: 5 additions & 5 deletions autopilot/staleness_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import logging
import os
import sys
from datetime import datetime, timezone
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

Expand Down Expand Up @@ -204,7 +204,7 @@ def generate_comment(
cost = result.get("usage", {})
self._cost_log.append(
{
"timestamp": datetime.now(timezone.utc).isoformat(),
"timestamp": datetime.now(UTC).isoformat(),
"repo": repo_full_name,
"tier": tier,
"usage": cost,
Expand All @@ -231,7 +231,7 @@ def scan_repos(self, repos: list[dict[str, Any]]) -> list[dict[str, Any]]:
repo_full_name, pr_number, pr_title, pr_url, age_days, tier
"""
stale: list[dict[str, Any]] = []
now = datetime.now(timezone.utc)
now = datetime.now(UTC)

for repo_cfg in repos:
owner = repo_cfg["owner"]
Expand All @@ -243,7 +243,7 @@ def scan_repos(self, repos: list[dict[str, Any]]) -> list[dict[str, Any]]:
for pr in pulls:
created = pr.created_at
if created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
created = created.replace(tzinfo=UTC)
age_days = (now - created).days
tier = self.assign_tier(age_days)
if tier > 0:
Expand Down Expand Up @@ -291,7 +291,7 @@ def process_stale_prs(
Returns a summary dict with counts and per-PR actions taken.
"""
state = self.load_state()
now_iso = datetime.now(timezone.utc).isoformat()
now_iso = datetime.now(UTC).isoformat()
actions: list[dict[str, Any]] = []

for pr_info in stale_prs:
Expand Down
Loading