From af2a0ea1908fe36d075bff1347f1217192a90a20 Mon Sep 17 00:00:00 2001 From: ProfRandom92 Date: Fri, 22 May 2026 15:35:56 +0200 Subject: [PATCH 1/2] feat: add agent artifact bundle --- docs/codex_skills/artifact_validation.md | 1 + scripts/agent_artifact_bundle.py | 105 +++++++++++++++++++++ tests/test_agent_artifact_bundle.py | 114 +++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 scripts/agent_artifact_bundle.py create mode 100644 tests/test_agent_artifact_bundle.py diff --git a/docs/codex_skills/artifact_validation.md b/docs/codex_skills/artifact_validation.md index eb4c933..5bee0cf 100644 --- a/docs/codex_skills/artifact_validation.md +++ b/docs/codex_skills/artifact_validation.md @@ -7,6 +7,7 @@ Create or update deterministic artifact checks without changing benchmark meanin ## When to use Use for artifacts under `artifacts/`, generator scripts under `scripts/`, regeneration parity tests, stable JSON checks, and committed artifact reproducibility. +Use `scripts/agent_artifact_bundle.py` for deterministic evidence bundles that combine safe gate status with explicit validation evidence. ## Allowed actions diff --git a/scripts/agent_artifact_bundle.py b/scripts/agent_artifact_bundle.py new file mode 100644 index 0000000..2754a4c --- /dev/null +++ b/scripts/agent_artifact_bundle.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Build a deterministic local evidence bundle for AI-assisted work.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.safe_pr_gate import GateState, collect_gate_state, evaluate_gate + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a deterministic local evidence bundle for AI-assisted work.") + parser.add_argument("--allow-main", action="store_true", help="Allow bundle generation on main.") + parser.add_argument( + "--validation-command", + action="append", + default=[], + help="Validation command executed for the bundle evidence. May be repeated.", + ) + parser.add_argument( + "--validation-result", + action="append", + default=[], + help="Validation result corresponding to each command. May be repeated.", + ) + parser.add_argument( + "--mcp-context-output-ref", + help="Optional reference to a previously generated MCP context tool output.", + ) + return parser.parse_args(argv) + + +def _error_response(exc: RuntimeError) -> dict[str, Any]: + return { + "error": { + "message": str(exc), + "type": exc.__class__.__name__, + }, + "ok": False, + "result": "ERROR", + } + + +def _build_validation_evidence(commands: list[str], results: list[str]) -> list[dict[str, str]]: + if len(commands) != len(results): + raise RuntimeError( + "validation command/result count mismatch: " + f"{len(commands)} command(s), {len(results)} result(s)" + ) + return [{"command": command, "result": result} for command, result in zip(commands, results)] + + +def build_agent_artifact_bundle( + state: GateState, + *, + allow_main: bool, + validation_commands: list[str], + validation_results: list[str], + mcp_context_output_ref: str | None = None, +) -> dict[str, Any]: + if state.branch == "main" and not allow_main: + raise RuntimeError("main branch is not allowed for agent artifact bundling") + + safe_pr_gate_result = evaluate_gate(state) + bundle: dict[str, Any] = { + "branch": state.branch, + "changed_files": list(state.changed_paths), + "ok": True, + "result": "PASS", + "safe_pr_gate": safe_pr_gate_result.to_dict(), + "validation_evidence": _build_validation_evidence(validation_commands, validation_results), + } + if mcp_context_output_ref is not None: + bundle["mcp_context_output_ref"] = mcp_context_output_ref + return bundle + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + try: + state = collect_gate_state() + bundle = build_agent_artifact_bundle( + state, + allow_main=args.allow_main, + validation_commands=list(args.validation_command), + validation_results=list(args.validation_result), + mcp_context_output_ref=args.mcp_context_output_ref, + ) + sys.stdout.write(json.dumps(bundle, indent=2, sort_keys=True) + "\n") + return 0 + except RuntimeError as exc: + sys.stdout.write(json.dumps(_error_response(exc), indent=2, sort_keys=True) + "\n") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_agent_artifact_bundle.py b/tests/test_agent_artifact_bundle.py new file mode 100644 index 0000000..0130ecf --- /dev/null +++ b/tests/test_agent_artifact_bundle.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json + +import pytest + +import scripts.agent_artifact_bundle as agent_artifact_bundle +from scripts.safe_pr_gate import GateState, evaluate_gate + + +def test_build_agent_artifact_bundle_is_deterministic_and_includes_optional_metadata() -> None: + state = GateState( + branch="feat/agent-artifact-bundle", + status_short=(" M docs/example.md",), + changed_paths=("docs/example.md",), + ) + + bundle = agent_artifact_bundle.build_agent_artifact_bundle( + state, + allow_main=False, + validation_commands=["python -m compileall -q scripts/agent_artifact_bundle.py", "pytest tests/test_agent_artifact_bundle.py -q"], + validation_results=["pass", "pass"], + mcp_context_output_ref="artifacts/mcp_context_layer_example.json", + ) + + assert bundle == { + "branch": "feat/agent-artifact-bundle", + "changed_files": ["docs/example.md"], + "mcp_context_output_ref": "artifacts/mcp_context_layer_example.json", + "ok": True, + "result": "PASS", + "safe_pr_gate": evaluate_gate(state).to_dict(), + "validation_evidence": [ + { + "command": "python -m compileall -q scripts/agent_artifact_bundle.py", + "result": "pass", + }, + { + "command": "pytest tests/test_agent_artifact_bundle.py -q", + "result": "pass", + }, + ], + } + + first = json.dumps(bundle, indent=2, sort_keys=True) + second = json.dumps(bundle, indent=2, sort_keys=True) + assert first == second + + +def test_build_agent_artifact_bundle_rejects_main_without_allow_main() -> None: + state = GateState(branch="main", status_short=(), changed_paths=()) + + with pytest.raises(RuntimeError, match="main branch is not allowed for agent artifact bundling"): + agent_artifact_bundle.build_agent_artifact_bundle( + state, + allow_main=False, + validation_commands=[], + validation_results=[], + ) + + +def test_main_emits_deterministic_json_and_omits_optional_reference(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + state = GateState( + branch="feat/agent-artifact-bundle", + status_short=(), + changed_paths=("scripts/agent_artifact_bundle.py",), + ) + monkeypatch.setattr(agent_artifact_bundle, "collect_gate_state", lambda: state) + + exit_code = agent_artifact_bundle.main( + [ + "--validation-command", + "python -m compileall -q scripts/agent_artifact_bundle.py", + "--validation-result", + "pass", + ] + ) + output = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert output == { + "branch": "feat/agent-artifact-bundle", + "changed_files": ["scripts/agent_artifact_bundle.py"], + "ok": True, + "result": "PASS", + "safe_pr_gate": evaluate_gate(state).to_dict(), + "validation_evidence": [ + { + "command": "python -m compileall -q scripts/agent_artifact_bundle.py", + "result": "pass", + } + ], + } + assert "mcp_context_output_ref" not in output + + +def test_main_reports_main_branch_as_deterministic_error_json( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(agent_artifact_bundle, "collect_gate_state", lambda: GateState(branch="main", status_short=(), changed_paths=())) + + exit_code = agent_artifact_bundle.main([]) + output = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert output == { + "error": { + "message": "main branch is not allowed for agent artifact bundling", + "type": "RuntimeError", + }, + "ok": False, + "result": "ERROR", + } From 7497aa29cece8dbb0b93aa57828decebc6ecddfe Mon Sep 17 00:00:00 2001 From: ProfRandom92 Date: Fri, 22 May 2026 15:54:48 +0200 Subject: [PATCH 2/2] fix: reflect safe gate status in agent bundle --- scripts/agent_artifact_bundle.py | 7 ++-- tests/test_agent_artifact_bundle.py | 54 +++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/scripts/agent_artifact_bundle.py b/scripts/agent_artifact_bundle.py index 2754a4c..ff42825 100644 --- a/scripts/agent_artifact_bundle.py +++ b/scripts/agent_artifact_bundle.py @@ -70,11 +70,12 @@ def build_agent_artifact_bundle( raise RuntimeError("main branch is not allowed for agent artifact bundling") safe_pr_gate_result = evaluate_gate(state) + bundle_ok = safe_pr_gate_result.ok bundle: dict[str, Any] = { "branch": state.branch, "changed_files": list(state.changed_paths), - "ok": True, - "result": "PASS", + "ok": bundle_ok, + "result": "PASS" if bundle_ok else "FAIL", "safe_pr_gate": safe_pr_gate_result.to_dict(), "validation_evidence": _build_validation_evidence(validation_commands, validation_results), } @@ -95,7 +96,7 @@ def main(argv: list[str] | None = None) -> int: mcp_context_output_ref=args.mcp_context_output_ref, ) sys.stdout.write(json.dumps(bundle, indent=2, sort_keys=True) + "\n") - return 0 + return 0 if bundle["ok"] else 1 except RuntimeError as exc: sys.stdout.write(json.dumps(_error_response(exc), indent=2, sort_keys=True) + "\n") return 1 diff --git a/tests/test_agent_artifact_bundle.py b/tests/test_agent_artifact_bundle.py index 0130ecf..2932af1 100644 --- a/tests/test_agent_artifact_bundle.py +++ b/tests/test_agent_artifact_bundle.py @@ -11,8 +11,8 @@ def test_build_agent_artifact_bundle_is_deterministic_and_includes_optional_metadata() -> None: state = GateState( branch="feat/agent-artifact-bundle", - status_short=(" M docs/example.md",), - changed_paths=("docs/example.md",), + status_short=(), + changed_paths=(), ) bundle = agent_artifact_bundle.build_agent_artifact_bundle( @@ -25,7 +25,7 @@ def test_build_agent_artifact_bundle_is_deterministic_and_includes_optional_meta assert bundle == { "branch": "feat/agent-artifact-bundle", - "changed_files": ["docs/example.md"], + "changed_files": [], "mcp_context_output_ref": "artifacts/mcp_context_layer_example.json", "ok": True, "result": "PASS", @@ -59,6 +59,26 @@ def test_build_agent_artifact_bundle_rejects_main_without_allow_main() -> None: ) +def test_build_agent_artifact_bundle_reflects_safe_gate_failure() -> None: + state = GateState( + branch="feat/agent-artifact-bundle", + status_short=(" M docs/example.md",), + changed_paths=("docs/example.md",), + ) + + bundle = agent_artifact_bundle.build_agent_artifact_bundle( + state, + allow_main=False, + validation_commands=["python -m compileall -q scripts/agent_artifact_bundle.py"], + validation_results=["pass"], + ) + + assert bundle["ok"] is False + assert bundle["result"] == "FAIL" + assert bundle["safe_pr_gate"]["ok"] is False + assert bundle["safe_pr_gate"]["result"] == "FAIL" + + def test_main_emits_deterministic_json_and_omits_optional_reference(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: state = GateState( branch="feat/agent-artifact-bundle", @@ -94,6 +114,34 @@ def test_main_emits_deterministic_json_and_omits_optional_reference(monkeypatch: assert "mcp_context_output_ref" not in output +def test_main_returns_failure_exit_code_when_safe_gate_fails( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + state = GateState( + branch="feat/agent-artifact-bundle", + status_short=(" M docs/example.md",), + changed_paths=("docs/example.md",), + ) + monkeypatch.setattr(agent_artifact_bundle, "collect_gate_state", lambda: state) + + exit_code = agent_artifact_bundle.main( + [ + "--validation-command", + "python -m compileall -q scripts/agent_artifact_bundle.py", + "--validation-result", + "pass", + ] + ) + output = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert output["ok"] is False + assert output["result"] == "FAIL" + assert output["safe_pr_gate"]["ok"] is False + assert output["safe_pr_gate"]["result"] == "FAIL" + + def test_main_reports_main_branch_as_deterministic_error_json( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],