diff --git a/AGENTS.md b/AGENTS.md index bb7ad00..45ec5b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,14 +46,13 @@ or makes a public SDK/provider/MCP stability promise. The real execution-host path is now selectable on macOS and proves an honest `weak` `process-group` posture at compose time; the current real GitHub Forge/GitHub Issues path includes blocked-PR surfacing, held-merge replay safety, landing-path redaction, and origin-bearing work-source provenance — -the success-path real Forge `open-pr` smoke has been rerun on the current checkout, while -held-merge and commit-status/comment block-surfacing legs remain unit-proven with real-effect -evidence still owed. +the success-path real Forge `open-pr`, protected-branch held merge, commit-status/comment +block-surfacing, and resume idempotency smokes have been captured on the current checkout. Strong no-phone-home evidence remains future work. EVRUN-partial is recorded; P11 also records a blocked EVRUN-full capture attempt plus a later successful combined EVRUN-full smoke: real GitHub Issues, real Codex app-server, real host attestation, real GitHub Forge `open-pr`, and records -integrity on the disposable target repo. Strong no-phone-home, held-merge/idempotency, hosted, and -Windows evidence remain future work. +integrity on the disposable target repo. Strong no-phone-home, hosted, and Windows evidence remain +future work. ## Commands diff --git a/README.md b/README.md index 885ae75..065a298 100644 --- a/README.md +++ b/README.md @@ -35,16 +35,13 @@ checkout. The package split and the private Codex app-server transport are now i inside this private workspace; neither creates a shipped public API. The real execution-host path now exercises a local macOS process-group probe and reports an honest `weak` containment posture. -Current evidence proves a scoped real-provider path with a scripted agent leg -(`EVRUN-partial`). P11 also captured an EVRUN-full attempt: the narrow Codex app-server and -real-host smokes pass outside the Codex sandbox, but the combined real GitHub path could not run -without sandbox credentials and an integrity key. The current real GitHub Forge/GitHub Issues path -also carries blocked-PR surfacing, held-merge replay safety, landing-path redaction, and -per-candidate origin-bearing provenance — the success-path real Forge `open-pr` smoke has been -rerun on the current checkout, while held-merge and commit-status/comment block-surfacing legs -remain unit-proven with real-effect evidence still owed. `EVRUN-full`, remote execution, public -package publication, and a full Codex-driven agent leg remain future work; strong/no-phone-home -execution-host evidence remains in that later proof step. +Current evidence proves scoped real-provider paths with a scripted agent leg (`EVRUN-partial` and +P05 follow-up smokes). P11 also captured a combined EVRUN-full smoke with real GitHub Issues, real +Codex app-server, real host attestation, real GitHub Forge `open-pr`, records integrity, and +redaction on the disposable target repo. The current real GitHub Forge path now carries real-effect +evidence for blocked-PR surfacing, held-merge posture, landing-path redaction, repeated-effect +idempotency, and per-candidate origin-bearing provenance. Remote execution, public package +publication, and strong/no-phone-home execution-host evidence remain future work. ## Quick Start diff --git a/docs/README.md b/docs/README.md index ad706c2..b63c76d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,9 +25,8 @@ scoped to **Jig the execution engine**. Current implementation truth includes a private setup command that emits validated owner configuration, plus a private real GitHub Forge/GitHub Issues path with blocked-PR surfacing, held-merge replay safety, landing-path redaction, and origin-bearing candidate provenance — -the success-path real Forge `open-pr` smoke has been rerun on the current checkout, while -held-merge and commit-status/comment block-surfacing legs remain unit-proven with real-effect -evidence still owed; +the success-path real Forge `open-pr`, protected-branch held merge, commit-status/comment +block-surfacing, and resume idempotency smokes have been captured on the current checkout; phase-table PR-number status updates still live in delivery docs. Preserve active design history in place: diff --git a/docs/archive/reviews/2026-07-06-p04-p12-target-state-review.md b/docs/archive/reviews/2026-07-06-p04-p12-target-state-review.md index 746eb66..0f4e3e0 100644 --- a/docs/archive/reviews/2026-07-06-p04-p12-target-state-review.md +++ b/docs/archive/reviews/2026-07-06-p04-p12-target-state-review.md @@ -363,28 +363,27 @@ all four were not delivered. > `98993ff` (`test: capture evrun full smoke evidence (#79)`), using only post-audit repo > evidence now present on `main`. -| ID | Status | Current-main closure evidence | -| --- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| F1 | fixed | PR #72 / `0dfb9d0` (`test: restore branch-coverage headroom above the gate floor`) added the missing branch-coverage headroom back into the test corpus, and current [`verification.md`](../../delivery/target-state-implementation/verification.md) now codifies both the 90.5% headroom rule and the post-merge watch that this finding said were missing. | -| F2 | fixed | PR #74 / `81d5664` (`fix: support live parked decisions`) rewired the live parked decision path in `sdk.ts`, `harness.ts`, `projection.ts`, and `resume.ts`; current tests include `P09-F2: decide routes a live parked run instead of refusing` in `packages/jig-sdk/tests/sdk.unit.test.ts`. | -| F3 | partially closed | PR #78 / `dcffa56` (`docs: record p05 real forge smoke rerun`) added a current-checkout real Forge/work-source rerun under [`docs/design/evidence/2026-07-06-p05-real-forge-smoke-rerun.md`](../../design/evidence/2026-07-06-p05-real-forge-smoke-rerun.md) and updated front-door status docs to qualify the shipped claim. The same evidence and current `README.md`/`AGENTS.md`/`docs/README.md` remain explicit that held-merge plus commit-status/comment block-surfacing real-effect proof is still open, so `MERGE-5` is not fully closed. | -| F4 | fixed | PR #79 / `98993ff` updated P11 from an internally contradictory blocked-capture posture to a reconciled combined-smoke posture in [`phases/11-evrun-full-evidence.md`](../../delivery/target-state-implementation/phases/11-evrun-full-evidence.md), with per-AC current dispositions that distinguish what is now met from what remains stronger-probe debt. The finding's contradiction is closed without overclaiming no-phone-home or idempotency. | -| F5 | fixed | PR #77 / `a78f67c` (`docs: harden target-state delivery tracking`) corrected the stale track header, phase-table rows, and P11/P12 status lines on current `main`, and tightened the delivery-foundation/status-flip checks so the daisy-chain failure mode is no longer the documented workflow. | -| F6 | fixed | PR #77 / `a78f67c` updated [`docs/design/contracts/driving.md`](../../design/contracts/driving.md) to the shipped 10-verb operator-control vocabulary (`start`, `preview`, `watch`, `inspect`, `ask-why`, `notice-ack`, `notice-snooze`, `decide`, `stop`, `export`), matching ADR 0033 and the current `JigOperatorControlPort`. | -| F7 | fixed | PR #77 / `a78f67c` removed the stale "resume placement is unsettled" posture from the track README and now states the settled split directly: resume lives on `JigRecoverySurface`, not on the operator-control port or MCP surface. | -| F8 | fixed | PR #75 / `1e23b5f` (`fix: exercise negative egress probe`) replaced the hardcoded `negativeEgressProbePassed: false` behavior with an exercised probe result path in `packages/jig-sdk/src/providers/real/confinement.ts`; current Phase 04 post-merge notes record that remediation explicitly. | -| F9 | fixed | PR #75 / `1e23b5f` also closed the latent substrate-validation hole: current `createRealExecutionHost()` now refuses probe substrate requests when no approved manifest is supplied and validates every declared request before exercising the probe. | -| F10 | fixed | PR #76 / `a66747d` (`feat: surface export attribution in observations`) added export attribution to inspect/ask-why and covered it with current CLI and SDK tests, including `P10-F10: custom-dir export remains attributable from inspect and run-level ask-why`. | -| F11 | fixed | PR #76 / `a66747d` added the missing second-process watch integration test (`P08-F11: watch observes a live run from a second process and then the finished record`) and updated the phase doc to state the shipped watch vocabulary honestly: LIVE posture is projected into snapshot signals rather than a literal `stalled` status string. | -| F12 | fixed | PR #77 / `a78f67c` added the missing process teeth to current delivery rules: explicit review-arrival gate, post-merge main-branch check watch, 90.5% coverage headroom, and a rewritten same-PR `merged (#N)` status-flip rule in [`verification.md`](../../delivery/target-state-implementation/verification.md). The historical incidents remain part of the audit record, but the process gap identified by F12 is now addressed in the governing docs. | +| ID | Status | Current-main closure evidence | +| --- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| F1 | fixed | PR #72 / `0dfb9d0` (`test: restore branch-coverage headroom above the gate floor`) added the missing branch-coverage headroom back into the test corpus, and current [`verification.md`](../../delivery/target-state-implementation/verification.md) now codifies both the 90.5% headroom rule and the post-merge watch that this finding said were missing. | +| F2 | fixed | PR #74 / `81d5664` (`fix: support live parked decisions`) rewired the live parked decision path in `sdk.ts`, `harness.ts`, `projection.ts`, and `resume.ts`; current tests include `P09-F2: decide routes a live parked run instead of refusing` in `packages/jig-sdk/tests/sdk.unit.test.ts`. | +| F3 | fixed | This follow-up added [`docs/design/evidence/2026-07-06-p05-held-merge-smoke.md`](../../design/evidence/2026-07-06-p05-held-merge-smoke.md), which records a real protected-branch `done-not-landed` / `held-by-review` merge plus readable PR/status/comment block surfacing against `agentic-workflow-kit/jig-smoke-target`. The earlier PR #78 / `dcffa56` success-path `open-pr` rerun remains provenance for the first half of the evidence gap. `MERGE-5` is now backed by real-effect evidence. | +| F4 | fixed | PR #79 / `98993ff` updated P11 from an internally contradictory blocked-capture posture to a reconciled combined-smoke posture in [`phases/11-evrun-full-evidence.md`](../../delivery/target-state-implementation/phases/11-evrun-full-evidence.md), with per-AC current dispositions that distinguish what is now met from what remains stronger-probe debt. The finding's contradiction is closed without overclaiming no-phone-home or idempotency. | +| F5 | fixed | PR #77 / `a78f67c` (`docs: harden target-state delivery tracking`) corrected the stale track header, phase-table rows, and P11/P12 status lines on current `main`, and tightened the delivery-foundation/status-flip checks so the daisy-chain failure mode is no longer the documented workflow. | +| F6 | fixed | PR #77 / `a78f67c` updated [`docs/design/contracts/driving.md`](../../design/contracts/driving.md) to the shipped 10-verb operator-control vocabulary (`start`, `preview`, `watch`, `inspect`, `ask-why`, `notice-ack`, `notice-snooze`, `decide`, `stop`, `export`), matching ADR 0033 and the current `JigOperatorControlPort`. | +| F7 | fixed | PR #77 / `a78f67c` removed the stale "resume placement is unsettled" posture from the track README and now states the settled split directly: resume lives on `JigRecoverySurface`, not on the operator-control port or MCP surface. | +| F8 | fixed | PR #75 / `1e23b5f` (`fix: exercise negative egress probe`) replaced the hardcoded `negativeEgressProbePassed: false` behavior with an exercised probe result path in `packages/jig-sdk/src/providers/real/confinement.ts`; current Phase 04 post-merge notes record that remediation explicitly. | +| F9 | fixed | PR #75 / `1e23b5f` also closed the latent substrate-validation hole: current `createRealExecutionHost()` now refuses probe substrate requests when no approved manifest is supplied and validates every declared request before exercising the probe. | +| F10 | fixed | PR #76 / `a66747d` (`feat: surface export attribution in observations`) added export attribution to inspect/ask-why and covered it with current CLI and SDK tests, including `P10-F10: custom-dir export remains attributable from inspect and run-level ask-why`. | +| F11 | fixed | PR #76 / `a66747d` added the missing second-process watch integration test (`P08-F11: watch observes a live run from a second process and then the finished record`) and updated the phase doc to state the shipped watch vocabulary honestly: LIVE posture is projected into snapshot signals rather than a literal `stalled` status string. | +| F12 | fixed | PR #77 / `a78f67c` added the missing process teeth to current delivery rules: explicit review-arrival gate, post-merge main-branch check watch, 90.5% coverage headroom, and a rewritten same-PR `merged (#N)` status-flip rule in [`verification.md`](../../delivery/target-state-implementation/verification.md). The historical incidents remain part of the audit record, but the process gap identified by F12 is now addressed in the governing docs. | ### Remaining open debt after the remediation wave -- F3 is only partially closed. Current docs and the P05 rerun record still say the success-path - real Forge `open-pr` smoke is proven, while held-merge and commit-status/comment block-surfacing - legs remain unit-proven or otherwise unevidenced on a real-effect path. -- P11 evidence is stronger than it was at `a239986`, but it is still not a claim that - no-phone-home or repeated-effect idempotency is proven. Current P11 docs, evidence index, root - `README.md`, repo `AGENTS.md`, and track `verification.md` all continue to treat those as open - evidence or owner-deferral work, so this closeout does not advance P13/P14 beyond what current - `main` already states. +- SEC-2 strong no-phone-home remains open. Current P11 docs, evidence index, root `README.md`, + repo `AGENTS.md`, and track `verification.md` continue to treat adversarial no-phone-home as open + evidence or owner-deferral work, so this closeout does not advance P13/P14 beyond that remaining + gate. +- RESUME-3 repeated-effect idempotency is now backed by + [`docs/design/evidence/2026-07-06-resume-idempotency-smoke.md`](../../design/evidence/2026-07-06-resume-idempotency-smoke.md) + for the real `open-pr` path. diff --git a/docs/delivery/target-state-implementation/README.md b/docs/delivery/target-state-implementation/README.md index b83b026..4f983bd 100644 --- a/docs/delivery/target-state-implementation/README.md +++ b/docs/delivery/target-state-implementation/README.md @@ -60,8 +60,8 @@ Verified against the repo after Phase 02 workspace split: includes the private production Codex app-server transport: `bootstrap.ts` composes `createProductionCodexAgentSession`, and driver selection supports `agent: 'codex'`. The remaining limitation is evidence, not absence of implementation: P11 now records the combined - real Codex / real GitHub `open-pr` smoke, while stronger no-phone-home and repeated-effect - idempotency evidence remain open. The real Execution host is now selectable on macOS and + real Codex / real GitHub `open-pr` smoke and a later real repeated-effect idempotency smoke, + while stronger no-phone-home evidence remains open. The real Execution host is now selectable on macOS and exercises a local `process-group` confinement probe that reports an honest proven `weak` posture at compose time. - **Conformance and controlled doubles** now live in `packages/jig-testkit` @@ -78,9 +78,9 @@ Verified against the repo after Phase 02 workspace split: - **Evidence**: EVRUN-partial is committed ([evidence index](../../design/evidence/README.md)) — one real work-source → forge → records-integrity run with a **scripted** agent leg. P11 also records the - earlier blocked EVRUN-full attempt and the later combined real Codex / real-host / real GitHub - `open-pr` smoke. Strong no-phone-home, held-merge/idempotency, hosted, and Windows evidence - remain open. + earlier blocked EVRUN-full attempt, the later combined real Codex / real-host / real GitHub + `open-pr` smoke, and the follow-up held-merge / resume idempotency smokes. Strong + no-phone-home, hosted, and Windows evidence remain open. ## Desired target state @@ -103,9 +103,9 @@ Verified against the repo after Phase 02 workspace split: - **Observability** ([guarantee 5](../../product/guarantees.md#5-full-observability)): watch, notices with acknowledge/snooze, ask-why, and write-once redacted export, all answered from the run's own records. -- **Evidence**: EVRUN-full combined smoke committed; remaining no-phone-home/idempotency evidence - captured or explicitly deferred; contract-freeze readiness prepared for the contract owner's T14 - decision. +- **Evidence**: EVRUN-full combined smoke committed; repeated-effect idempotency captured; remaining + no-phone-home evidence captured or explicitly deferred; contract-freeze readiness prepared for the + contract owner's T14 decision. - **Docs**: README, AGENTS.md, and docs indexes state the shipped surface truthfully at every phase boundary. @@ -346,16 +346,13 @@ boundary enforcement from P02 on, docs checks, and the definition of "delivered" Per the escalation rule (product owns what/why; design reconciles to product; delivery names conflicts rather than resolving them): -- **`MERGE-5` is partially proven; block-surfacing real-effect evidence is still owed.** The - product guarantees blocked-PR surfacing. Surfacing is now reachable from normal runner wiring, - and the success-path real Forge `open-pr` smoke was rerun on the current checkout - (`docs/design/evidence/2026-07-06-p05-real-forge-smoke-rerun.md`). The commit-status/comment - block-surfacing leg remains unit-proven, the earlier commit-status write/read attempt failed - with an HTTP 403, and held-merge posture was never observed against a protected target — the - EVRUN-partial-style smoke records explicitly do not claim `MERGE-5` end to end. Additional - real-effect proof remains required before the claim is fully proven; see the review record - `docs/archive/reviews/2026-07-06-p04-p12-target-state-review.md` for the observed failure - detail. +- **`MERGE-5` has real-effect evidence.** The product guarantees blocked-PR surfacing. Surfacing is + reachable from normal runner wiring, the success-path real Forge `open-pr` smoke was rerun on the + current checkout (`docs/design/evidence/2026-07-06-p05-real-forge-smoke-rerun.md`), and the + follow-up held-merge smoke observed a protected-branch `done-not-landed` outcome plus readable + PR/status/comment block surfacing + (`docs/design/evidence/2026-07-06-p05-held-merge-smoke.md`). Earlier partial records remain + provenance for the previous gap, not the current claim. - **GUARD-2 pause shape is an open design question** (distinct sub-state vs. reusing `parked`, raised in [`plan-intake.md`](../../design/core/plan-intake.md#open-questions); [`authorization.md`](../../design/core/authorization.md) explicitly declines to invent a new diff --git a/docs/delivery/target-state-implementation/phases/11-evrun-full-evidence.md b/docs/delivery/target-state-implementation/phases/11-evrun-full-evidence.md index 1954f09..7bbf47b 100644 --- a/docs/delivery/target-state-implementation/phases/11-evrun-full-evidence.md +++ b/docs/delivery/target-state-implementation/phases/11-evrun-full-evidence.md @@ -12,8 +12,9 @@ Issues work source, real Codex agent over the app-server transport, real executi containment, real GitHub Forge landing, integrity and redaction active — plus the adversarial probes EVRUN-partial could not claim (no-phone-home, multi-run idempotency). P11 first landed an honest blocked capture attempt; a later follow-up captured the combined real GitHub / real Codex / -real-host `open-pr` smoke. Strong no-phone-home, held-merge replay, and multi-run idempotency -remain open. Product code changes are not this phase's purpose. +real-host `open-pr` smoke, and this follow-up captured multi-run idempotency against a real +`open-pr` effect. Strong no-phone-home remains open. Product code changes are not this phase's +purpose. ## Background @@ -55,8 +56,9 @@ gated on this evidence path. This phase is the proof step: it converts "implemen - A successful capture would prove `SEC-2` (no-phone-home proven, not asserted), `RESUME-3` (no double effect against real systems), and `MERGE-2`/`FENCE-3` observations on a real path. - The later combined smoke removes the original "real Codex plus real GitHub path unproven" - blocker. P13/P14 status claims still need either stronger no-phone-home/idempotency evidence or - an explicit owner decision to defer those gates. + blocker, and the RESUME-3 smoke removes the repeated real-effect idempotency blocker. P13/P14 + status claims still need either stronger no-phone-home evidence or an explicit owner decision to + defer that gate. ## Technical Requirements @@ -87,8 +89,8 @@ gated on this evidence path. This phase is the proof step: it converts "implemen - **Requires:** P03 and P04 (hard). - **Benefits from:** P05 (block-surfacing and held-merge paths worth capturing in the same sweep). -- **Would unlock:** P13 (hard) and P14 status claims only after the remaining no-phone-home / - idempotency evidence is captured or explicitly deferred by the owner. +- **Would unlock:** P13 (hard) and P14 status claims only after the remaining no-phone-home + evidence is captured or explicitly deferred by the owner. - **Parallel:** P07–P10, P12 may proceed concurrently. ## Acceptance Criteria @@ -110,8 +112,8 @@ do not lower the bar for the stronger probes. **Current disposition: partially met.** The combined smoke demonstrates real Codex editing through the owned app-server transport, real macOS execution-host attestation with honest `weak` strength, real GitHub Issues / Forge effects, records integrity, and redaction. The - adversarial no-phone-home observation remains `ambiguous`, and multi-run idempotency against a - repeated real landed effect remains unproven. + RESUME-3 smoke demonstrates multi-run idempotency against a repeated real `open-pr` effect. The + adversarial no-phone-home observation remains `ambiguous`. 3. Hosted/remote operation and Windows behavior are explicitly restated as out of scope in the Limitations section. 4. README/AGENTS/product status lines about EVRUN reflect the new boundary — no claim exceeds diff --git a/docs/design/evidence/2026-07-06-p05-held-merge-smoke.md b/docs/design/evidence/2026-07-06-p05-held-merge-smoke.md new file mode 100644 index 0000000..f933060 --- /dev/null +++ b/docs/design/evidence/2026-07-06-p05-held-merge-smoke.md @@ -0,0 +1,117 @@ +--- +title: "2026-07-06 - P05 held-merge and block-surfacing smoke" +status: applied - coordinator privileged run 2026-07-06 +date: 2026-07-06 +--- + +# 2026-07-06 - P05 Held-Merge and Block-Surfacing Smoke + +## Probe + +This record captures a real GitHub Forge held-merge and blocked-story surfacing run against the +disposable sandbox repository `agentic-workflow-kit/jig-smoke-target`. The sandbox `main` branch +was made public and protected before the run: one approving review is required, admins are +enforced, and no required status checks are configured. That posture is intentionally left in +place as the future fixture state. + +The smoke used the opt-in scenario in +`packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts`. It first opened a sandbox PR +for a temporary branch, then attempted a real `merge` landing. GitHub refused the squash merge +because branch protection required review, and Jig recorded the landing as `done-not-landed` with +`mergeability: held-by-review`. The same run then exercised the real block-surfacing path and +verified that the PR, failed commit status, and PR comment were readable through GitHub. + +## Tool Version Pins + +| Tool | Version | +| ----------------- | ------------------------------------------ | +| Node.js | `v26.4.0` | +| pnpm | `11.9.0` | +| GitHub CLI (`gh`) | `2.87.2 (2026-02-20)` | +| Git | `2.53.0` | +| Host | `Darwin 25.5.0 RELEASE_ARM64_T6000 arm64` | +| Jig base checkout | `38c08da22563db6b06eb17315b9a46d81554f147` | +| Git transport | HTTPS with token-backed environment config | + +## Invocation + +The operator supplied GitHub credentials and the records-integrity key via environment variables. +Secret values were not printed. + +```bash +GITHUB_TOKEN="$(gh auth token)" \ +EVRUN_HELD_MERGE_SMOKE=1 \ +EVRUN_REAL_EFFECT_KEEP_ARTIFACTS=1 \ +JIG_GITHUB_ISSUES_REPOSITORY=agentic-workflow-kit/jig-smoke-target \ +JIG_GITHUB_ISSUES_LABEL=jig-candidate \ +JIG_RECORDS_INTEGRITY_KEY="" \ +JIG_RECORDS_INTEGRITY_KEY_ID="held-merge-smoke" \ +EVRUN_HELD_MERGE_EVIDENCE_OUT=/private/tmp/jig-held-merge-smoke-evidence.json \ +corepack pnpm exec vitest run packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts +``` + +Result: + +```text +Test Files 1 passed (1) +Tests 1 passed | 1 skipped (2) +Duration 10.59s +``` + +## Result Summary + +| Field | Value | +| ------------------------- | -------------------------------------------------------------------------------------- | +| Run ID | `run-p05-held-merge-smoke-1783358861177-61e9c545-37c3-4751-b058-bd58f594f7de` | +| Run status | `failure` (expected blocked-story tail after held merge) | +| Sandbox PR | `https://github.com/agentic-workflow-kit/jig-smoke-target/pull/20` | +| Held story | `HELD-MERGE` | +| Held outcome | `done-not-landed` | +| Mergeability | `held-by-review` | +| Branch protection | `enforceAdmins: true`, `requiredApprovingReviewCount: 1`, `requiredStatusChecks: null` | +| Block-surfacing story | `BLOCK-SURFACE` | +| Commit status observed | `jig/block` / `failure` | +| PR block comment observed | `true` | +| Integrity status | `verified` | +| `run.json` hash | `1b37e102440f23f4bd63a0bd9eae2fcaa3d64ce603adc8cc61e056aa7463b0b8` | +| `integrity` hash | `b56ef34a5b309363324d5dbcef0a06ad969b085c8d7e5baa0d65a4f7b3e043c0` | + +## Observed Live Classifier Fix + +The first live attempts exposed two GitHub refusal strings that were not classified correctly by +the previous adapter: + +- `the base branch policy prohibits the merge` +- `At least 1 approving review is required by reviewers with write access` + +This PR pins both messages in `packages/jig-sdk/tests/providers.real-forge.p7.unit.test.ts` and +classifies them as `held-by-review`. + +## Cleanup + +The successful smoke-created sandbox pull request was closed after capture and its branch was +deleted: `https://github.com/agentic-workflow-kit/jig-smoke-target/pull/20`. Diagnostic sandbox +PRs from failed capture attempts were also closed and their branches deleted. Branch protection on +`main` was intentionally left in place. + +## Redaction + +No tokens or credentials appear in this record. The sandbox repository name and PR URL are +intentionally cited. The smoke asserts serialized run artifacts and emitted evidence facts do not +include GitHub tokens, `GH_TOKEN`, `GITHUB_TOKEN`, token-backed Git config values, credential +URLs, or the records-integrity key. + +## Supported IDs + +- Supports `MERGE-5` for real blocked-PR surfacing: the run observed the PR, commit-status, and + comment legs against the disposable sandbox. +- Supports `MERGE-4` held-merge posture against a real protected branch: the run observed + `done-not-landed` and `held-by-review` without forcing a merge. +- Supports the front-door claim that held-merge and block-surfacing are real-effect proven, not + only unit-proven. + +## Limitations + +This is a scripted-agent smoke, not a real Codex app-server run. It does not prove SEC-2 +no-phone-home behavior, hosted/remote execution, Windows behavior, or package-publication posture. +It uses a disposable sandbox repository only; no production repository was touched. diff --git a/docs/design/evidence/2026-07-06-resume-idempotency-smoke.md b/docs/design/evidence/2026-07-06-resume-idempotency-smoke.md new file mode 100644 index 0000000..8da8967 --- /dev/null +++ b/docs/design/evidence/2026-07-06-resume-idempotency-smoke.md @@ -0,0 +1,109 @@ +--- +title: "2026-07-06 - RESUME-3 real-effect idempotency smoke" +status: applied - coordinator privileged run 2026-07-06 +date: 2026-07-06 +--- + +# 2026-07-06 - RESUME-3 Real-Effect Idempotency Smoke + +## Probe + +This record captures a real repeated-effect idempotency smoke against the disposable sandbox +repository `agentic-workflow-kit/jig-smoke-target`. The first pass created one real GitHub +`open-pr` landing for a temporary branch, then stopped at a safe unattended-park checkpoint on the +second story. The resume pass re-read the hosted branch head, recorded +`runner-action.skipped-repeated-effect` with `reason: already-landed`, and verified that the +sandbox still had exactly one PR for the branch. + +The smoke used the opt-in scenario in +`packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts`. + +## Tool Version Pins + +| Tool | Version | +| ----------------- | ------------------------------------------ | +| Node.js | `v26.4.0` | +| pnpm | `11.9.0` | +| GitHub CLI (`gh`) | `2.87.2 (2026-02-20)` | +| Git | `2.53.0` | +| Host | `Darwin 25.5.0 RELEASE_ARM64_T6000 arm64` | +| Jig base checkout | `38c08da22563db6b06eb17315b9a46d81554f147` | +| Git transport | HTTPS with token-backed environment config | + +## Invocation + +The operator supplied GitHub credentials and the records-integrity key via environment variables. +Secret values were not printed. + +```bash +GITHUB_TOKEN="$(gh auth token)" \ +EVRUN_RESUME_IDEMPOTENCY_SMOKE=1 \ +EVRUN_REAL_EFFECT_KEEP_ARTIFACTS=1 \ +JIG_GITHUB_ISSUES_REPOSITORY=agentic-workflow-kit/jig-smoke-target \ +JIG_GITHUB_ISSUES_LABEL=jig-candidate \ +JIG_RECORDS_INTEGRITY_KEY="" \ +JIG_RECORDS_INTEGRITY_KEY_ID="resume-idempotency-smoke" \ +EVRUN_RESUME_IDEMPOTENCY_EVIDENCE_OUT=/private/tmp/jig-resume-idempotency-smoke-evidence.json \ +corepack pnpm exec vitest run packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts +``` + +Result: + +```text +Test Files 1 passed (1) +Tests 1 passed | 1 skipped (2) +Duration 10.52s +``` + +## Result Summary + +| Field | Value | +| ----------------------------- | --------------------------------------------------------------------------------- | +| Run ID | `run-resume-idempotency-smoke-1783358911285-0e28c7f6-35de-462a-b1e8-7dac141d3b7d` | +| Final run status | `failure` (expected parked second story after repeated-effect check) | +| Branch | `resume-idempotency-smoke-1783358904804-7f3f44a7` | +| First-pass status | `failure` | +| First-pass stop reason | `unattended-park` | +| First-pass checkpoint | `after:RESUME-PARKED.parked` | +| First-pass PR count | `1` | +| Resume status | `failure` | +| Resume repeated-effect reason | `already-landed` | +| Repeated action | `open-pr` | +| Target ref | `refs/heads/resume-idempotency-smoke-1783358904804-7f3f44a7` | +| Target head | `9e5a8daa86f6672cb4549ef076b3dcc79a04201d` | +| Open-PR events for story | `1` | +| Post-resume PR count | `1` | +| Integrity status | `verified` | +| `run.json` hash | `627d7840fe76ea20edf26263f6a24865e580231de6159e45a5737ec2c6749f53` | +| `integrity` hash | `03ad8aafbc7fe069bc4ea028776cfa253e036b6b43c07c0aeb75c76a4863b053` | + +## Cleanup + +The smoke-created sandbox pull request was closed after capture and its branch was deleted: +`https://github.com/agentic-workflow-kit/jig-smoke-target/pull/22`. Branch protection on sandbox +`main` was left in place. + +## Redaction + +No tokens or credentials appear in this record. The sandbox repository name and temporary branch +name are intentionally cited. The smoke asserts serialized run artifacts and emitted evidence facts +do not include GitHub tokens, `GH_TOKEN`, `GITHUB_TOKEN`, token-backed Git config values, +credential URLs, or the records-integrity key. + +## Supported IDs + +- Supports `RESUME-3`: resume against a previously landed real `open-pr` effect recorded + `runner-action.skipped-repeated-effect` with `reason: already-landed`. +- Supports the no-double-effect claim for this real Forge path: only one PR existed for the branch + before and after resume, and the run record contains one `open-pr` landing event for the landed + story. +- Supports P11-AC-2 for the repeated-effect idempotency leg. + +## Limitations + +This is a scripted-agent smoke, not a real Codex app-server run. It proves repeated-effect +idempotency for a real GitHub `open-pr` landing, not for every possible landing action. The second +story intentionally parks to satisfy the resume precondition, so the final run status is +`failure`. It does not prove SEC-2 no-phone-home behavior, hosted/remote execution, Windows +behavior, or package-publication posture. It uses a disposable sandbox repository only; no +production repository was touched. diff --git a/docs/design/evidence/README.md b/docs/design/evidence/README.md index d026004..14696f2 100644 --- a/docs/design/evidence/README.md +++ b/docs/design/evidence/README.md @@ -29,6 +29,8 @@ guarantee, invariant, or decision IDs the evidence supports. | 2026-07-06 | [EVRUN-full capture attempt](./2026-07-06-evrun-full-capture-attempt.md) | P11 capture attempt showing successful narrow Codex app-server and real-host smokes, plus the exact missing-prerequisite blocker that kept the combined smoke unproven at that point. | | 2026-07-06 | [EVRUN-full combined smoke](./2026-07-06-evrun-full-smoke.md) | Successful combined GitHub Issues -> real Codex app-server -> real host -> GitHub Forge `open-pr` -> records-integrity smoke, with stronger no-phone-home/idempotency probes still open. | | 2026-07-06 | [P05 real Forge smoke rerun](./2026-07-06-p05-real-forge-smoke-rerun.md) | Current-checkout rerun of the real GitHub Issues -> GitHub Forge `open-pr` smoke against the disposable sandbox, with records integrity verified and explicit MERGE-5 limitations. | +| 2026-07-06 | [P05 held-merge and block-surfacing smoke](./2026-07-06-p05-held-merge-smoke.md) | Real protected-branch held merge plus PR/status/comment block surfacing against the disposable sandbox, closing the MERGE-5 real-effect evidence gap. | +| 2026-07-06 | [RESUME-3 real-effect idempotency smoke](./2026-07-06-resume-idempotency-smoke.md) | Real `open-pr` landing followed by resume no-op recognition with `already-landed`, proving the repeated-effect idempotency leg against the disposable sandbox. | ## N1a deferred probes @@ -47,5 +49,6 @@ The EVRUN-partial record was enough for the org M7 exit decision, but it is not 2026-07-06 P11 capture attempt records the original missing-prerequisite blocker. The later 2026-07-06 EVRUN-full combined smoke proves the combined real Codex, real GitHub, real execution-host, records-integrity, and redaction path for an operator-initiated disposable -`open-pr` run. Strong adversarial no-phone-home evidence, multi-run idempotency against a repeated -real effect, hosted/remote operation, and Windows behavior remain unproven. +`open-pr` run. The RESUME-3 smoke proves multi-run idempotency against a repeated real `open-pr` +effect. Strong adversarial no-phone-home evidence, hosted/remote operation, and Windows behavior +remain unproven. diff --git a/docs/design/security-model.md b/docs/design/security-model.md index 5b3ff43..a5e64e8 100644 --- a/docs/design/security-model.md +++ b/docs/design/security-model.md @@ -326,9 +326,11 @@ consolidation: - **EVRUN-full is only partially closed.** EVRUN-partial remains the M7 baseline, and the later EVRUN-full combined smoke proves the real Codex / real-host / real GitHub `open-pr` path against - a disposable sandbox. It still does not prove adversarial no-phone-home behavior or multi-run - idempotency against a repeated real effect ([`evidence/README.md`](./evidence/README.md); - [`evidence/2026-07-06-evrun-full-smoke.md`](./evidence/2026-07-06-evrun-full-smoke.md)). + a disposable sandbox. A later RESUME-3 smoke proves multi-run idempotency against a repeated real + `open-pr` effect. The remaining EVRUN evidence gap is adversarial no-phone-home behavior + ([`evidence/README.md`](./evidence/README.md); + [`evidence/2026-07-06-evrun-full-smoke.md`](./evidence/2026-07-06-evrun-full-smoke.md); + [`evidence/2026-07-06-resume-idempotency-smoke.md`](./evidence/2026-07-06-resume-idempotency-smoke.md)). - **No-phone-home is not yet adversarially probed.** The design requires proven, not asserted, confinement, but red-team adversarial probing of the no-phone-home boundary remains later integration work, not yet exercised ([`contracts/providers.md`](./contracts/providers.md) diff --git a/packages/jig-sdk/src/providers/real/forge.ts b/packages/jig-sdk/src/providers/real/forge.ts index e0a6cc5..dfa08dd 100644 --- a/packages/jig-sdk/src/providers/real/forge.ts +++ b/packages/jig-sdk/src/providers/real/forge.ts @@ -140,19 +140,13 @@ function mergeabilityForMessage( } if ( - normalized.includes('cannot be merged cleanly') || - normalized.includes('not mergeable') || - normalized.includes('merge conflict') || - normalized.includes('conflict') - ) { - return 'held-by-conflict'; - } - - if ( + normalized.includes('base branch policy') || + normalized.includes('policy prohibits the merge') || normalized.includes('protected branch') || normalized.includes('branch protection') || normalized.includes('required review') || normalized.includes('required approving review') || + normalized.includes('approving review is required') || normalized.includes('required status') || normalized.includes('required checks') || normalized.includes('review required') @@ -160,6 +154,15 @@ function mergeabilityForMessage( return 'held-by-review'; } + if ( + normalized.includes('cannot be merged cleanly') || + normalized.includes('not mergeable') || + normalized.includes('merge conflict') || + normalized.includes('conflict') + ) { + return 'held-by-conflict'; + } + return null; } diff --git a/packages/jig-sdk/tests/providers.real-forge.p7.unit.test.ts b/packages/jig-sdk/tests/providers.real-forge.p7.unit.test.ts index d1ed04e..9130333 100644 --- a/packages/jig-sdk/tests/providers.real-forge.p7.unit.test.ts +++ b/packages/jig-sdk/tests/providers.real-forge.p7.unit.test.ts @@ -709,6 +709,77 @@ test('P05 MERGE-4: a held merge records done-not-landed instead of forcing merge ); }); +test('P05 MERGE-5: base branch policy refusal is classified as review-held before conflict wording', async () => { + const execute: CommandExecutor = async (file, args) => { + if (file === 'gh' && args.join(' ') === 'pr view --json number,url,headRefName,headRefOid,baseRefName') { + return { + stdout: JSON.stringify({ + number: 18, + url: 'https://github.example/pull/18', + headRefName: 'phase-7', + headRefOid: 'held-head', + baseRefName: 'main', + }), + }; + } + + if (file === 'gh' && args.join(' ') === 'pr merge --squash --delete-branch=false') { + throw new Error( + [ + 'Command failed: gh pr merge --squash --delete-branch=false', + 'X Pull request agentic-workflow-kit/jig-smoke-target#18 is not mergeable: the base branch policy prohibits the merge.', + 'To have the pull request merged after all the requirements have been met, add the `--auto` flag.', + ].join('\n'), + ); + } + + throw new Error(`unexpected command: ${file} ${args.join(' ')}`); + }; + + const outcome = await createGitHubCommandTransport(execute).mergePullRequest({ + storyId: 'STORY-1', + action: 'merge', + }); + + assert.strictEqual(outcome.mergeability, 'held-by-review'); + assert.strictEqual(outcome.targetHead, 'held-head'); +}); + +test('P05 MERGE-5: approving-review GraphQL refusal is classified as review-held', async () => { + const execute: CommandExecutor = async (file, args) => { + if (file === 'gh' && args.join(' ') === 'pr view --json number,url,headRefName,headRefOid,baseRefName') { + return { + stdout: JSON.stringify({ + number: 19, + url: 'https://github.example/pull/19', + headRefName: 'phase-7', + headRefOid: 'review-held-head', + baseRefName: 'main', + }), + }; + } + + if (file === 'gh' && args.join(' ') === 'pr merge --squash --delete-branch=false') { + throw new Error( + [ + 'Command failed: gh pr merge --squash --delete-branch=false', + 'GraphQL: At least 1 approving review is required by reviewers with write access. (mergePullRequest)', + ].join('\n'), + ); + } + + throw new Error(`unexpected command: ${file} ${args.join(' ')}`); + }; + + const outcome = await createGitHubCommandTransport(execute).mergePullRequest({ + storyId: 'STORY-1', + action: 'merge', + }); + + assert.strictEqual(outcome.mergeability, 'held-by-review'); + assert.strictEqual(outcome.targetHead, 'review-held-head'); +}); + test('P7-AC-3: a land-then-relaunch is recognized from the records and is a recorded no-op', async () => { const calls: string[] = []; const priorEvents: RunEvent[] = [ diff --git a/packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts b/packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts new file mode 100644 index 0000000..0d858a1 --- /dev/null +++ b/packages/jig-sdk/tests/smoke/evrun-held-merge-resume.smoke.test.ts @@ -0,0 +1,581 @@ +import assert from 'node:assert'; +import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, test } from 'vitest'; +import { composeReferenceRun } from '../../src/bootstrap.js'; +import { LocalHarness } from '../../src/harness.js'; +import { validatePlanForScheduling } from '../../src/intake.js'; +import { INTEGRITY_FILE, verifyIntegritySidecar } from '../../src/integrity.js'; +import { RecordManager } from '../../src/records.js'; +import { resumeRunLoaded } from '../../src/resume.js'; +import type { ConfigDoc, PlanInstance, PolicyDoc, RunEvent } from '../../src/types.js'; +import { + emitJsonEvidenceFacts, + readSingleRunRecord, + SMOKE_REPOSITORY, + SMOKE_SEED_ISSUE_LABEL, + sha256, +} from './evrun-smoke.js'; + +const HELD_CHANGED_FILE = 'held-merge-smoke.txt'; +const HELD_BLOCKED_FILE = 'held-merge-blocked-smoke.txt'; +const RESUME_CHANGED_FILE = 'resume-idempotency-smoke.txt'; +const RESUME_RULE_FILE = 'policy/resume-idempotency-rule.json'; +const cleanupDirs: string[] = []; +const GIT_CONFIG_ENV_KEYS = [ + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_KEY_0', + 'GIT_CONFIG_VALUE_0', + 'GIT_CONFIG_KEY_1', + 'GIT_CONFIG_VALUE_1', +] as const; + +const HELD_PLAN_INSTANCE = { + plan: { + id: 'p05-held-merge-smoke', + version: 'execution-plan-shape-v0', + stories: [ + { + id: 'HELD-MERGE', + title: 'Exercise a protected-branch held merge', + scope: [HELD_CHANGED_FILE], + authority: { + requests: ['edit-files', 'run-checks'], + }, + }, + { + id: 'BLOCK-SURFACE', + title: 'Surface a blocked story on the same real GitHub branch', + scope: [HELD_BLOCKED_FILE], + authority: { + requests: ['edit-files', 'run-checks'], + }, + }, + ], + }, +} satisfies PlanInstance; + +const HELD_SCRIPTED_OUTPUT = { + stories: [ + { + storyId: 'HELD-MERGE', + outcome: 'success', + evidence: { + category: 'automated-check', + name: 'scripted-held-merge-smoke', + result: 'passed', + }, + changedFiles: [HELD_CHANGED_FILE], + }, + { + storyId: 'BLOCK-SURFACE', + outcome: 'success', + evidence: { + category: 'automated-check', + name: 'scripted-block-surface-smoke', + result: 'failed', + }, + changedFiles: [HELD_BLOCKED_FILE], + }, + ], +} satisfies Record; + +const RESUME_PLAN_INSTANCE = { + plan: { + id: 'resume-idempotency-smoke', + version: 'execution-plan-shape-v0', + stories: [ + { + id: 'RESUME-LANDED', + title: 'Create one real open-pr landing before a safe stopped checkpoint', + scope: [RESUME_CHANGED_FILE], + authority: { + requests: ['edit-files', 'run-checks'], + }, + }, + { + id: 'RESUME-PARKED', + title: 'Park on a rule-governing file after the real landing', + scope: [RESUME_RULE_FILE], + authority: { + requests: ['edit-files', 'run-checks'], + }, + }, + ], + }, +} satisfies PlanInstance; + +const RESUME_SCRIPTED_OUTPUT = { + stories: [ + { + storyId: 'RESUME-LANDED', + outcome: 'success', + evidence: { + category: 'automated-check', + name: 'scripted-resume-landing-smoke', + result: 'passed', + }, + changedFiles: [RESUME_CHANGED_FILE], + }, + { + storyId: 'RESUME-PARKED', + outcome: 'success', + evidence: { + category: 'automated-check', + name: 'scripted-resume-park-smoke', + result: 'passed', + }, + changedFiles: [RESUME_RULE_FILE], + }, + ], +} satisfies Record; + +afterEach(() => { + if (process.env.EVRUN_REAL_EFFECT_KEEP_ARTIFACTS) { + return; + } + for (const dir of cleanupDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function execGit(args: string[], cwd: string): string { + return execFileSync('git', args, { + cwd, + encoding: 'utf8', + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function execGhJson(args: string[], cwd: string): T { + return JSON.parse( + execFileSync('gh', args, { + cwd, + encoding: 'utf8', + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }), + ) as T; +} + +function restoreEnv(snapshot: Record): void { + for (const [key, value] of Object.entries(snapshot)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +function gitBasicAuthHeader(token: string): string { + const encoded = Buffer.from(`x-access-token:${token}`, 'utf8').toString('base64'); + return `AUTHORIZATION: basic ${encoded}`; +} + +function configureGitHttpsToken(token: string): void { + process.env.GIT_CONFIG_COUNT = '2'; + process.env.GIT_CONFIG_KEY_0 = 'credential.helper'; + process.env.GIT_CONFIG_VALUE_0 = ''; + process.env.GIT_CONFIG_KEY_1 = 'http.https://github.com/.extraHeader'; + process.env.GIT_CONFIG_VALUE_1 = gitBasicAuthHeader(token); + process.env.GIT_TERMINAL_PROMPT = '0'; +} + +function provisionCheckout(input: { + prefix: string; + fileName: string; + fileLines: string[]; + commitMessage: string; + createPullRequest?: boolean; +}): { + checkoutDir: string; + branchName: string; +} { + assert.ok(process.env.GITHUB_TOKEN, 'set GITHUB_TOKEN before provisioning the real-effect smoke checkout'); + const tempRoot = mkdtempSync(join(tmpdir(), `jig-${input.prefix}-`)); + cleanupDirs.push(tempRoot); + const checkoutDir = join(tempRoot, 'jig-smoke-target'); + const branchName = `${input.prefix}-${Date.now()}-${randomUUID().slice(0, 8)}`; + + process.env.GH_TOKEN = process.env.GITHUB_TOKEN; + configureGitHttpsToken(process.env.GITHUB_TOKEN); + + execFileSync( + 'git', + ['-c', 'credential.helper=', 'clone', `https://github.com/${SMOKE_REPOSITORY}.git`, checkoutDir], + { + encoding: 'utf8', + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + execGit(['checkout', '-b', branchName], checkoutDir); + execGit(['config', 'user.name', 'jig smoke runner'], checkoutDir); + execGit(['config', 'user.email', 'jig-smoke-runner@example.invalid'], checkoutDir); + + writeFileSync(join(checkoutDir, input.fileName), [...input.fileLines, `branch=${branchName}`, ''].join('\n')); + execGit(['add', input.fileName], checkoutDir); + execGit(['commit', '-m', input.commitMessage], checkoutDir); + execGit(['push', '--set-upstream', 'origin', `HEAD:${branchName}`], checkoutDir); + if (input.createPullRequest) { + execFileSync('gh', ['pr', 'create', '--fill'], { + cwd: checkoutDir, + encoding: 'utf8', + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } + + return { checkoutDir, branchName }; +} + +function smokeConfig(recordDir: string): ConfigDoc { + return { + runner: { + mode: 'local-real-provider-smoke', + recordDir, + }, + drivers: { + agent: 'scripted-stub', + executionHost: 'local', + forge: 'github', + workSource: 'reference', + }, + }; +} + +function smokePolicy(extraRules: NonNullable['rules'] = {}): PolicyDoc { + return { + version: 'policy-v0', + policy: { + id: 'real-effect-tail-smoke-policy', + rules: { + allowLocalDryRun: true, + ...extraRules, + }, + }, + }; +} + +function findEvent(record: { events: RunEvent[] }, family: string, storyId?: string): RunEvent { + const event = record.events.find( + (candidate) => candidate.family === family && (!storyId || candidate.storyId === storyId), + ); + assert.ok(event, `expected ${family}${storyId ? ` for ${storyId}` : ''}`); + return event; +} + +function findLandingHeldEvent(record: { events: RunEvent[] }, storyId: string): RunEvent { + const event = record.events.find( + (candidate) => + candidate.family === 'story.done' && + candidate.storyId === storyId && + typeof candidate.mergeability === 'string' && + typeof candidate.targetHead === 'string', + ); + assert.ok(event, `expected held landing story.done for ${storyId}`); + return event; +} + +function uniquePrNumbers(record: { events: RunEvent[] }): number[] { + return [ + ...new Set( + record.events + .map((event) => event.prNumber) + .filter((prNumber): prNumber is number => typeof prNumber === 'number'), + ), + ]; +} + +function assertRequiredEnv(): void { + assert.ok(process.env.GITHUB_TOKEN, 'set GITHUB_TOKEN for the sandbox GitHub smoke run'); + assert.strictEqual( + process.env.JIG_GITHUB_ISSUES_REPOSITORY ?? SMOKE_REPOSITORY, + SMOKE_REPOSITORY, + 'real-effect smoke must target the sandbox repo', + ); + assert.strictEqual( + process.env.JIG_GITHUB_ISSUES_LABEL ?? SMOKE_SEED_ISSUE_LABEL, + SMOKE_SEED_ISSUE_LABEL, + 'unexpected GitHub Issues label override', + ); + assert.ok(process.env.JIG_RECORDS_INTEGRITY_KEY, 'set JIG_RECORDS_INTEGRITY_KEY for the sidecar'); +} + +function branchProtectionSummary(cwd: string): Record { + const protection = execGhJson>( + ['api', `repos/${SMOKE_REPOSITORY}/branches/main/protection`], + cwd, + ); + const reviews = protection.required_pull_request_reviews as Record | undefined; + const admins = protection.enforce_admins as Record | undefined; + return { + enforceAdmins: admins?.enabled === true, + requiredApprovingReviewCount: reviews?.required_approving_review_count, + dismissStaleReviews: reviews?.dismiss_stale_reviews, + requiredStatusChecks: protection.required_status_checks ?? null, + }; +} + +function assertBlockSurfaceOnGitHub(input: { cwd: string; prNumber: number; targetHead: string }): { + statusContext: string; + statusState: string; + commentFound: boolean; +} { + const statuses = execGhJson>>( + ['api', `repos/${SMOKE_REPOSITORY}/commits/${input.targetHead}/statuses`], + input.cwd, + ); + const blockStatus = statuses.find((status) => status.context === 'jig/block'); + assert.ok(blockStatus, 'expected readable jig/block commit status on the surfaced head'); + assert.strictEqual(blockStatus.state, 'failure'); + + const comments = execGhJson>>( + ['api', `repos/${SMOKE_REPOSITORY}/issues/${input.prNumber}/comments`], + input.cwd, + ); + const commentFound = comments.some( + (comment) => typeof comment.body === 'string' && comment.body.includes('evidence-gate-failed'), + ); + assert.strictEqual(commentFound, true, 'expected block-surfacing PR comment'); + + return { + statusContext: String(blockStatus.context), + statusState: String(blockStatus.state), + commentFound, + }; +} + +function listPrsForBranch(cwd: string, branchName: string): Array> { + return execGhJson>>( + [ + 'pr', + 'list', + '--repo', + SMOKE_REPOSITORY, + '--head', + branchName, + '--state', + 'all', + '--json', + 'number,url,headRefName,state', + ], + cwd, + ); +} + +describe.skipIf(!process.env.EVRUN_HELD_MERGE_SMOKE)('P05 held-merge real-effect smoke', () => { + test('observes protected-branch held merge and real block-surfacing PR, status, and comment', async () => { + assertRequiredEnv(); + const previousCwd = process.cwd(); + const envSnapshot = { + GIT_ASKPASS: process.env.GIT_ASKPASS, + GIT_TERMINAL_PROMPT: process.env.GIT_TERMINAL_PROMPT, + GH_TOKEN: process.env.GH_TOKEN, + ...Object.fromEntries(GIT_CONFIG_ENV_KEYS.map((key) => [key, process.env[key]])), + }; + const recordDir = mkdtempSync(join(tmpdir(), 'jig-held-merge-records-')); + cleanupDirs.push(recordDir); + + try { + const { checkoutDir, branchName } = provisionCheckout({ + prefix: 'held-merge-smoke', + fileName: HELD_CHANGED_FILE, + fileLines: ['P05 held merge smoke scripted edit', 'landing-leg=protected-branch-merge'], + commitMessage: 'test: held merge smoke edit', + createPullRequest: true, + }); + process.chdir(checkoutDir); + const config = smokeConfig(recordDir); + const policy = smokePolicy(); + const composed = await composeReferenceRun({ + config, + planInstance: HELD_PLAN_INSTANCE, + scriptedOutput: HELD_SCRIPTED_OUTPUT, + }); + const harness = new LocalHarness(composed.agent, new RecordManager({ redaction: composed.redaction }), null, { + forge: composed.forge, + blockSurface: composed.blockSurface, + landingAction: 'merge', + }); + + const status = await harness.run(validatePlanForScheduling(HELD_PLAN_INSTANCE), config, policy); + assert.strictEqual(status, 'failure'); + + const { runDir, record, runJsonBytes, integrityJsonBytes } = readSingleRunRecord(recordDir); + const held = findLandingHeldEvent(record, 'HELD-MERGE'); + assert.strictEqual(held.outcome, 'done-not-landed'); + assert.strictEqual(held.mergeability, 'held-by-review'); + assert.strictEqual(held.targetRef, `refs/heads/${branchName}`); + assert.strictEqual(typeof held.targetHead, 'string'); + + const openedPr = findEvent(record, 'runner-action.opened-pr', 'BLOCK-SURFACE'); + const postedStatus = findEvent(record, 'runner-action.posted-status', 'BLOCK-SURFACE'); + findEvent(record, 'runner-action.posted-comment', 'BLOCK-SURFACE'); + assert.strictEqual(typeof openedPr.prNumber, 'number'); + assert.strictEqual(postedStatus.targetHead, openedPr.targetHead); + assert.strictEqual(typeof postedStatus.targetHead, 'string'); + const surface = assertBlockSurfaceOnGitHub({ + cwd: checkoutDir, + prNumber: openedPr.prNumber as number, + targetHead: postedStatus.targetHead as string, + }); + + const integrityStatus = verifyIntegritySidecar(runDir, { expected: true }).status; + assert.strictEqual(integrityStatus, 'verified'); + assert.ok(readFileSync(join(runDir, INTEGRITY_FILE), 'utf8').includes('"hmac"')); + + if (process.env.EVRUN_HELD_MERGE_EVIDENCE_OUT) { + emitJsonEvidenceFacts(process.env.EVRUN_HELD_MERGE_EVIDENCE_OUT, { + runId: record.run.id, + runStatus: record.run.status, + branchName, + branchProtection: branchProtectionSummary(checkoutDir), + heldMerge: { + storyId: held.storyId, + outcome: held.outcome, + mergeability: held.mergeability, + targetRef: held.targetRef, + targetHead: held.targetHead, + prNumber: held.prNumber, + prUrl: held.prUrl, + }, + blockSurface: { + storyId: openedPr.storyId, + prNumber: openedPr.prNumber, + prUrl: openedPr.prUrl, + statusContext: surface.statusContext, + statusState: surface.statusState, + commentFound: surface.commentFound, + }, + integrityStatus, + runJsonSha256: sha256(runJsonBytes), + integrityJsonSha256: sha256(integrityJsonBytes), + }); + } + } finally { + process.chdir(previousCwd); + restoreEnv(envSnapshot); + } + }, 120_000); +}); + +describe.skipIf(!process.env.EVRUN_RESUME_IDEMPOTENCY_SMOKE)('RESUME-3 real-effect idempotency smoke', () => { + test('resuming after a real open-pr landing records already-landed and does not create a second PR', async () => { + assertRequiredEnv(); + const previousCwd = process.cwd(); + const envSnapshot = { + GIT_ASKPASS: process.env.GIT_ASKPASS, + GIT_TERMINAL_PROMPT: process.env.GIT_TERMINAL_PROMPT, + GH_TOKEN: process.env.GH_TOKEN, + ...Object.fromEntries(GIT_CONFIG_ENV_KEYS.map((key) => [key, process.env[key]])), + }; + const recordDir = mkdtempSync(join(tmpdir(), 'jig-resume-idempotency-records-')); + cleanupDirs.push(recordDir); + + try { + const { checkoutDir, branchName } = provisionCheckout({ + prefix: 'resume-idempotency-smoke', + fileName: RESUME_CHANGED_FILE, + fileLines: ['RESUME-3 idempotency smoke scripted edit', 'landing-leg=open-pr-before-resume'], + commitMessage: 'test: resume idempotency smoke edit', + }); + process.chdir(checkoutDir); + const config = smokeConfig(recordDir); + const policy = smokePolicy({ + ruleGoverningSurfaces: [RESUME_RULE_FILE], + }); + const composed = await composeReferenceRun({ + config, + planInstance: RESUME_PLAN_INSTANCE, + scriptedOutput: RESUME_SCRIPTED_OUTPUT, + }); + const harness = new LocalHarness(composed.agent, new RecordManager({ redaction: composed.redaction }), null, { + forge: composed.forge, + blockSurface: composed.blockSurface, + landingAction: 'open-pr', + }); + + const firstStatus = await harness.run(validatePlanForScheduling(RESUME_PLAN_INSTANCE), config, policy); + assert.strictEqual(firstStatus, 'failure'); + const firstRecord = readSingleRunRecord(recordDir).record; + const firstPrs = listPrsForBranch(checkoutDir, branchName); + assert.strictEqual(firstPrs.length, 1, 'first pass should create exactly one sandbox PR for the branch'); + assert.ok(findEvent(firstRecord, 'runner-action.opened-pr', 'RESUME-LANDED')); + const stopped = findEvent(firstRecord, 'run.stopped'); + assert.strictEqual(stopped.reason, 'unattended-park'); + assert.strictEqual(stopped.checkpoint, 'after:RESUME-PARKED.parked'); + + const resumeStatus = await resumeRunLoaded({ + runDir: readSingleRunRecord(recordDir).runDir, + scriptedOutput: RESUME_SCRIPTED_OUTPUT, + config, + policy, + planInstance: RESUME_PLAN_INSTANCE, + workSourceTransport: { + fetchCandidates: async () => [ + { + sourceSystem: 'github-issues', + identifier: 'resume-idempotency-smoke-local-candidate', + planInstance: RESUME_PLAN_INSTANCE, + }, + ], + }, + }); + assert.strictEqual(resumeStatus, 'failure'); + + const { record, runJsonBytes, integrityJsonBytes, runDir } = readSingleRunRecord(recordDir); + const skipped = findEvent(record, 'runner-action.skipped-repeated-effect', 'RESUME-LANDED'); + assert.strictEqual(skipped.reason, 'already-landed'); + assert.strictEqual(skipped.action, 'open-pr'); + assert.strictEqual(skipped.landingKind, 'open-pr'); + const openedPrEvents = record.events.filter( + (event) => event.family === 'runner-action.opened-pr' && event.storyId === 'RESUME-LANDED', + ); + assert.strictEqual(openedPrEvents.length, 1, 'resume must not record a second open-pr landing'); + const prNumbers = uniquePrNumbers(record); + assert.strictEqual(prNumbers.length, 1, 'run record should mention one sandbox PR number'); + const prsAfterResume = listPrsForBranch(checkoutDir, branchName); + assert.strictEqual(prsAfterResume.length, 1, 'resume must not create a second sandbox PR for the branch'); + + const integrityStatus = verifyIntegritySidecar(runDir, { expected: true }).status; + assert.strictEqual(integrityStatus, 'verified'); + assert.ok(readFileSync(join(runDir, INTEGRITY_FILE), 'utf8').includes('"hmac"')); + + if (process.env.EVRUN_RESUME_IDEMPOTENCY_EVIDENCE_OUT) { + emitJsonEvidenceFacts(process.env.EVRUN_RESUME_IDEMPOTENCY_EVIDENCE_OUT, { + runId: record.run.id, + runStatus: record.run.status, + branchName, + firstPass: { + status: firstStatus, + stoppedReason: stopped.reason, + checkpoint: stopped.checkpoint, + prCountForBranch: firstPrs.length, + }, + resume: { + status: resumeStatus, + skippedReason: skipped.reason, + action: skipped.action, + landingKind: skipped.landingKind, + targetRef: skipped.targetRef, + targetHead: skipped.targetHead, + openPrEventCountForLandingStory: openedPrEvents.length, + prCountForBranchAfterResume: prsAfterResume.length, + }, + integrityStatus, + runJsonSha256: sha256(runJsonBytes), + integrityJsonSha256: sha256(integrityJsonBytes), + }); + } + } finally { + process.chdir(previousCwd); + restoreEnv(envSnapshot); + } + }, 120_000); +}); diff --git a/packages/jig-sdk/tests/smoke/evrun-smoke.ts b/packages/jig-sdk/tests/smoke/evrun-smoke.ts index 284bd3b..f4f09f6 100644 --- a/packages/jig-sdk/tests/smoke/evrun-smoke.ts +++ b/packages/jig-sdk/tests/smoke/evrun-smoke.ts @@ -52,7 +52,7 @@ export interface EvRunEvidenceFacts { }; } -function sha256(bytes: Buffer): string { +export function sha256(bytes: Buffer): string { return createHash('sha256').update(bytes).digest('hex'); } @@ -83,6 +83,12 @@ export function assertSerializedTokenClean(serialized: string): void { assert.doesNotMatch(serialized, /https:\/\/[^:\s/]+:[^@\s/]+@/i); } +export function emitJsonEvidenceFacts(outPath: string, facts: Record): void { + const serialized = `${JSON.stringify(facts, null, 2)}\n`; + assertSerializedTokenClean(serialized); + writeFileSync(outPath, serialized); +} + function assertArtifactTokenClean(runDir: string): { runJson: string; runJsonBytes: Buffer;