diff --git a/README.md b/README.md index db81148..c9026f6 100644 --- a/README.md +++ b/README.md @@ -96,10 +96,18 @@ pnpm install --frozen-lockfile pnpm check ``` -`pnpm check` runs formatting validation plus the eval package's deterministic checks: -skill/profile static checks, Vitest fixture tests, and the seeded `enforce-architecture` -dependency-cruiser evals. Manual or model-graded evals use the root `pnpm eval:*` commands backed -by `@agentic-workflow-kit/eval-kit`. +`pnpm check` is the fast repository gate. It runs only offline/static/local deterministic checks: +formatting validation, root-visible contract checks, adapter import and syntax validation, fixture +manifest validation, Vitest fixture tests, and the seeded `enforce-architecture` dependency-cruiser +evals. It must not run Promptfoo provider calls, Codex/OpenAI generation, LLM judge coverage, +pairwise judging, long replay suites, or anything requiring auth, network, model calls, or manual +calibration. + +Before significant changes to skills, methodology templates, fixtures, or eval logic, run the +deterministic local eval slice that matches the change. Use `pnpm eval:case -- --case +--candidate ` for a targeted product-to-design case, then widen back to `pnpm check`. +Manual model-assisted evals use direct `pnpm exec eval-kit ...` commands and remain advisory outside +CI. ## Methodology model diff --git a/docs/design/evaluation-strategy.md b/docs/design/evaluation-strategy.md index f87f970..4906990 100644 --- a/docs/design/evaluation-strategy.md +++ b/docs/design/evaluation-strategy.md @@ -35,9 +35,37 @@ The design principle is: - Do not treat one public reference design as the only correct answer. - Do not use broad 1-10 scores as release gates. - Do not allow an LLM judge verdict to override deterministic source or schema failures. +- Do not put Promptfoo, Codex/OpenAI generation, LLM judge coverage, pairwise judging, long replay + suites, or any auth/network/model-assisted eval into `pnpm check` or CI. - Do not add private application names or private repository facts to eval fixtures. - Do not require every eval case to exercise every skill. +## Gate Policy + +The global eval policy has three lanes: + +| Lane | Purpose | Runs in `pnpm check` / CI | +| -------------------- | -------------------------------------------- | ------------------------- | +| Fast repository gate | Offline/static/local deterministic checks | Yes | +| Local on-demand | Deterministic case runs before major changes | No, run explicitly | +| Manual advisory | Promptfoo/Codex/model-assisted evals | No | + +`pnpm check` is limited to format/lint checks, static docs/schema validation, adapter import and +syntax validation, fixture manifest validation, local unit tests for graders/helpers, +non-online deterministic structural checks, and seeded local fixture checks that do not call +external providers. + +Local on-demand deterministic evals should run before significant changes to skills, methodology +templates, fixtures, graders, or eval policy. Prefer the smallest targeted `pnpm eval:case` or +individual deterministic eval command that exercises the changed behavior, then widen back to +`pnpm check`. + +Manual/advisory evals cover model-assisted candidate generation, pointwise LLM judge coverage, +pairwise judging, long product-to-design or product-to-plan session evals, expensive full-case replay +suites, and anything requiring auth, network, model calls, or manual calibration. These methods must +remain disabled by default in `evals/eval-kit.config.json` and require an intentional local edit +before use. + ## Evaluation Pyramid Cheap, deterministic checks sit at the base. Expensive product-to-design case replays sit at the @@ -258,6 +286,9 @@ comparison anchors, not answer keys. ### LLM Semantic Judge +LLM semantic judging is manual and advisory. It is never part of `pnpm check` or CI, and its config +method stays disabled by default. + Use an LLM judge only where deterministic and fact graders cannot reasonably decide: - coherence of bounded contexts; @@ -411,13 +442,16 @@ Non-functional requirements: ## Verification Plan -1. Keep `pnpm check` as the required repository gate. +1. Keep `pnpm check` as the required repository gate and keep it offline/static/local + deterministic-only. 2. Add static checks when a new case format is introduced. 3. Add at least one deliberately failing fixture for every deterministic rule. -4. Run a first golden-set pass manually before trusting any LLM-judge verdict. -5. Re-run pairwise regression cases before changing methodology templates, review rubrics, or - orchestration behavior. -6. Review eval failures as design signals first; only relax rubrics after confirming the expected +4. Run targeted deterministic `pnpm eval:case` checks before significant skill, methodology, + fixture, grader, or eval-policy changes. +5. Run a first golden-set pass manually before trusting any LLM-judge verdict. +6. Re-run pairwise regression cases manually before accepting methodology-template, review-rubric, + or orchestration behavior changes that need semantic comparison. +7. Review eval failures as design signals first; only relax rubrics after confirming the expected case fact is wrong or too narrow. ## Reviewer Workflow diff --git a/docs/design/promptfoo-codex-eval-pilot.md b/docs/design/promptfoo-codex-eval-pilot.md index 10ceecb..a40a216 100644 --- a/docs/design/promptfoo-codex-eval-pilot.md +++ b/docs/design/promptfoo-codex-eval-pilot.md @@ -2,11 +2,15 @@ ## Goal -Build a small, repeatable eval pilot that proves `technical-design` can run an actual design task, -grade the generated design, judge it semantically, and produce a readable final report. +Build a small, repeatable manual eval pilot that proves `technical-design` can run an actual design +task, grade the generated design, judge it semantically, and produce a readable final report. The pilot starts with one case: `case-tiny-laundry-pickup-v1`. +This pilot is advisory. It must stay outside `pnpm check`, CI, and required contributor gates because +it uses local auth, Promptfoo provider calls, Codex/OpenAI model-assisted generation, and LLM +judging. + ## What We Need The pilot needs one end-to-end eval flow: @@ -44,6 +48,8 @@ judge rubric, result bundle convention, and final reporting story. false-fail behavior. - The reference design is a comparison anchor, not a certified great solution or exact-output target. - Model-graded commands remain manual and outside `pnpm check`. +- Model-assisted methods remain disabled by default and require an intentional local config edit + before each manual run. ## Tooling Decision @@ -145,9 +151,10 @@ Root-level docs may still describe the evaluation strategy, because that strateg product is designed and governed. Executable eval assets, fixture data, generated outputs, schemas, and eval-specific implementation docs belong in `evals/`. -The root package stays small. It keeps `check` as the public deterministic repo gate and exposes -only the basic deterministic eval-kit commands. Model-assisted commands remain direct `eval-kit` -CLI usage because they require local auth, explicit calibration, and human interpretation: +The root package stays small. It keeps `check` as the public deterministic repo gate and exposes only +offline/static/local deterministic scripts such as `pnpm eval:case`, `pnpm eval:unit`, and +`pnpm eval:enforce`. Model-assisted commands remain direct `eval-kit` CLI usage because they require +local auth, explicit calibration, and human interpretation: ```text pnpm eval:doctor @@ -159,8 +166,9 @@ pnpm exec eval-kit judge-pairwise --config evals/eval-kit.config.json pnpm exec eval-kit report --config evals/eval-kit.config.json ``` -The root `pnpm check` command remains the public repository gate. It should run formatting and the -eval package's deterministic check. +The root `pnpm check` command remains the public repository gate. It should run formatting, +root-visible static checks, adapter import and syntax validation, fixture validation, local unit +tests, and seeded deterministic fixture checks only. ## Planned Suite Shape @@ -251,8 +259,11 @@ The final report command combines all run bundles and states: ## Acceptance Criteria - `pnpm check` remains green and deterministic-only. +- `pnpm check` does not invoke generation, LLM judge coverage, pairwise judging, Promptfoo provider + calls, network/auth/model calls, long replay suites, or manual calibration. - The pilot does not require or document `OPENAI_API_KEY`. - `codex login status` is checked before model-graded runs. +- Model-assisted methods remain disabled by default in `evals/eval-kit.config.json`. - Promptfoo, not shell wrappers, owns provider execution, assertions, and JSON/HTML result exports. - Candidate generation produces a non-empty Markdown design for `case-tiny-laundry-pickup-v1`. - The existing deterministic grader runs against the generated candidate. diff --git a/evals/README.md b/evals/README.md index 907f2f7..b26ab25 100644 --- a/evals/README.md +++ b/evals/README.md @@ -60,7 +60,7 @@ The product-to-design cases graded by `eval:case`: - `case-openfeature-evaluation-api-v1` - `case-tiny-laundry-pickup-v1` -## Running checks +## Running Checks The repository gate: @@ -68,6 +68,13 @@ The repository gate: pnpm check ``` +`pnpm check` is intentionally fast, offline, and deterministic. It is limited to formatting, +static docs/schema validation, adapter import and syntax validation, fixture manifest validation, +local unit tests for graders/helpers, seeded local fixture checks, and the deterministic +`enforce-architecture` eval. It does not call Promptfoo providers, Codex/OpenAI models, LLM judges, +pairwise judging, long product-to-design replay suites, or anything that needs auth, network, model +calls, or manual calibration. + Individual eval checks: ```bash @@ -82,6 +89,9 @@ Run a single deterministic case: pnpm eval:case -- --case case-tiny-laundry-pickup-v1 --candidate evals/cases/case-tiny-laundry-pickup-v1/reference-design.md ``` +Before significant skill, methodology, fixture, or grader changes, run the smallest deterministic +case or eval slice that exercises the changed behavior, then widen to `pnpm check` before review. + ## Reading the evals 1. `eval-kit.config.json` — suite config; start here to understand what runs. @@ -90,10 +100,10 @@ pnpm eval:case -- --case case-tiny-laundry-pickup-v1 --candidate evals/cases/cas 4. `cases/README.md` — authoring contract for product-to-design cases. 5. `results/README.md` — conventions for generated run outputs. -## Manual Product-to-Design Flow +## Manual Model-Assisted Product-to-Design Flow Manual model-graded runs use Promptfoo with the Codex App Server provider. They are advisory only -and not part of `pnpm check`. +and must stay outside `pnpm check`, CI, and required contributor gates. Check local auth: diff --git a/package.json b/package.json index 88a7239..863616e 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "eval:enforce": "node scripts/run_enforce_eval.mjs" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.1", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2", "ajv": "^8.17.1", "dependency-cruiser": "^18.0.0", "prettier": "^3.6.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f12c80..0d91af4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.1 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89 + specifier: github:agentic-workflow-kit/eval-kit#v0.1.2 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974 ajv: specifier: ^8.17.1 version: 8.20.0 @@ -32,9 +32,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89': - resolution: {gitHosted: true, integrity: sha512-4sSEGmu2oZ0wiz3V0TEQ6qzq/G+YBppcfbKT3QRCWmakRx7kzqY93VFl5q9Wfw6GLUZ8OsETv8yiCQs3sGQZ3Q==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89} - version: 0.1.1 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': + resolution: {gitHosted: true, integrity: sha512-XNdPutRH/06q2cI2jxaZCBgh583JVoAc8W9uipxnpgNzcovcSY8TK5feFnk4L32RG5oGdl8wFGzpnXlFsAaRnA==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974} + version: 0.1.2 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -4202,7 +4202,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': dependencies: ajv: 8.20.0