diff --git a/CHANGELOG.md b/CHANGELOG.md index 944000f..62f6c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,20 @@ documented with migration notes. - Additional docs for suite-specific adoption. - Better compatibility tests for Promptfoo variable contracts. +## [0.1.4] - 2026-07-04 + +### Fixed + +- Made model-assisted CLI commands fail closed unless their method is explicitly enabled in config. +- Added regression coverage for omitted model-assisted method config and method entries without + `enabled`. + +### Notes + +- Deterministic `run-case` and manual `report` remain compatible unless explicitly disabled. +- No npm package is published. +- Consumers may pin `github:agentic-workflow-kit/eval-kit#v0.1.4`. + ## [0.1.3] - 2026-07-04 ### Added @@ -77,7 +91,8 @@ documented with migration notes. - Suite-specific presets remain deferred. - Consumer repos own their own semantics, prompts, cases, and pass/fail policies. -[Unreleased]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.3...main +[Unreleased]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.4...main +[0.1.4]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.3...v0.1.4 [0.1.3]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.2...v0.1.3 [0.1.2]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.0...v0.1.1 diff --git a/README.md b/README.md index 9cce4e3..5819f13 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Shared evaluation infrastructure for `agentic-workflow-kit` repositories. ```json { "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3" + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4" } } ``` @@ -70,7 +70,7 @@ Install from a Git tag in a consumer repo: ```json { "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3" + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4" }, "scripts": { "eval:doctor": "eval-kit doctor --config evals/eval-kit.config.json", @@ -191,6 +191,7 @@ v0.1.0 v0.1.1 v0.1.2 v0.1.3 +v0.1.4 v0.2.0 ``` diff --git a/docs/design/consumer-integration.md b/docs/design/consumer-integration.md index cae5127..617a9b1 100644 --- a/docs/design/consumer-integration.md +++ b/docs/design/consumer-integration.md @@ -9,7 +9,7 @@ Consumer repos should adopt eval-kit through a pinned Git tag and keep their eva ```json { "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3" + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4" } } ``` diff --git a/docs/guides/consumer-integration.md b/docs/guides/consumer-integration.md index 7cdd389..bffde6b 100644 --- a/docs/guides/consumer-integration.md +++ b/docs/guides/consumer-integration.md @@ -18,7 +18,7 @@ If you cannot state the eval goal, do not bootstrap a suite yet. Empty harnesses ```json { "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3" + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4" } } ``` diff --git a/docs/guides/quickstart.md b/docs/guides/quickstart.md index 14d37ab..a6a0fbc 100644 --- a/docs/guides/quickstart.md +++ b/docs/guides/quickstart.md @@ -7,7 +7,7 @@ This guide adds a generic deterministic eval suite to a consumer repo. ```json { "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3" + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4" } } ``` diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 2971c04..3ac936a 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -22,6 +22,15 @@ Run-producing commands fail closed when their configured method is explicitly di Disabled commands exit before Codex auth checks, Promptfoo execution, model provider calls, adapter hooks, or result artifact writes. +Model-assisted commands also require explicit enablement in the selected config before required +model/candidate arguments are validated: + +| Command | Required enablement | +| ---------------- | ------------------------------------- | +| `generate` | `methods.generate.enabled=true` | +| `judge-coverage` | `methods.judge_coverage.enabled=true` | +| `judge-pairwise` | `methods.judge_pairwise.enabled=true` | + ## `init` Create a deterministic generic eval skeleton. @@ -110,7 +119,7 @@ pnpm exec eval-kit generate \ ``` Requires Promptfoo and local Codex auth. This is manual/advisory evidence, not a default CI gate. -Fails closed before those checks when `methods.generate.enabled` is explicitly `false`. +Fails closed before those checks unless `methods.generate.enabled` is explicitly `true`. ## `judge-coverage` @@ -127,7 +136,7 @@ pnpm exec eval-kit judge-coverage \ [--config ] ``` -Fails closed when `methods.judge_coverage.enabled` is explicitly `false`. +Fails closed unless `methods.judge_coverage.enabled` is explicitly `true`. ## `judge-pairwise` @@ -151,7 +160,7 @@ pnpm exec eval-kit judge-pairwise \ [--config ] ``` -Fails closed when `methods.judge_pairwise.enabled` is explicitly `false`. +Fails closed unless `methods.judge_pairwise.enabled` is explicitly `true`. ## `report` diff --git a/docs/reference/config.md b/docs/reference/config.md index 147b6bb..6fee69c 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -69,11 +69,12 @@ lanes: - `run-case` is deterministic local on-demand evidence for semantic case portfolios. Run it before significant changes, but do not make long run-producing portfolios a default CI gate. - `generate`, `judge_coverage`, and `judge_pairwise` are manual/advisory model-assisted methods. - They require explicit local setup and must not require auth, network, Promptfoo provider calls, - Codex/OpenAI calls, LLM judging, or manual calibration in `pnpm check`. -- Run-producing commands fail closed when their method's `enabled` flag is explicitly `false`, - before Codex auth checks, Promptfoo execution, model provider calls, adapter hooks, or result - artifact writes. + They require explicit local setup, must be explicitly enabled in the selected config, and must not + require auth, network, Promptfoo provider calls, Codex/OpenAI calls, LLM judging, or manual + calibration in `pnpm check`. +- Run-producing commands fail closed before Codex auth checks, Promptfoo execution, model provider + calls, adapter hooks, or result artifact writes when disabled. Model-assisted commands also fail + closed unless their method's `enabled` flag is explicitly `true`. ## Path rules diff --git a/docs/reference/release-process.md b/docs/reference/release-process.md index ef0378e..d427df0 100644 --- a/docs/reference/release-process.md +++ b/docs/reference/release-process.md @@ -34,7 +34,7 @@ Consumers depend on tags like: Title: ```text -chore(release): v0.1.3 +chore(release): v0.1.4 ``` Required changes: @@ -63,18 +63,18 @@ git checkout main git pull --ff-only git rev-parse HEAD -git tag -a v0.1.3 -m "v0.1.3" -git push origin v0.1.3 +git tag -a v0.1.4 -m "v0.1.4" +git push origin v0.1.4 ``` Verify: ```bash -git rev-parse v0.1.3^{} -git show --no-patch --decorate v0.1.3 +git rev-parse v0.1.4^{} +git show --no-patch --decorate v0.1.4 ``` -`v0.1.3^{}` must point to the release commit. With an annotated tag, `git rev-parse v0.1.3` +`v0.1.4^{}` must point to the release commit. With an annotated tag, `git rev-parse v0.1.4` returns the tag object; `^{}` dereferences to the commit. ## GitHub Release @@ -93,7 +93,7 @@ For each consumer repo: ```json { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3" + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4" } ``` @@ -108,7 +108,7 @@ pnpm check 3. Run consumer smoke commands, for example in `technical-design`: ```bash -pnpm eval:case -- --case case-tiny-laundry-pickup-v1 --candidate evals/cases/case-tiny-laundry-pickup-v1/reference-design.md --run-id verify-eval-kit-v0.1.3 +pnpm eval:case -- --case case-tiny-laundry-pickup-v1 --candidate evals/cases/case-tiny-laundry-pickup-v1/reference-design.md --run-id verify-eval-kit-v0.1.4 ``` 4. Open a PR with dependency, lockfile, and any compatibility fixes. @@ -120,7 +120,7 @@ Do not move the tag. Create a new patch release: ```text -v0.1.3 -> v0.1.4 +v0.1.4 -> v0.1.5 ``` Then open consumer bump PRs. diff --git a/package.json b/package.json index 21d7d71..0829e31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentic-workflow-kit/eval-kit", - "version": "0.1.3", + "version": "0.1.4", "description": "Portable eval runner primitives for local eval suites.", "private": true, "type": "module", diff --git a/src/cli.mjs b/src/cli.mjs index 2cf8300..fd0fa48 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -17,12 +17,29 @@ import { validateFixtures, } from "./sdk.mjs"; -const requireEnabledMethod = (config, methodKey, commandName) => { - if (config.raw.methods?.[methodKey]?.enabled === false) { +const modelAssistedMethodKeys = new Set([ + "generate", + "judge_coverage", + "judge_pairwise", +]); + +const requireEnabledMethod = ( + config, + methodKey, + commandName, + { explicitEnableRequired = false } = {}, +) => { + const enabled = config.raw.methods?.[methodKey]?.enabled; + if (enabled === false) { throw new Error( `${commandName} is disabled by methods.${methodKey}.enabled=false`, ); } + if (explicitEnableRequired && enabled !== true) { + throw new Error( + `${commandName} requires methods.${methodKey}.enabled=true`, + ); + } }; const runProducingMethodByCommand = { @@ -36,7 +53,9 @@ const runProducingMethodByCommand = { const requireEnabledCommandMethod = (config, commandName) => { const methodKey = runProducingMethodByCommand[commandName]; if (methodKey) { - requireEnabledMethod(config, methodKey, commandName); + requireEnabledMethod(config, methodKey, commandName, { + explicitEnableRequired: modelAssistedMethodKeys.has(methodKey), + }); } }; diff --git a/src/sdk.mjs b/src/sdk.mjs index b935084..7fd3521 100644 --- a/src/sdk.mjs +++ b/src/sdk.mjs @@ -15,7 +15,7 @@ import { assertContainedPath, assertSafeId, toPosixPath } from "./paths.mjs"; const DEFAULT_SANDBOX_MODE = "read-only"; const DEFAULT_APPROVAL_POLICY = "never"; const RANDOMIZATION_METHOD = "sha256-seed-parity-v1"; -const EVAL_KIT_VERSION = "0.1.3"; +const EVAL_KIT_VERSION = "0.1.4"; const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); diff --git a/tests/cli.test.mjs b/tests/cli.test.mjs index 793104e..59387a1 100644 --- a/tests/cli.test.mjs +++ b/tests/cli.test.mjs @@ -36,6 +36,33 @@ const writeConfigWithDisabledMethod = (methodKey) => { return tempConfigPath; }; +const writeConfigOmittingMethod = (methodKey) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-kit-cli-")); + tempDirs.push(tempDir); + + const config = JSON.parse(fs.readFileSync(configPath, "utf8")); + delete config.methods[methodKey]; + + const tempConfigPath = path.join(tempDir, "eval-kit.config.json"); + fs.writeFileSync(tempConfigPath, `${JSON.stringify(config, null, 2)}\n`); + return tempConfigPath; +}; + +const writeConfigWithMethodWithoutEnabled = (methodKey) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-kit-cli-")); + tempDirs.push(tempDir); + + const config = JSON.parse(fs.readFileSync(configPath, "utf8")); + config.methods = { + ...config.methods, + [methodKey]: {}, + }; + + const tempConfigPath = path.join(tempDir, "eval-kit.config.json"); + fs.writeFileSync(tempConfigPath, `${JSON.stringify(config, null, 2)}\n`); + return tempConfigPath; +}; + const runCli = (args) => spawnSync(process.execPath, [cliPath, ...args], { cwd: packageRoot, @@ -75,4 +102,50 @@ describe("eval-kit CLI", () => { expect(result.stderr).not.toContain("missing required argument"); }, ); + + it.each([ + ["generate", "generate"], + ["judge-coverage", "judge_coverage"], + ["judge-pairwise", "judge_pairwise"], + ])( + "fails closed when %s method config is omitted", + (commandName, methodKey) => { + const omittedConfigPath = + methodKey in JSON.parse(fs.readFileSync(configPath, "utf8")).methods + ? writeConfigOmittingMethod(methodKey) + : configPath; + + const result = runCli([commandName, "--config", omittedConfigPath]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + `${commandName} requires methods.${methodKey}.enabled=true`, + ); + expect(result.stderr).not.toContain("missing required argument"); + }, + ); + + it.each([ + ["generate", "generate"], + ["judge-coverage", "judge_coverage"], + ["judge-pairwise", "judge_pairwise"], + ])( + "fails closed when %s method config omits enabled", + (commandName, methodKey) => { + const missingEnabledConfigPath = + writeConfigWithMethodWithoutEnabled(methodKey); + + const result = runCli([ + commandName, + "--config", + missingEnabledConfigPath, + ]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + `${commandName} requires methods.${methodKey}.enabled=true`, + ); + expect(result.stderr).not.toContain("missing required argument"); + }, + ); });