chore(release): v0.1.4

CHANGELOG.md

@@ -12,6 +12,20 @@ documented with migration notes.
 - Additional docs for suite-specific adoption.
 - Better compatibility tests for Promptfoo variable contracts.
 
+## [0.1.4] - 2026-07-04
+
+### Fixed
+
+- Made model-assisted CLI commands fail closed unless their method is explicitly enabled in config.
+- Added regression coverage for omitted model-assisted method config and method entries without
+  `enabled`.
+
+### Notes
+
+- Deterministic `run-case` and manual `report` remain compatible unless explicitly disabled.
+- No npm package is published.
+- Consumers may pin `github:agentic-workflow-kit/eval-kit#v0.1.4`.
+
 ## [0.1.3] - 2026-07-04
 
 ### Added
@@ -77,7 +91,8 @@ documented with migration notes.
 - Suite-specific presets remain deferred.
 - Consumer repos own their own semantics, prompts, cases, and pass/fail policies.
 
-[Unreleased]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.3...main
+[Unreleased]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.4...main
+[0.1.4]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.3...v0.1.4
 [0.1.3]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.2...v0.1.3
 [0.1.2]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.1...v0.1.2
 [0.1.1]: https://github.com/agentic-workflow-kit/eval-kit/compare/v0.1.0...v0.1.1

README.md

@@ -11,7 +11,7 @@ Shared evaluation infrastructure for `agentic-workflow-kit` repositories.
 ```json
 {
   "devDependencies": {
-    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3"
+    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4"
   }
 }
 ```
@@ -70,7 +70,7 @@ Install from a Git tag in a consumer repo:
 ```json
 {
   "devDependencies": {
-    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3"
+    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4"
   },
   "scripts": {
     "eval:doctor": "eval-kit doctor --config evals/eval-kit.config.json",
@@ -191,6 +191,7 @@ v0.1.0
 v0.1.1
 v0.1.2
 v0.1.3
+v0.1.4
 v0.2.0
 ```
 

docs/design/consumer-integration.md

@@ -9,7 +9,7 @@ Consumer repos should adopt eval-kit through a pinned Git tag and keep their eva
 ```json
 {
   "devDependencies": {
-    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3"
+    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4"
   }
 }
 ```

docs/guides/consumer-integration.md

@@ -18,7 +18,7 @@ If you cannot state the eval goal, do not bootstrap a suite yet. Empty harnesses
 ```json
 {
   "devDependencies": {
-    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3"
+    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4"
   }
 }
 ```

docs/guides/quickstart.md

@@ -7,7 +7,7 @@ This guide adds a generic deterministic eval suite to a consumer repo.
 ```json
 {
   "devDependencies": {
-    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3"
+    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4"
   }
 }
 ```

docs/reference/cli.md

@@ -22,6 +22,15 @@ Run-producing commands fail closed when their configured method is explicitly di
 Disabled commands exit before Codex auth checks, Promptfoo execution, model provider calls, adapter
 hooks, or result artifact writes.
 
+Model-assisted commands also require explicit enablement in the selected config before required
+model/candidate arguments are validated:
+
+| Command          | Required enablement                   |
+| ---------------- | ------------------------------------- |
+| `generate`       | `methods.generate.enabled=true`       |
+| `judge-coverage` | `methods.judge_coverage.enabled=true` |
+| `judge-pairwise` | `methods.judge_pairwise.enabled=true` |
+
 ## `init`
 
 Create a deterministic generic eval skeleton.
@@ -110,7 +119,7 @@ pnpm exec eval-kit generate \
 ```
 
 Requires Promptfoo and local Codex auth. This is manual/advisory evidence, not a default CI gate.
-Fails closed before those checks when `methods.generate.enabled` is explicitly `false`.
+Fails closed before those checks unless `methods.generate.enabled` is explicitly `true`.
 
 ## `judge-coverage`
 
@@ -127,7 +136,7 @@ pnpm exec eval-kit judge-coverage \
   [--config <path>]
 ```
 
-Fails closed when `methods.judge_coverage.enabled` is explicitly `false`.
+Fails closed unless `methods.judge_coverage.enabled` is explicitly `true`.
 
 ## `judge-pairwise`
 
@@ -151,7 +160,7 @@ pnpm exec eval-kit judge-pairwise \
   [--config <path>]
 ```
 
-Fails closed when `methods.judge_pairwise.enabled` is explicitly `false`.
+Fails closed unless `methods.judge_pairwise.enabled` is explicitly `true`.
 
 ## `report`
 

docs/reference/config.md

@@ -69,11 +69,12 @@ lanes:
 - `run-case` is deterministic local on-demand evidence for semantic case portfolios. Run it before
   significant changes, but do not make long run-producing portfolios a default CI gate.
 - `generate`, `judge_coverage`, and `judge_pairwise` are manual/advisory model-assisted methods.
-  They require explicit local setup and must not require auth, network, Promptfoo provider calls,
-  Codex/OpenAI calls, LLM judging, or manual calibration in `pnpm check`.
-- Run-producing commands fail closed when their method's `enabled` flag is explicitly `false`,
-  before Codex auth checks, Promptfoo execution, model provider calls, adapter hooks, or result
-  artifact writes.
+  They require explicit local setup, must be explicitly enabled in the selected config, and must not
+  require auth, network, Promptfoo provider calls, Codex/OpenAI calls, LLM judging, or manual
+  calibration in `pnpm check`.
+- Run-producing commands fail closed before Codex auth checks, Promptfoo execution, model provider
+  calls, adapter hooks, or result artifact writes when disabled. Model-assisted commands also fail
+  closed unless their method's `enabled` flag is explicitly `true`.
 
 ## Path rules
 

docs/reference/release-process.md

@@ -34,7 +34,7 @@ Consumers depend on tags like:
 Title:
 
 ```text
-chore(release): v0.1.3
+chore(release): v0.1.4
 ```
 
 Required changes:
@@ -63,18 +63,18 @@ git checkout main
 git pull --ff-only
 git rev-parse HEAD
 
-git tag -a v0.1.3 -m "v0.1.3"
-git push origin v0.1.3
+git tag -a v0.1.4 -m "v0.1.4"
+git push origin v0.1.4
 ```
 
 Verify:
 
 ```bash
-git rev-parse v0.1.3^{}
-git show --no-patch --decorate v0.1.3
+git rev-parse v0.1.4^{}
+git show --no-patch --decorate v0.1.4
 ```
 
-`v0.1.3^{}` must point to the release commit. With an annotated tag, `git rev-parse v0.1.3`
+`v0.1.4^{}` must point to the release commit. With an annotated tag, `git rev-parse v0.1.4`
 returns the tag object; `^{}` dereferences to the commit.
 
 ## GitHub Release
@@ -93,7 +93,7 @@ For each consumer repo:
 
 ```json
 {
-  "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3"
+  "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4"
 }
 ```
 
@@ -108,7 +108,7 @@ pnpm check
 3. Run consumer smoke commands, for example in `technical-design`:
 
 ```bash
-pnpm eval:case -- --case case-tiny-laundry-pickup-v1 --candidate evals/cases/case-tiny-laundry-pickup-v1/reference-design.md --run-id verify-eval-kit-v0.1.3
+pnpm eval:case -- --case case-tiny-laundry-pickup-v1 --candidate evals/cases/case-tiny-laundry-pickup-v1/reference-design.md --run-id verify-eval-kit-v0.1.4
 ```
 
 4. Open a PR with dependency, lockfile, and any compatibility fixes.
@@ -120,7 +120,7 @@ Do not move the tag.
 Create a new patch release:
 
 ```text
-v0.1.3 -> v0.1.4
+v0.1.4 -> v0.1.5
 ```
 
 Then open consumer bump PRs.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentic-workflow-kit/eval-kit",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Portable eval runner primitives for local eval suites.",
   "private": true,
   "type": "module",

src/cli.mjs

@@ -17,12 +17,29 @@ import {
   validateFixtures,
 } from "./sdk.mjs";
 
-const requireEnabledMethod = (config, methodKey, commandName) => {
-  if (config.raw.methods?.[methodKey]?.enabled === false) {
+const modelAssistedMethodKeys = new Set([
+  "generate",
+  "judge_coverage",
+  "judge_pairwise",
+]);
+
+const requireEnabledMethod = (
+  config,
+  methodKey,
+  commandName,
+  { explicitEnableRequired = false } = {},
+) => {
+  const enabled = config.raw.methods?.[methodKey]?.enabled;
+  if (enabled === false) {
     throw new Error(
       `${commandName} is disabled by methods.${methodKey}.enabled=false`,
     );
   }
+  if (explicitEnableRequired && enabled !== true) {
+    throw new Error(
+      `${commandName} requires methods.${methodKey}.enabled=true`,
+    );
+  }
 };
 
 const runProducingMethodByCommand = {
@@ -36,7 +53,9 @@ const runProducingMethodByCommand = {
 const requireEnabledCommandMethod = (config, commandName) => {
   const methodKey = runProducingMethodByCommand[commandName];
   if (methodKey) {
-    requireEnabledMethod(config, methodKey, commandName);
+    requireEnabledMethod(config, methodKey, commandName, {
+      explicitEnableRequired: modelAssistedMethodKeys.has(methodKey),
+    });
   }
 };
 

src/sdk.mjs

@@ -15,7 +15,7 @@ import { assertContainedPath, assertSafeId, toPosixPath } from "./paths.mjs";
 const DEFAULT_SANDBOX_MODE = "read-only";
 const DEFAULT_APPROVAL_POLICY = "never";
 const RANDOMIZATION_METHOD = "sha256-seed-parity-v1";
-const EVAL_KIT_VERSION = "0.1.3";
+const EVAL_KIT_VERSION = "0.1.4";
 
 const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 

tests/cli.test.mjs

@@ -36,6 +36,33 @@ const writeConfigWithDisabledMethod = (methodKey) => {
   return tempConfigPath;
 };
 
+const writeConfigOmittingMethod = (methodKey) => {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-kit-cli-"));
+  tempDirs.push(tempDir);
+
+  const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
+  delete config.methods[methodKey];
+
+  const tempConfigPath = path.join(tempDir, "eval-kit.config.json");
+  fs.writeFileSync(tempConfigPath, `${JSON.stringify(config, null, 2)}\n`);
+  return tempConfigPath;
+};
+
+const writeConfigWithMethodWithoutEnabled = (methodKey) => {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-kit-cli-"));
+  tempDirs.push(tempDir);
+
+  const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
+  config.methods = {
+    ...config.methods,
+    [methodKey]: {},
+  };
+
+  const tempConfigPath = path.join(tempDir, "eval-kit.config.json");
+  fs.writeFileSync(tempConfigPath, `${JSON.stringify(config, null, 2)}\n`);
+  return tempConfigPath;
+};
+
 const runCli = (args) =>
   spawnSync(process.execPath, [cliPath, ...args], {
     cwd: packageRoot,
@@ -75,4 +102,50 @@ describe("eval-kit CLI", () => {
       expect(result.stderr).not.toContain("missing required argument");
     },
   );
+
+  it.each([
+    ["generate", "generate"],
+    ["judge-coverage", "judge_coverage"],
+    ["judge-pairwise", "judge_pairwise"],
+  ])(
+    "fails closed when %s method config is omitted",
+    (commandName, methodKey) => {
+      const omittedConfigPath =
+        methodKey in JSON.parse(fs.readFileSync(configPath, "utf8")).methods
+          ? writeConfigOmittingMethod(methodKey)
+          : configPath;
+
+      const result = runCli([commandName, "--config", omittedConfigPath]);
+
+      expect(result.status).toBe(1);
+      expect(result.stderr).toContain(
+        `${commandName} requires methods.${methodKey}.enabled=true`,
+      );
+      expect(result.stderr).not.toContain("missing required argument");
+    },
+  );
+
+  it.each([
+    ["generate", "generate"],
+    ["judge-coverage", "judge_coverage"],
+    ["judge-pairwise", "judge_pairwise"],
+  ])(
+    "fails closed when %s method config omits enabled",
+    (commandName, methodKey) => {
+      const missingEnabledConfigPath =
+        writeConfigWithMethodWithoutEnabled(methodKey);
+
+      const result = runCli([
+        commandName,
+        "--config",
+        missingEnabledConfigPath,
+      ]);
+
+      expect(result.status).toBe(1);
+      expect(result.stderr).toContain(
+        `${commandName} requires methods.${methodKey}.enabled=true`,
+      );
+      expect(result.stderr).not.toContain("missing required argument");
+    },
+  );
 });