From 3ee7c6b721094715b948443eab74e57811d54af9 Mon Sep 17 00:00:00 2001 From: Arye Kogan Date: Sat, 4 Jul 2026 06:39:23 +0300 Subject: [PATCH] test(evals): add technical-design bad pointwise fixtures --- .../candidate-bad-pointwise.md | 13 ++++++ .../candidate-bad-pointwise.md | 13 ++++++ .../candidate-bad-pointwise.md | 13 ++++++ .../candidate-bad-pointwise.md | 13 ++++++ .../candidate-bad-pointwise.md | 13 ++++++ .../candidate-bad-pointwise.md | 13 ++++++ .../candidate-bad-pointwise.md | 13 ++++++ evals/model-judge-calibration-notes.md | 36 +++++++++++----- evals/tests/public-reference-cases.test.mjs | 42 +++++++++++++++++++ 9 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 evals/cases/case-aerial-delivery-shipping-v1/candidate-bad-pointwise.md create mode 100644 evals/cases/case-cloudevents-core-contract-v1/candidate-bad-pointwise.md create mode 100644 evals/cases/case-customer-credit-order-saga-v1/candidate-bad-pointwise.md create mode 100644 evals/cases/case-fineract-loan-lifecycle-v1/candidate-bad-pointwise.md create mode 100644 evals/cases/case-kubernetes-sidecar-containers-v1/candidate-bad-pointwise.md create mode 100644 evals/cases/case-openfeature-evaluation-api-v1/candidate-bad-pointwise.md create mode 100644 evals/cases/case-tiny-laundry-pickup-v1/candidate-bad-pointwise.md diff --git a/evals/cases/case-aerial-delivery-shipping-v1/candidate-bad-pointwise.md b/evals/cases/case-aerial-delivery-shipping-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..cf371ea --- /dev/null +++ b/evals/cases/case-aerial-delivery-shipping-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design uses Shipping for shipping-facing workflow coordination, Delivery for active pickup and delivery execution, Package for package tagging, Scheduler for assignment policy, Accounts for business registration records, and Drone Management for drone availability. + +## Intentional Calibration Defect + +Delivery owns long-term delivery history. + +This contradicts the source-visible retention boundary: active delivery execution may be separate from long-term delivery history ownership. diff --git a/evals/cases/case-cloudevents-core-contract-v1/candidate-bad-pointwise.md b/evals/cases/case-cloudevents-core-contract-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..f8382e2 --- /dev/null +++ b/evals/cases/case-cloudevents-core-contract-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design keeps a CloudEvents Event Contract as the interoperability seam, supports structured and binary JSON encodings, and allows domain-specific payloads to travel between producers and consumers. + +## Intentional Calibration Defect + +CloudEvents defines event-sourcing lifecycle. + +This invents event-store and event-sourcing ownership that the contract fixture explicitly keeps out of scope. diff --git a/evals/cases/case-customer-credit-order-saga-v1/candidate-bad-pointwise.md b/evals/cases/case-customer-credit-order-saga-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..2b51190 --- /dev/null +++ b/evals/cases/case-customer-credit-order-saga-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design has an Ordering boundary that creates orders in `PENDING`, sends `reserveCredit` to Customer Credit, and records `APPROVED`, `REJECTED`, `UNKNOWN_CUSTOMER`, and `INSUFFICIENT_CREDIT` outcomes. Messaging and Integration Infrastructure carries commands and asynchronous replies. + +## Intentional Calibration Defect + +The design uses 2PC for consistency between Order Service and Customer Service. + +This contradicts the local-transaction saga model and the separate service persistence boundary. diff --git a/evals/cases/case-fineract-loan-lifecycle-v1/candidate-bad-pointwise.md b/evals/cases/case-fineract-loan-lifecycle-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..ca55aec --- /dev/null +++ b/evals/cases/case-fineract-loan-lifecycle-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design separates Loan Account Lifecycle, Loan Product Configuration, Loan Charges, and Accounting Integration. Loan Account Lifecycle tracks submitted, approved, active, closed, and overpaid states while reading loan product terms. + +## Intentional Calibration Defect + +Loan Account Lifecycle owns product configuration. + +This collapses loan product template authority into the account lifecycle boundary. diff --git a/evals/cases/case-kubernetes-sidecar-containers-v1/candidate-bad-pointwise.md b/evals/cases/case-kubernetes-sidecar-containers-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..885ddc3 --- /dev/null +++ b/evals/cases/case-kubernetes-sidecar-containers-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design treats sidecar behavior as part of Pod lifecycle semantics, keeps init-container ordering visible, and discusses Job completion, readiness, resource accounting, and termination ordering. + +## Intentional Calibration Defect + +Sidecars are ordinary app containers. + +This contradicts the source-visible rule that sidecars are restartable init containers under `initContainers` with `restartPolicy: Always`. diff --git a/evals/cases/case-openfeature-evaluation-api-v1/candidate-bad-pointwise.md b/evals/cases/case-openfeature-evaluation-api-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..b8cdb24 --- /dev/null +++ b/evals/cases/case-openfeature-evaluation-api-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design keeps OpenFeature as a vendor-agnostic evaluation API with providers as backend adapters, no-op default behavior, hook lifecycle stages, and provider readiness/error/configuration/stale events. + +## Intentional Calibration Defect + +OpenFeature owns flag management UI. + +This invents product-management scope that belongs outside the evaluation API seam. diff --git a/evals/cases/case-tiny-laundry-pickup-v1/candidate-bad-pointwise.md b/evals/cases/case-tiny-laundry-pickup-v1/candidate-bad-pointwise.md new file mode 100644 index 0000000..a1ec96f --- /dev/null +++ b/evals/cases/case-tiny-laundry-pickup-v1/candidate-bad-pointwise.md @@ -0,0 +1,13 @@ +# Expected-Bad Pointwise Candidate + +This candidate is intentionally defective for manual pointwise calibration. It is not a reference design and should remain adverse. + +## Candidate Design + +The design has Scheduling own booking lifecycle, overlap detection, cancellation rules, appliance availability reads, maintenance hold reads, and the public booking APIs. + +## Intentional Calibration Defect + +Identity owns scheduling conflict logic. + +This contradicts the source-visible boundary: identity may prove resident verification, but scheduling conflict decisions stay with the booking lifecycle owner. diff --git a/evals/model-judge-calibration-notes.md b/evals/model-judge-calibration-notes.md index a43f378..80f1576 100644 --- a/evals/model-judge-calibration-notes.md +++ b/evals/model-judge-calibration-notes.md @@ -8,15 +8,15 @@ Deterministic eval verdicts remain authoritative. Pointwise model-judge evidence These notes summarize the existing local provider-backed runs from July 4, 2026. No fresh provider calls were made for this curated summary. -| Field | Value | -| ----------------- | -------------------------------------------- | -| Config | `evals/eval-kit.model-judge.config.json` | -| Model | `gpt-5.4` | -| Provider | `openai:codex-app-server` | -| Reasoning effort | `medium` | -| Prompt version | `pointwise-prompt-v1` | -| Rubric version | `pointwise-coverage-rubric-v1` | -| Candidate profile | expected-good `reference-design.md` fixtures | +| Field | Value | +| ----------------- | --------------------------------------------------------------------------------------------------------------------- | +| Config | `evals/eval-kit.model-judge.config.json` | +| Model | `gpt-5.4` | +| Provider | `openai:codex-app-server` | +| Reasoning effort | `medium` | +| Prompt version | `pointwise-prompt-v1` | +| Rubric version | `pointwise-coverage-rubric-v1` | +| Candidate profile | expected-good `reference-design.md` fixtures; expected-bad candidates now committed for future false-pass calibration | ## Calibration Summary @@ -32,7 +32,7 @@ These notes summarize the existing local provider-backed runs from July 4, 2026. ## Human Calibration Notes -- False pass risk: not yet calibrated for technical-design because this run set contains expected-good pointwise candidates only. Phase C4 is the planned follow-up for expected-bad pointwise calibration fixtures. +- False pass risk: not yet provider-calibrated for technical-design because the summarized provider run set contains expected-good pointwise candidates only. Phase C4 added committed expected-bad pointwise candidates for future false-pass calibration. - False fail risk: no false-fail evidence observed in the expected-good run set. - Ambiguity: no `partial` or `unknown` verdicts appeared in these runs. - Verbosity bias: no evidence that longer wording was required for coverage, but expected-bad fixtures are needed before drawing stronger conclusions. @@ -41,4 +41,18 @@ These notes summarize the existing local provider-backed runs from July 4, 2026. ## Decision -Keep the technical-design pointwise lane manual and advisory. The current expected-good evidence is usable for smoke calibration, but false-pass detection remains untested until Phase C4 adds expected-bad pointwise calibration fixtures. +Keep the technical-design pointwise lane manual and advisory. The current expected-good evidence is usable for smoke calibration. The committed expected-bad candidates below are ready for a future provider-backed false-pass calibration pass, but they do not add fresh model evidence by themselves. + +## Expected-Bad Candidate Set + +These candidates are committed for manual pointwise false-pass calibration. They intentionally contain technical-design-specific semantic defects only and are expected to remain adverse. + +| Case | Candidate | Targeted defect | +| --------------------------------------- | ------------------------------------------------------------------------------ | -------------------------------------------------- | +| `case-aerial-delivery-shipping-v1` | `evals/cases/case-aerial-delivery-shipping-v1/candidate-bad-pointwise.md` | Delivery owns long-term delivery history. | +| `case-cloudevents-core-contract-v1` | `evals/cases/case-cloudevents-core-contract-v1/candidate-bad-pointwise.md` | CloudEvents owns event-sourcing lifecycle. | +| `case-customer-credit-order-saga-v1` | `evals/cases/case-customer-credit-order-saga-v1/candidate-bad-pointwise.md` | Order and Customer services use 2PC. | +| `case-fineract-loan-lifecycle-v1` | `evals/cases/case-fineract-loan-lifecycle-v1/candidate-bad-pointwise.md` | Loan Account Lifecycle owns product configuration. | +| `case-kubernetes-sidecar-containers-v1` | `evals/cases/case-kubernetes-sidecar-containers-v1/candidate-bad-pointwise.md` | Sidecars are ordinary app containers. | +| `case-openfeature-evaluation-api-v1` | `evals/cases/case-openfeature-evaluation-api-v1/candidate-bad-pointwise.md` | OpenFeature owns flag management UI. | +| `case-tiny-laundry-pickup-v1` | `evals/cases/case-tiny-laundry-pickup-v1/candidate-bad-pointwise.md` | Identity owns scheduling conflict logic. | diff --git a/evals/tests/public-reference-cases.test.mjs b/evals/tests/public-reference-cases.test.mjs index e1b754b..f49cfff 100644 --- a/evals/tests/public-reference-cases.test.mjs +++ b/evals/tests/public-reference-cases.test.mjs @@ -44,6 +44,37 @@ const newPublicCaseIds = [ "case-fineract-loan-lifecycle-v1", ]; +const expectedBadPointwiseCandidates = [ + { + caseId: "case-aerial-delivery-shipping-v1", + findingId: "CTX-005", + }, + { + caseId: "case-cloudevents-core-contract-v1", + findingId: "FACT-005", + }, + { + caseId: "case-customer-credit-order-saga-v1", + findingId: "FACT-008", + }, + { + caseId: "case-fineract-loan-lifecycle-v1", + findingId: "CTX-001", + }, + { + caseId: "case-kubernetes-sidecar-containers-v1", + findingId: "FACT-001", + }, + { + caseId: "case-openfeature-evaluation-api-v1", + findingId: "FACT-006", + }, + { + caseId: "case-tiny-laundry-pickup-v1", + findingId: "FACT-011", + }, +]; + describe("public reference cases", () => { it("accepts the new public reference designs", () => { for (const caseId of newPublicCaseIds) { @@ -56,6 +87,17 @@ describe("public reference cases", () => { } }); + it("keeps committed expected-bad pointwise candidates adverse", () => { + for (const { caseId, findingId } of expectedBadPointwiseCandidates) { + const candidate = readText(caseId, "candidate-bad-pointwise.md"); + const findings = findingsForCaseCandidate(caseId, candidate); + + expect(candidate, caseId).toContain("Expected-Bad Pointwise Candidate"); + expect(verdictFor(findings, findingId), caseId).toBe("contradicted"); + expect(verdictForFindings(findings), caseId).toBe("red"); + } + }); + it("passes the case rubric into pointwise judge variables", async () => { const config = loadConfig(path.join(packageRoot, "eval-kit.config.json")); const caseId = "case-tiny-laundry-pickup-v1";