Fix shipped evidence scoring

packages/core/src/__fixtures__/synthetic/strong-ai-pm.md

@@ -0,0 +1,68 @@
+# Alex Morgan
+
+AI Product Manager · 6+ years building LLM-powered features
+
+San Francisco · alex@example.test · linkedin.com/in/alex-morgan · github.com/alexmorgan
+
+---
+
+## Summary
+
+Product manager focused on shipping AI features that real users keep using.
+Owned the LLM-powered assistant and the prompt evaluation pipeline.
+Comfortable with embeddings, RAG, guardrails, hallucination checks, and A/B
+testing prompt variants. Run the AI PM function alongside engineering,
+design, and research.
+
+---
+
+## Experience
+
+### Senior AI Product Manager — Lyra Labs
+**Jan 2023 – Present**
+
+- Shipped an LLM assistant to 180k MAU; NPS 42 → 58 in 90 days.
+- Owned the prompt evaluation pipeline: 800+ golden cases, regression
+  caught 12 quality drops before release.
+- Drove a 3-month RAG rollout that reduced hallucination rate from 9.4%
+  to 1.6%.
+- Led a 6-person squad (3 eng, 1 design, 1 research, 1 PM); shipped 14
+  experiments.
+- Owned activation funnel: lifted D1 retention from 21% to 34% via
+  onboarding rewrite.
+- Built guardrails that auto-block 96% of unsafe outputs before they
+  reach users.
+
+### Product Manager — Northwind Health
+**Mar 2020 – Dec 2022**
+
+- Owned the patient-triage feature used by 600k patients/month.
+- Shipped a triage model integration; cut average response time from
+  14 min → 90 sec.
+- Ran 28 experiments with 95% statistical significance gating.
+- Defined OKRs quarterly; coordinated with clinical, engineering, and
+  compliance.
+
+### Associate PM — Brightline
+**Jul 2018 – Feb 2020**
+
+- Owned the activation funnel; lifted 7-day retention from 12% to 23%.
+- Wrote 60+ PRDs; ran 14 user research studies.
+
+---
+
+## Skills
+
+LLM, RAG, GPT, Claude, Gemini, embeddings, vector search, prompt
+engineering, fine-tuning, evaluation, guardrails, hallucination
+mitigation, A/B testing, hypothesis, experiment design, Amplitude,
+Mixpanel, Linear, Jira, roadmap, OKRs, KPIs, activation, retention,
+funnel, agile, sprint, user research.
+
+---
+
+## Links
+
+- github.com/alexmorgan (3 OSS repos, 1.4k stars total)
+- linkedin.com/in/alex-morgan
+- alexmorgan.substack.com (essay on evaluation-first AI products)

packages/core/src/__fixtures__/synthetic/strong-backend.md

@@ -0,0 +1,61 @@
+# Jamie Patel
+
+Staff Backend Engineer
+
+Berlin · jamie@example.test · github.com/jamiepatel · linkedin.com/in/jamiepatel
+
+---
+
+## Summary
+
+Backend engineer with 9 years of experience building distributed systems
+that ship. Focus on Go, Postgres, Kafka, Kubernetes, and observability.
+Recently led a payment-platform rewrite that cut p99 latency from 1.4s to
+220ms.
+
+---
+
+## Experience
+
+### Staff Backend Engineer — Halcyon Pay
+**Mar 2022 – Present**
+
+- Owned the ledger service rewrite in Go; p99 latency 1.4s → 220ms.
+- Led the migration from Postgres 13 → Postgres 16 with zero-downtime
+  cutover.
+- Built Kafka-backed event sourcing layer; reduced duplicate-charge
+  incidents from 0.7% to 0.04% of transactions.
+- Designed the multi-region failover; cutover completed in <14s with no
+  data loss.
+- On-call rotation for 14 services; 99.98% uptime maintained across 18
+  months.
+
+### Senior Backend Engineer — Beacon Logistics
+**Jun 2019 – Feb 2022**
+
+- Owned the routing engine (Go + gRPC); cut average dispatch time from
+  6m → 38s.
+- Replaced RabbitMQ → NATS JetStream; p99 latency improved 4x.
+- Wrote 240+ integration tests; CI went from 11min → 3min.
+
+### Backend Engineer — Northstar Cloud
+**Aug 2016 – May 2019**
+
+- Built the auth service in Go (Redis, Postgres); handled 8k req/s
+  sustained.
+
+---
+
+## Skills
+
+Go, Python, TypeScript, Postgres, Kafka, NATS, Redis, Kubernetes, Docker,
+Terraform, gRPC, REST, GraphQL, observability, Prometheus, Grafana,
+OpenTelemetry, distributed systems, microservices, event sourcing, CQRS,
+SRE, CI/CD.
+
+---
+
+## Links
+
+- github.com/jamiepatel (8 OSS repos, 2.1k stars total)
+- blog.jamiepatel.example (deep-dives on payments infra)

packages/core/src/__fixtures__/synthetic/weak-generic.md

@@ -0,0 +1,38 @@
+# Casey Lee
+
+Software engineer
+
+casey@example.test
+
+---
+
+## About me
+
+Hardworking and passionate software engineer with strong communication
+skills. A team player who is a self-starter and loves to leverage
+synergies to drive cross-functional alignment. Passionate about AI and
+excited to deliver cutting-edge solutions. Defined product vision and
+aligned stakeholders.
+
+## Experience
+
+Worked on various projects.
+
+- Responsible for backend things
+- Helped the frontend team
+- Did some cloud work
+- Wrote some code
+
+Previous job:
+
+- Worked on features
+- Improved performance
+- Worked with the team
+
+## Education
+
+BS in Computer Science
+
+## Skills
+
+JavaScript, Git, teamwork, communication, hard worker

packages/core/src/__tests__/shipped-evidence.test.ts

@@ -0,0 +1,86 @@
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+import { evaluate } from "../evaluator/index.js";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const synthetic = resolve(__dirname, "../__fixtures__/synthetic");
+const fixtures = resolve(__dirname, "../__fixtures__");
+
+const strongAIPM = readFileSync(resolve(synthetic, "strong-ai-pm.md"), "utf-8");
+const strongBackend = readFileSync(resolve(synthetic, "strong-backend.md"), "utf-8");
+const weakGeneric = readFileSync(resolve(synthetic, "weak-generic.md"), "utf-8");
+const existingStrong = readFileSync(resolve(fixtures, "sample-cv-strong.md"), "utf-8");
+const existingWeak = readFileSync(resolve(fixtures, "sample-cv-weak.md"), "utf-8");
+
+async function shippedScore(cv: string): Promise<number> {
+  const result = await evaluate({
+    cv: { content: cv, format: "markdown" },
+  });
+  const dim = result.dimensions.find((d) => d.name === "Shipped Evidence");
+  if (!dim) throw new Error("Shipped Evidence dimension missing");
+  return dim.score;
+}
+
+describe("scoreShippedEvidence", () => {
+  it("scores a strong AI PM CV 4 or 5 out of 5", async () => {
+    const score = await shippedScore(strongAIPM);
+    expect(score).toBeGreaterThanOrEqual(4);
+    expect(score).toBeLessThanOrEqual(5);
+  });
+
+  it("scores a strong backend CV 4 or 5 out of 5", async () => {
+    const score = await shippedScore(strongBackend);
+    expect(score).toBeGreaterThanOrEqual(4);
+    expect(score).toBeLessThanOrEqual(5);
+  });
+
+  it("scores a weak generic CV 1 or 2 out of 5", async () => {
+    const score = await shippedScore(weakGeneric);
+    expect(score).toBeGreaterThanOrEqual(1);
+    expect(score).toBeLessThanOrEqual(2);
+  });
+
+  it("preserves the existing strong fixture at 4 or 5", async () => {
+    // Regression check: the existing sample-cv-strong already scored 3/5
+    // under the narrow pattern; the broader heuristic should rate it at
+    // least as high.
+    const score = await shippedScore(existingStrong);
+    expect(score).toBeGreaterThanOrEqual(4);
+    expect(score).toBeLessThanOrEqual(5);
+  });
+
+  it("keeps the existing weak fixture at 1 or 2", async () => {
+    // Regression check: the existing sample-cv-weak should not be
+    // inflated by the new heuristic.
+    const score = await shippedScore(existingWeak);
+    expect(score).toBeGreaterThanOrEqual(1);
+    expect(score).toBeLessThanOrEqual(2);
+  });
+
+  it("ranks the strong AI PM strictly higher than the weak generic CV", async () => {
+    const strong = await shippedScore(strongAIPM);
+    const weak = await shippedScore(weakGeneric);
+    expect(strong).toBeGreaterThan(weak);
+  });
+
+  it("ranks the strong backend strictly higher than the weak generic CV", async () => {
+    const strong = await shippedScore(strongBackend);
+    const weak = await shippedScore(weakGeneric);
+    expect(strong).toBeGreaterThan(weak);
+  });
+
+  it("does not artificially inflate other dimensions for the strong AI PM", async () => {
+    // Quantified Impact, Tooling Visibility, ATS, Keyword Match, and
+    // Public Proof should match their pre-change values for the synthetic
+    // CVs (no regression outside Shipped Evidence).
+    const result = await evaluate({
+      cv: { content: strongAIPM, format: "markdown" },
+    });
+    const dims = Object.fromEntries(result.dimensions.map((d) => [d.name, d.score]));
+    expect(dims["Quantified Impact"]).toBe(5);
+    expect(dims["ATS Compatibility"]).toBe(5);
+    expect(dims["Public Proof"]).toBe(3);
+  });
+});