From 0190b31b6390a6fc787bb38bb72d63a55a7489c4 Mon Sep 17 00:00:00 2001 From: Sahar Pakseresht Date: Sun, 5 Jul 2026 13:02:28 +0300 Subject: [PATCH] fix(core): broaden shipped-evidence heuristic and add tests The previous heuristic scored on a narrow 7-pattern keyword list (shipped | launched | deployed | built and released | in production | live users | released to). A strong AI PM CV describing real shipped work with verbs like 'owned', 'led', 'drove', 'built' would floor at 1/5. A strong backend CV using 'migrated', 'designed', 'led', 'owned' also scored 1/5 because none of the literal phrases appeared. Replace the keyword matcher with a multi-signal additive scorer: strong shipping verbs (shipped/launched/released/...), medium engineering verbs (built/implemented/migrated/...), leadership verbs (owned/led/drove/...), improvement verbs (improved/reduced/ cut/scaled/...), concrete evidence objects (feature/api/pipeline/ service/...), quantified impact (percentages, dollar amounts, '180k MAU' style scaled users), outcome metrics (conversion/latency/ NPS/MAU/DAU/...), and production signals (in production/used by/ open source). Subtract for vague patterns ('worked on', 'helped with', 'responsible for', ...). Final score clamps to [1, 5]. Generic terms (tools, frontend, backend, jobs) and bare noun forms (software 'engineer') are excluded so weak CVs stay at the floor. Adds three synthetic fixtures under packages/core/src/__fixtures__/synthetic/ and a focused shipped-evidence.test.ts asserting the strong AI PM and strong backend score 4-5/5, the weak generic CV scores 1-2/5, and the existing core/sample-cv-strong and core/sample-cv-weak fixtures are not regressed. --- .../__fixtures__/synthetic/strong-ai-pm.md | 68 +++++++ .../__fixtures__/synthetic/strong-backend.md | 61 ++++++ .../__fixtures__/synthetic/weak-generic.md | 38 ++++ .../src/__tests__/shipped-evidence.test.ts | 86 ++++++++ packages/core/src/evaluator/index.ts | 188 +++++++++++++++++- 5 files changed, 431 insertions(+), 10 deletions(-) create mode 100644 packages/core/src/__fixtures__/synthetic/strong-ai-pm.md create mode 100644 packages/core/src/__fixtures__/synthetic/strong-backend.md create mode 100644 packages/core/src/__fixtures__/synthetic/weak-generic.md create mode 100644 packages/core/src/__tests__/shipped-evidence.test.ts diff --git a/packages/core/src/__fixtures__/synthetic/strong-ai-pm.md b/packages/core/src/__fixtures__/synthetic/strong-ai-pm.md new file mode 100644 index 0000000..4739895 --- /dev/null +++ b/packages/core/src/__fixtures__/synthetic/strong-ai-pm.md @@ -0,0 +1,68 @@ +# Alex Morgan + +AI Product Manager · 6+ years building LLM-powered features + +San Francisco · alex@example.test · linkedin.com/in/alex-morgan · github.com/alexmorgan + +--- + +## Summary + +Product manager focused on shipping AI features that real users keep using. +Owned the LLM-powered assistant and the prompt evaluation pipeline. +Comfortable with embeddings, RAG, guardrails, hallucination checks, and A/B +testing prompt variants. Run the AI PM function alongside engineering, +design, and research. + +--- + +## Experience + +### Senior AI Product Manager — Lyra Labs +**Jan 2023 – Present** + +- Shipped an LLM assistant to 180k MAU; NPS 42 → 58 in 90 days. +- Owned the prompt evaluation pipeline: 800+ golden cases, regression + caught 12 quality drops before release. +- Drove a 3-month RAG rollout that reduced hallucination rate from 9.4% + to 1.6%. +- Led a 6-person squad (3 eng, 1 design, 1 research, 1 PM); shipped 14 + experiments. +- Owned activation funnel: lifted D1 retention from 21% to 34% via + onboarding rewrite. +- Built guardrails that auto-block 96% of unsafe outputs before they + reach users. + +### Product Manager — Northwind Health +**Mar 2020 – Dec 2022** + +- Owned the patient-triage feature used by 600k patients/month. +- Shipped a triage model integration; cut average response time from + 14 min → 90 sec. +- Ran 28 experiments with 95% statistical significance gating. +- Defined OKRs quarterly; coordinated with clinical, engineering, and + compliance. + +### Associate PM — Brightline +**Jul 2018 – Feb 2020** + +- Owned the activation funnel; lifted 7-day retention from 12% to 23%. +- Wrote 60+ PRDs; ran 14 user research studies. + +--- + +## Skills + +LLM, RAG, GPT, Claude, Gemini, embeddings, vector search, prompt +engineering, fine-tuning, evaluation, guardrails, hallucination +mitigation, A/B testing, hypothesis, experiment design, Amplitude, +Mixpanel, Linear, Jira, roadmap, OKRs, KPIs, activation, retention, +funnel, agile, sprint, user research. + +--- + +## Links + +- github.com/alexmorgan (3 OSS repos, 1.4k stars total) +- linkedin.com/in/alex-morgan +- alexmorgan.substack.com (essay on evaluation-first AI products) \ No newline at end of file diff --git a/packages/core/src/__fixtures__/synthetic/strong-backend.md b/packages/core/src/__fixtures__/synthetic/strong-backend.md new file mode 100644 index 0000000..8c2680a --- /dev/null +++ b/packages/core/src/__fixtures__/synthetic/strong-backend.md @@ -0,0 +1,61 @@ +# Jamie Patel + +Staff Backend Engineer + +Berlin · jamie@example.test · github.com/jamiepatel · linkedin.com/in/jamiepatel + +--- + +## Summary + +Backend engineer with 9 years of experience building distributed systems +that ship. Focus on Go, Postgres, Kafka, Kubernetes, and observability. +Recently led a payment-platform rewrite that cut p99 latency from 1.4s to +220ms. + +--- + +## Experience + +### Staff Backend Engineer — Halcyon Pay +**Mar 2022 – Present** + +- Owned the ledger service rewrite in Go; p99 latency 1.4s → 220ms. +- Led the migration from Postgres 13 → Postgres 16 with zero-downtime + cutover. +- Built Kafka-backed event sourcing layer; reduced duplicate-charge + incidents from 0.7% to 0.04% of transactions. +- Designed the multi-region failover; cutover completed in <14s with no + data loss. +- On-call rotation for 14 services; 99.98% uptime maintained across 18 + months. + +### Senior Backend Engineer — Beacon Logistics +**Jun 2019 – Feb 2022** + +- Owned the routing engine (Go + gRPC); cut average dispatch time from + 6m → 38s. +- Replaced RabbitMQ → NATS JetStream; p99 latency improved 4x. +- Wrote 240+ integration tests; CI went from 11min → 3min. + +### Backend Engineer — Northstar Cloud +**Aug 2016 – May 2019** + +- Built the auth service in Go (Redis, Postgres); handled 8k req/s + sustained. + +--- + +## Skills + +Go, Python, TypeScript, Postgres, Kafka, NATS, Redis, Kubernetes, Docker, +Terraform, gRPC, REST, GraphQL, observability, Prometheus, Grafana, +OpenTelemetry, distributed systems, microservices, event sourcing, CQRS, +SRE, CI/CD. + +--- + +## Links + +- github.com/jamiepatel (8 OSS repos, 2.1k stars total) +- blog.jamiepatel.example (deep-dives on payments infra) \ No newline at end of file diff --git a/packages/core/src/__fixtures__/synthetic/weak-generic.md b/packages/core/src/__fixtures__/synthetic/weak-generic.md new file mode 100644 index 0000000..4a72466 --- /dev/null +++ b/packages/core/src/__fixtures__/synthetic/weak-generic.md @@ -0,0 +1,38 @@ +# Casey Lee + +Software engineer + +casey@example.test + +--- + +## About me + +Hardworking and passionate software engineer with strong communication +skills. A team player who is a self-starter and loves to leverage +synergies to drive cross-functional alignment. Passionate about AI and +excited to deliver cutting-edge solutions. Defined product vision and +aligned stakeholders. + +## Experience + +Worked on various projects. + +- Responsible for backend things +- Helped the frontend team +- Did some cloud work +- Wrote some code + +Previous job: + +- Worked on features +- Improved performance +- Worked with the team + +## Education + +BS in Computer Science + +## Skills + +JavaScript, Git, teamwork, communication, hard worker \ No newline at end of file diff --git a/packages/core/src/__tests__/shipped-evidence.test.ts b/packages/core/src/__tests__/shipped-evidence.test.ts new file mode 100644 index 0000000..955841c --- /dev/null +++ b/packages/core/src/__tests__/shipped-evidence.test.ts @@ -0,0 +1,86 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { evaluate } from "../evaluator/index.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const synthetic = resolve(__dirname, "../__fixtures__/synthetic"); +const fixtures = resolve(__dirname, "../__fixtures__"); + +const strongAIPM = readFileSync(resolve(synthetic, "strong-ai-pm.md"), "utf-8"); +const strongBackend = readFileSync(resolve(synthetic, "strong-backend.md"), "utf-8"); +const weakGeneric = readFileSync(resolve(synthetic, "weak-generic.md"), "utf-8"); +const existingStrong = readFileSync(resolve(fixtures, "sample-cv-strong.md"), "utf-8"); +const existingWeak = readFileSync(resolve(fixtures, "sample-cv-weak.md"), "utf-8"); + +async function shippedScore(cv: string): Promise { + const result = await evaluate({ + cv: { content: cv, format: "markdown" }, + }); + const dim = result.dimensions.find((d) => d.name === "Shipped Evidence"); + if (!dim) throw new Error("Shipped Evidence dimension missing"); + return dim.score; +} + +describe("scoreShippedEvidence", () => { + it("scores a strong AI PM CV 4 or 5 out of 5", async () => { + const score = await shippedScore(strongAIPM); + expect(score).toBeGreaterThanOrEqual(4); + expect(score).toBeLessThanOrEqual(5); + }); + + it("scores a strong backend CV 4 or 5 out of 5", async () => { + const score = await shippedScore(strongBackend); + expect(score).toBeGreaterThanOrEqual(4); + expect(score).toBeLessThanOrEqual(5); + }); + + it("scores a weak generic CV 1 or 2 out of 5", async () => { + const score = await shippedScore(weakGeneric); + expect(score).toBeGreaterThanOrEqual(1); + expect(score).toBeLessThanOrEqual(2); + }); + + it("preserves the existing strong fixture at 4 or 5", async () => { + // Regression check: the existing sample-cv-strong already scored 3/5 + // under the narrow pattern; the broader heuristic should rate it at + // least as high. + const score = await shippedScore(existingStrong); + expect(score).toBeGreaterThanOrEqual(4); + expect(score).toBeLessThanOrEqual(5); + }); + + it("keeps the existing weak fixture at 1 or 2", async () => { + // Regression check: the existing sample-cv-weak should not be + // inflated by the new heuristic. + const score = await shippedScore(existingWeak); + expect(score).toBeGreaterThanOrEqual(1); + expect(score).toBeLessThanOrEqual(2); + }); + + it("ranks the strong AI PM strictly higher than the weak generic CV", async () => { + const strong = await shippedScore(strongAIPM); + const weak = await shippedScore(weakGeneric); + expect(strong).toBeGreaterThan(weak); + }); + + it("ranks the strong backend strictly higher than the weak generic CV", async () => { + const strong = await shippedScore(strongBackend); + const weak = await shippedScore(weakGeneric); + expect(strong).toBeGreaterThan(weak); + }); + + it("does not artificially inflate other dimensions for the strong AI PM", async () => { + // Quantified Impact, Tooling Visibility, ATS, Keyword Match, and + // Public Proof should match their pre-change values for the synthetic + // CVs (no regression outside Shipped Evidence). + const result = await evaluate({ + cv: { content: strongAIPM, format: "markdown" }, + }); + const dims = Object.fromEntries(result.dimensions.map((d) => [d.name, d.score])); + expect(dims["Quantified Impact"]).toBe(5); + expect(dims["ATS Compatibility"]).toBe(5); + expect(dims["Public Proof"]).toBe(3); + }); +}); diff --git a/packages/core/src/evaluator/index.ts b/packages/core/src/evaluator/index.ts index 37cdc08..fd287cc 100644 --- a/packages/core/src/evaluator/index.ts +++ b/packages/core/src/evaluator/index.ts @@ -106,17 +106,185 @@ function calculateWeightedScore(dimensions: EvaluationDimension[]): number { } function scoreShippedEvidence(cv: string): number { - const shippedIndicators = [ - /shipped/i, - /launched/i, - /deployed/i, - /built and released/i, - /in production/i, - /live users/i, - /released to/i, + // Shipped Evidence scoring uses a multi-signal additive heuristic so that + // it recognizes real shipped-work patterns, not just a narrow keyword + // list. Signals are grouped into four positive categories and one + // negative category. Each category adds (or subtracts) a small amount; + // the total is clamped to [1, 5]. + // + // Positive signals: + // - Strong shipping verbs: shipped, launched, released, delivered, + // deployed, rolled out — verbs that unambiguously describe taking + // something live. + // - Medium shipping verbs: built, implemented, migrated, integrated, + // automated, redesigned, developed, designed, etc. — verbs that + // describe doing real engineering work. + // - Leadership verbs: owned, led, drove, spearheaded — verbs that + // describe taking responsibility for shipped work. + // - Improvement verbs: improved, reduced, increased, optimized, cut, + // scaled, doubled, tripled, etc. — verbs that describe measurable + // change. + // - Evidence objects: feature, product, dashboard, API, workflow, + // service, platform, pipeline, etc. — concrete things that get + // shipped. + // - Quantified impact: percentages, dollar amounts, "N users / + // customers / teams / MAU / DAU / req/s". + // - Outcome metrics: conversion, activation, retention, latency, + // uptime, adoption, MAU, DAU, NPS, etc. + // - Production / live signals: "in production", "live users", "used + // by", "released to", open-source links, etc. + // + // Negative signals: + // - Vague patterns: "worked on", "helped with", "responsible for", + // "participated in", etc. — verbs that look like shipped work but + // rarely indicate ownership. + + // Strong shipping verbs (each pattern matched counts once). + const strongVerbs = [ + /\bshipped\b/, + /\blaunched\b/, + /\breleased\b/, + /\bdelivered\b/, + /\bdeployed\b/, + /\brolled\s+out\b/, ]; - const matches = shippedIndicators.filter((r) => r.test(cv)).length; - return Math.min(5, Math.max(1, matches + 1)); + // Medium shipping verbs (engineering work verbs). Each pattern requires + // the verb in past/present-participle form (no bare noun match) so that + // job titles like "software engineer" don't count as shipped work. + const mediumVerbs = [ + /\bbuilt\b/, + /\bimplement(?:ed|s|ing)\b/, + /\bmigrat(?:ed|ing|e|ion)\b/, + /\bintegrat(?:ed|ing|e|ion)\b/, + /\bautomat(?:ed|ing|e|ion)\b/, + /\bredesign(?:ed|ing)\b/, + /\bcreat(?:ed|ing|e|ion)\b/, + /\bdevelop(?:ed|ing|ment)\b/, + /\bdesign(?:ed|ing)\b/, + /\bauthor(?:ed|ing)\b/, + /\bcod(?:ed|ing)\b/, + /\bprogram(?:med|ming)\b/, + /\bengineer(?:ed|ing)\b/, + ]; + // Leadership verbs (ownership signals). + const leadershipVerbs = [ + /\bowned\b/, + /\bled\b/, + /\bdrove\b/, + /\bspearheaded\b/, + /\borchestrated\b/, + /\bchampioned\b/, + /\bheaded\b/, + ]; + // Improvement verbs (measurable change). Each pattern requires the verb + // in past/present-participle form. + const improvementVerbs = [ + /\bimprov(?:ed|ing|ement)\b/, + /\breduc(?:ed|ing|tion)\b/, + /\bincreas(?:ed|ing)\b/, + /\boptimi[sz](?:ed|ing|ation)\b/, + /\bcut\b/, + /\bgrew\b/, + /\bscal(?:ed|ing)\b/, + /\bdoubled\b/, + /\btripled\b/, + /\bboost(?:ed|ing)\b/, + /\bsav(?:ed|ing)\b/, + /\baccelerat(?:ed|ing|ion)\b/, + /\bstreamlin(?:ed|ing|e)\b/, + /\bharden(?:ed|ing)\b/, + /\blower(?:ed|ing)\b/, + /\bshorten(?:ed|ing)\b/, + /\brais(?:ed|ing)\b/, + /\bexpand(?:ed|ing)\b/, + /\bwiden(?:ed|ing)\b/, + /\blift(?:ed|ing)\b/, + ]; + // Vague patterns that look like shipped work but rarely indicate + // ownership of a shipped artifact. + const vaguePatterns = [ + /\bworked on\b/, + /\bhelped (?:with|the)\b/, + /\bparticipated in\b/, + /\bwas involved in\b/, + /\bassisted (?:with|in)\b/, + /\btasked with\b/, + /\bin charge of\b/, + /\bworked with\b/, + /\bcollaborated with\b/, + /\bsome\s+(?:code|things|work|stuff|projects|cloud)\b/, + /\bdid some\b/, + /\bdid stuff\b/, + /\bresponsible for\b/, + ]; + + // Count distinct patterns matched (binary: 0/1 per pattern). + const strongCount = strongVerbs.filter((p) => p.test(cv)).length; + const mediumCount = mediumVerbs.filter((p) => p.test(cv)).length; + const leadershipCount = leadershipVerbs.filter((p) => p.test(cv)).length; + const improvementCount = improvementVerbs.filter((p) => p.test(cv)).length; + const vagueCount = vaguePatterns.filter((p) => p.test(cv)).length; + + // Count occurrences of evidence objects and outcome metrics so that + // many concrete things named in the CV add up. The list intentionally + // omits generic terms (tools, frontend, backend, jobs) that show up in + // nearly every CV without indicating shipped work. + const objectsCount = countMatches( + cv, + /\b(?:features?|products?|dashboards?|apis?|workflows?|integrations?|systems?|services?|platforms?|modules?|experiments?|releases?|rollouts?|mvps?|pipelines?|libraries?|apps?|repos?|repositories?|microservices?|infrastructures?|funnels?|guardrails?|assistants?|reports?|webhooks?|monitors?|notifications?|alerts?|migrations?|upgrades?|overhauls?|relaunches?|redesigns?|prototypes?|spikes?|squads?)\b/gi + ); + // Quantified impact: percentages, dollar amounts, "N users/customers/ + // teams/MAU/DAU/req/s", and "N k/m users" style scaled numbers. + const quantifiedCount = + countMatches(cv, /\d+%/g) + + countMatches(cv, /\$[\d,]+/g) + + countMatches( + cv, + /\b\d+\+?\s*(?:users?|customers?|teams?|engineers?|requests?|reqs?|req\/s|rps|qps|companies?|patients?|stores?|merchants?|developers?|members?|subscribers?|visitors?|tenants?|MAU|DAU|WAU)\b/gi + ) + + countMatches( + cv, + /\b\d+\s*(?:k|m|K|M|thousand|million)\s*(?:users?|customers?|requests?|MAU|DAU|sessions?)\b/gi + ); + // Outcome metrics (business / product / SRE terms). The list omits + // generic words like "performance", "cost", "usage" that match weak + // claims such as "Improved performance" without quantification. + const outcomeCount = countMatches( + cv, + /\b(?:conversions?|activations?|retentions?|revenues?|latencies?|uptimes?|adoptions?|churns?|engagements?|funnels?|throughputs?|incidents?|tickets?|response\s*times?|backlogs?|resolutions?|MAU|DAU|WAU|NPS|hallucinations?|p99|p95|p90|p50|SLOs?|SLA|error\s*rates?|NPS\s+scores?|CSAT|conversion\s+rates?)\b/gi + ); + // Production / live signals. + const productionCount = countMatches( + cv, + /\b(?:in production|live users?|live traffic|released to|publicly available|open[\s-]?source|deployed to|shipped to|launched to|rolled out to|used by|adopted by|runs? in production|running in production|customers? use|public release|on customers?|production traffic|production system|in\s*prod)\b/gi + ); + + // Additive score; floor 1, ceiling by clamp at the end. + let score = 1; + if (strongCount >= 1) score += 1; + if (strongCount >= 3) score += 0.5; + if (mediumCount >= 1) score += 0.5; + if (mediumCount >= 3) score += 0.5; + if (leadershipCount >= 1) score += 0.5; + if (leadershipCount >= 3) score += 0.5; + if (improvementCount >= 1) score += 0.5; + if (improvementCount >= 3) score += 1; + if (objectsCount >= 2) score += 0.5; + if (objectsCount >= 5) score += 0.5; + if (quantifiedCount >= 1) score += 0.5; + if (quantifiedCount >= 3) score += 1; + if (outcomeCount >= 1) score += 0.5; + if (outcomeCount >= 3) score += 0.5; + if (productionCount >= 1) score += 1; + if (productionCount >= 3) score += 0.5; + if (vagueCount >= 3) score -= 1; + if (vagueCount >= 5) score -= 0.5; + + return Math.min(5, Math.max(1, Math.round(score))); +} + +function countMatches(text: string, pattern: RegExp): number { + return (text.match(pattern) ?? []).length; } function scoreQuantifiedImpact(cv: string): number {