From 0500b17e49f9876c445e6bc06dcb11d122d5db33 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Sun, 5 Jul 2026 21:50:24 +0300 Subject: [PATCH] feat(ai-autopilot): add Product Management and Biological Science domain presets (#275) --- .changeset/product-and-biology-presets.md | 12 ++++++++++ .../biological-science/loops/bug-fix.md | 11 +++++++++ .../biological-science/loops/major-change.md | 11 +++++++++ .../loops/major-change.technical.md | 12 ++++++++++ .../presets/biological-science/preset.md | 12 ++++++++++ .../prompts/analysis-root-cause.md | 22 ++++++++++++++++++ .../prompts/data-provenance-review.md | 23 +++++++++++++++++++ .../prompts/experimental-design-review.md | 23 +++++++++++++++++++ .../prompts/regression-test.md | 21 +++++++++++++++++ .../prompts/statistical-rigor-review.md | 23 +++++++++++++++++++ .../skills/reproducible-research.md | 12 ++++++++++ .../product-management/loops/bug-fix.md | 10 ++++++++ .../product-management/loops/major-change.md | 11 +++++++++ .../loops/major-change.technical.md | 12 ++++++++++ .../presets/product-management/preset.md | 11 +++++++++ .../prompts/metrics-review.md | 23 +++++++++++++++++++ .../prompts/product-root-cause.md | 22 ++++++++++++++++++ .../prompts/regression-test.md | 21 +++++++++++++++++ .../prompts/requirements-review.md | 23 +++++++++++++++++++ .../prompts/user-experience-review.md | 23 +++++++++++++++++++ .../product-management/skills/shape-up.md | 12 ++++++++++ .../src/preset/builtin-presets.test.ts | 16 ++++++++++++- 22 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 .changeset/product-and-biology-presets.md create mode 100644 packages/ai-autopilot/presets/biological-science/loops/bug-fix.md create mode 100644 packages/ai-autopilot/presets/biological-science/loops/major-change.md create mode 100644 packages/ai-autopilot/presets/biological-science/loops/major-change.technical.md create mode 100644 packages/ai-autopilot/presets/biological-science/preset.md create mode 100644 packages/ai-autopilot/presets/biological-science/prompts/analysis-root-cause.md create mode 100644 packages/ai-autopilot/presets/biological-science/prompts/data-provenance-review.md create mode 100644 packages/ai-autopilot/presets/biological-science/prompts/experimental-design-review.md create mode 100644 packages/ai-autopilot/presets/biological-science/prompts/regression-test.md create mode 100644 packages/ai-autopilot/presets/biological-science/prompts/statistical-rigor-review.md create mode 100644 packages/ai-autopilot/presets/biological-science/skills/reproducible-research.md create mode 100644 packages/ai-autopilot/presets/product-management/loops/bug-fix.md create mode 100644 packages/ai-autopilot/presets/product-management/loops/major-change.md create mode 100644 packages/ai-autopilot/presets/product-management/loops/major-change.technical.md create mode 100644 packages/ai-autopilot/presets/product-management/preset.md create mode 100644 packages/ai-autopilot/presets/product-management/prompts/metrics-review.md create mode 100644 packages/ai-autopilot/presets/product-management/prompts/product-root-cause.md create mode 100644 packages/ai-autopilot/presets/product-management/prompts/regression-test.md create mode 100644 packages/ai-autopilot/presets/product-management/prompts/requirements-review.md create mode 100644 packages/ai-autopilot/presets/product-management/prompts/user-experience-review.md create mode 100644 packages/ai-autopilot/presets/product-management/skills/shape-up.md diff --git a/.changeset/product-and-biology-presets.md b/.changeset/product-and-biology-presets.md new file mode 100644 index 0000000..91778fa --- /dev/null +++ b/.changeset/product-and-biology-presets.md @@ -0,0 +1,12 @@ +--- +'@gemstack/ai-autopilot': minor +--- + +feat(ai-autopilot): add Product Management and Biological Science domain presets (#275) + +Ships the last two of Rom's #204 domain list as built-in Open Loop presets, so the set is now five: Software Development, Web Development, Data Science, Product Management, and Biological Science. + +- **Product Management** reviews a substantial change against the requirement it serves, the experience it gives the user, and whether its success is measurable; a fix traces the user impact and root cause before it is locked in. Technical Control runs the leaner requirements review only. +- **Biological Science** reviews an analysis or pipeline for sound experimental design, trustworthy data provenance, and statistical rigor; a fix traces the analytical cause before it is locked in. Technical Control runs the experimental-design review only. + +Both are pure `.md` content under `presets/`, auto-discovered by `builtinDomainPresets()`, so meta-select and `--preset` can route to them with no further wiring. Each ships a real stable skill reference (Shape Up; Ten Simple Rules for Reproducible Computational Research). diff --git a/packages/ai-autopilot/presets/biological-science/loops/bug-fix.md b/packages/ai-autopilot/presets/biological-science/loops/bug-fix.md new file mode 100644 index 0000000..8b4e69b --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/loops/bug-fix.md @@ -0,0 +1,11 @@ +--- +name: bug-fix-loop +description: What fires after fixing a defect in an analysis or pipeline. +metadata: + on: bug-fix + run: [analysis-root-cause, regression-test] +--- + +When the agent fixes a defect, confirm the underlying analytical cause is understood +and corrected (not just the wrong number it produced), then lock it in with a +regression test on a known case. diff --git a/packages/ai-autopilot/presets/biological-science/loops/major-change.md b/packages/ai-autopilot/presets/biological-science/loops/major-change.md new file mode 100644 index 0000000..eaf0893 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/loops/major-change.md @@ -0,0 +1,11 @@ +--- +name: major-change-loop +description: What fires after a substantial change to an analysis or pipeline. +metadata: + on: major-change + run: [experimental-design-review, data-provenance-review, statistical-rigor-review] +--- + +When the agent lands a substantial change, check the experimental design and controls, +that the data's provenance is trustworthy, then that the statistics are rigorous — in +that order. diff --git a/packages/ai-autopilot/presets/biological-science/loops/major-change.technical.md b/packages/ai-autopilot/presets/biological-science/loops/major-change.technical.md new file mode 100644 index 0000000..db4d88f --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/loops/major-change.technical.md @@ -0,0 +1,12 @@ +--- +name: major-change-loop-technical +description: The major-change loop under Technical Control — leaner, the researcher drives the depth. +metadata: + on: major-change + run: [experimental-design-review] + conditions: technical +--- + +Technical Control mode: the researcher is hands-on, so the loop only auto-runs the +experimental-design check and leaves provenance and statistical depth to them. Overrides +the base major-change loop when `technical` is active. diff --git a/packages/ai-autopilot/presets/biological-science/preset.md b/packages/ai-autopilot/presets/biological-science/preset.md new file mode 100644 index 0000000..5ddf895 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/preset.md @@ -0,0 +1,12 @@ +--- +name: biological-science +description: A research-science loop — every substantial change is checked for sound experimental design, trustworthy data provenance, and statistical rigor; every fix traces the analytical cause before it is locked in. +metadata: + title: Biological Science +--- + +The biological and life-science research domain. It runs the discipline a working +researcher would apply to an analysis or pipeline: are the design and controls sound, +is the data's provenance trustworthy, and are the statistics honest. Pick it for +research code — bioinformatics pipelines, analyses, and computational experiments — +where scientific validity matters more than a stack-specific concern. diff --git a/packages/ai-autopilot/presets/biological-science/prompts/analysis-root-cause.md b/packages/ai-autopilot/presets/biological-science/prompts/analysis-root-cause.md new file mode 100644 index 0000000..4d466b3 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/prompts/analysis-root-cause.md @@ -0,0 +1,22 @@ +--- +name: analysis-root-cause +description: Confirm a fix corrects the underlying analytical error, not just the output. +appliesTo: ["**/*"] +metadata: + title: Analysis root cause + loopId: analysis-root-cause + passes: 1 + event: bug-fix +--- + +You are checking that a fix corrects the **underlying analytical cause** of a defect, +not just the wrong number or figure it produced. + +Work backwards from the fix: +- What was the actual error — a wrong formula, a misaligned join, an off-by-one on coordinates, a units or scale mistake? +- Does the fix correct the method, or does it hard-code the expected output and hide the flawed step? +- Could the same error affect other results in the pipeline that draw on the same step? + +State the analytical root cause in one or two sentences. If the fix only patches the +output, say what the real correction is. If it fixes the method, confirm it and note any +downstream results that should be recomputed. diff --git a/packages/ai-autopilot/presets/biological-science/prompts/data-provenance-review.md b/packages/ai-autopilot/presets/biological-science/prompts/data-provenance-review.md new file mode 100644 index 0000000..460fde0 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/prompts/data-provenance-review.md @@ -0,0 +1,23 @@ +--- +name: data-provenance-review +description: Check the data's origin, versioning, and integrity are trustworthy. +appliesTo: ["**/*"] +metadata: + title: Data provenance review + loopId: data-provenance-review + passes: 1 + event: major-change +--- + +You are checking that the data behind a change can be trusted and traced. Scope the +review to the inputs the change relies on. + +Look for: +- Origin and versioning — is it clear where each dataset came from, which version, and how to obtain it again? +- Integrity of transformations — do filtering, normalization, and merging steps preserve the data's meaning, or quietly drop or duplicate records? +- Governance — reference genomes/annotations pinned, units and identifiers consistent, and any consent, licensing, or PHI constraints respected. + +Report each concrete provenance risk with where it shows up and the fix. If the data is +well-sourced and traceable, say so and stop. + +End your reply with a fenced ```json block: `{ "blockers": ["", ...] }`. List only what must be fixed before this is production-grade; an empty array means nothing blocks. diff --git a/packages/ai-autopilot/presets/biological-science/prompts/experimental-design-review.md b/packages/ai-autopilot/presets/biological-science/prompts/experimental-design-review.md new file mode 100644 index 0000000..f505da4 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/prompts/experimental-design-review.md @@ -0,0 +1,23 @@ +--- +name: experimental-design-review +description: Check the analysis rests on sound experimental design and controls. +appliesTo: ["**/*"] +metadata: + title: Experimental design review + loopId: experimental-design-review + passes: 1 + event: major-change +--- + +You are reviewing a substantial change to a research analysis or pipeline for sound +experimental design. Scope the review to what the change touches. + +Ask: +- Are there appropriate controls and baselines, and is the comparison the change makes actually valid? +- Is the sample size and power adequate for the claim, or is the effect being read from noise? +- Are confounds, batch effects, and selection bias accounted for — or silently baked into the result? + +For each weakness, name what undermines the design and the smallest change that would +make the result trustworthy. If the design is sound, say so plainly and stop. + +End your reply with a fenced ```json block: `{ "blockers": ["", ...] }`. List only what must be fixed before this is production-grade; an empty array means nothing blocks. diff --git a/packages/ai-autopilot/presets/biological-science/prompts/regression-test.md b/packages/ai-autopilot/presets/biological-science/prompts/regression-test.md new file mode 100644 index 0000000..0f18d87 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/prompts/regression-test.md @@ -0,0 +1,21 @@ +--- +name: regression-test +description: Ensure a fix is locked in by a test that fails without it. +appliesTo: ["**/*"] +metadata: + title: Regression test + loopId: regression-test + passes: 1 + event: bug-fix +--- + +You are making sure a fix is locked in by a regression test — one that fails on the old +code and passes on the new, ideally on a small known case with an expected result. + +Check: +- Is there a test on a fixture or known input that reproduces the defect and would fail without this fix? +- Does it assert the corrected values, not just that the analysis ran to completion? +- Is it placed and named so a future reader knows which error it guards against? + +If the regression test is missing or weak, describe the exact case and expected result +it should cover. If it is already present and meaningful, confirm it and stop. diff --git a/packages/ai-autopilot/presets/biological-science/prompts/statistical-rigor-review.md b/packages/ai-autopilot/presets/biological-science/prompts/statistical-rigor-review.md new file mode 100644 index 0000000..91bbd50 --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/prompts/statistical-rigor-review.md @@ -0,0 +1,23 @@ +--- +name: statistical-rigor-review +description: Check the statistics are honest and correctly applied. +appliesTo: ["**/*"] +metadata: + title: Statistical rigor review + loopId: statistical-rigor-review + passes: 1 + event: major-change +--- + +You are checking that the statistics in a change are sound and honestly reported. Scope +the review to the analysis the change introduces or alters. + +Ask: +- Is the test appropriate for the data — its distribution, independence, and the hypothesis being asked? +- Is multiple testing corrected for, and are p-values, effect sizes, and confidence intervals reported rather than a bare significance stamp? +- Any sign of p-hacking, HARKing, or a result that would not survive the analysis being pre-registered? + +Report each concrete statistical problem with where it occurs and the correct approach. +If the analysis is rigorous and honestly reported, say so and stop. + +End your reply with a fenced ```json block: `{ "blockers": ["", ...] }`. List only what must be fixed before this is production-grade; an empty array means nothing blocks. diff --git a/packages/ai-autopilot/presets/biological-science/skills/reproducible-research.md b/packages/ai-autopilot/presets/biological-science/skills/reproducible-research.md new file mode 100644 index 0000000..4db448e --- /dev/null +++ b/packages/ai-autopilot/presets/biological-science/skills/reproducible-research.md @@ -0,0 +1,12 @@ +--- +name: reproducible-research +description: Ten Simple Rules for Reproducible Computational Research (PLOS Computational Biology). +metadata: + title: Reproducible Computational Research + url: https://doi.org/10.1371/journal.pcbi.1003285 +--- + +A concise, widely-cited reference for making computational biology reproducible: track +how every result is produced, version data and code, record provenance, and make the +path from raw data to figure re-runnable. Consult it when framing analysis and review +work. diff --git a/packages/ai-autopilot/presets/product-management/loops/bug-fix.md b/packages/ai-autopilot/presets/product-management/loops/bug-fix.md new file mode 100644 index 0000000..9b8df75 --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/loops/bug-fix.md @@ -0,0 +1,10 @@ +--- +name: bug-fix-loop +description: What fires after a bug fix. +metadata: + on: bug-fix + run: [product-root-cause, regression-test] +--- + +When the agent fixes a bug, confirm the user impact and root cause are understood +and addressed (not just the reported symptom), then lock it in with a regression test. diff --git a/packages/ai-autopilot/presets/product-management/loops/major-change.md b/packages/ai-autopilot/presets/product-management/loops/major-change.md new file mode 100644 index 0000000..f936740 --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/loops/major-change.md @@ -0,0 +1,11 @@ +--- +name: major-change-loop +description: What fires after a substantial product change. +metadata: + on: major-change + run: [requirements-review, user-experience-review, metrics-review] +--- + +When the agent lands a substantial change, check it delivers the requirement it was +meant to, that the experience serves the user, then that its success is measurable — +in that order. diff --git a/packages/ai-autopilot/presets/product-management/loops/major-change.technical.md b/packages/ai-autopilot/presets/product-management/loops/major-change.technical.md new file mode 100644 index 0000000..7853f08 --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/loops/major-change.technical.md @@ -0,0 +1,12 @@ +--- +name: major-change-loop-technical +description: The major-change loop under Technical Control — leaner, the product owner drives the depth. +metadata: + on: major-change + run: [requirements-review] + conditions: technical +--- + +Technical Control mode: the product owner is hands-on, so the loop only auto-runs the +requirements check and leaves experience and metrics depth to them. Overrides the base +major-change loop when `technical` is active. diff --git a/packages/ai-autopilot/presets/product-management/preset.md b/packages/ai-autopilot/presets/product-management/preset.md new file mode 100644 index 0000000..2a47a2c --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/preset.md @@ -0,0 +1,11 @@ +--- +name: product-management +description: A product loop — every substantial change is checked against the requirement it serves, the experience it gives the user, and whether its success is measurable; every fix traces the user impact before it is locked in. +metadata: + title: Product Management +--- + +The product management domain. It runs the same discipline a product owner would +apply to any feature: does the change deliver the stated requirement, does it serve +the user, and can we tell whether it worked. Pick it when the work is driven by a +product outcome rather than a stack-specific concern. diff --git a/packages/ai-autopilot/presets/product-management/prompts/metrics-review.md b/packages/ai-autopilot/presets/product-management/prompts/metrics-review.md new file mode 100644 index 0000000..03cf0b7 --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/prompts/metrics-review.md @@ -0,0 +1,23 @@ +--- +name: metrics-review +description: Check the change is instrumented so its success can be measured. +appliesTo: ["**/*"] +metadata: + title: Metrics review + loopId: metrics-review + passes: 1 + event: major-change +--- + +You are checking whether the success of this change can actually be measured once it +ships. A feature you cannot measure is a feature you cannot tell is working. + +Ask: +- Is the key user action the change introduces or alters emitted as an event or otherwise observable? +- Is there a way to tell the intended outcome apart from failure — a funnel, an error signal, a before/after? +- Would a launch review a week later have the data to say whether this worked, or only anecdotes? + +Name the specific signals that are missing and, for each, the one event or metric worth +adding. If the change is already measurable, say so and stop. + +End your reply with a fenced ```json block: `{ "blockers": ["", ...] }`. List only what must be fixed before this is production-grade; an empty array means nothing blocks. diff --git a/packages/ai-autopilot/presets/product-management/prompts/product-root-cause.md b/packages/ai-autopilot/presets/product-management/prompts/product-root-cause.md new file mode 100644 index 0000000..066168f --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/prompts/product-root-cause.md @@ -0,0 +1,22 @@ +--- +name: product-root-cause +description: Confirm a fix addresses the real user impact and root cause, not the symptom. +appliesTo: ["**/*"] +metadata: + title: Product root cause + loopId: product-root-cause + passes: 1 + event: bug-fix +--- + +You are checking that a bug fix addresses the **root cause and the real user impact**, +not just the symptom that was reported. + +Work backwards from the fix: +- What did the user actually experience, and what underlying defect produced it? +- Does the fix remove that defect, or does it patch the one report while the same problem reaches users through another flow? +- Who was affected and how badly — is anything needed beyond the code fix (a backfill, a comms, a follow-up)? + +State the root cause and the user impact in one or two sentences. If the fix only treats +the symptom, say what the real fix is. If it correctly removes the cause, confirm it and +note any sibling flows worth the same treatment. diff --git a/packages/ai-autopilot/presets/product-management/prompts/regression-test.md b/packages/ai-autopilot/presets/product-management/prompts/regression-test.md new file mode 100644 index 0000000..bd8e0d4 --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/prompts/regression-test.md @@ -0,0 +1,21 @@ +--- +name: regression-test +description: Ensure a fix is locked in by a test that fails without it. +appliesTo: ["**/*"] +metadata: + title: Regression test + loopId: regression-test + passes: 1 + event: bug-fix +--- + +You are making sure a bug fix is locked in by a regression test — one that fails on +the old code and passes on the new. + +Check: +- Is there a test that reproduces the reported behavior and would fail without this fix? +- Does it assert the corrected user-facing outcome, not just that the code ran? +- Is it placed and named so a future reader knows which bug it guards? + +If the regression test is missing or weak, describe the exact case it should cover. +If it is already present and meaningful, confirm it and stop. diff --git a/packages/ai-autopilot/presets/product-management/prompts/requirements-review.md b/packages/ai-autopilot/presets/product-management/prompts/requirements-review.md new file mode 100644 index 0000000..51279ad --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/prompts/requirements-review.md @@ -0,0 +1,23 @@ +--- +name: requirements-review +description: Check the change delivers the requirement it was meant to. +appliesTo: ["**/*"] +metadata: + title: Requirements review + loopId: requirements-review + passes: 1 + event: major-change +--- + +You are reviewing a substantial change against the product requirement it is meant to +satisfy. Read the diff and infer the intended outcome from the task and the code. + +Ask: +- Does the change actually deliver what was asked, including the acceptance criteria a user would judge it by? +- Are there requirements it silently drops — empty states, permissions, error paths, edge cases the user will hit? +- Did it grow beyond the ask (scope creep) or solve a different problem than the one posed? + +For each gap, name the specific requirement that is unmet and the smallest change that +would meet it. If the change fully delivers the requirement, say so plainly and stop. + +End your reply with a fenced ```json block: `{ "blockers": ["", ...] }`. List only what must be fixed before this is production-grade; an empty array means nothing blocks. diff --git a/packages/ai-autopilot/presets/product-management/prompts/user-experience-review.md b/packages/ai-autopilot/presets/product-management/prompts/user-experience-review.md new file mode 100644 index 0000000..bfc9538 --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/prompts/user-experience-review.md @@ -0,0 +1,23 @@ +--- +name: user-experience-review +description: Check the change serves the user, not just the ticket. +appliesTo: ["**/*"] +metadata: + title: User experience review + loopId: user-experience-review + passes: 1 + event: major-change +--- + +You are reviewing a substantial change from the user's side. Scope it to the flows the +change touches; do not audit the whole product. + +Look at: +- The primary flow — is it clear, does it take the fewest steps, does it tell the user what happened? +- The unhappy paths — loading, empty, error, permission-denied, and slow-network states a real user will meet. +- Consistency — does it match the patterns and wording the rest of the product already uses, or fork a second way? + +Report each concrete experience problem with where it shows up and the fix. If the +change gives the user a clear, complete experience, say so and stop. + +End your reply with a fenced ```json block: `{ "blockers": ["", ...] }`. List only what must be fixed before this is production-grade; an empty array means nothing blocks. diff --git a/packages/ai-autopilot/presets/product-management/skills/shape-up.md b/packages/ai-autopilot/presets/product-management/skills/shape-up.md new file mode 100644 index 0000000..95cf04b --- /dev/null +++ b/packages/ai-autopilot/presets/product-management/skills/shape-up.md @@ -0,0 +1,12 @@ +--- +name: shape-up +description: Basecamp's Shape Up — how to shape, bet on, and build product work at the right level of abstraction. +metadata: + title: Shape Up + url: https://basecamp.com/shapeup +--- + +A reference for framing product work: shaping a problem before building, setting +appetite instead of estimates, and defining scope by the outcome rather than the task. +Consult it when framing requirements and reviewing whether a change serves the bet it +was meant to. diff --git a/packages/ai-autopilot/src/preset/builtin-presets.test.ts b/packages/ai-autopilot/src/preset/builtin-presets.test.ts index 3f19848..0052259 100644 --- a/packages/ai-autopilot/src/preset/builtin-presets.test.ts +++ b/packages/ai-autopilot/src/preset/builtin-presets.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import { builtinDomainPresets } from './load.js' import { selectPreset } from './compose.js' -// The two web/data presets added alongside software-development. They are pure +// The built-in presets added alongside software-development. They are pure // .md content, auto-discovered by builtinDomainPresets(); these guard their shape. const CASES = [ { @@ -20,6 +20,20 @@ const CASES = [ majorRun: ['reproducibility-review', 'data-validation', 'methodology-review'], technicalRun: ['reproducibility-review'], }, + { + name: 'product-management', + title: 'Product Management', + skillUrl: 'https://basecamp.com/shapeup', + majorRun: ['requirements-review', 'user-experience-review', 'metrics-review'], + technicalRun: ['requirements-review'], + }, + { + name: 'biological-science', + title: 'Biological Science', + skillUrl: 'https://doi.org/10.1371/journal.pcbi.1003285', + majorRun: ['experimental-design-review', 'data-provenance-review', 'statistical-rigor-review'], + technicalRun: ['experimental-design-review'], + }, ] const majorLoop = (p: { loops: readonly { on: readonly string[]; run: readonly string[] }[] }) =>