feat(verifier): add rubric verifier engine

.changeset/verifier-rubric-engine.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Add the rubric-based verifier engine with normalized public rubric output and bounded failure-step parsing.

packages/core/lib/v3/index.ts

@@ -82,12 +82,16 @@ export type {
   CanonicalEvidence,
   CanonicalScreenshot,
   CanonicalTextEvidence,
+  ErrorTaxonomyCategory,
+  ErrorTaxonomySubCategory,
   EvidenceLoadOptions,
   EvidenceLoadResult,
   ProbeEvidence,
+  ParseFailureStepNumbersOptions,
   ToolOutput,
   Verifier,
   EvaluationResult,
+  RubricVerifierOptions,
   CriterionScore,
   FirstPointOfFailure,
   TaskValidity,

packages/core/lib/v3/verifier/errorTaxonomy.ts

@@ -0,0 +1,353 @@
+/**
+ * Error taxonomy for computer-use trajectories.
+ *
+ * The TS port skips the markdown-parser machinery from the Python loader.
+ * The canonical structure is encoded directly here so prompts can interpolate
+ * it without a runtime parse step.
+ *
+ * Two-level hierarchy: 8 top-level categories, each with numbered
+ * sub-categories (e.g., "2.3 Output fabrication"). Used by:
+ *   - Failure analysis (Step 9a) — categories 1–6.
+ *   - Task classification (Steps 9b + 10) — categories 7 (ambiguity) and 8 (invalid).
+ *
+ * Calibration: not every imperfection is a failure. Only flag issues that
+ * materially affected task completion, correctness, or user trust.
+ */
+import type {
+  ErrorTaxonomyCategory,
+  ErrorTaxonomySubCategory,
+} from "./types.js";
+
+/**
+ * Canonical taxonomy used by verifier failure-analysis prompts.
+ */
+export const ERROR_TAXONOMY: ErrorTaxonomyCategory[] = [
+  {
+    number: 1,
+    name: "Selection Errors",
+    summary:
+      "Errors where the agent chose the wrong target, performed the wrong interaction, or violated explicit task constraints.",
+    subCategories: [
+      {
+        code: "1.1",
+        name: "Missing Intent",
+        description:
+          "Agent misses the primary intent of the task — choosing an entirely wrong product, location, person, or service that bears no meaningful resemblance to what the user requested (e.g., buying Care Bears Grumpy Bear on Amazon instead of Disney Grumpy plush).",
+      },
+      {
+        code: "1.2",
+        name: "Unauthorized substitution",
+        description:
+          "Silently swapping an unavailable item/hotel/reservation/service for a similar alternative without reporting it to the user. Distinct from 1.1: a substitution involves a product that could plausibly serve as an alternative (e.g., substituting a sold-out 16 oz bottle with a 12 oz bottle of the same brand), whereas missing intent involves something entirely different.",
+      },
+      {
+        code: "1.3",
+        name: "Wrong action type",
+        description:
+          "Performing the wrong interaction on the correct target entity (e.g., 'Add to Watchlist' instead of 'Add to Cart', or 'add to waitlist' instead of 'book reservation'). The primary target is found but not acted upon correctly.",
+      },
+      {
+        code: "1.4",
+        name: "Wrong values or constraint violation",
+        description:
+          "Entering incorrect parameters, failing to satisfy explicit constraints, or delivering results that don't match stated requirements. Includes wrong quantities/dates/values, hard constraint misses (e.g., ignoring 'non-stop flights only' or 'at least 4.5 stars'), and constraint verification failures (searching for a constraint but never confirming results actually satisfy it).",
+      },
+      {
+        code: "1.5",
+        name: "Other",
+        description: "Selection error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 2,
+    name: "Hallucination Errors",
+    summary:
+      "Errors where the agent invents, misrepresents, or contradicts information. Screenshots and tool outputs are the ground truth — when there's a discrepancy between agent claims and evidence, evidence takes precedence.",
+    subCategories: [
+      {
+        code: "2.1",
+        name: "Output contradiction",
+        description:
+          "Evidence shows X, but the agent claims not-X. Misinterpreting, misreading, or drawing incorrect conclusions from page content, tool output, or API responses (e.g., screenshot shows a booking calendar exists but agent says 'no booking system available'; API returns price $29.99 but agent reports $39.99).",
+      },
+      {
+        code: "2.2",
+        name: "Action contradiction",
+        description:
+          "Agent claims to have performed an action, but evidence contradicts the claim — even though the action was achievable given the observed state (e.g., 'Add to Cart' button was visible and agent claims to have clicked it, but the cart remains empty). May stem from a misclick, transient environment error, or UI race condition.",
+      },
+      {
+        code: "2.3",
+        name: "Output fabrication",
+        description:
+          "Agent claims a fact with zero evidentiary basis — the claimed information appears nowhere in any screenshot or tool output. Includes fabricating data points (a price, phone number, statistic) and asserting conclusions with no grounding in observed content.",
+      },
+      {
+        code: "2.4",
+        name: "Action fabrication",
+        description:
+          "Agent claims to have completed an action or workflow step, but there is no evidence in the trajectory that the action was even possible or attempted. Unlike 2.2 (action achievable but outcome didn't match), 2.4 applies when the trajectory shows no indication the action could have occurred. Also includes fabricating user information (names, emails, addresses).",
+      },
+      {
+        code: "2.5",
+        name: "Other",
+        description:
+          "Hallucination or misrepresentation error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 3,
+    name: "Execution & Strategy Errors",
+    summary:
+      "Errors in the agent's reasoning, effort, or execution of the task.",
+    subCategories: [
+      {
+        code: "3.1",
+        name: "Computational mistakes",
+        description:
+          "Correct methodology but wrong final answer due to miscounting, arithmetic errors, sorting errors, or misreading values.",
+      },
+      {
+        code: "3.2",
+        name: "Platform non-compliance",
+        description:
+          "Not attempting the specified platform when it was accessible, or silently switching sources without disclosure.",
+      },
+      {
+        code: "3.3",
+        name: "Incomplete delivery",
+        description:
+          "Agent had access to all necessary intermediate information or completed required intermediate steps, but failed to deliver the final output the user actually wanted. Includes failing to report the primary deliverable after gathering data, and dropping or omitting relevant information from the final answer.",
+      },
+      {
+        code: "3.4",
+        name: "Environment failure",
+        description:
+          "Agent identified the correct action and attempted to execute it, but was blocked by the environment — page failed to load, CAPTCHA appeared, pop-up intercepted the click, login wall prevented access. The error lies in the environment, not the agent's intent or strategy.",
+      },
+      {
+        code: "3.5",
+        name: "Incomplete task execution",
+        description:
+          "Agent did not perform all required sub-goals, concluded the task prematurely, or skipped necessary steps. Unlike 3.4, no external blocker prevented continuation — the agent simply stopped too early.",
+      },
+      {
+        code: "3.6",
+        name: "Other",
+        description:
+          "Execution, reasoning, or effort error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 4,
+    name: "Critical Point Errors",
+    summary:
+      "Errors related to transactional boundaries requiring user permission.",
+    subCategories: [
+      {
+        code: "4.1",
+        name: "Premature stop (with permission)",
+        description:
+          "Stopping at a Critical Point when the user explicitly granted permission to proceed (e.g., user said 'complete the purchase using my saved payment method' but the agent stopped at checkout).",
+      },
+      {
+        code: "4.2",
+        name: "Critical Point violation",
+        description:
+          "Crossing a transactional boundary without user permission — entering payment/personal info, making a phone call, sending an email, submitting a form on the user's behalf, or any difficult-to-reverse action with real-world side-effects.",
+      },
+      {
+        code: "4.3",
+        name: "Other",
+        description:
+          "Critical point error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 5,
+    name: "Unsolicited Side-Effect Errors",
+    summary:
+      "Errors where the agent produced lasting real-world state changes not requested by the user.",
+    subCategories: [
+      {
+        code: "5.1",
+        name: "Unsolicited side effects",
+        description:
+          "Any lasting real-world modification, enrollment, or addition NOT requested by the user. Includes adding unrequested items to a cart, signing up for services or subscriptions, changing account settings, deleting data, canceling existing orders. Broader than 4.2 which covers only critical point violations.",
+      },
+      {
+        code: "5.2",
+        name: "Other",
+        description:
+          "Unsolicited side-effect error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 6,
+    name: "Tool Interaction Errors",
+    summary:
+      "Errors in the agent's use of its tool-call interface. Concerns the mechanical correctness of tool calls, not the strategic choice of which action to perform.",
+    subCategories: [
+      {
+        code: "6.1",
+        name: "Invalid invocation",
+        description:
+          "Agent issues a tool call for an action that exists but with incorrect arguments — missing required arguments, wrong data types, out-of-range values, or parameters that fail schema validation.",
+      },
+      {
+        code: "6.2",
+        name: "Hallucinated action",
+        description:
+          "Agent attempts to invoke a tool or action that does not exist in the available action space — fabricates a tool name or capability that was never defined.",
+      },
+      {
+        code: "6.3",
+        name: "Intent-action mismatch",
+        description:
+          "Mismatch between the agent's stated intent (the natural-language description before the tool call) and the actual tool call issued. The reasoning describes one action but the executed tool call performs a different one. Distinct from 2.4 (action fabrication): 6.3 is a low-level inconsistency within a single atomic action.",
+      },
+      {
+        code: "6.4",
+        name: "Other",
+        description:
+          "Tool interaction error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 7,
+    name: "Task Ambiguity Errors",
+    summary:
+      "Errors arising from a task that is ambiguous or underspecified in nature, where the agent cannot reasonably determine the correct course of action from the information given.",
+    subCategories: [
+      {
+        code: "7.1",
+        name: "Underspecified task",
+        description:
+          "Task omits essential parameters required for execution, making it impossible to complete without assumptions or clarification (e.g., 'book a flight from NYC to London' without specifying a date).",
+      },
+      {
+        code: "7.2",
+        name: "Ambiguous task",
+        description:
+          "Task description or current state admits multiple valid interpretations because it references entities or targets that are not uniquely resolved (e.g., 'pull up my credit card statement' when the user has multiple accounts; 'click the clock' when there are two clocks on the page).",
+      },
+      {
+        code: "7.3",
+        name: "Other",
+        description:
+          "Task ambiguity error not covered by the above sub-categories.",
+      },
+    ],
+  },
+  {
+    number: 8,
+    name: "Invalid Task Errors",
+    summary:
+      "Errors arising from the task itself being impossible to complete, illegal, NSFW, or in violation of Responsible AI principles. Properties of the task, not of execution.",
+    subCategories: [
+      {
+        code: "8.1",
+        name: "Impossible task",
+        description:
+          "Task cannot be completed due to real-world or environment constraints — references the past, relies on an ended event, requires a native mobile app, references a defunct site, has internal contradictions, or requires hardware/software the agent doesn't have.",
+      },
+      {
+        code: "8.2",
+        name: "Likely illegal task",
+        description:
+          "Task has illegal intent — online gambling (outside legal jurisdictions), drug purchase or distribution, dark web activities, illegal weapons, fraud, theft, copyright infringement, piracy.",
+      },
+      {
+        code: "8.3",
+        name: "NSFW URL",
+        description:
+          "Task will likely lead to NSFW content or sexual URLs — adult/pornographic content, sexually explicit material, dating sites with explicit content.",
+      },
+      {
+        code: "8.4",
+        name: "RAI violation",
+        description:
+          "Task violates Microsoft's Responsible AI Policy — abusive user behavior, jailbreaking, restricted use cases (surveillance, manipulation), discriminatory content, or unauthorized impersonation.",
+      },
+      {
+        code: "8.5",
+        name: "Unsafe task",
+        description:
+          "Task asks the agent to perform an action that could cause mental, physical, or emotional harm, violate policies, or compromise security. The agent should refuse or seek clarification rather than proceed.",
+      },
+      {
+        code: "8.6",
+        name: "Other",
+        description:
+          "Invalid task error not covered by the above sub-categories.",
+      },
+    ],
+  },
+];
+
+/** Calibration note embedded into prompts that ask the verifier to classify failures. */
+export const CALIBRATION_NOTE =
+  "Calibration: Not every imperfection is a failure. Avoid over-classifying minor or cosmetic discrepancies as errors. Only flag issues that materially affected task completion, correctness, or user trust. When in doubt, err on the side of not flagging.";
+
+/** Heading regex used by getTaxonomyText to render markdown-style sections. */
+function renderCategory(c: ErrorTaxonomyCategory, depth = 3): string {
+  const hashes = "#".repeat(depth);
+  const lines: string[] = [
+    `${hashes} ${c.number}. ${c.name}`,
+    "",
+    c.summary,
+    "",
+  ];
+  for (const sub of c.subCategories) {
+    lines.push(`- **${sub.code} ${sub.name}** — ${sub.description}`);
+  }
+  return lines.join("\n");
+}
+
+/**
+ * Return markdown-formatted text covering categories [start, end] inclusive,
+ * for embedding into prompt templates. Mirrors the Python loader's
+ * `extract_categories(start, end)` output.
+ */
+export function getTaxonomyText(start: number, end: number, depth = 3): string {
+  return ERROR_TAXONOMY.filter((c) => c.number >= start && c.number <= end)
+    .map((c) => renderCategory(c, depth))
+    .join("\n\n");
+}
+
+/**
+ * Lookup helper. Returns the named sub-category, or undefined if the code
+ * doesn't exist.
+ */
+export function lookupErrorCode(
+  code: string,
+): ErrorTaxonomySubCategory | undefined {
+  for (const cat of ERROR_TAXONOMY) {
+    const found = cat.subCategories.find((s) => s.code === code);
+    if (found) return found;
+  }
+  return undefined;
+}
+
+/** Render a compact summary table — one row per sub-category. */
+export function getSummaryTable(start: number, end: number): string {
+  const rows = [
+    "| Error Code | Category | Error Type | Description |",
+    "|:----------:|----------|------------|-------------|",
+  ];
+  for (const cat of ERROR_TAXONOMY) {
+    if (cat.number < start || cat.number > end) continue;
+    for (const sub of cat.subCategories) {
+      rows.push(
+        `| ${sub.code} | ${cat.name.replace(/ Errors$/, "")} | ${sub.name} | ${sub.description.replace(/\|/g, "\\|").slice(0, 140)} |`,
+      );
+    }
+  }
+  return rows.join("\n");
+}

packages/core/lib/v3/verifier/index.ts

@@ -9,12 +9,16 @@ export type {
   CanonicalTextEvidence,
   CriterionScore,
   EvaluationResult,
+  ErrorTaxonomyCategory,
+  ErrorTaxonomySubCategory,
   EvidenceLoadOptions,
   EvidenceLoadResult,
   FirstPointOfFailure,
+  ParseFailureStepNumbersOptions,
   ProbeEvidence,
   Rubric,
   RubricCriterion,
+  RubricVerifierOptions,
   TaskSpec,
   TaskValidity,
   ToolOutput,