add variables to agent

.changeset/clear-hands-attend.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Add unified variables support across act and agent with a single VariableValue type

packages/core/lib/prompt.ts

@@ -1,4 +1,5 @@
 import { ChatMessage } from "./v3/llm/LLMClient.js";
+import type { Variables } from "./v3/types/public/agent.js";
 
 export function buildUserInstructionsString(
   userProvidedInstructions?: string,
@@ -170,7 +171,7 @@ Return the element that matches the instruction if it exists. Otherwise, return
 export function buildActPrompt(
   action: string,
   supportedActions: string[],
-  variables?: Record<string, string>,
+  variables?: Variables,
 ): string {
   // Base instruction
   let instruction = `Find the most relevant element to perform an action on given the following action: ${action}.  
@@ -215,7 +216,7 @@ export function buildStepTwoPrompt(
   originalUserAction: string,
   previousAction: string,
   supportedActions: string[],
-  variables?: Record<string, string>,
+  variables?: Variables,
 ): string {
   // Base instruction
   let instruction = `

packages/core/lib/v3/agent/prompts/agentSystemPrompt.ts

@@ -1,4 +1,4 @@
-import type { AgentToolMode } from "../../types/public/agent.js";
+import type { AgentToolMode, Variables } from "../../types/public/agent.js";
 
 export interface AgentSystemPromptOptions {
   url: string;
@@ -9,6 +9,8 @@ export interface AgentSystemPromptOptions {
   isBrowserbase?: boolean;
   /** Tools to exclude from the system prompt */
   excludeTools?: string[];
+  /** Variables available to the agent for use in act/type tools */
+  variables?: Variables;
 }
 
 /**
@@ -122,6 +124,7 @@ export function buildAgentSystemPrompt(
     systemInstructions,
     isBrowserbase = false,
     excludeTools,
+    variables,
   } = options;
   const localeDate = new Date().toLocaleDateString();
   const isoDate = new Date().toISOString();
@@ -203,6 +206,30 @@ export function buildAgentSystemPrompt(
     ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\n  `
     : "";
 
+  // Build variables section only if variables are provided
+  const hasVariables = variables && Object.keys(variables).length > 0;
+  const variableToolsNote = isHybridMode
+    ? "Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields."
+    : "Use %variableName% syntax in the act or fillForm tool's value/action fields.";
+  const variablesSection = hasVariables
+    ? `<variables>
+    <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>
+    <usage>${variableToolsNote}</usage>
+    <example>To type a password, use: type %password% into the password field</example>
+    ${Object.entries(variables)
+      .map(([name, v]) => {
+        const description =
+          typeof v === "object" && v !== null && "value" in v
+            ? v.description
+            : undefined;
+        return description
+          ? `<variable name="${name}">${description}</variable>`
+          : `<variable name="${name}" />`;
+      })
+      .join("\n    ")}
+  </variables>`
+    : "";
+
   return `<system>
   <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>
   ${customInstructionsBlock}<task>
@@ -234,6 +261,7 @@ export function buildAgentSystemPrompt(
     ${commonStrategyItems}
   </strategy>
   ${roadblocksSection}
+  ${variablesSection}
   <completion>
     <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>
     <examples>

packages/core/lib/v3/agent/tools/act.ts

@@ -2,18 +2,23 @@ import { tool } from "ai";
 import { z } from "zod";
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
-import type { AgentModelConfig } from "../../types/public/agent.js";
+import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
 
-export const actTool = (v3: V3, executionModel?: string | AgentModelConfig) =>
-  tool({
+export const actTool = (
+  v3: V3,
+  executionModel?: string | AgentModelConfig,
+  variables?: Variables,
+) => {
+  const hasVariables = variables && Object.keys(variables).length > 0;
+  const actionDescription = hasVariables
+    ? `Describe what to click or type, e.g. "click the Login button" or "type %variableName% into the input". Available variables: ${Object.keys(variables).join(", ")}`
+    : 'Describe what to click or type, e.g. "click the Login button" or "type "John" into the first name input"';
+
+  return tool({
     description:
       "Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.",
     inputSchema: z.object({
-      action: z
-        .string()
-        .describe(
-          'Describe what to click or type, e.g. "click the Login button" or "type "John" into the first name input"',
-        ),
+      action: z.string().describe(actionDescription),
     }),
     execute: async ({ action }) => {
       try {
@@ -28,7 +33,9 @@ export const actTool = (v3: V3, executionModel?: string | AgentModelConfig) =>
             },
           },
         });
-        const options = executionModel ? { model: executionModel } : undefined;
+        const options = executionModel
+          ? { model: executionModel, variables }
+          : { variables };
         const result = await v3.act(action, options);
         const actions = (result.actions as Action[] | undefined) ?? [];
         v3.recordAgentReplayStep({
@@ -57,3 +64,4 @@ export const actTool = (v3: V3, executionModel?: string | AgentModelConfig) =>
       }
     },
   });
+};

packages/core/lib/v3/agent/tools/fillFormVision.ts

@@ -5,13 +5,24 @@ import type { Action } from "../../types/public/methods.js";
 import type {
   FillFormVisionToolResult,
   ModelOutputContentItem,
+  Variables,
 } from "../../types/public/agent.js";
 import { processCoordinates } from "../utils/coordinateNormalization.js";
 import { ensureXPath } from "../utils/xpath.js";
 import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+import { substituteVariables } from "../utils/variables.js";
 
-export const fillFormVisionTool = (v3: V3, provider?: string) =>
-  tool({
+export const fillFormVisionTool = (
+  v3: V3,
+  provider?: string,
+  variables?: Variables,
+) => {
+  const hasVariables = variables && Object.keys(variables).length > 0;
+  const valueDescription = hasVariables
+    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+    : "Text to type into the target field";
+
+  return tool({
     description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL
 
 CRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)
@@ -38,7 +49,7 @@ MANDATORY USE CASES (always use fillFormVision for these):
               .describe(
                 "Description of the typing action, e.g. 'type foo into the bar field'",
               ),
-            value: z.string().describe("Text to type into the target field"),
+            value: z.string().describe(valueDescription),
             coordinates: z
               .object({
                 x: z.number(),
@@ -53,7 +64,8 @@ MANDATORY USE CASES (always use fillFormVision for these):
       try {
         const page = await v3.context.awaitActivePage();
 
-        // Process coordinates for each field
+        // Process coordinates and substitute variables for each field
+        // Keep original values (with %tokens%) for logging/caching, substituted values for typing
         const processedFields = fields.map((field) => {
           const processed = processCoordinates(
             field.coordinates.x,
@@ -63,6 +75,8 @@ MANDATORY USE CASES (always use fillFormVision for these):
           );
           return {
             ...field,
+            originalValue: field.value, // Keep original with %tokens% for cache
+            value: substituteVariables(field.value, variables),
             coordinates: { x: processed.x, y: processed.y },
           };
         });
@@ -73,7 +87,7 @@ MANDATORY USE CASES (always use fillFormVision for these):
           level: 1,
           auxiliary: {
             arguments: {
-              value: JSON.stringify({ fields, processedFields }),
+              value: JSON.stringify({ fields }), // Don't log substituted values
               type: "object",
             },
           },
@@ -95,14 +109,15 @@ MANDATORY USE CASES (always use fillFormVision for these):
           await page.type(field.value);
 
           // Build Action with XPath for deterministic replay (only when caching)
+          // Use originalValue (with %tokens%) so cache stores references, not sensitive values
           if (shouldCollectXpath) {
             const normalizedXpath = ensureXPath(xpath);
             if (normalizedXpath) {
               actions.push({
                 selector: normalizedXpath,
                 description: field.action,
                 method: "type",
-                arguments: [field.value],
+                arguments: [field.originalValue],
               });
             }
           }
@@ -169,3 +184,4 @@ MANDATORY USE CASES (always use fillFormVision for these):
       };
     },
   });
+};

packages/core/lib/v3/agent/tools/fillform.ts

@@ -2,13 +2,19 @@ import { tool } from "ai";
 import { z } from "zod";
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
-import type { AgentModelConfig } from "../../types/public/agent.js";
+import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
 
 export const fillFormTool = (
   v3: V3,
   executionModel?: string | AgentModelConfig,
-) =>
-  tool({
+  variables?: Variables,
+) => {
+  const hasVariables = variables && Object.keys(variables).length > 0;
+  const valueDescription = hasVariables
+    ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+    : "Text to type into the target";
+
+  return tool({
     description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
     inputSchema: z.object({
       fields: z
@@ -19,7 +25,7 @@ export const fillFormTool = (
               .describe(
                 'Description of typing action, e.g. "type foo into the email field"',
               ),
-            value: z.string().describe("Text to type into the target"),
+            value: z.string().describe(valueDescription),
           }),
         )
         .min(1, "Provide at least one field to fill"),
@@ -48,7 +54,8 @@ export const fillFormTool = (
       const completed = [] as unknown[];
       const replayableActions: Action[] = [];
       for (const res of observeResults) {
-        const actResult = await v3.act(res);
+        const actOptions = variables ? { variables } : undefined;
+        const actResult = await v3.act(res, actOptions);
         completed.push(actResult);
         if (Array.isArray(actResult.actions)) {
           replayableActions.push(...(actResult.actions as Action[]));
@@ -67,3 +74,4 @@ export const fillFormTool = (
       };
     },
   });
+};

packages/core/lib/v3/agent/tools/index.ts

@@ -22,6 +22,7 @@ import type { LogLine } from "../../types/public/logs.js";
 import type {
   AgentToolMode,
   AgentModelConfig,
+  Variables,
 } from "../../types/public/agent.js";
 
 export interface V3AgentToolOptions {
@@ -42,6 +43,11 @@ export interface V3AgentToolOptions {
    * These tools will be filtered out after mode-based filtering.
    */
   excludeTools?: string[];
+  /**
+   * Variables available to the agent for use in act/type tools.
+   * When provided, these tools will have an optional useVariable field.
+   */
+  variables?: Variables;
 }
 
 /**
@@ -83,23 +89,24 @@ export function createAgentTools(v3: V3, options?: V3AgentToolOptions) {
   const mode = options?.mode ?? "dom";
   const provider = options?.provider;
   const excludeTools = options?.excludeTools;
+  const variables = options?.variables;
 
   const allTools: ToolSet = {
-    act: actTool(v3, executionModel),
+    act: actTool(v3, executionModel, variables),
     ariaTree: ariaTreeTool(v3),
     click: clickTool(v3, provider),
     clickAndHold: clickAndHoldTool(v3, provider),
     dragAndDrop: dragAndDropTool(v3, provider),
     extract: extractTool(v3, executionModel),
-    fillForm: fillFormTool(v3, executionModel),
-    fillFormVision: fillFormVisionTool(v3, provider),
+    fillForm: fillFormTool(v3, executionModel, variables),
+    fillFormVision: fillFormVisionTool(v3, provider, variables),
     goto: gotoTool(v3),
     keys: keysTool(v3),
     navback: navBackTool(v3),
     screenshot: screenshotTool(v3),
     scroll: mode === "hybrid" ? scrollVisionTool(v3, provider) : scrollTool(v3),
     think: thinkTool(),
-    type: typeTool(v3, provider),
+    type: typeTool(v3, provider, variables),
     wait: waitTool(v3, mode),
   };
 

packages/core/lib/v3/agent/tools/type.ts

@@ -5,13 +5,20 @@ import type { Action } from "../../types/public/methods.js";
 import type {
   TypeToolResult,
   ModelOutputContentItem,
+  Variables,
 } from "../../types/public/agent.js";
 import { processCoordinates } from "../utils/coordinateNormalization.js";
 import { ensureXPath } from "../utils/xpath.js";
 import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+import { substituteVariables } from "../utils/variables.js";
 
-export const typeTool = (v3: V3, provider?: string) =>
-  tool({
+export const typeTool = (v3: V3, provider?: string, variables?: Variables) => {
+  const hasVariables = variables && Object.keys(variables).length > 0;
+  const textDescription = hasVariables
+    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+    : "The text to type into the element";
+
+  return tool({
     description:
       "Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
     inputSchema: z.object({
@@ -20,7 +27,7 @@ export const typeTool = (v3: V3, provider?: string) =>
         .describe(
           "Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description",
         ),
-      text: z.string().describe("The text to type into the element"),
+      text: z.string().describe(textDescription),
       coordinates: z
         .array(z.number())
         .describe("The (x, y) coordinates to type into the element"),
@@ -39,6 +46,9 @@ export const typeTool = (v3: V3, provider?: string) =>
           v3,
         );
 
+        // Substitute any %variableName% tokens in the text
+        const actualText = substituteVariables(text, variables);
+
         v3.logger({
           category: "agent",
           message: `Agent calling tool: type`,
@@ -57,7 +67,7 @@ export const typeTool = (v3: V3, provider?: string) =>
           returnXpath: shouldCollectXpath,
         });
 
-        await page.type(text);
+        await page.type(actualText);
 
         const screenshotBase64 = await waitAndCaptureScreenshot(page);
 
@@ -83,7 +93,7 @@ export const typeTool = (v3: V3, provider?: string) =>
         return {
           success: true,
           describe,
-          text,
+          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM
           screenshotBase64,
         };
       } catch (error) {
@@ -128,3 +138,4 @@ export const typeTool = (v3: V3, provider?: string) =>
       };
     },
   });
+};