feat(core): implement stable selectors (id, css, data-attributes)

packages/core/lib/v3/cache/ActCache.ts

@@ -292,6 +292,18 @@ export class ActCache {
         return true;
       }
 
+      if (orig.id !== next.id) {
+        return true;
+      }
+
+      if (orig.cssSelector !== next.cssSelector) {
+        return true;
+      }
+
+      if (JSON.stringify(orig.attributes) !== JSON.stringify(next.attributes)) {
+        return true;
+      }
+
       if ((orig.method ?? "") !== (next.method ?? "")) {
         return true;
       }

packages/core/lib/v3/handlers/actHandler.ts

@@ -51,6 +51,7 @@ export class ActHandler {
     inferenceTimeMs: number,
   ) => void;
   private readonly defaultDomSettleTimeoutMs?: number;
+  private readonly preferredSelectorType?: "id" | "css" | "xpath";
 
   constructor(
     llmClient: LLMClient,
@@ -69,6 +70,7 @@ export class ActHandler {
       inferenceTimeMs: number,
     ) => void,
     defaultDomSettleTimeoutMs?: number,
+    preferredSelectorType?: "id" | "css" | "xpath",
   ) {
     this.llmClient = llmClient;
     this.defaultModelName = defaultModelName;
@@ -79,6 +81,7 @@ export class ActHandler {
     this.selfHeal = !!selfHeal;
     this.onMetrics = onMetrics;
     this.defaultDomSettleTimeoutMs = defaultDomSettleTimeoutMs;
+    this.preferredSelectorType = preferredSelectorType;
   }
 
   private recordActMetrics(response: ActInferenceResponse): void {
@@ -96,12 +99,18 @@ export class ActHandler {
     instruction,
     domElements,
     xpathMap,
+    idMap,
+    cssSelectorMap,
+    attributesMap,
     llmClient,
     requireMethodAndArguments = true,
   }: {
     instruction: string;
     domElements: string;
     xpathMap: Record<string, string>;
+    idMap: Record<string, string>;
+    cssSelectorMap: Record<string, string>;
+    attributesMap: Record<string, Record<string, string>>;
     llmClient: LLMClient;
     requireMethodAndArguments?: boolean;
   }): Promise<{ action?: Action; response: ActInferenceResponse }> {
@@ -119,6 +128,9 @@ export class ActHandler {
     const normalized = normalizeActInferenceElement(
       response.element as ActInferenceElement | undefined,
       xpathMap,
+      idMap,
+      cssSelectorMap,
+      attributesMap,
       requireMethodAndArguments,
     );
 
@@ -150,10 +162,13 @@ export class ActHandler {
       this.defaultDomSettleTimeoutMs,
     );
     ensureTimeRemaining();
-    const { combinedTree, combinedXpathMap } = await captureHybridSnapshot(
-      page,
-      { experimental: true },
-    );
+    const {
+      combinedTree,
+      combinedXpathMap,
+      combinedIdMap,
+      combinedCssSelectorMap,
+      combinedAttributesMap,
+    } = await captureHybridSnapshot(page, { experimental: true });
 
     const actInstruction = buildActPrompt(
       instruction,
@@ -167,6 +182,9 @@ export class ActHandler {
         instruction: actInstruction,
         domElements: combinedTree,
         xpathMap: combinedXpathMap,
+        idMap: combinedIdMap,
+        cssSelectorMap: combinedCssSelectorMap,
+        attributesMap: combinedAttributesMap,
         llmClient,
       });
 
@@ -202,10 +220,15 @@ export class ActHandler {
 
     // Take a new focused snapshot and observe again
     ensureTimeRemaining();
-    const { combinedTree: combinedTree2, combinedXpathMap: combinedXpathMap2 } =
-      await captureHybridSnapshot(page, {
-        experimental: true,
-      });
+    const {
+      combinedTree: combinedTree2,
+      combinedXpathMap: combinedXpathMap2,
+      combinedIdMap: combinedIdMap2,
+      combinedCssSelectorMap: combinedCssSelectorMap2,
+      combinedAttributesMap: combinedAttributesMap2,
+    } = await captureHybridSnapshot(page, {
+      experimental: true,
+    });
 
     let diffedTree = diffCombinedTrees(combinedTree, combinedTree2);
     if (!diffedTree.trim()) {
@@ -234,6 +257,9 @@ export class ActHandler {
       instruction: stepTwoInstructions,
       domElements: diffedTree,
       xpathMap: combinedXpathMap2,
+      idMap: combinedIdMap2,
+      cssSelectorMap: combinedCssSelectorMap2,
+      attributesMap: combinedAttributesMap2,
       llmClient,
     });
 
@@ -302,23 +328,26 @@ export class ActHandler {
     const resolvedArgs =
       substituteVariablesInArguments(action.arguments, variables) ?? [];
 
+    const bestSelector = getBestSelector(action, this.preferredSelectorType);
+
     try {
       ensureTimeRemaining?.();
       await performUnderstudyMethod(
         page,
         page.mainFrame(),
         method,
-        action.selector,
+        bestSelector,
         resolvedArgs,
         settleTimeout,
       );
       return {
         success: true,
-        message: `Action [${method}] performed successfully on selector: ${action.selector}`,
+        message: `Action [${method}] performed successfully on selector: ${bestSelector}`,
         actionDescription: action.description || `action (${method})`,
         actions: [
           {
-            selector: action.selector,
+            ...action,
+            selector: bestSelector,
             description: action.description || `action (${method})`,
             method,
             arguments: placeholderArgs,
@@ -357,10 +386,15 @@ export class ActHandler {
 
           // Take a fresh snapshot and ask for a new actionable element
           ensureTimeRemaining?.();
-          const { combinedTree, combinedXpathMap } =
-            await captureHybridSnapshot(page, {
-              experimental: true,
-            });
+          const {
+            combinedTree,
+            combinedXpathMap,
+            combinedIdMap,
+            combinedCssSelectorMap,
+            combinedAttributesMap,
+          } = await captureHybridSnapshot(page, {
+            experimental: true,
+          });
 
           const instruction = buildActPrompt(
             actCommand,
@@ -374,6 +408,9 @@ export class ActHandler {
               instruction,
               domElements: combinedTree,
               xpathMap: combinedXpathMap,
+              idMap: combinedIdMap,
+              cssSelectorMap: combinedCssSelectorMap,
+              attributesMap: combinedAttributesMap,
               llmClient: effectiveClient,
               requireMethodAndArguments: false,
             });
@@ -390,9 +427,12 @@ export class ActHandler {
           }
 
           // Retry with original method/args but new selector from fallback
-          let newSelector = action.selector;
-          if (fallbackAction?.selector) {
-            newSelector = fallbackAction.selector;
+          let newSelector = bestSelector;
+          if (fallbackAction) {
+            newSelector = getBestSelector(
+              fallbackAction,
+              this.preferredSelectorType,
+            );
           }
 
           ensureTimeRemaining?.();
@@ -411,6 +451,7 @@ export class ActHandler {
             actionDescription: action.description || `action (${method})`,
             actions: [
               {
+                ...(fallbackAction ?? action),
                 selector: newSelector,
                 description: action.description || `action (${method})`,
                 method,
@@ -443,9 +484,37 @@ export class ActHandler {
   }
 }
 
+function getBestSelector(
+  action: Action,
+  preferredSelectorType?: "id" | "css" | "xpath",
+): string {
+  // 1. Strict Backward Compatibility: If no preference is set, use the original selector (XPath)
+  if (!preferredSelectorType) {
+    return action.selector;
+  }
+
+  // 2. Try explicit preference
+  if (preferredSelectorType === "id" && action.id) return `#${action.id}`;
+  if (preferredSelectorType === "css" && action.cssSelector)
+    return action.cssSelector;
+  if (preferredSelectorType === "xpath") return action.selector;
+
+  // 3. Fallback for Opt-in users:
+  // If the user opted into stable selectors (by setting any preference),
+  // but the specific preferred one wasn't found, try other stable options.
+  if (action.id) return `#${action.id}`;
+  if (action.cssSelector) return action.cssSelector;
+
+  // 4. Final fallback
+  return action.selector;
+}
+
 function normalizeActInferenceElement(
   element: ActInferenceElement | undefined,
   xpathMap: Record<string, string>,
+  idMap: Record<string, string>,
+  cssSelectorMap: Record<string, string>,
+  attributesMap: Record<string, Record<string, string>>,
   requireMethodAndArguments = true,
 ): Action | undefined {
   if (!element) {
@@ -476,6 +545,9 @@ function normalizeActInferenceElement(
     method,
     arguments: hasArgs ? args : undefined,
     selector: `xpath=${trimmed}`,
+    id: idMap[elementId as EncodedId],
+    cssSelector: cssSelectorMap[elementId as EncodedId],
+    attributes: attributesMap[elementId as EncodedId],
   } as Action;
 }
 

packages/core/lib/v3/handlers/observeHandler.ts

@@ -98,6 +98,9 @@ export class ObserveHandler {
 
     const combinedTree = snapshot.combinedTree;
     const combinedXpathMap = snapshot.combinedXpathMap ?? {};
+    const combinedIdMap = snapshot.combinedIdMap ?? {};
+    const combinedCssSelectorMap = snapshot.combinedCssSelectorMap ?? {};
+    const combinedAttributesMap = snapshot.combinedAttributesMap ?? {};
 
     v3Logger({
       category: "observation",
@@ -148,20 +151,18 @@ export class ObserveHandler {
             return {
               ...rest,
               selector: `xpath=${trimmedXpath}`,
-            } as {
-              description: string;
-              method?: string;
-              arguments?: string[];
-              selector: string;
-            };
+              id: combinedIdMap[lookUpIndex],
+              cssSelector: combinedCssSelectorMap[lookUpIndex],
+              attributes: combinedAttributesMap[lookUpIndex],
+            } as Action;
           }
           // shadow-root fallback:
           return {
             description: "an element inside a shadow DOM",
             method: "not-supported",
             arguments: [],
             selector: "not-supported",
-          };
+          } as Action;
         }),
       )
     ).filter(<T>(e: T | undefined): e is T => e !== undefined);

packages/core/lib/v3/tests/v3.bb.config.ts

@@ -6,9 +6,11 @@ import path from "path";
 dotenv.config();
 
 const rootEnvPath = path.resolve(__dirname, "../../../.env");
+// @ts-ignore
 dotenv.config({ path: rootEnvPath, override: false });
 
 const localTestEnvPath = path.resolve(__dirname, ".env");
+// @ts-ignore
 dotenv.config({ path: localTestEnvPath, override: false });
 
 export const v3BBTestConfig: V3Options = {

packages/core/lib/v3/tests/v3.bb.playwright.config.ts

@@ -7,6 +7,7 @@ dotenv.config();
 
 // Try loading from repo root (packages/core/lib/v3/tests -> repo root = 5 levels up)
 const repoRootEnvPath = path.resolve(__dirname, "../../../../../.env");
+// @ts-ignore
 dotenv.config({ path: repoRootEnvPath, override: false });
 
 // Set TEST_ENV before tests run

packages/core/lib/v3/tests/v3.dynamic.config.ts

@@ -6,9 +6,11 @@ import path from "path";
 dotenv.config();
 
 const rootEnvPath = path.resolve(__dirname, "../../../.env");
+// @ts-ignore
 dotenv.config({ path: rootEnvPath, override: false });
 
 const localTestEnvPath = path.resolve(__dirname, ".env");
+// @ts-ignore
 dotenv.config({ path: localTestEnvPath, override: false });
 
 // Determine environment from TEST_ENV variable

packages/core/lib/v3/tests/v3.local.playwright.config.ts

@@ -7,6 +7,7 @@ dotenv.config();
 
 // Try loading from repo root (packages/core/lib/v3/tests -> repo root = 5 levels up)
 const repoRootEnvPath = path.resolve(__dirname, "../../../../../.env");
+// @ts-ignore
 dotenv.config({ path: repoRootEnvPath, override: false });
 
 // Set TEST_ENV before tests run

packages/core/lib/v3/types/private/snapshot.ts

@@ -28,6 +28,12 @@ export type HybridSnapshot = {
   combinedXpathMap: Record<string, string>;
   /** EncodedId -> URL extracted from AX properties. */
   combinedUrlMap: Record<string, string>;
+  /** EncodedId -> ID attribute. */
+  combinedIdMap: Record<string, string>;
+  /** EncodedId -> CSS Selector. */
+  combinedCssSelectorMap: Record<string, string>;
+  /** EncodedId -> Attributes. */
+  combinedAttributesMap: Record<string, Record<string, string>>;
   /** Per-frame payloads expose the original relative data for debugging. */
   perFrame?: PerFrameSnapshot[];
 };
@@ -37,6 +43,9 @@ export type PerFrameSnapshot = {
   outline: string;
   xpathMap: Record<string, string>;
   urlMap: Record<string, string>;
+  idMap: Record<string, string>;
+  cssSelectorMap: Record<string, string>;
+  attributesMap: Record<string, Record<string, string>>;
 };
 
 /**
@@ -48,6 +57,7 @@ export type SessionDomIndex = {
   absByBe: Map<number, string>;
   tagByBe: Map<number, string>;
   scrollByBe: Map<number, boolean>;
+  attributesByBe: Map<number, Record<string, string>>;
   docRootOf: Map<number, number>;
   contentDocRootByIframe: Map<number, number>;
 };
@@ -57,6 +67,9 @@ export type FrameDomMaps = {
   xpathMap: Record<string, string>;
   scrollableMap: Record<string, boolean>;
   urlMap: Record<string, string>;
+  idMap: Record<string, string>;
+  cssSelectorMap: Record<string, string>;
+  attributesMap: Record<string, Record<string, string>>;
 };
 
 export type ResolvedLocation = {

packages/core/lib/v3/types/public/methods.ts

@@ -28,6 +28,9 @@ export interface Action {
   description: string;
   method?: string;
   arguments?: string[];
+  cssSelector?: string;
+  id?: string;
+  attributes?: Record<string, string>;
 }
 
 export interface HistoryEntry {

packages/core/lib/v3/types/public/options.ts

@@ -49,4 +49,5 @@ export interface V3Options {
   cacheDir?: string;
   domSettleTimeout?: number;
   disableAPI?: boolean;
+  preferredSelectorType?: "id" | "css" | "xpath";
 }