diff --git a/packages/core/lib/v3/handlers/extractHandler.ts b/packages/core/lib/v3/handlers/extractHandler.ts index 772c4abbe..6311226e7 100644 --- a/packages/core/lib/v3/handlers/extractHandler.ts +++ b/packages/core/lib/v3/handlers/extractHandler.ts @@ -115,6 +115,7 @@ export class ExtractHandler { page, selector, ignoreSelectors, + selectAll, timeout, model, } = params; @@ -135,6 +136,7 @@ export class ExtractHandler { experimental: this.experimental, focusSelector: focusSelector || undefined, ignoreSelectors, + selectAll, }); ensureTimeRemaining(); @@ -157,6 +159,7 @@ export class ExtractHandler { experimental: this.experimental, focusSelector: focusSelector, ignoreSelectors, + selectAll, }); v3Logger({ diff --git a/packages/core/lib/v3/handlers/observeHandler.ts b/packages/core/lib/v3/handlers/observeHandler.ts index 54565e6bd..5731cb005 100644 --- a/packages/core/lib/v3/handlers/observeHandler.ts +++ b/packages/core/lib/v3/handlers/observeHandler.ts @@ -70,6 +70,7 @@ export class ObserveHandler { timeout, selector, ignoreSelectors, + selectAll, model, variables, } = params; @@ -104,6 +105,7 @@ export class ObserveHandler { experimental: this.experimental, focusSelector: focusSelector || undefined, ignoreSelectors, + selectAll, }); const combinedTree = snapshot.combinedTree; @@ -149,39 +151,52 @@ export class ObserveHandler { // Map elementIds -> selectors via combinedXpathMap const elementsWithSelectors = ( await Promise.all( - observationResponse.elements.map(async (element) => { - const { elementId, ...rest } = element; // rest may or may not have method/arguments - if (typeof elementId === "string" && elementId.includes("-")) { - const lookUpIndex = elementId as EncodedId; - const xpath = combinedXpathMap[lookUpIndex]; - const trimmedXpath = trimTrailingTextNode(xpath); - if (!trimmedXpath) return undefined; - - // For dragAndDrop, convert element ID in arguments to xpath (target element) - let resolvedArgs = rest.arguments; - if ( - rest.method === "dragAndDrop" && - Array.isArray(rest.arguments) && - rest.arguments.length > 0 - ) { - const targetArg = rest.arguments[0]; - // Check if argument looks like an element ID (e.g., "1-67") + observationResponse.elements.map( + async (element: (typeof observationResponse.elements)[number]) => { + const { elementId, ...rest } = element; // rest may or may not have method/arguments + if (typeof elementId === "string" && elementId.includes("-")) { + const lookUpIndex = elementId as EncodedId; + const xpath = combinedXpathMap[lookUpIndex]; + const trimmedXpath = trimTrailingTextNode(xpath); + if (!trimmedXpath) return undefined; + + // For dragAndDrop, convert element ID in arguments to xpath (target element) + let resolvedArgs = rest.arguments; if ( - typeof targetArg === "string" && - /^\d+-\d+$/.test(targetArg) + rest.method === "dragAndDrop" && + Array.isArray(rest.arguments) && + rest.arguments.length > 0 ) { - const argXpath = combinedXpathMap[targetArg as EncodedId]; - const trimmedArgXpath = trimTrailingTextNode(argXpath); - if (trimmedArgXpath) { - resolvedArgs = [ - `xpath=${trimmedArgXpath}`, - ...rest.arguments.slice(1), - ]; + const targetArg = rest.arguments[0]; + // Check if argument looks like an element ID (e.g., "1-67") + if ( + typeof targetArg === "string" && + /^\d+-\d+$/.test(targetArg) + ) { + const argXpath = combinedXpathMap[targetArg as EncodedId]; + const trimmedArgXpath = trimTrailingTextNode(argXpath); + if (trimmedArgXpath) { + resolvedArgs = [ + `xpath=${trimmedArgXpath}`, + ...rest.arguments.slice(1), + ]; + } else { + // Target element lookup failed, filter out this action + v3Logger({ + category: "observation", + message: "dragAndDrop target element lookup failed", + level: 0, + auxiliary: { + targetElementId: { value: targetArg, type: "string" }, + sourceElementId: { value: elementId, type: "string" }, + }, + }); + return undefined; + } } else { - // Target element lookup failed, filter out this action v3Logger({ category: "observation", - message: "dragAndDrop target element lookup failed", + message: "dragAndDrop target element invalid ID format", level: 0, auxiliary: { targetElementId: { value: targetArg, type: "string" }, @@ -190,39 +205,28 @@ export class ObserveHandler { }); return undefined; } - } else { - v3Logger({ - category: "observation", - message: "dragAndDrop target element invalid ID format", - level: 0, - auxiliary: { - targetElementId: { value: targetArg, type: "string" }, - sourceElementId: { value: elementId, type: "string" }, - }, - }); - return undefined; } - } + return { + ...rest, + arguments: resolvedArgs, + selector: `xpath=${trimmedXpath}`, + } as { + description: string; + method?: string; + arguments?: string[]; + selector: string; + }; + } + // shadow-root fallback: return { - ...rest, - arguments: resolvedArgs, - selector: `xpath=${trimmedXpath}`, - } as { - description: string; - method?: string; - arguments?: string[]; - selector: string; + description: "an element inside a shadow DOM", + method: "not-supported", + arguments: [], + selector: "not-supported", }; - } - // shadow-root fallback: - return { - description: "an element inside a shadow DOM", - method: "not-supported", - arguments: [], - selector: "not-supported", - }; - }), + }, + ), ) ).filter((e: T | undefined): e is T => e !== undefined); diff --git a/packages/core/lib/v3/types/private/handlers.ts b/packages/core/lib/v3/types/private/handlers.ts index 22d98ed53..2140e503e 100644 --- a/packages/core/lib/v3/types/private/handlers.ts +++ b/packages/core/lib/v3/types/private/handlers.ts @@ -18,6 +18,7 @@ export interface ExtractHandlerParams { timeout?: number; selector?: string; ignoreSelectors?: string[]; + selectAll?: boolean; page: Page; } @@ -28,6 +29,7 @@ export interface ObserveHandlerParams { timeout?: number; selector?: string; ignoreSelectors?: string[]; + selectAll?: boolean; page: Page; } diff --git a/packages/core/lib/v3/types/private/snapshot.ts b/packages/core/lib/v3/types/private/snapshot.ts index 0d3a6f2a1..b3d794221 100644 --- a/packages/core/lib/v3/types/private/snapshot.ts +++ b/packages/core/lib/v3/types/private/snapshot.ts @@ -7,6 +7,8 @@ export type SnapshotOptions = { * Supports XPath (prefixed with `xpath=` or starting with `/`) and CSS with iframe hops via `>>`. */ focusSelector?: string; + /** When true, scope to every element matched by `focusSelector` instead of only the first match. */ + selectAll?: boolean; /** * Exclude matching elements and their descendants from the captured snapshot. * Each selector may be XPath (prefixed with `xpath=` or starting with `/`) or CSS. @@ -112,6 +114,7 @@ export type A11yNode = { export type A11yOptions = { focusSelector?: string; isIgnoredBackendNode?: (backendNodeId: number) => boolean; + selectAll?: boolean; experimental: boolean; tagNameMap: Record; scrollableMap: Record; diff --git a/packages/core/lib/v3/types/public/api.ts b/packages/core/lib/v3/types/public/api.ts index 3ed21db8f..d1d97733a 100644 --- a/packages/core/lib/v3/types/public/api.ts +++ b/packages/core/lib/v3/types/public/api.ts @@ -517,6 +517,11 @@ export const ExtractOptionsSchema = z "Selectors for elements and subtrees that should be excluded from extraction", example: ["nav", ".cookie-banner", "#sidebar-ads"], }), + selectAll: z.boolean().optional().meta({ + description: + "When true, apply selector scoping to all matching elements instead of only the first match", + example: true, + }), }) .optional() .meta({ id: "ExtractOptions" }); @@ -597,6 +602,11 @@ export const ObserveOptionsSchema = z "Selectors for elements and subtrees that should be excluded from observation", example: ["nav", ".cookie-banner", "#sidebar-ads"], }), + selectAll: z.boolean().optional().meta({ + description: + "When true, apply selector scoping to all matching elements instead of only the first match", + example: true, + }), }) .optional() .meta({ id: "ObserveOptions" }); diff --git a/packages/core/lib/v3/types/public/methods.ts b/packages/core/lib/v3/types/public/methods.ts index a86cbafaf..d046ca878 100644 --- a/packages/core/lib/v3/types/public/methods.ts +++ b/packages/core/lib/v3/types/public/methods.ts @@ -55,6 +55,7 @@ export interface ExtractOptions { timeout?: number; selector?: string; ignoreSelectors?: string[]; + selectAll?: boolean; page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page; /** * Override the instance-level serverCache setting for this request. @@ -78,6 +79,7 @@ export interface ObserveOptions { timeout?: number; selector?: string; ignoreSelectors?: string[]; + selectAll?: boolean; page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page; /** * Override the instance-level serverCache setting for this request. diff --git a/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts b/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts index a8d70b056..24d0afcbf 100644 --- a/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts +++ b/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts @@ -7,6 +7,8 @@ import type { } from "../../../types/private/snapshot.js"; import { resolveObjectIdForCss, + resolveObjectIdsForCss, + resolveObjectIdsForXPath, resolveObjectIdForXPath, } from "./focusSelectors.js"; import { formatTreeLine, normaliseSpaces } from "./treeFormatUtils.js"; @@ -48,34 +50,97 @@ export async function a11yForFrame( if (!sel) return nodes; try { const looksLikeXPath = /^xpath=/i.test(sel) || sel.startsWith("/"); - const objectId = looksLikeXPath - ? await resolveObjectIdForXPath(session, sel, frameId) - : await resolveObjectIdForCss(session, sel, frameId); - if (!objectId) return nodes; - const desc = await session.send<{ node?: { backendNodeId?: number } }>( - "DOM.describeNode", - { objectId }, + let objectIds: string[] = []; + + if (opts.selectAll) { + if (looksLikeXPath) { + objectIds = await resolveObjectIdsForXPath(session, sel, frameId); + } else { + objectIds = await resolveObjectIdsForCss(session, sel, frameId); + } + } else { + if (looksLikeXPath) { + const objId = await resolveObjectIdForXPath(session, sel, frameId); + objectIds = objId ? [objId] : []; + } else { + const objId = await resolveObjectIdForCss(session, sel, frameId); + objectIds = objId ? [objId] : []; + } + } + + if (objectIds.length === 0) return nodes; + + const targetBackendNodeIds = new Set(); + + for (const objectId of objectIds) { + try { + const desc = await session.send<{ + node?: { backendNodeId?: number }; + }>("DOM.describeNode", { objectId }); + + const backendNodeId = desc.node?.backendNodeId; + if (typeof backendNodeId === "number") { + targetBackendNodeIds.add(backendNodeId); + } + } catch { + // Keep any successfully resolved matches instead of falling back to an + // unscoped tree because one candidate object went stale. + } finally { + await session + .send("Runtime.releaseObject", { objectId }) + .catch(() => {}); + } + } + + if (targetBackendNodeIds.size === 0) return nodes; + + const nodeById = new Map(nodes.map((node) => [node.nodeId, node])); + + const matchedTargets = nodes.filter( + (node) => + typeof node.backendDOMNodeId === "number" && + targetBackendNodeIds.has(node.backendDOMNodeId), ); - const be = desc.node?.backendNodeId; - if (typeof be !== "number") return nodes; - const target = nodes.find((n) => n.backendDOMNodeId === be); - if (!target) return nodes; + + if (matchedTargets.length === 0) return nodes; + + const topLevelTargets = matchedTargets.filter((target) => { + let parentId = target.parentId; + while (parentId) { + const parent = nodeById.get(parentId); + if (!parent) break; + if ( + typeof parent.backendDOMNodeId === "number" && + targetBackendNodeIds.has(parent.backendDOMNodeId) + ) { + return false; + } + parentId = parent.parentId; + } + return true; + }); + scopeApplied = true; - const keep = new Set([target.nodeId]); - const queue: Protocol.Accessibility.AXNode[] = [target]; + const keep = new Set(topLevelTargets.map((target) => target.nodeId)); + const queue = [...topLevelTargets]; + while (queue.length) { const cur = queue.shift()!; for (const id of cur.childIds ?? []) { if (keep.has(id)) continue; keep.add(id); - const child = nodes.find((n) => n.nodeId === id); + const child = nodeById.get(id); if (child) queue.push(child); } } + + const topLevelTargetIds = new Set( + topLevelTargets.map((target) => target.nodeId), + ); return nodes .filter((n) => keep.has(n.nodeId)) .map((n) => - n.nodeId === target.nodeId ? { ...n, parentId: undefined } : n, + topLevelTargetIds.has(n.nodeId) ? { ...n, parentId: undefined } : n, ); } catch { return nodes; diff --git a/packages/core/lib/v3/understudy/a11y/snapshot/capture.ts b/packages/core/lib/v3/understudy/a11y/snapshot/capture.ts index da432c52c..3871331b5 100644 --- a/packages/core/lib/v3/understudy/a11y/snapshot/capture.ts +++ b/packages/core/lib/v3/understudy/a11y/snapshot/capture.ts @@ -235,6 +235,7 @@ export async function tryScopedSnapshot( ownerSessionIndexForFrame(page, targetFrameId, sessionToIndex), exclusionIntervalsByFrame, ), + selectAll: options?.selectAll, tagNameMap, experimental: options?.experimental ?? false, scrollableMap, diff --git a/packages/core/lib/v3/understudy/a11y/snapshot/focusSelectors.ts b/packages/core/lib/v3/understudy/a11y/snapshot/focusSelectors.ts index 257ce8cb2..2e5ca553f 100644 --- a/packages/core/lib/v3/understudy/a11y/snapshot/focusSelectors.ts +++ b/packages/core/lib/v3/understudy/a11y/snapshot/focusSelectors.ts @@ -205,6 +205,16 @@ export async function resolveObjectIdForXPath( xpath: string, frameId?: string, ): Promise { + const objectIds = await resolveObjectIdsForXPath(session, xpath, frameId, 1); + return objectIds[0] ?? null; +} + +export async function resolveObjectIdsForXPath( + session: CDPSessionLike, + xpath: string, + frameId?: string, + limit = Number.POSITIVE_INFINITY, +): Promise { let contextId: number | undefined; try { if (frameId) { @@ -217,21 +227,13 @@ export async function resolveObjectIdForXPath( } catch { contextId = undefined; } - const expr = buildLocatorInvocation("resolveXPathMainWorld", [ - JSON.stringify(xpath), - "0", - ]); - const { result, exceptionDetails } = await session.send<{ - result: { objectId?: string | undefined }; - exceptionDetails?: Protocol.Runtime.ExceptionDetails; - }>("Runtime.evaluate", { - expression: expr, - returnByValue: false, + return resolveIndexedObjectIds( + session, contextId, - awaitPromise: true, - }); - if (exceptionDetails) return null; - return result?.objectId ?? null; + "resolveXPathMainWorld", + xpath, + limit, + ); } /** Resolve a CSS selector (supports '>>' within the same frame only) to a Runtime objectId. */ @@ -240,6 +242,16 @@ export async function resolveObjectIdForCss( selector: string, frameId?: string, ): Promise { + const objectIds = await resolveObjectIdsForCss(session, selector, frameId, 1); + return objectIds[0] ?? null; +} + +export async function resolveObjectIdsForCss( + session: CDPSessionLike, + selector: string, + frameId?: string, + limit = Number.POSITIVE_INFINITY, +): Promise { let contextId: number | undefined; try { if (frameId) { @@ -252,32 +264,65 @@ export async function resolveObjectIdForCss( } catch { contextId = undefined; } - const primaryExpr = buildLocatorInvocation("resolveCssSelector", [ - JSON.stringify(selector), - "0", - ]); - const fallbackExpr = buildLocatorInvocation("resolveCssSelectorPierce", [ - JSON.stringify(selector), - "0", - ]); - const evaluate = async (expression: string): Promise => { + const primary = await resolveIndexedObjectIds( + session, + contextId, + "resolveCssSelector", + selector, + limit, + ); + + if (primary.length > 0) return primary; + return resolveIndexedObjectIds( + session, + contextId, + "resolveCssSelectorPierce", + selector, + limit, + ); +} + +async function resolveIndexedObjectIds( + session: CDPSessionLike, + contextId: number | undefined, + resolverName: + | "resolveCssSelector" + | "resolveCssSelectorPierce" + | "resolveXPathMainWorld", + selector: string, + limit: number, +): Promise { + const normalizedLimit = + Number.isFinite(limit) && limit > 0 + ? Math.floor(limit) + : Number.MAX_SAFE_INTEGER; + + const objectIds: string[] = []; + + for (let index = 0; index < normalizedLimit; index += 1) { + const expr = buildLocatorInvocation(resolverName, [ + JSON.stringify(selector), + String(index), + ]); + const { result, exceptionDetails } = await session.send<{ result: { objectId?: string | undefined }; exceptionDetails?: Protocol.Runtime.ExceptionDetails; }>("Runtime.evaluate", { - expression, + expression: expr, returnByValue: false, contextId, awaitPromise: true, }); - if (exceptionDetails) return null; - return result?.objectId ?? null; - }; + if (exceptionDetails) return objectIds; + const objectId = result?.objectId ?? null; + + if (!objectId) break; + objectIds.push(objectId); + } - const primary = await evaluate(primaryExpr); - if (primary) return primary; - return evaluate(fallbackExpr); + return objectIds; } export function listChildrenOf( diff --git a/packages/core/lib/v3/v3.ts b/packages/core/lib/v3/v3.ts index 634c9ca7c..f1a638ee1 100644 --- a/packages/core/lib/v3/v3.ts +++ b/packages/core/lib/v3/v3.ts @@ -1443,6 +1443,7 @@ export class V3 { timeout: options?.timeout, selector: options?.selector, ignoreSelectors: options?.ignoreSelectors, + selectAll: options?.selectAll, page, }; let result: z.infer | { pageText: string }; @@ -1467,6 +1468,7 @@ export class V3 { instruction, selector: options?.selector, ignoreSelectors: options?.ignoreSelectors, + selectAll: options?.selectAll, timeout: options?.timeout, schema: historySchemaDescriptor, }, @@ -1517,6 +1519,7 @@ export class V3 { timeout: options?.timeout, selector: options?.selector, ignoreSelectors: options?.ignoreSelectors, + selectAll: options?.selectAll, page: page!, }; @@ -1540,6 +1543,7 @@ export class V3 { variables: options?.variables, selector: options?.selector, ignoreSelectors: options?.ignoreSelectors, + selectAll: options?.selectAll, timeout: options?.timeout, }, results, diff --git a/packages/core/tests/unit/api-client-observe-variables.test.ts b/packages/core/tests/unit/api-client-observe-variables.test.ts index 3ca74b489..744ff1f0c 100644 --- a/packages/core/tests/unit/api-client-observe-variables.test.ts +++ b/packages/core/tests/unit/api-client-observe-variables.test.ts @@ -68,6 +68,7 @@ describe("StagehandAPIClient variable serialization", () => { await client.observe({ instruction: "find the field where %username% should be entered", options: { + selectAll: true, variables: { username: { value: "john@example.com", @@ -84,6 +85,7 @@ describe("StagehandAPIClient variable serialization", () => { args: { instruction: "find the field where %username% should be entered", options: { + selectAll: true, variables: { username: { value: "john@example.com", @@ -99,6 +101,42 @@ describe("StagehandAPIClient variable serialization", () => { }); }); + it("preserves selectAll when sending the extract request", async () => { + const client = new StagehandAPIClient({ + apiKey: "bb-test", + logger: vi.fn(), + }); + const executeMock = vi.fn().mockResolvedValue({ extraction: "ok" }); + + ( + client as unknown as { + execute: typeof executeMock; + } + ).execute = executeMock; + + await client.extract({ + instruction: "extract all product cards", + options: { + selector: ".card", + selectAll: true, + }, + }); + + expect(executeMock).toHaveBeenCalledWith({ + method: "extract", + args: { + instruction: "extract all product cards", + schema: undefined, + options: { + selector: ".card", + selectAll: true, + }, + frameId: undefined, + }, + serverCache: undefined, + }); + }); + it("preserves rich variables when sending the agentExecute request", async () => { const client = new StagehandAPIClient({ apiKey: "bb-test", diff --git a/packages/core/tests/unit/public-api/public-types.test.ts b/packages/core/tests/unit/public-api/public-types.test.ts index 8b8655654..e25208863 100644 --- a/packages/core/tests/unit/public-api/public-types.test.ts +++ b/packages/core/tests/unit/public-api/public-types.test.ts @@ -153,6 +153,7 @@ describe("Stagehand public API types", () => { timeout?: number; selector?: string; ignoreSelectors?: string[]; + selectAll?: boolean; page?: Stagehand.AnyPage; serverCache?: boolean; }; @@ -169,6 +170,7 @@ describe("Stagehand public API types", () => { timeout?: number; selector?: string; ignoreSelectors?: string[]; + selectAll?: boolean; page?: Stagehand.AnyPage; serverCache?: boolean; }; diff --git a/packages/core/tests/unit/snapshot-a11y-resolvers.test.ts b/packages/core/tests/unit/snapshot-a11y-resolvers.test.ts index 3a412a749..8be4cbb6f 100644 --- a/packages/core/tests/unit/snapshot-a11y-resolvers.test.ts +++ b/packages/core/tests/unit/snapshot-a11y-resolvers.test.ts @@ -111,9 +111,11 @@ describe("a11yForFrame", () => { const resolveSpy = vi .spyOn(focusSelectors, "resolveObjectIdForXPath") .mockResolvedValue("object-1"); + const resolveAllSpy = vi.spyOn(focusSelectors, "resolveObjectIdsForXPath"); const opts: A11yOptions = { focusSelector: "xpath=//a", + selectAll: false, experimental: false, tagNameMap: { "enc-101": "a" }, scrollableMap: {}, @@ -125,9 +127,142 @@ describe("a11yForFrame", () => { expect(result.scopeApplied).toBe(true); expect(result.outline).not.toContain("RootWebArea"); expect(resolveSpy).toHaveBeenCalled(); + expect(resolveAllSpy).not.toHaveBeenCalled(); resolveSpy.mockRestore(); }); + it("scopes the tree to all matched selector roots when selectAll is true", async () => { + const nodes: Protocol.Accessibility.AXNode[] = [ + { + nodeId: "1", + role: { type: stringType, value: "RootWebArea" }, + backendDOMNodeId: 100, + childIds: ["2", "3"], + ignored: false, + }, + { + nodeId: "2", + role: { type: stringType, value: "link" }, + name: { type: stringType, value: "Docs" }, + backendDOMNodeId: 101, + parentId: "1", + childIds: [], + ignored: false, + }, + { + nodeId: "3", + role: { type: stringType, value: "link" }, + name: { type: stringType, value: "Blog" }, + backendDOMNodeId: 102, + parentId: "1", + childIds: [], + ignored: false, + }, + ]; + const session = new MockCDPSession({ + ...baseHandlers, + "Accessibility.getFullAXTree": async () => ({ nodes }), + "DOM.describeNode": async ({ objectId }) => ({ + node: { + backendNodeId: + objectId === "object-1" ? 101 : objectId === "object-2" ? 102 : 999, + }, + }), + }); + + vi.spyOn(focusSelectors, "resolveObjectIdForCss").mockResolvedValue(null); + const resolveAllSpy = vi + .spyOn(focusSelectors, "resolveObjectIdsForCss") + .mockResolvedValue(["object-1", "object-2"]); + + const opts: A11yOptions = { + focusSelector: ".card", + selectAll: true, + experimental: false, + tagNameMap: { "enc-101": "a", "enc-102": "a" }, + scrollableMap: {}, + encode: (backend) => `enc-${backend}`, + }; + + const result = await a11yForFrame(session, "frame-1", opts); + + expect(result.scopeApplied).toBe(true); + expect(result.outline).toContain("Docs"); + expect(result.outline).toContain("Blog"); + expect(resolveAllSpy).toHaveBeenCalledWith(session, ".card", "frame-1"); + }); + + it("keeps successful selectAll matches when one describeNode call fails", async () => { + const nodes: Protocol.Accessibility.AXNode[] = [ + { + nodeId: "1", + role: { type: stringType, value: "RootWebArea" }, + backendDOMNodeId: 100, + childIds: ["2", "3"], + ignored: false, + }, + { + nodeId: "2", + role: { type: stringType, value: "link" }, + name: { type: stringType, value: "Docs" }, + backendDOMNodeId: 101, + parentId: "1", + childIds: [], + ignored: false, + }, + { + nodeId: "3", + role: { type: stringType, value: "link" }, + name: { type: stringType, value: "Blog" }, + backendDOMNodeId: 102, + parentId: "1", + childIds: [], + ignored: false, + }, + ]; + const session = new MockCDPSession({ + ...baseHandlers, + "Accessibility.getFullAXTree": async () => ({ nodes }), + "DOM.describeNode": async ({ objectId }) => { + if (objectId === "object-2") { + throw new Error("stale object"); + } + + return { + node: { + backendNodeId: 101, + }, + }; + }, + }); + + vi.spyOn(focusSelectors, "resolveObjectIdForCss").mockResolvedValue(null); + vi.spyOn(focusSelectors, "resolveObjectIdsForCss").mockResolvedValue([ + "object-1", + "object-2", + ]); + + const opts: A11yOptions = { + focusSelector: ".card", + selectAll: true, + experimental: false, + tagNameMap: { "enc-101": "a", "enc-102": "a" }, + scrollableMap: {}, + encode: (backend) => `enc-${backend}`, + }; + + const result = await a11yForFrame(session, "frame-1", opts); + + expect(result.scopeApplied).toBe(true); + expect(result.outline).toContain("Docs"); + expect(result.outline).not.toContain("Blog"); + expect(result.outline).not.toContain("RootWebArea"); + expect(session.callsFor("Runtime.releaseObject")).toEqual([ + { params: { objectId: "object-1" } }, + { params: { objectId: "object-2" } }, + ]); + }); + it("falls back to full tree when resolveObjectId throws", async () => { const session = new MockCDPSession({ ...baseHandlers, @@ -295,6 +430,50 @@ describe("resolveObjectIdForCss", () => { }); }); +describe("resolveObjectIdsForCss", () => { + it("collects all primary matches before stopping", async () => { + let call = 0; + const session = new MockCDPSession({ + "Runtime.evaluate": async () => { + call += 1; + if (call === 1) return { result: { objectId: "css-1" } }; + if (call === 2) return { result: { objectId: "css-2" } }; + return { result: {} }; + }, + }); + + const objectIds = await focusSelectors.resolveObjectIdsForCss( + session, + ".card", + undefined, + ); + + expect(objectIds).toEqual(["css-1", "css-2"]); + }); + + it("preserves already-resolved matches when a later evaluation errors", async () => { + let call = 0; + const session = new MockCDPSession({ + "Runtime.evaluate": async () => { + call += 1; + if (call === 1) return { result: { objectId: "css-1" } }; + return { + result: {}, + exceptionDetails: { exception: { description: "fail" } }, + }; + }, + }); + + const objectIds = await focusSelectors.resolveObjectIdsForCss( + session, + ".card", + undefined, + ); + + expect(objectIds).toEqual(["css-1"]); + }); +}); + describe("tryScopedSnapshot", () => { const ordinal = (frameId: string) => (frameId === "frame-1" ? 0 : 1); const context: FrameContext = { @@ -394,6 +573,43 @@ describe("tryScopedSnapshot", () => { expect(a11ySpy).toHaveBeenCalled(); }); + it("forwards selectAll to the a11y scoping helper", async () => { + const session = new MockCDPSession({}); + vi.spyOn(domTree, "domMapsForSession").mockResolvedValue({ + tagNameMap: { "1-10": "div" }, + xpathMap: { "1-10": "/div[1]" }, + scrollableMap: {}, + }); + const a11ySpy = vi.spyOn(a11yTree, "a11yForFrame").mockResolvedValue({ + outline: "[1-10] div", + urlMap: {}, + scopeApplied: true, + } as AccessibilityTreeResult); + vi.spyOn(focusSelectors, "resolveCssFocusFrameAndTail").mockResolvedValue({ + targetFrameId: "frame-2", + tailSelector: ".card", + absPrefix: "", + }); + + await tryScopedSnapshot( + makePage(session), + { focusSelector: ".card", selectAll: true }, + context, + true, + new Map(), + new Map(), + ); + + expect(a11ySpy).toHaveBeenCalledWith( + session, + "frame-2", + expect.objectContaining({ + focusSelector: ".card", + selectAll: true, + }), + ); + }); + it("returns null and logs fallback when scope is not applied", async () => { const session = new MockCDPSession({}); vi.spyOn(domTree, "domMapsForSession").mockResolvedValue({ diff --git a/packages/server-v3/openapi.v3.yaml b/packages/server-v3/openapi.v3.yaml index 4b82d6d56..5c37bf2f0 100644 --- a/packages/server-v3/openapi.v3.yaml +++ b/packages/server-v3/openapi.v3.yaml @@ -622,6 +622,11 @@ components: type: array items: type: string + selectAll: + description: When true, apply selector scoping to all matching elements instead + of only the first match + example: true + type: boolean ExtractRequest: type: object properties: @@ -695,6 +700,11 @@ components: type: array items: type: string + selectAll: + description: When true, apply selector scoping to all matching elements instead + of only the first match + example: true + type: boolean ObserveRequest: type: object properties: @@ -1630,6 +1640,11 @@ components: type: array items: type: string + selectAll: + description: When true, apply selector scoping to all matching elements instead + of only the first match + example: true + type: boolean additionalProperties: false ExtractResultOutput: type: object @@ -1681,6 +1696,11 @@ components: type: array items: type: string + selectAll: + description: When true, apply selector scoping to all matching elements instead + of only the first match + example: true + type: boolean additionalProperties: false ObserveResultOutput: type: object diff --git a/packages/server-v3/tests/integration/v3/extract.test.ts b/packages/server-v3/tests/integration/v3/extract.test.ts index b0b71674a..fd4daa1f6 100644 --- a/packages/server-v3/tests/integration/v3/extract.test.ts +++ b/packages/server-v3/tests/integration/v3/extract.test.ts @@ -216,6 +216,52 @@ describe("POST /v1/sessions/:id/extract (V3)", () => { ); }); + it("should extract with selector and selectAll options", async () => { + const url = getBaseUrl(); + + interface ExtractResponse { + success: boolean; + data?: { result: Record; actionId?: string }; + } + + const ctx = await fetchWithContext( + `${url}/v1/sessions/${sessionId}/extract`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + instruction: "extract the link information", + schema: { + type: "object", + properties: { + href: { type: "string" }, + text: { type: "string" }, + }, + }, + options: { + selector: "a", + selectAll: true, + }, + }), + }, + ); + + assertFetchStatus( + ctx, + HTTP_OK, + "Extract with selector and selectAll should succeed", + ); + assertFetchOk(ctx.body !== null, "Response should have body", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk(!!ctx.body.data, "Response should have data", ctx); + assertFetchOk(!!ctx.body.data.result, "Response should have result", ctx); + assert.equal( + typeof ctx.body.data.result, + "object", + "Result should be an object", + ); + }); + it("should extract with instruction only (no schema)", async () => { const url = getBaseUrl(); diff --git a/packages/server-v3/tests/integration/v3/observe.test.ts b/packages/server-v3/tests/integration/v3/observe.test.ts index 2f6b71549..0f35119bb 100644 --- a/packages/server-v3/tests/integration/v3/observe.test.ts +++ b/packages/server-v3/tests/integration/v3/observe.test.ts @@ -156,6 +156,48 @@ describe("POST /v1/sessions/:id/observe (V3)", () => { ); }); + it("should observe with selector and selectAll options", async () => { + const url = getBaseUrl(); + + interface ObserveResponse { + success: boolean; + data?: { result: unknown[]; actionId?: string }; + } + + const ctx = await fetchWithContext( + `${url}/v1/sessions/${sessionId}/observe`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + instruction: "Find any link on the page", + options: { + selector: "a", + selectAll: true, + }, + }), + }, + ); + + assertFetchStatus( + ctx, + HTTP_OK, + "Observe with selector and selectAll should succeed", + ); + assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + ctx.body.data !== undefined, + "Response should have data", + ctx, + ); + assertFetchOk( + Array.isArray(ctx.body.data.result), + "Result should be an array of observed elements", + ctx, + ); + }); + it("should observe with variables option", async () => { const url = getBaseUrl();