From 5000623fef3a15fada4e810fda1c00400e33aa21 Mon Sep 17 00:00:00 2001 From: alexzaetoro Date: Thu, 11 Jun 2026 09:41:39 +0300 Subject: [PATCH 1/3] fix(ios): recover sparse snapshot trees via element queries On some apps (notably React Native), the public XCUIElement.snapshot() traversal collapses to only structural containers (application/window/ other) even though the screen is rendering content with resolvable controls. The runner returned a 2-3 node tree, which broke snapshot, selector resolution, and react-native dismiss-overlay. Detect the sparse-tree signature in snapshotFast and recover through the existing query-based flat traversal (XCUIElementQuery), the same path the collapsed-tab fallback already trusts. The recovered payload is marked truncated and carries an explanatory message; recovery is skipped when queries reveal nothing more than the sparse tree. Adds unit tests for the degenerate-tree detector. --- CHANGELOG.md | 4 + .../RunnerTests+Snapshot.swift | 133 +++++++++++++++++- 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb9b8a37..d23e2c625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- iOS: `snapshot` now recovers from a sparse accessibility tree. When the public `XCUIElement.snapshot()` traversal collapses to only structural containers (application/window/other) while the screen is rendering content — common on React Native apps — the runner falls back to a query-based (`XCUIElementQuery`) flat traversal so `testID`s and on-screen controls remain visible instead of returning a 2–3 node tree. The recovered payload is marked `truncated` and carries an explanatory `message`. + ## 0.15.0 - Breaking: `apps` discovery and public app-list helpers now default to user-installed apps. Use `--all` or `filter: 'all'` to include system/OEM apps. diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 5b90f056e..4ff17b36b 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -11,6 +11,16 @@ extension RunnerTests { private static let rawSnapshotMaxNodes = 5_000 private static let rawSnapshotTooLargeHint = "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." + private static let degenerateTreeRecoveryMessage = + "Recovered the snapshot through accessibility element queries: XCTest's snapshot tree was sparse (only structural containers) while the screen has rendered content. This is common on React Native apps, where XCUIElementQuery still resolves controls the public snapshot omits. Nodes below are a flattened, query-resolved view of the on-screen controls." + // Structural container types carry no actionable content on their own; a tree made up of only + // these (plus the application/window roots) is the signature of a sparse snapshot. + private static let structuralOnlyNodeTypes: Set = [ + "Application", + "Window", + "Other", + "ScrollView" + ] private static let collapsedTabCandidateTypes: Set = [ .button, .link, @@ -243,10 +253,12 @@ extension RunnerTests { } - return DataPayload( - nodes: applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes), - truncated: false - ) + let resolvedNodes = applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes) + if snapshotNodesAreDegenerate(resolvedNodes), + let recovered = snapshotQueryRecovery(app: app, options: options) { + return recovered + } + return DataPayload(nodes: resolvedNodes, truncated: false) } func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { @@ -380,6 +392,52 @@ extension RunnerTests { return DataPayload(nodes: nodes, truncated: truncated) } + /// Detects the "sparse snapshot" signature: the public `XCUIElement.snapshot()` traversal + /// produced only structural container nodes (application/window/other/scrollView) with no + /// label, identifier, value, or hittable control — even though the app is foregrounded and + /// rendering content. React Native apps hit this regularly: the public snapshot API collapses + /// the host view's subtree, while `XCUIElementQuery` still resolves the underlying controls. + private func snapshotNodesAreDegenerate(_ nodes: [SnapshotNode]) -> Bool { + guard !nodes.isEmpty else { return false } + for node in nodes { + let hasContent = (node.label?.isEmpty == false) + || (node.identifier?.isEmpty == false) + || (node.value?.isEmpty == false) + if hasContent || node.hittable { return false } + // A concretely typed control (Button, StaticText, TextField, …) means the tree is not + // degenerate, even when it happens to carry no resolved content this frame. + if !Self.structuralOnlyNodeTypes.contains(node.type) { return false } + } + return true + } + + /// Recovers a usable tree for a sparse snapshot by reusing the query-based flat traversal + /// (the same `XCUIElementQuery` path the collapsed-tab fallback already trusts). Returns `nil` + /// when queries see nothing more than the sparse tree, so the caller keeps the original result. + private func snapshotQueryRecovery( + app: XCUIApplication, + options: SnapshotOptions + ) -> DataPayload? { + let recovery = snapshotFlatInteractive( + app: app, + options: SnapshotOptions( + interactiveOnly: false, + compact: options.compact, + depth: options.depth ?? Int.max, + scope: options.scope, + raw: false + ) + ) + // nodes[0] is always the synthetic Application root; require at least one real control beyond it. + guard let nodes = recovery.nodes, nodes.count > 1 else { return nil } + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_DEGENERATE_TREE_RECOVERED=%d", nodes.count) + return DataPayload( + message: Self.degenerateTreeRecoveryMessage, + nodes: nodes, + truncated: true + ) + } + private func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) @@ -554,6 +612,73 @@ extension RunnerTests { XCTAssertEqual(currentBundleId, "com.example.app") } + private func makeTestSnapshotNode( + index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + value: String? = nil, + hittable: Bool = false + ) -> SnapshotNode { + SnapshotNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: value, + rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), + enabled: true, + focused: nil, + selected: nil, + hittable: hittable, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + func testSnapshotNodesAreDegenerateDetectsStructuralOnlyTree() { + let nodes = [ + makeTestSnapshotNode(index: 0, type: "Application"), + makeTestSnapshotNode(index: 1, type: "Window"), + makeTestSnapshotNode(index: 2, type: "Other") + ] + + XCTAssertTrue(snapshotNodesAreDegenerate(nodes)) + } + + func testSnapshotNodesAreNotDegenerateWhenContentPresent() { + let nodes = [ + makeTestSnapshotNode(index: 0, type: "Application"), + makeTestSnapshotNode(index: 1, type: "StaticText", label: "Welcome back!") + ] + + XCTAssertFalse(snapshotNodesAreDegenerate(nodes)) + } + + func testSnapshotNodesAreNotDegenerateWhenHittableControlPresent() { + let nodes = [ + makeTestSnapshotNode(index: 0, type: "Application"), + makeTestSnapshotNode(index: 1, type: "Other", hittable: true) + ] + + XCTAssertFalse(snapshotNodesAreDegenerate(nodes)) + } + + func testSnapshotNodesAreNotDegenerateWhenTypedControlPresent() { + let nodes = [ + makeTestSnapshotNode(index: 0, type: "Application"), + makeTestSnapshotNode(index: 1, type: "Button") + ] + + XCTAssertFalse(snapshotNodesAreDegenerate(nodes)) + } + + func testSnapshotNodesAreNotDegenerateWhenEmpty() { + XCTAssertFalse(snapshotNodesAreDegenerate([])) + } + private func compactInteractiveRootNode(rect: CGRect) -> SnapshotNode { SnapshotNode( index: 0, From a6d1acc76f8012cb5c3428445a15f4114e0f3114 Mon Sep 17 00:00:00 2001 From: alexzaetoro Date: Thu, 11 Jun 2026 09:46:05 +0300 Subject: [PATCH 2/3] docs(ios): trim snapshot-recovery comments to essential rationale --- .../RunnerTests+Snapshot.swift | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 4ff17b36b..a735a5b7e 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -13,8 +13,6 @@ extension RunnerTests { "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." private static let degenerateTreeRecoveryMessage = "Recovered the snapshot through accessibility element queries: XCTest's snapshot tree was sparse (only structural containers) while the screen has rendered content. This is common on React Native apps, where XCUIElementQuery still resolves controls the public snapshot omits. Nodes below are a flattened, query-resolved view of the on-screen controls." - // Structural container types carry no actionable content on their own; a tree made up of only - // these (plus the application/window roots) is the signature of a sparse snapshot. private static let structuralOnlyNodeTypes: Set = [ "Application", "Window", @@ -392,11 +390,9 @@ extension RunnerTests { return DataPayload(nodes: nodes, truncated: truncated) } - /// Detects the "sparse snapshot" signature: the public `XCUIElement.snapshot()` traversal - /// produced only structural container nodes (application/window/other/scrollView) with no - /// label, identifier, value, or hittable control — even though the app is foregrounded and - /// rendering content. React Native apps hit this regularly: the public snapshot API collapses - /// the host view's subtree, while `XCUIElementQuery` still resolves the underlying controls. + // A tree of only structural containers with no content or hittable control is the signature of + // a sparse snapshot: the public XCUIElement.snapshot() API collapses the subtree (common on + // React Native) while XCUIElementQuery still resolves the underlying controls. private func snapshotNodesAreDegenerate(_ nodes: [SnapshotNode]) -> Bool { guard !nodes.isEmpty else { return false } for node in nodes { @@ -404,16 +400,13 @@ extension RunnerTests { || (node.identifier?.isEmpty == false) || (node.value?.isEmpty == false) if hasContent || node.hittable { return false } - // A concretely typed control (Button, StaticText, TextField, …) means the tree is not - // degenerate, even when it happens to carry no resolved content this frame. if !Self.structuralOnlyNodeTypes.contains(node.type) { return false } } return true } - /// Recovers a usable tree for a sparse snapshot by reusing the query-based flat traversal - /// (the same `XCUIElementQuery` path the collapsed-tab fallback already trusts). Returns `nil` - /// when queries see nothing more than the sparse tree, so the caller keeps the original result. + // Recovers a sparse snapshot via the query-based flat traversal (the same XCUIElementQuery path + // collapsedTabFallbackNodes uses). Returns nil when queries see nothing more than the sparse tree. private func snapshotQueryRecovery( app: XCUIApplication, options: SnapshotOptions @@ -428,7 +421,6 @@ extension RunnerTests { raw: false ) ) - // nodes[0] is always the synthetic Application root; require at least one real control beyond it. guard let nodes = recovery.nodes, nodes.count > 1 else { return nil } NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_DEGENERATE_TREE_RECOVERED=%d", nodes.count) return DataPayload( From e9ca48b3cdb9b66949d4722a6f83de0c5e78e0c4 Mon Sep 17 00:00:00 2001 From: alexzaetoro Date: Thu, 11 Jun 2026 10:09:44 +0300 Subject: [PATCH 3/3] Update RunnerTests --- .../AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index a735a5b7e..eabd7c1a5 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -390,9 +390,6 @@ extension RunnerTests { return DataPayload(nodes: nodes, truncated: truncated) } - // A tree of only structural containers with no content or hittable control is the signature of - // a sparse snapshot: the public XCUIElement.snapshot() API collapses the subtree (common on - // React Native) while XCUIElementQuery still resolves the underlying controls. private func snapshotNodesAreDegenerate(_ nodes: [SnapshotNode]) -> Bool { guard !nodes.isEmpty else { return false } for node in nodes { @@ -405,8 +402,6 @@ extension RunnerTests { return true } - // Recovers a sparse snapshot via the query-based flat traversal (the same XCUIElementQuery path - // collapsedTabFallbackNodes uses). Returns nil when queries see nothing more than the sparse tree. private func snapshotQueryRecovery( app: XCUIApplication, options: SnapshotOptions