Skip to content

Commit d40f3ac

Browse files
gdealmeida1885gdealmeida1885
committed
Fix directional relative selectors to prefer closest element over deepest
- Change below/above/leftOf/rightOf selection from DeepestMatchingElement to candidates[0] (closest by distance, clickable-preferred) - Keep deepest element behavior for non-directional filters (childOf, etc.) - Add regression tests for all three drivers (WDA, Appium, UIAutomator2)
1 parent 20549b2 commit d40f3ac

6 files changed

Lines changed: 220 additions & 4 deletions

File tree

‎pkg/driver/appium/driver.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -693,7 +693,14 @@ func (d *Driver) findElementRelativeWithElements(sel flow.Selector, allElements
693693
return nil, fmt.Errorf("no candidates after sorting")
694694
}
695695

696-
selected := SelectByIndex(candidates, sel.Index)
696+
var selected *ParsedElement
697+
if sel.Index == "" && (filterType == filterBelow || filterType == filterAbove || filterType == filterLeftOf || filterType == filterRightOf) {
698+
// Directional filters sort candidates by distance. Pick the closest
699+
// (first) element to match Maestro's .firstOrNull() behavior.
700+
selected = candidates[0]
701+
} else {
702+
selected = SelectByIndex(candidates, sel.Index)
703+
}
697704

698705
// If element isn't clickable, try to find a clickable parent
699706
// This handles React Native pattern where text nodes aren't clickable but containers are

‎pkg/driver/appium/driver_test.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,6 +1264,74 @@ func TestFindElementRelativeWithElementsContainsDescendants(t *testing.T) {
12641264
}
12651265
}
12661266

1267+
// mockAppiumServerForRelativeDepthTest creates a server with elements that test
1268+
// distance vs. depth selection in directional relative selectors.
1269+
func mockAppiumServerForRelativeDepthTest() *httptest.Server {
1270+
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1271+
w.Header().Set("Content-Type", "application/json")
1272+
path := r.URL.Path
1273+
1274+
if strings.HasSuffix(path, "/source") {
1275+
writeJSON(w, map[string]interface{}{
1276+
"value": `<?xml version="1.0" encoding="UTF-8"?>
1277+
<hierarchy rotation="0">
1278+
<android.widget.FrameLayout bounds="[0,0][1080,2340]">
1279+
<android.widget.TextView text="Email Address" bounds="[100,100][500,130]"/>
1280+
<android.widget.EditText text="email input" clickable="true" enabled="true" bounds="[100,140][500,180]"/>
1281+
<android.widget.FrameLayout bounds="[100,300][500,500]">
1282+
<android.widget.FrameLayout bounds="[100,300][500,500]">
1283+
<android.widget.FrameLayout bounds="[100,300][500,500]">
1284+
<android.widget.TextView text="deep link" clickable="true" enabled="true" bounds="[100,350][500,380]"/>
1285+
</android.widget.FrameLayout>
1286+
</android.widget.FrameLayout>
1287+
</android.widget.FrameLayout>
1288+
</android.widget.FrameLayout>
1289+
</hierarchy>`,
1290+
})
1291+
return
1292+
}
1293+
1294+
if strings.Contains(path, "/window/rect") {
1295+
writeJSON(w, map[string]interface{}{
1296+
"value": map[string]interface{}{"width": 1080.0, "height": 2340.0, "x": 0.0, "y": 0.0},
1297+
})
1298+
return
1299+
}
1300+
1301+
writeJSON(w, map[string]interface{}{"value": nil})
1302+
}))
1303+
}
1304+
1305+
// TestFindElementRelativePrefersClosestOverDeepest verifies that directional
1306+
// relative selectors pick the closest element by distance rather than the
1307+
// deepest in the DOM.
1308+
func TestFindElementRelativePrefersClosestOverDeepest(t *testing.T) {
1309+
server := mockAppiumServerForRelativeDepthTest()
1310+
defer server.Close()
1311+
driver := createTestAppiumDriver(server)
1312+
1313+
source, _ := driver.client.Source()
1314+
elements, platform, _ := ParsePageSource(source)
1315+
1316+
sel := flow.Selector{
1317+
Below: &flow.Selector{Text: "Email Address"},
1318+
}
1319+
1320+
info, err := driver.findElementRelativeWithElements(sel, elements, platform)
1321+
if err != nil {
1322+
t.Fatalf("Expected success, got: %v", err)
1323+
}
1324+
if info == nil {
1325+
t.Fatal("Expected element info")
1326+
}
1327+
1328+
// The closest element below "Email Address" (bottom at y=130) is the
1329+
// EditText at y=140, not the deeply-nested TextView at y=350.
1330+
if info.Bounds.Y != 140 {
1331+
t.Errorf("Expected element at y=140, got y=%d", info.Bounds.Y)
1332+
}
1333+
}
1334+
12671335
// TestFindElementRelativeWithNestedRelative tests nested relative selector
12681336
func TestFindElementRelativeWithNestedRelative(t *testing.T) {
12691337
server := mockAppiumServerForRelativeElements()

‎pkg/driver/uiautomator2/driver.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -783,7 +783,14 @@ func (d *Driver) resolveRelativeSelector(sel flow.Selector) (*core.ElementInfo,
783783
// Prioritize clickable elements
784784
candidates = SortClickableFirst(candidates)
785785

786-
selected := SelectByIndex(candidates, sel.Index)
786+
var selected *ParsedElement
787+
if sel.Index == "" && (filterType == filterBelow || filterType == filterAbove || filterType == filterLeftOf || filterType == filterRightOf) {
788+
// Directional filters sort candidates by distance. Pick the closest
789+
// (first) element to match Maestro's .firstOrNull() behavior.
790+
selected = candidates[0]
791+
} else {
792+
selected = SelectByIndex(candidates, sel.Index)
793+
}
787794

788795
// If element isn't clickable, try to find a clickable parent
789796
// This handles React Native pattern where text nodes aren't clickable but containers are
@@ -872,7 +879,14 @@ func (d *Driver) findElementRelativeWithElements(sel flow.Selector, allElements
872879
// Prioritize clickable elements
873880
candidates = SortClickableFirst(candidates)
874881

875-
selected := SelectByIndex(candidates, sel.Index)
882+
var selected *ParsedElement
883+
if sel.Index == "" && (filterType == filterBelow || filterType == filterAbove || filterType == filterLeftOf || filterType == filterRightOf) {
884+
// Directional filters sort candidates by distance. Pick the closest
885+
// (first) element to match Maestro's .firstOrNull() behavior.
886+
selected = candidates[0]
887+
} else {
888+
selected = SelectByIndex(candidates, sel.Index)
889+
}
876890

877891
// If element isn't clickable, try to find a clickable parent
878892
// This handles React Native pattern where text nodes aren't clickable but containers are

‎pkg/driver/uiautomator2/driver_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2010,6 +2010,52 @@ func TestTapOnRelativeSelectorBelow(t *testing.T) {
20102010
}
20112011
}
20122012

2013+
// TestResolveRelativeSelectorPrefersClosestOverDeepest verifies that directional
2014+
// relative selectors pick the closest element by distance rather than the
2015+
// deepest in the DOM.
2016+
func TestResolveRelativeSelectorPrefersClosestOverDeepest(t *testing.T) {
2017+
pageSource := `<?xml version="1.0" encoding="UTF-8"?>
2018+
<hierarchy>
2019+
<node text="Email Address" bounds="[100,100][500,130]" class="android.widget.TextView" />
2020+
<node text="email input" bounds="[100,140][500,180]" class="android.widget.EditText" clickable="true" enabled="true" />
2021+
<node bounds="[100,300][500,500]" class="android.widget.FrameLayout">
2022+
<node bounds="[100,300][500,500]" class="android.widget.FrameLayout">
2023+
<node bounds="[100,300][500,500]" class="android.widget.FrameLayout">
2024+
<node text="deep link" bounds="[100,350][500,380]" class="android.widget.TextView" clickable="true" enabled="true" />
2025+
</node>
2026+
</node>
2027+
</node>
2028+
</hierarchy>`
2029+
2030+
server := setupMockServer(t, map[string]func(w http.ResponseWriter, r *http.Request){
2031+
"GET /source": func(w http.ResponseWriter, r *http.Request) {
2032+
writeJSON(w, map[string]interface{}{"value": pageSource})
2033+
},
2034+
})
2035+
defer server.Close()
2036+
2037+
client := newMockHTTPClient(server.URL)
2038+
driver := New(client.Client, nil, nil)
2039+
2040+
sel := flow.Selector{
2041+
Below: &flow.Selector{Text: "Email Address"},
2042+
}
2043+
2044+
info, err := driver.resolveRelativeSelector(sel)
2045+
if err != nil {
2046+
t.Fatalf("Expected success, got: %v", err)
2047+
}
2048+
if info == nil {
2049+
t.Fatal("Expected element info")
2050+
}
2051+
2052+
// The closest element below "Email Address" (bottom at y=130) is the
2053+
// EditText at y=140, not the deeply-nested TextView at y=350.
2054+
if info.Bounds.Y != 140 {
2055+
t.Errorf("Expected element at y=140, got y=%d", info.Bounds.Y)
2056+
}
2057+
}
2058+
20132059
func TestTapOnRelativeSelectorClickError(t *testing.T) {
20142060
pageSource := `<?xml version="1.0" encoding="UTF-8"?>
20152061
<hierarchy>

‎pkg/driver/wda/driver.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,7 +655,14 @@ func (d *Driver) resolveRelativeSelector(sel flow.Selector, allElements []*Parse
655655
// Prioritize clickable/interactive elements
656656
candidates = SortClickableFirst(candidates)
657657

658-
selected := SelectByIndex(candidates, sel.Index)
658+
var selected *ParsedElement
659+
if sel.Index == "" && (filterType == filterBelow || filterType == filterAbove || filterType == filterLeftOf || filterType == filterRightOf) {
660+
// Directional filters sort candidates by distance. Pick the closest
661+
// (first) element to match Maestro's .firstOrNull() behavior.
662+
selected = candidates[0]
663+
} else {
664+
selected = SelectByIndex(candidates, sel.Index)
665+
}
659666

660667
return &core.ElementInfo{
661668
Text: selected.Label,

‎pkg/driver/wda/driver_test.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1569,6 +1569,80 @@ func TestResolveRelativeSelectorContainsDescendants(t *testing.T) {
15691569
}
15701570
}
15711571

1572+
// mockWDAServerForRelativeDepthTest creates a server with elements that test
1573+
// distance vs. depth selection in directional relative selectors.
1574+
// The page source has a close TextField (depth 2) and a far-but-deeply-nested
1575+
// Link (depth 5) below the anchor. The correct behavior is to select the closer one.
1576+
func mockWDAServerForRelativeDepthTest() *httptest.Server {
1577+
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1578+
w.Header().Set("Content-Type", "application/json")
1579+
path := r.URL.Path
1580+
1581+
if strings.HasSuffix(path, "/source") {
1582+
jsonResponse(w, map[string]interface{}{
1583+
"value": `<?xml version="1.0" encoding="UTF-8"?>
1584+
<AppiumAUT>
1585+
<XCUIElementTypeApplication type="XCUIElementTypeApplication" name="TestApp" enabled="true" visible="true" x="0" y="0" width="390" height="844">
1586+
<XCUIElementTypeStaticText type="XCUIElementTypeStaticText" label="Email Address" enabled="true" visible="true" x="50" y="100" width="290" height="30"/>
1587+
<XCUIElementTypeTextField type="XCUIElementTypeTextField" label="email input" enabled="true" visible="true" x="50" y="140" width="290" height="40"/>
1588+
<XCUIElementTypeOther type="XCUIElementTypeOther" enabled="true" visible="true" x="50" y="300" width="290" height="100">
1589+
<XCUIElementTypeOther type="XCUIElementTypeOther" enabled="true" visible="true" x="50" y="300" width="290" height="100">
1590+
<XCUIElementTypeOther type="XCUIElementTypeOther" enabled="true" visible="true" x="50" y="300" width="290" height="100">
1591+
<XCUIElementTypeLink type="XCUIElementTypeLink" label="deep link" enabled="true" visible="true" x="50" y="350" width="290" height="30"/>
1592+
</XCUIElementTypeOther>
1593+
</XCUIElementTypeOther>
1594+
</XCUIElementTypeOther>
1595+
</XCUIElementTypeApplication>
1596+
</AppiumAUT>`,
1597+
})
1598+
return
1599+
}
1600+
1601+
if strings.Contains(path, "/window/size") {
1602+
jsonResponse(w, map[string]interface{}{
1603+
"value": map[string]interface{}{"width": 390.0, "height": 844.0},
1604+
})
1605+
return
1606+
}
1607+
1608+
jsonResponse(w, map[string]interface{}{"status": 0})
1609+
}))
1610+
}
1611+
1612+
// TestResolveRelativeSelectorPrefersClosestOverDeepest verifies that directional
1613+
// relative selectors (below/above/leftOf/rightOf) pick the closest element by
1614+
// distance rather than the deepest in the DOM. This matches Maestro's
1615+
// .firstOrNull() behavior on the distance-sorted candidate list.
1616+
func TestResolveRelativeSelectorPrefersClosestOverDeepest(t *testing.T) {
1617+
server := mockWDAServerForRelativeDepthTest()
1618+
defer server.Close()
1619+
driver := createTestDriver(server)
1620+
1621+
source, _ := driver.client.Source()
1622+
elements, _ := ParsePageSource(source)
1623+
1624+
sel := flow.Selector{
1625+
Below: &flow.Selector{Text: "Email Address"},
1626+
}
1627+
1628+
info, err := driver.resolveRelativeSelector(sel, elements)
1629+
if err != nil {
1630+
t.Fatalf("Expected success, got: %v", err)
1631+
}
1632+
if info == nil {
1633+
t.Fatal("Expected element info")
1634+
}
1635+
1636+
// The closest element below "Email Address" (bottom at y=130) is the
1637+
// TextField at y=140, not the deeply-nested Link at y=350 (depth 5).
1638+
if info.Text != "email input" {
1639+
t.Errorf("Expected closest element 'email input', got '%s'", info.Text)
1640+
}
1641+
if info.Bounds.Y != 140 {
1642+
t.Errorf("Expected element at y=140, got y=%d", info.Bounds.Y)
1643+
}
1644+
}
1645+
15721646
// TestEraseTextWithActiveElement tests eraseText with active element
15731647
func TestEraseTextWithActiveElement(t *testing.T) {
15741648
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

0 commit comments

Comments
 (0)