Skip to content

Commit 3d812b7

Browse files
rcholicrcholic
authored
Merge pull request #41 from SentienceAPI/click_xy
new features for click by rect box;semantic wait
2 parents 44124da + a3895dc commit 3d812b7

File tree

7 files changed

+818
-15
lines changed

7 files changed

+818
-15
lines changed

‎README.md‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ const firstRow = query(snap, 'bbox.y<600');
195195

196196
### Actions - Interact with Elements
197197
- **`click(browser, elementId)`** - Click element by ID
198+
- **`clickRect(browser, rect)`** - Click at center of rectangle (coordinate-based)
198199
- **`typeText(browser, elementId, text)`** - Type into input fields
199200
- **`press(browser, key)`** - Press keyboard keys (Enter, Escape, Tab, etc.)
200201

@@ -209,18 +210,55 @@ console.log(`Duration: ${result.duration_ms}ms`);
209210
console.log(`URL changed: ${result.url_changed}`);
210211
```
211212

213+
**Coordinate-based clicking:**
214+
```typescript
215+
import { clickRect } from './src';
216+
217+
// Click at center of rectangle (x, y, width, height)
218+
await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 });
219+
220+
// With visual highlight (default: red border for 2 seconds)
221+
await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 }, true, 2.0);
222+
223+
// Using element's bounding box
224+
const snap = await snapshot(browser);
225+
const element = find(snap, 'role=button');
226+
if (element) {
227+
await clickRect(browser, {
228+
x: element.bbox.x,
229+
y: element.bbox.y,
230+
w: element.bbox.width,
231+
h: element.bbox.height
232+
});
233+
}
234+
```
235+
212236
### Wait & Assertions
213-
- **`waitFor(browser, selector, timeout?)`** - Wait for element to appear
237+
- **`waitFor(browser, selector, timeout?, interval?, useApi?)`** - Wait for element to appear
214238
- **`expect(browser, selector)`** - Assertion helper with fluent API
215239

216240
**Examples:**
217241
```typescript
218-
// Wait for element
242+
// Wait for element (auto-detects optimal interval based on API usage)
219243
const result = await waitFor(browser, 'role=button text="Submit"', 10000);
220244
if (result.found) {
221245
console.log(`Found after ${result.duration_ms}ms`);
222246
}
223247

248+
// Use local extension with fast polling (250ms interval)
249+
const result = await waitFor(browser, 'role=button', 5000, undefined, false);
250+
251+
// Use remote API with network-friendly polling (1500ms interval)
252+
const result = await waitFor(browser, 'role=button', 5000, undefined, true);
253+
254+
// Custom interval override
255+
const result = await waitFor(browser, 'role=button', 5000, 500, false);
256+
257+
// Semantic wait conditions
258+
await waitFor(browser, 'clickable=true', 5000); // Wait for clickable element
259+
await waitFor(browser, 'importance>100', 5000); // Wait for important element
260+
await waitFor(browser, 'role=link visible=true', 5000); // Wait for visible link
261+
224262
// Assertions
225263
await expect(browser, 'role=button text="Submit"').toExist(5000);
226264
await expect(browser, 'role=heading').toBeVisible();

‎examples/click-rect-demo.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/**
2+
* Example: Using clickRect for coordinate-based clicking with visual feedback
3+
*/
4+
5+
import { SentienceBrowser, snapshot, find, clickRect, BBox } from '../src/index';
6+
7+
async function main() {
8+
// Get API key from environment variable (optional - uses free tier if not set)
9+
const apiKey = process.env.SENTIENCE_API_KEY as string | undefined;
10+
11+
const browser = new SentienceBrowser(apiKey, undefined, false);
12+
13+
try {
14+
await browser.start();
15+
16+
// Navigate to example.com
17+
await browser.getPage().goto('https://example.com');
18+
await browser.getPage().waitForLoadState('networkidle');
19+
20+
console.log('=== clickRect Demo ===\n');
21+
22+
// Example 1: Click using rect object
23+
console.log('1. Clicking at specific coordinates (100, 100) with size 50x30');
24+
console.log(' (You should see a red border highlight for 2 seconds)');
25+
let result = await clickRect(browser, { x: 100, y: 100, w: 50, h: 30 });
26+
console.log(` Result: success=${result.success}, outcome=${result.outcome}`);
27+
console.log(` Duration: ${result.duration_ms}ms\n`);
28+
29+
// Wait a bit
30+
await browser.getPage().waitForTimeout(1000);
31+
32+
// Example 2: Click using element's bbox
33+
console.log('2. Clicking using element\'s bounding box');
34+
const snap = await snapshot(browser);
35+
const link = find(snap, 'role=link');
36+
37+
if (link) {
38+
console.log(` Found link: '${link.text}' at (${link.bbox.x}, ${link.bbox.y})`);
39+
console.log(' Clicking at center of element\'s bbox...');
40+
result = await clickRect(browser, {
41+
x: link.bbox.x,
42+
y: link.bbox.y,
43+
w: link.bbox.width,
44+
h: link.bbox.height
45+
});
46+
console.log(` Result: success=${result.success}, outcome=${result.outcome}`);
47+
console.log(` URL changed: ${result.url_changed}\n`);
48+
49+
// Navigate back if needed
50+
if (result.url_changed) {
51+
await browser.getPage().goto('https://example.com');
52+
await browser.getPage().waitForLoadState('networkidle');
53+
}
54+
}
55+
56+
// Example 3: Click without highlight (for headless/CI)
57+
console.log('3. Clicking without visual highlight');
58+
result = await clickRect(browser, { x: 200, y: 200, w: 40, h: 20 }, false);
59+
console.log(` Result: success=${result.success}\n`);
60+
61+
// Example 4: Custom highlight duration
62+
console.log('4. Clicking with custom highlight duration (3 seconds)');
63+
result = await clickRect(browser, { x: 300, y: 300, w: 60, h: 40 }, true, 3.0);
64+
console.log(` Result: success=${result.success}`);
65+
console.log(' (Red border should stay visible for 3 seconds)\n');
66+
67+
// Example 5: Click with snapshot capture
68+
console.log('5. Clicking and capturing snapshot after action');
69+
result = await clickRect(
70+
browser,
71+
{ x: 150, y: 150, w: 50, h: 30 },
72+
true,
73+
2.0,
74+
true
75+
);
76+
if (result.snapshot_after) {
77+
console.log(` Snapshot captured: ${result.snapshot_after.elements.length} elements found`);
78+
console.log(` URL: ${result.snapshot_after.url}\n`);
79+
}
80+
81+
// Example 6: Using BBox object
82+
console.log('6. Clicking using BBox object');
83+
const bbox: BBox = { x: 250, y: 250, width: 45, height: 25 };
84+
result = await clickRect(browser, bbox);
85+
console.log(` Result: success=${result.success}\n`);
86+
87+
console.log('✅ clickRect demo complete!');
88+
console.log('\nNote: clickRect uses Playwright\'s native mouse.click() for realistic');
89+
console.log('event simulation, triggering hover, focus, mousedown, mouseup sequences.');
90+
} catch (e: any) {
91+
console.error(`❌ Error: ${e.message}`);
92+
console.error(e.stack);
93+
} finally {
94+
await browser.close();
95+
}
96+
}
97+
98+
main().catch(console.error);
99+

‎examples/semantic-wait-demo.ts‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/**
2+
* Example: Semantic waitFor using query DSL
3+
* Demonstrates waiting for elements using semantic selectors
4+
*/
5+
6+
import { SentienceBrowser, snapshot, find, waitFor, click } from '../src/index';
7+
8+
async function main() {
9+
// Get API key from environment variable (optional - uses free tier if not set)
10+
const apiKey = process.env.SENTIENCE_API_KEY as string | undefined;
11+
12+
const browser = new SentienceBrowser(apiKey, undefined, false);
13+
14+
try {
15+
await browser.start();
16+
17+
// Navigate to example.com
18+
await browser.getPage().goto('https://example.com', { waitUntil: 'domcontentloaded' });
19+
20+
console.log('=== Semantic waitFor Demo ===\n');
21+
22+
// Example 1: Wait for element by role
23+
console.log('1. Waiting for link element (role=link)');
24+
let waitResult = await waitFor(browser, 'role=link', 5000);
25+
if (waitResult.found) {
26+
console.log(` ✅ Found after ${waitResult.duration_ms}ms`);
27+
console.log(` Element: '${waitResult.element?.text}' (id: ${waitResult.element?.id})`);
28+
} else {
29+
console.log(` ❌ Not found (timeout: ${waitResult.timeout})`);
30+
}
31+
console.log();
32+
33+
// Example 2: Wait for element by role and text
34+
console.log('2. Waiting for link with specific text');
35+
waitResult = await waitFor(browser, 'role=link text~"Example"', 5000);
36+
if (waitResult.found) {
37+
console.log(` ✅ Found after ${waitResult.duration_ms}ms`);
38+
console.log(` Element text: '${waitResult.element?.text}'`);
39+
} else {
40+
console.log(' ❌ Not found');
41+
}
42+
console.log();
43+
44+
// Example 3: Wait for clickable element
45+
console.log('3. Waiting for clickable element');
46+
waitResult = await waitFor(browser, 'clickable=true', 5000);
47+
if (waitResult.found) {
48+
console.log(` ✅ Found clickable element after ${waitResult.duration_ms}ms`);
49+
console.log(` Role: ${waitResult.element?.role}`);
50+
console.log(` Text: '${waitResult.element?.text}'`);
51+
console.log(` Is clickable: ${waitResult.element?.visual_cues.is_clickable}`);
52+
} else {
53+
console.log(' ❌ Not found');
54+
}
55+
console.log();
56+
57+
// Example 4: Wait for element with importance threshold
58+
console.log('4. Waiting for important element (importance > 100)');
59+
waitResult = await waitFor(browser, 'importance>100', 5000);
60+
if (waitResult.found) {
61+
console.log(` ✅ Found important element after ${waitResult.duration_ms}ms`);
62+
console.log(` Importance: ${waitResult.element?.importance}`);
63+
console.log(` Role: ${waitResult.element?.role}`);
64+
} else {
65+
console.log(' ❌ Not found');
66+
}
67+
console.log();
68+
69+
// Example 5: Wait and then click
70+
console.log('5. Wait for element, then click it');
71+
waitResult = await waitFor(browser, 'role=link', 5000);
72+
if (waitResult.found && waitResult.element) {
73+
console.log(' ✅ Found element, clicking...');
74+
const clickResult = await click(browser, waitResult.element.id);
75+
console.log(` Click result: success=${clickResult.success}, outcome=${clickResult.outcome}`);
76+
if (clickResult.url_changed) {
77+
console.log(` ✅ Navigation occurred: ${browser.getPage().url()}`);
78+
}
79+
} else {
80+
console.log(' ❌ Element not found, cannot click');
81+
}
82+
console.log();
83+
84+
// Example 6: Using local extension (fast polling)
85+
console.log('6. Using local extension with auto-optimized interval');
86+
console.log(' When useApi=false, interval auto-adjusts to 250ms (fast)');
87+
waitResult = await waitFor(browser, 'role=link', 5000, undefined, false);
88+
if (waitResult.found) {
89+
console.log(` ✅ Found after ${waitResult.duration_ms}ms`);
90+
console.log(' (Used local extension, polled every 250ms)');
91+
}
92+
console.log();
93+
94+
// Example 7: Using remote API (slower polling)
95+
console.log('7. Using remote API with auto-optimized interval');
96+
console.log(' When useApi=true, interval auto-adjusts to 1500ms (network-friendly)');
97+
if (apiKey) {
98+
waitResult = await waitFor(browser, 'role=link', 5000, undefined, true);
99+
if (waitResult.found) {
100+
console.log(` ✅ Found after ${waitResult.duration_ms}ms`);
101+
console.log(' (Used remote API, polled every 1500ms)');
102+
}
103+
} else {
104+
console.log(' ⚠️ Skipped (no API key set)');
105+
}
106+
console.log();
107+
108+
// Example 8: Custom interval override
109+
console.log('8. Custom interval override (manual control)');
110+
console.log(' You can still specify custom interval if needed');
111+
waitResult = await waitFor(browser, 'role=link', 5000, 500, false);
112+
if (waitResult.found) {
113+
console.log(` ✅ Found after ${waitResult.duration_ms}ms`);
114+
console.log(' (Custom interval: 500ms)');
115+
}
116+
console.log();
117+
118+
// Example 9: Wait for visible element (not occluded)
119+
console.log('9. Waiting for visible element (not occluded)');
120+
waitResult = await waitFor(browser, 'role=link visible=true', 5000);
121+
if (waitResult.found) {
122+
console.log(` ✅ Found visible element after ${waitResult.duration_ms}ms`);
123+
console.log(` Is occluded: ${waitResult.element?.is_occluded}`);
124+
console.log(` In viewport: ${waitResult.element?.in_viewport}`);
125+
}
126+
console.log();
127+
128+
console.log('✅ Semantic waitFor demo complete!');
129+
console.log('\nNote: waitFor uses the semantic query DSL to find elements.');
130+
console.log('This is more robust than CSS selectors because it understands');
131+
console.log('the semantic meaning of elements (role, text, clickability, etc.).');
132+
} catch (e: any) {
133+
console.error(`❌ Error: ${e.message}`);
134+
console.error(e.stack);
135+
} finally {
136+
await browser.close();
137+
}
138+
}
139+
140+
main().catch(console.error);
141+

0 commit comments

Comments
 (0)