Skip to content

Commit 24bf8f1

Browse files
rcholicrcholic
committed
new features for click by rect box;semantic wait
1 parent 4fdedad commit 24bf8f1

File tree

3 files changed

+378
-21
lines changed

3 files changed

+378
-21
lines changed

‎sentience/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from .models import Snapshot, Element, BBox, Viewport, ActionResult, WaitResult
77
from .snapshot import snapshot
88
from .query import query, find
9-
from .actions import click, type_text, press
9+
from .actions import click, type_text, press, click_rect
1010
from .wait import wait_for
1111
from .expect import expect
1212
from .inspector import Inspector, inspect
@@ -31,6 +31,7 @@
3131
"click",
3232
"type_text",
3333
"press",
34+
"click_rect",
3435
"wait_for",
3536
"expect",
3637
"Inspector",

‎sentience/actions.py‎

Lines changed: 261 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,27 @@
33
"""
44

55
import time
6-
from typing import Optional
6+
from typing import Optional, Dict, Any
77
from .browser import SentienceBrowser
8-
from .models import ActionResult, Snapshot
8+
from .models import ActionResult, Snapshot, BBox
99
from .snapshot import snapshot
1010

1111

12-
def click(browser: SentienceBrowser, element_id: int, take_snapshot: bool = False) -> ActionResult:
12+
def click(
13+
browser: SentienceBrowser,
14+
element_id: int,
15+
use_mouse: bool = True,
16+
take_snapshot: bool = False,
17+
) -> ActionResult:
1318
"""
14-
Click an element by ID
19+
Click an element by ID using hybrid approach (mouse simulation by default)
1520
1621
Args:
1722
browser: SentienceBrowser instance
1823
element_id: Element ID from snapshot
19-
take_snapshot: Whether to take snapshot after action (optional in Week 1)
24+
use_mouse: If True, use Playwright's mouse.click() at element center (hybrid approach).
25+
If False, use JS-based window.sentience.click() (legacy).
26+
take_snapshot: Whether to take snapshot after action
2027
2128
Returns:
2229
ActionResult
@@ -27,22 +34,84 @@ def click(browser: SentienceBrowser, element_id: int, take_snapshot: bool = Fals
2734
start_time = time.time()
2835
url_before = browser.page.url
2936

30-
# Call extension click method
31-
success = browser.page.evaluate(
32-
"""
33-
(id) => {
34-
return window.sentience.click(id);
35-
}
36-
""",
37-
element_id,
38-
)
37+
if use_mouse:
38+
# Hybrid approach: Get element bbox from snapshot, calculate center, use mouse.click()
39+
try:
40+
snap = snapshot(browser)
41+
element = None
42+
for el in snap.elements:
43+
if el.id == element_id:
44+
element = el
45+
break
46+
47+
if element:
48+
# Calculate center of element bbox
49+
center_x = element.bbox.x + element.bbox.width / 2
50+
center_y = element.bbox.y + element.bbox.height / 2
51+
# Use Playwright's native mouse click for realistic simulation
52+
try:
53+
browser.page.mouse.click(center_x, center_y)
54+
success = True
55+
except Exception:
56+
# If navigation happens, mouse.click might fail, but that's OK
57+
# The click still happened, just check URL change
58+
success = True
59+
else:
60+
# Fallback to JS click if element not found in snapshot
61+
try:
62+
success = browser.page.evaluate(
63+
"""
64+
(id) => {
65+
return window.sentience.click(id);
66+
}
67+
""",
68+
element_id,
69+
)
70+
except Exception:
71+
# Navigation might have destroyed context, assume success if URL changed
72+
success = True
73+
except Exception:
74+
# Fallback to JS click on error
75+
try:
76+
success = browser.page.evaluate(
77+
"""
78+
(id) => {
79+
return window.sentience.click(id);
80+
}
81+
""",
82+
element_id,
83+
)
84+
except Exception:
85+
# Navigation might have destroyed context, assume success if URL changed
86+
success = True
87+
else:
88+
# Legacy JS-based click
89+
success = browser.page.evaluate(
90+
"""
91+
(id) => {
92+
return window.sentience.click(id);
93+
}
94+
""",
95+
element_id,
96+
)
3997

4098
# Wait a bit for navigation/DOM updates
41-
browser.page.wait_for_timeout(500)
99+
try:
100+
browser.page.wait_for_timeout(500)
101+
except Exception:
102+
# Navigation might have happened, context destroyed
103+
pass
42104

43105
duration_ms = int((time.time() - start_time) * 1000)
44-
url_after = browser.page.url
45-
url_changed = url_before != url_after
106+
107+
# Check if URL changed (handle navigation gracefully)
108+
try:
109+
url_after = browser.page.url
110+
url_changed = url_before != url_after
111+
except Exception:
112+
# Context destroyed due to navigation - assume URL changed
113+
url_after = url_before
114+
url_changed = True
46115

47116
# Determine outcome
48117
outcome: Optional[str] = None
@@ -56,7 +125,11 @@ def click(browser: SentienceBrowser, element_id: int, take_snapshot: bool = Fals
56125
# Optional snapshot after
57126
snapshot_after: Optional[Snapshot] = None
58127
if take_snapshot:
59-
snapshot_after = snapshot(browser)
128+
try:
129+
snapshot_after = snapshot(browser)
130+
except Exception:
131+
# Navigation might have destroyed context
132+
pass
60133

61134
return ActionResult(
62135
success=success,
@@ -174,3 +247,173 @@ def press(browser: SentienceBrowser, key: str, take_snapshot: bool = False) -> A
174247
snapshot_after=snapshot_after,
175248
)
176249

250+
251+
def _highlight_rect(browser: SentienceBrowser, rect: Dict[str, float], duration_sec: float = 2.0) -> None:
252+
"""
253+
Highlight a rectangle with a red border overlay
254+
255+
Args:
256+
browser: SentienceBrowser instance
257+
rect: Dictionary with x, y, width (w), height (h) keys
258+
duration_sec: How long to show the highlight (default: 2 seconds)
259+
"""
260+
if not browser.page:
261+
return
262+
263+
# Create a unique ID for this highlight
264+
highlight_id = f"sentience_highlight_{int(time.time() * 1000)}"
265+
266+
# Combine all arguments into a single object for Playwright
267+
args = {
268+
"rect": {
269+
"x": rect["x"],
270+
"y": rect["y"],
271+
"w": rect["w"],
272+
"h": rect["h"],
273+
},
274+
"highlightId": highlight_id,
275+
"durationSec": duration_sec,
276+
}
277+
278+
# Inject CSS and create overlay element
279+
browser.page.evaluate(
280+
"""
281+
(args) => {
282+
const { rect, highlightId, durationSec } = args;
283+
// Create overlay div
284+
const overlay = document.createElement('div');
285+
overlay.id = highlightId;
286+
overlay.style.position = 'fixed';
287+
overlay.style.left = `${rect.x}px`;
288+
overlay.style.top = `${rect.y}px`;
289+
overlay.style.width = `${rect.w}px`;
290+
overlay.style.height = `${rect.h}px`;
291+
overlay.style.border = '3px solid red';
292+
overlay.style.borderRadius = '2px';
293+
overlay.style.boxSizing = 'border-box';
294+
overlay.style.pointerEvents = 'none';
295+
overlay.style.zIndex = '999999';
296+
overlay.style.backgroundColor = 'rgba(255, 0, 0, 0.1)';
297+
overlay.style.transition = 'opacity 0.3s ease-out';
298+
299+
document.body.appendChild(overlay);
300+
301+
// Remove after duration
302+
setTimeout(() => {
303+
overlay.style.opacity = '0';
304+
setTimeout(() => {
305+
if (overlay.parentNode) {
306+
overlay.parentNode.removeChild(overlay);
307+
}
308+
}, 300); // Wait for fade-out transition
309+
}, durationSec * 1000);
310+
}
311+
""",
312+
args,
313+
)
314+
315+
316+
def click_rect(
317+
browser: SentienceBrowser,
318+
rect: Dict[str, float],
319+
highlight: bool = True,
320+
highlight_duration: float = 2.0,
321+
take_snapshot: bool = False,
322+
) -> ActionResult:
323+
"""
324+
Click at the center of a rectangle using Playwright's native mouse simulation.
325+
This uses a hybrid approach: calculates center coordinates and uses mouse.click()
326+
for realistic event simulation (triggers hover, focus, mousedown, mouseup).
327+
328+
Args:
329+
browser: SentienceBrowser instance
330+
rect: Dictionary with x, y, width (w), height (h) keys, or BBox object
331+
highlight: Whether to show a red border highlight when clicking (default: True)
332+
highlight_duration: How long to show the highlight in seconds (default: 2.0)
333+
take_snapshot: Whether to take snapshot after action
334+
335+
Returns:
336+
ActionResult
337+
338+
Example:
339+
>>> click_rect(browser, {"x": 100, "y": 200, "w": 50, "h": 30})
340+
>>> # Or using BBox object
341+
>>> from sentience import BBox
342+
>>> bbox = BBox(x=100, y=200, width=50, height=30)
343+
>>> click_rect(browser, {"x": bbox.x, "y": bbox.y, "w": bbox.width, "h": bbox.height})
344+
"""
345+
if not browser.page:
346+
raise RuntimeError("Browser not started. Call browser.start() first.")
347+
348+
# Handle BBox object or dict
349+
if isinstance(rect, BBox):
350+
x = rect.x
351+
y = rect.y
352+
w = rect.width
353+
h = rect.height
354+
else:
355+
x = rect.get("x", 0)
356+
y = rect.get("y", 0)
357+
w = rect.get("w") or rect.get("width", 0)
358+
h = rect.get("h") or rect.get("height", 0)
359+
360+
if w <= 0 or h <= 0:
361+
return ActionResult(
362+
success=False,
363+
duration_ms=0,
364+
outcome="error",
365+
error={"code": "invalid_rect", "reason": "Rectangle width and height must be positive"},
366+
)
367+
368+
start_time = time.time()
369+
url_before = browser.page.url
370+
371+
# Calculate center of rectangle
372+
center_x = x + w / 2
373+
center_y = y + h / 2
374+
375+
# Show highlight before clicking (if enabled)
376+
if highlight:
377+
_highlight_rect(browser, {"x": x, "y": y, "w": w, "h": h}, highlight_duration)
378+
# Small delay to ensure highlight is visible
379+
browser.page.wait_for_timeout(50)
380+
381+
# Use Playwright's native mouse click for realistic simulation
382+
# This triggers hover, focus, mousedown, mouseup sequences
383+
try:
384+
browser.page.mouse.click(center_x, center_y)
385+
success = True
386+
except Exception as e:
387+
success = False
388+
error_msg = str(e)
389+
390+
# Wait a bit for navigation/DOM updates
391+
browser.page.wait_for_timeout(500)
392+
393+
duration_ms = int((time.time() - start_time) * 1000)
394+
url_after = browser.page.url
395+
url_changed = url_before != url_after
396+
397+
# Determine outcome
398+
outcome: Optional[str] = None
399+
if url_changed:
400+
outcome = "navigated"
401+
elif success:
402+
outcome = "dom_updated"
403+
else:
404+
outcome = "error"
405+
406+
# Optional snapshot after
407+
snapshot_after: Optional[Snapshot] = None
408+
if take_snapshot:
409+
snapshot_after = snapshot(browser)
410+
411+
return ActionResult(
412+
success=success,
413+
duration_ms=duration_ms,
414+
outcome=outcome,
415+
url_changed=url_changed,
416+
snapshot_after=snapshot_after,
417+
error=None if success else {"code": "click_failed", "reason": error_msg if not success else "Click failed"},
418+
)
419+

0 commit comments

Comments
 (0)