Skip to content

Commit a89fe1c

Browse files
rcholicrcholic
authored
Merge pull request #142 from SentienceAPI/grouping_overlay
Grouping overlay, showGrid
2 parents affd8cc + a65c27d commit a89fe1c

File tree

5 files changed

+345
-29
lines changed

5 files changed

+345
-29
lines changed

‎examples/show_grid_examples.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
"""
2+
Example: Grid Overlay Visualization
3+
4+
Demonstrates how to use the grid overlay feature to visualize detected grids
5+
on a webpage, including highlighting specific grids and identifying the dominant group.
6+
"""
7+
8+
import os
9+
import time
10+
11+
from sentience import SentienceBrowser, snapshot
12+
from sentience.models import SnapshotOptions
13+
14+
15+
def main():
16+
# Get API key from environment variable (optional - uses free tier if not set)
17+
api_key = os.environ.get("SENTIENCE_API_KEY")
18+
19+
# Use VPS IP directly if domain is not configured
20+
# Replace with your actual domain once DNS is set up: api_url="https://api.sentienceapi.com"
21+
api_url = os.environ.get("SENTIENCE_API_URL", "http://15.204.243.91:9000")
22+
23+
try:
24+
with SentienceBrowser(api_key=api_key, api_url=api_url, headless=False) as browser:
25+
# Navigate to a page with grid layouts (e.g., product listings, article feeds)
26+
browser.page.goto("https://example.com", wait_until="domcontentloaded")
27+
time.sleep(2) # Wait for page to fully load
28+
29+
print("=" * 60)
30+
print("Example 1: Show all detected grids")
31+
print("=" * 60)
32+
# Show all grids (all in purple)
33+
snap = snapshot(browser, SnapshotOptions(show_grid=True, use_api=True))
34+
print(f"✅ Found {len(snap.elements)} elements")
35+
print(" Purple borders appear around all detected grids for 5 seconds")
36+
time.sleep(6) # Wait to see the overlay
37+
38+
print("\n" + "=" * 60)
39+
print("Example 2: Highlight a specific grid in red")
40+
print("=" * 60)
41+
# Get grid information first
42+
grids = snap.get_grid_bounds()
43+
if grids:
44+
print(f"✅ Found {len(grids)} grids:")
45+
for grid in grids:
46+
print(
47+
f" Grid {grid.grid_id}: {grid.item_count} items, "
48+
f"{grid.row_count}x{grid.col_count} rows/cols, "
49+
f"label: {grid.label or 'none'}"
50+
)
51+
52+
# Highlight the first grid in red
53+
if len(grids) > 0:
54+
target_grid_id = grids[0].grid_id
55+
print(f"\n Highlighting Grid {target_grid_id} in red...")
56+
snap = snapshot(
57+
browser,
58+
SnapshotOptions(
59+
show_grid=True,
60+
grid_id=target_grid_id, # This grid will be highlighted in red
61+
),
62+
)
63+
time.sleep(6) # Wait to see the overlay
64+
else:
65+
print(" ⚠️ No grids detected on this page")
66+
67+
print("\n" + "=" * 60)
68+
print("Example 3: Highlight the dominant group")
69+
print("=" * 60)
70+
# Find and highlight the dominant grid
71+
grids = snap.get_grid_bounds()
72+
dominant_grid = next((g for g in grids if g.is_dominant), None)
73+
74+
if dominant_grid:
75+
print(f"✅ Dominant group detected: Grid {dominant_grid.grid_id}")
76+
print(f" Label: {dominant_grid.label or 'none'}")
77+
print(f" Items: {dominant_grid.item_count}")
78+
print(f" Size: {dominant_grid.row_count}x{dominant_grid.col_count}")
79+
print(f"\n Highlighting dominant grid in red...")
80+
snap = snapshot(
81+
browser,
82+
SnapshotOptions(
83+
show_grid=True,
84+
grid_id=dominant_grid.grid_id, # Highlight dominant grid in red
85+
),
86+
)
87+
time.sleep(6) # Wait to see the overlay
88+
else:
89+
print(" ⚠️ No dominant group detected")
90+
91+
print("\n" + "=" * 60)
92+
print("Example 4: Combine element overlay and grid overlay")
93+
print("=" * 60)
94+
# Show both element borders and grid borders simultaneously
95+
snap = snapshot(
96+
browser,
97+
SnapshotOptions(
98+
show_overlay=True, # Show element borders (green/blue/red)
99+
show_grid=True, # Show grid borders (purple/orange/red)
100+
),
101+
)
102+
print("✅ Both overlays are now visible:")
103+
print(" - Element borders: Green (regular), Blue (primary), Red (target)")
104+
print(" - Grid borders: Purple (regular), Orange (dominant), Red (target)")
105+
time.sleep(6) # Wait to see the overlay
106+
107+
print("\n" + "=" * 60)
108+
print("Example 5: Grid information analysis")
109+
print("=" * 60)
110+
# Analyze grid structure
111+
grids = snap.get_grid_bounds()
112+
print(f"✅ Grid Analysis:")
113+
for grid in grids:
114+
dominant_indicator = "⭐ DOMINANT" if grid.is_dominant else ""
115+
print(f"\n Grid {grid.grid_id} {dominant_indicator}:")
116+
print(f" Label: {grid.label or 'none'}")
117+
print(f" Items: {grid.item_count}")
118+
print(f" Size: {grid.row_count} rows × {grid.col_count} cols")
119+
print(
120+
f" BBox: ({grid.bbox.x:.0f}, {grid.bbox.y:.0f}) "
121+
f"{grid.bbox.width:.0f}×{grid.bbox.height:.0f}"
122+
)
123+
print(f" Confidence: {grid.confidence}")
124+
125+
print("\n✅ All examples completed!")
126+
127+
except Exception as e:
128+
print(f"❌ Error: {e}")
129+
import traceback
130+
131+
traceback.print_exc()
132+
133+
134+
if __name__ == "__main__":
135+
main()

‎sentience/agent_runtime.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -342,8 +342,7 @@ def assert_done(
342342
Returns:
343343
True if task is complete (assertion passed), False otherwise
344344
"""
345-
ok = self.assertTrue(predicate, label=label, required=True)
346-
345+
ok = self.assert_(predicate, label=label, required=True)
347346
if ok:
348347
self._task_done = True
349348
self._task_done_label = label

‎sentience/models.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class GridInfo(BaseModel):
118118
label: str | None = (
119119
None # Optional inferred label (e.g., "product_grid", "search_results", "navigation")
120120
)
121+
is_dominant: bool = False # Whether this grid is the dominant group (main content area)
121122

122123

123124
class Snapshot(BaseModel):
@@ -190,10 +191,16 @@ def get_grid_bounds(self, grid_id: int | None = None) -> list[GridInfo]:
190191

191192
grid_infos = []
192193

194+
# First pass: compute all grid infos and count dominant group elements
195+
grid_dominant_counts = {}
193196
for gid, elements_in_grid in sorted(grid_elements.items()):
194197
if not elements_in_grid:
195198
continue
196199

200+
# Count dominant group elements in this grid
201+
dominant_count = sum(1 for elem in elements_in_grid if elem.in_dominant_group is True)
202+
grid_dominant_counts[gid] = (dominant_count, len(elements_in_grid))
203+
197204
# Compute bounding box
198205
min_x = min(elem.bbox.x for elem in elements_in_grid)
199206
min_y = min(elem.bbox.y for elem in elements_in_grid)
@@ -226,9 +233,42 @@ def get_grid_bounds(self, grid_id: int | None = None) -> list[GridInfo]:
226233
item_count=len(elements_in_grid),
227234
confidence=1.0,
228235
label=label,
236+
is_dominant=False, # Will be set below
229237
)
230238
)
231239

240+
# Second pass: identify dominant grid
241+
# The grid with the highest count (or highest percentage >= 50%) of dominant group elements
242+
if grid_dominant_counts:
243+
# Find grid with highest absolute count
244+
max_dominant_count = max(count for count, _ in grid_dominant_counts.values())
245+
if max_dominant_count > 0:
246+
# Find grid(s) with highest count
247+
dominant_grids = [
248+
gid
249+
for gid, (count, total) in grid_dominant_counts.items()
250+
if count == max_dominant_count
251+
]
252+
# If multiple grids tie, prefer the one with highest percentage
253+
if len(dominant_grids) > 1:
254+
dominant_grids.sort(
255+
key=lambda gid: (
256+
grid_dominant_counts[gid][0] / grid_dominant_counts[gid][1]
257+
if grid_dominant_counts[gid][1] > 0
258+
else 0
259+
),
260+
reverse=True,
261+
)
262+
# Mark the dominant grid
263+
dominant_gid = dominant_grids[0]
264+
# Only mark as dominant if it has >= 50% dominant group elements or >= 3 elements
265+
dominant_count, total_count = grid_dominant_counts[dominant_gid]
266+
if dominant_count >= 3 or (total_count > 0 and dominant_count / total_count >= 0.5):
267+
for grid_info in grid_infos:
268+
if grid_info.grid_id == dominant_gid:
269+
grid_info.is_dominant = True
270+
break
271+
232272
return grid_infos
233273

234274
@staticmethod
@@ -456,6 +496,10 @@ class SnapshotOptions(BaseModel):
456496
trace_path: str | None = None # Path to save trace (default: "trace_{timestamp}.json")
457497
goal: str | None = None # Optional goal/task description for the snapshot
458498
show_overlay: bool = False # Show visual overlay highlighting elements in browser
499+
show_grid: bool = False # Show visual overlay highlighting detected grids
500+
grid_id: int | None = (
501+
None # Optional grid ID to show specific grid (only used if show_grid=True)
502+
)
459503

460504
# API credentials (for browser-use integration without SentienceBrowser)
461505
sentience_api_key: str | None = None # Sentience API key for Pro/Enterprise features

0 commit comments

Comments
 (0)