Skip to content

Commit fcfb6f4

Browse files
rcholicrcholic
authored
Merge pull request #172 from SentienceAPI/tweaking_fixes2
Tweaking fixes: trace upload when exception + ffmpeg clip fix
2 parents e909bec + 939fc1b commit fcfb6f4

File tree

7 files changed

+307
-18
lines changed

7 files changed

+307
-18
lines changed

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ await runtime.enable_failure_artifacts(
129129
await runtime.record_action("CLICK")
130130
```
131131

132+
**Video clip generation (optional):** To generate MP4 video clips from captured frames, install [ffmpeg](https://ffmpeg.org/) (version 4.0 or later; version 5.1+ recommended for best compatibility). If ffmpeg is not installed, frames are still captured but no video clip is generated.
133+
132134
### Redaction callback (Phase 3)
133135

134136
Provide a user-defined callback to redact snapshots and decide whether to persist frames. The SDK does not implement image/video redaction.

‎sentience/agent.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,13 @@ def act( # noqa: C901
205205
pre_url=pre_url,
206206
)
207207

208+
# Track data collected during step execution for step_end emission on failure
209+
_step_snap_with_diff: Snapshot | None = None
210+
_step_pre_url: str | None = None
211+
_step_llm_response: LLMResponse | None = None
212+
_step_result: AgentActionResult | None = None
213+
_step_duration_ms: int = 0
214+
208215
for attempt in range(max_retries + 1):
209216
try:
210217
# 1. OBSERVE: Get refined semantic snapshot
@@ -254,6 +261,10 @@ def act( # noqa: C901
254261
error=snap.error,
255262
)
256263

264+
# Track for step_end emission on failure
265+
_step_snap_with_diff = snap_with_diff
266+
_step_pre_url = snap.url
267+
257268
# Update previous snapshot for next comparison
258269
self._previous_snapshot = snap
259270

@@ -311,6 +322,9 @@ def act( # noqa: C901
311322
# 3. THINK: Query LLM for next action
312323
llm_response = self.llm_handler.query_llm(context, goal)
313324

325+
# Track for step_end emission on failure
326+
_step_llm_response = llm_response
327+
314328
# Emit LLM query trace event if tracer is enabled
315329
if self.tracer:
316330
_safe_tracer_call(
@@ -358,6 +372,10 @@ def act( # noqa: C901
358372
cursor=result_dict.get("cursor"),
359373
)
360374

375+
# Track for step_end emission on failure
376+
_step_result = result
377+
_step_duration_ms = duration_ms
378+
361379
# Emit action execution trace event if tracer is enabled
362380
if self.tracer:
363381
post_url = self.browser.page.url if self.browser.page else None
@@ -539,6 +557,65 @@ def act( # noqa: C901
539557
time.sleep(1.0) # Brief delay before retry
540558
continue
541559
else:
560+
# Emit step_end with whatever data we collected before failure
561+
# This ensures diff_status and other fields are preserved in traces
562+
if self.tracer and _step_snap_with_diff is not None:
563+
post_url = self.browser.page.url if self.browser.page else None
564+
snapshot_digest = f"sha256:{self._compute_hash(f'{_step_pre_url}{_step_snap_with_diff.timestamp}')}"
565+
566+
# Build pre_elements from snap_with_diff (includes diff_status)
567+
snapshot_event_data = TraceEventBuilder.build_snapshot_event(
568+
_step_snap_with_diff
569+
)
570+
pre_elements = snapshot_event_data.get("elements", [])
571+
572+
# Build LLM data if available
573+
llm_data = None
574+
if _step_llm_response:
575+
llm_response_text = _step_llm_response.content
576+
llm_response_hash = f"sha256:{self._compute_hash(llm_response_text)}"
577+
llm_data = {
578+
"response_text": llm_response_text,
579+
"response_hash": llm_response_hash,
580+
"usage": {
581+
"prompt_tokens": _step_llm_response.prompt_tokens or 0,
582+
"completion_tokens": _step_llm_response.completion_tokens or 0,
583+
"total_tokens": _step_llm_response.total_tokens or 0,
584+
},
585+
}
586+
587+
# Build exec data (failure state)
588+
exec_data = {
589+
"success": False,
590+
"action": _step_result.action if _step_result else "error",
591+
"outcome": str(e),
592+
"duration_ms": _step_duration_ms,
593+
}
594+
595+
# Build step_end event for failed step
596+
step_end_data = TraceEventBuilder.build_step_end_event(
597+
step_id=step_id,
598+
step_index=self._step_count,
599+
goal=goal,
600+
attempt=attempt,
601+
pre_url=_step_pre_url,
602+
post_url=post_url,
603+
snapshot_digest=snapshot_digest,
604+
llm_data=llm_data,
605+
exec_data=exec_data,
606+
verify_data=None,
607+
pre_elements=pre_elements,
608+
)
609+
610+
_safe_tracer_call(
611+
self.tracer,
612+
"emit",
613+
self.verbose,
614+
"step_end",
615+
step_end_data,
616+
step_id=step_id,
617+
)
618+
542619
# Create error result
543620
error_result = AgentActionResult(
544621
success=False,
@@ -771,6 +848,13 @@ async def act( # noqa: C901
771848
pre_url=pre_url,
772849
)
773850

851+
# Track data collected during step execution for step_end emission on failure
852+
_step_snap_with_diff: Snapshot | None = None
853+
_step_pre_url: str | None = None
854+
_step_llm_response: LLMResponse | None = None
855+
_step_result: AgentActionResult | None = None
856+
_step_duration_ms: int = 0
857+
774858
for attempt in range(max_retries + 1):
775859
try:
776860
# 1. OBSERVE: Get refined semantic snapshot
@@ -823,6 +907,10 @@ async def act( # noqa: C901
823907
error=snap.error,
824908
)
825909

910+
# Track for step_end emission on failure
911+
_step_snap_with_diff = snap_with_diff
912+
_step_pre_url = snap.url
913+
826914
# Update previous snapshot for next comparison
827915
self._previous_snapshot = snap
828916

@@ -880,6 +968,9 @@ async def act( # noqa: C901
880968
# 3. THINK: Query LLM for next action
881969
llm_response = self.llm_handler.query_llm(context, goal)
882970

971+
# Track for step_end emission on failure
972+
_step_llm_response = llm_response
973+
883974
# Emit LLM query trace event if tracer is enabled
884975
if self.tracer:
885976
_safe_tracer_call(
@@ -926,6 +1017,10 @@ async def act( # noqa: C901
9261017
message=result_dict.get("message"),
9271018
)
9281019

1020+
# Track for step_end emission on failure
1021+
_step_result = result
1022+
_step_duration_ms = duration_ms
1023+
9291024
# Emit action execution trace event if tracer is enabled
9301025
if self.tracer:
9311026
post_url = self.browser.page.url if self.browser.page else None
@@ -1104,6 +1199,65 @@ async def act( # noqa: C901
11041199
await asyncio.sleep(1.0) # Brief delay before retry
11051200
continue
11061201
else:
1202+
# Emit step_end with whatever data we collected before failure
1203+
# This ensures diff_status and other fields are preserved in traces
1204+
if self.tracer and _step_snap_with_diff is not None:
1205+
post_url = self.browser.page.url if self.browser.page else None
1206+
snapshot_digest = f"sha256:{self._compute_hash(f'{_step_pre_url}{_step_snap_with_diff.timestamp}')}"
1207+
1208+
# Build pre_elements from snap_with_diff (includes diff_status)
1209+
snapshot_event_data = TraceEventBuilder.build_snapshot_event(
1210+
_step_snap_with_diff
1211+
)
1212+
pre_elements = snapshot_event_data.get("elements", [])
1213+
1214+
# Build LLM data if available
1215+
llm_data = None
1216+
if _step_llm_response:
1217+
llm_response_text = _step_llm_response.content
1218+
llm_response_hash = f"sha256:{self._compute_hash(llm_response_text)}"
1219+
llm_data = {
1220+
"response_text": llm_response_text,
1221+
"response_hash": llm_response_hash,
1222+
"usage": {
1223+
"prompt_tokens": _step_llm_response.prompt_tokens or 0,
1224+
"completion_tokens": _step_llm_response.completion_tokens or 0,
1225+
"total_tokens": _step_llm_response.total_tokens or 0,
1226+
},
1227+
}
1228+
1229+
# Build exec data (failure state)
1230+
exec_data = {
1231+
"success": False,
1232+
"action": _step_result.action if _step_result else "error",
1233+
"outcome": str(e),
1234+
"duration_ms": _step_duration_ms,
1235+
}
1236+
1237+
# Build step_end event for failed step
1238+
step_end_data = TraceEventBuilder.build_step_end_event(
1239+
step_id=step_id,
1240+
step_index=self._step_count,
1241+
goal=goal,
1242+
attempt=attempt,
1243+
pre_url=_step_pre_url,
1244+
post_url=post_url,
1245+
snapshot_digest=snapshot_digest,
1246+
llm_data=llm_data,
1247+
exec_data=exec_data,
1248+
verify_data=None,
1249+
pre_elements=pre_elements,
1250+
)
1251+
1252+
_safe_tracer_call(
1253+
self.tracer,
1254+
"emit",
1255+
self.verbose,
1256+
"step_end",
1257+
step_end_data,
1258+
step_id=step_id,
1259+
)
1260+
11071261
# Create error result
11081262
error_result = AgentActionResult(
11091263
success=False,

‎sentience/agent_runtime.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -583,7 +583,7 @@ def assert_done(
583583
True if task is complete (assertion passed), False otherwise
584584
"""
585585
# Convenience wrapper for assert_ with required=True
586-
ok = self.assert_(predicate, label=label, required=True)
586+
ok = self.assertTrue(predicate, label=label, required=True)
587587
if ok:
588588
self._task_done = True
589589
self._task_done_label = label

‎sentience/backends/snapshot.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,10 @@ async def _snapshot_via_api(
392392
# Step 1: Get raw data from local extension (always happens locally)
393393
raw_options: dict[str, Any] = {}
394394
if options.screenshot is not False:
395-
raw_options["screenshot"] = options.screenshot
395+
if hasattr(options.screenshot, "model_dump"):
396+
raw_options["screenshot"] = options.screenshot.model_dump()
397+
else:
398+
raw_options["screenshot"] = options.screenshot
396399

397400
# Call extension to get raw elements
398401
raw_result = await _eval_with_navigation_retry(

0 commit comments

Comments
 (0)