feat: Add suspend/resume evaluation sample with trajectory evaluators

pyproject.toml

@@ -5,7 +5,7 @@ description = "Python SDK that enables developers to build and deploy LangGraph
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"
 dependencies = [
-    "uipath>=2.4.0, <2.5.0",
+    "uipath",
     "uipath-runtime>=0.4.0, <0.5.0",
     "langgraph>=1.0.0, <2.0.0",
     "langchain-core>=1.2.5, <2.0.0",

samples/tool-calling-suspend-resume/README.md

@@ -0,0 +1,85 @@
+# Tool-Calling Suspend/Resume Agent
+
+A simple agent demonstrating the suspend/resume pattern for RPA process invocations using LangGraph's `interrupt()` function.
+
+## Overview
+
+This agent calls `interrupt(InvokeProcess(...))` to suspend execution while waiting for an external RPA process to complete. The evaluation runtime detects the suspension and extracts trigger information for resumption.
+
+## Features
+
+- Single-node graph demonstrating tool-calling suspend pattern
+- Uses LangGraph's `interrupt()` to suspend execution
+- Proper `InvokeProcess` structure for RPA invocation
+- Includes checkpointer (required for interrupts)
+- Comprehensive evaluation sets with trajectory validation
+
+## Running Evaluations
+
+```bash
+# Navigate to the sample directory
+cd samples/tool-calling-suspend-resume
+
+# Run evaluation
+uv run uipath eval graph evaluations/eval-sets/test_suspend_resume.json
+```
+
+## Evaluation Sets
+
+### `test_suspend_resume.json`
+Tests the actual suspend/resume behavior:
+- Validates agent calls `interrupt()` with proper `InvokeProcess` structure
+- Checks for suspension indicators in logs
+- Uses both LLM-based trajectory evaluator and contains-based evaluator
+
+### `test_with_evaluators.json`
+Tests evaluator execution after completion:
+- Modifies the agent to complete without suspending
+- Validates that evaluators run and produce scores
+- Useful for verifying evaluator configuration
+
+## Architecture
+
+```
+graph.py (LangGraph Agent)
+    ↓
+invoke_process_node → interrupt(InvokeProcess(...))
+    ↓
+SUSPENDS execution
+    ↓
+Runtime detects suspension
+    ↓
+Extracts triggers
+    ↓
+Skips evaluators (run after resume)
+```
+
+## Key Components
+
+- **graph.py**: Main agent with single node that calls `interrupt()`
+- **evaluations/**: Evaluation sets and evaluator configurations
+  - **eval-sets/**: Test cases for suspend and evaluator testing
+  - **evaluators/**: LLM trajectory and contains evaluator configs
+- **pyproject.toml**: Package metadata
+- **uipath.json**: Agent configuration
+
+## How It Works
+
+1. **Agent Execution**: Agent runs and reaches `interrupt(InvokeProcess(...))`
+2. **Suspension**: LangGraph raises interrupt, runtime detects `SUSPENDED` status
+3. **Trigger Extraction**: Runtime extracts trigger with process details
+4. **Evaluator Skip**: Evaluators are skipped during suspension
+5. **Resume** (when implemented): Process completes, agent resumes
+6. **Evaluator Execution**: Evaluators run on final output
+
+## Expected Output
+
+When running the evaluation, you should see:
+```
+🔴 DETECTED SUSPENSION → Runtime detects status change
+📋 Extracted N trigger(s) → Shows trigger details
+⏭️ Skipping evaluators → Explains why no evaluation
+✅ Passing through triggers → Shows trigger propagation
+```
+
+The evaluation result will show `status: SUSPENDED` with trigger information in the output JSON.

samples/tool-calling-suspend-resume/evaluations/eval-sets/test_simple.json

@@ -0,0 +1,23 @@
+{
+  "version": "1.0",
+  "id": "test-simple-suspend",
+  "name": "Simple Suspend Test",
+  "description": "Basic test for suspend/resume pattern without LLM evaluator",
+  "evaluatorRefs": [
+    "SuspendCheckEvaluator"
+  ],
+  "evaluations": [
+    {
+      "id": "test_suspend_basic",
+      "name": "Basic suspend test",
+      "inputs": {
+        "query": "Test suspend and resume with RPA process"
+      },
+      "evaluationCriterias": {
+        "SuspendCheckEvaluator": {
+          "searchText": "SUSPENDING EXECUTION"
+        }
+      }
+    }
+  ]
+}

samples/tool-calling-suspend-resume/evaluations/eval-sets/test_suspend_resume.json

@@ -0,0 +1,42 @@
+{
+  "version": "1.0",
+  "id": "test-suspend-resume-eval",
+  "name": "Suspend Resume Test",
+  "description": "Test evaluation set for suspend/resume pattern with RPA process invocation",
+  "evaluatorRefs": [
+    "SuspendResumeTrajectoryEvaluator",
+    "SuspendCheckEvaluator"
+  ],
+  "evaluations": [
+    {
+      "id": "test_suspend_basic",
+      "name": "Basic suspend test",
+      "inputs": {
+        "query": "Test suspend and resume with RPA process"
+      },
+      "evaluationCriterias": {
+        "SuspendResumeTrajectoryEvaluator": {
+          "expectedAgentBehavior": "The agent should call interrupt() to suspend execution while waiting for the TestProcess RPA process to complete. The execution trace should contain a GraphInterrupt exception with InvokeProcess details including process name 'TestProcess' and query parameter."
+        },
+        "SuspendCheckEvaluator": {
+          "searchText": "SUSPENDING EXECUTION"
+        }
+      }
+    },
+    {
+      "id": "test_suspend_complex",
+      "name": "Complex query suspend test",
+      "inputs": {
+        "query": "Process complex data structure with multiple parameters"
+      },
+      "evaluationCriterias": {
+        "SuspendResumeTrajectoryEvaluator": {
+          "expectedAgentBehavior": "The agent should call interrupt() to suspend execution with InvokeProcess request for TestProcess. The trace should show the suspend happened in the invoke_process node with proper InvokeProcess structure."
+        },
+        "SuspendCheckEvaluator": {
+          "searchText": "SUSPENDING EXECUTION"
+        }
+      }
+    }
+  ]
+}

samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-check.json

@@ -0,0 +1,15 @@
+{
+  "version": "1.0",
+  "id": "SuspendCheckEvaluator",
+  "description": "Checks if the execution contains suspend-related indicators in the output.",
+  "evaluatorTypeId": "uipath-contains",
+  "evaluatorConfig": {
+    "name": "SuspendCheckEvaluator",
+    "targetOutputKey": "logs",
+    "negated": false,
+    "ignoreCase": false,
+    "defaultEvaluationCriteria": {
+      "searchText": "SUSPENDING EXECUTION"
+    }
+  }
+}

samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-resume-trajectory.json

@@ -0,0 +1,15 @@
+{
+  "version": "1.0",
+  "id": "SuspendResumeTrajectoryEvaluator",
+  "description": "Evaluates the agent's execution trajectory for proper suspend/resume pattern with RPA process invocation.",
+  "evaluatorTypeId": "uipath-llm-judge-trajectory-similarity",
+  "evaluatorConfig": {
+    "name": "SuspendResumeTrajectoryEvaluator",
+    "model": "gpt-4.1-2025-04-14",
+    "prompt": "Evaluate the agent's execution trajectory for proper suspend/resume pattern implementation.\n\nExpected Agent Behavior: {{ExpectedAgentBehavior}}\nAgent Run History: {{AgentRunHistory}}\n\nAssess whether the agent:\n1. Called interrupt() to suspend execution at the appropriate point\n2. Created a proper InvokeProcess request with correct structure\n3. Included all necessary parameters (process name, query, etc.)\n4. Followed the expected execution flow for suspend/resume pattern\n\nProvide a score from 0-100 based on how well the agent followed the expected trajectory and properly implemented the suspend/resume pattern.",
+    "temperature": 0.0,
+    "defaultEvaluationCriteria": {
+      "expectedAgentBehavior": "The agent should call interrupt() to suspend execution while waiting for an RPA process to complete, with proper InvokeProcess details in the trace."
+    }
+  }
+}

samples/tool-calling-suspend-resume/graph.py

@@ -0,0 +1,94 @@
+"""Tool-calling agent demonstrating suspend/resume with RPA process invocation."""
+
+import logging
+
+from langgraph.constants import END, START
+from langgraph.graph import StateGraph
+from langgraph.types import interrupt
+from pydantic import BaseModel
+from uipath.platform.common import InvokeProcess
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+
+class Input(BaseModel):
+    """Input for the test agent."""
+
+    query: str
+
+
+class Output(BaseModel):
+    """Output from the test agent."""
+
+    result: str
+
+
+class State(BaseModel):
+    """Internal state for the agent."""
+
+    query: str
+    result: str = ""
+
+
+async def invoke_process_node(state: State) -> State:
+    """Node that invokes an RPA process and suspends execution."""
+    logger.info("=" * 80)
+    logger.info("AGENT NODE: Starting invoke_process_node")
+    logger.info(f"AGENT NODE: Received query: {state.query}")
+
+    # Create an InvokeProcess request
+    invoke_request = InvokeProcess(
+        name="TestProcess",
+        input_arguments={"query": state.query, "data": "test_data"},
+        process_folder_path="Shared",
+    )
+
+    logger.info(
+        f"AGENT NODE: Created InvokeProcess request: {invoke_request.model_dump()}"
+    )
+    logger.info("🔴 AGENT NODE: About to call interrupt() - SUSPENDING EXECUTION")
+    logger.info("=" * 80)
+
+    # Interrupt execution and capture the process output
+    # The runtime will detect this and return SUSPENDED status
+    # When resumed, the return value will contain the process output
+    process_output = interrupt(invoke_request)
+
+    # This code won't execute until the process completes and execution resumes
+    logger.info("=" * 80)
+    logger.info("🟢 AGENT NODE: Execution RESUMED after interrupt()")
+    logger.info("AGENT NODE: RPA process has completed")
+    logger.info(f"AGENT NODE: Process output: {process_output}")
+    logger.info(f"AGENT NODE: Returning result for query: {state.query}")
+    logger.info("=" * 80)
+
+    # Extract result from process output
+    result = (
+        process_output.get("result", "Process completed")
+        if isinstance(process_output, dict)
+        else str(process_output)
+    )
+    return State(query=state.query, result=result)
+
+
+# Build the graph
+builder = StateGraph(state_schema=State)
+
+# Add single node that invokes the process
+builder.add_node("invoke_process", invoke_process_node)
+
+# Connect: START -> invoke_process -> END
+builder.add_edge(START, "invoke_process")
+builder.add_edge("invoke_process", END)
+
+# Compile with checkpointer (required for interrupts to work)
+from langgraph.checkpoint.memory import MemorySaver
+
+checkpointer = MemorySaver()
+graph = builder.compile(checkpointer=checkpointer)

samples/tool-calling-suspend-resume/langgraph.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": ["."],
+  "graphs": {
+    "graph": "./graph.py:graph"
+  }
+}

samples/tool-calling-suspend-resume/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "tool-calling-suspend-resume"
+version = "0.1.0"
+description = "Tool-calling agent demonstrating suspend/resume with RPA process invocation"
+requires-python = ">=3.11"
+dependencies = []

samples/tool-calling-suspend-resume/uipath.json

@@ -0,0 +1,5 @@
+{
+  "runtimeOptions": {
+    "isConversational": false
+  }
+}