Skip to content

Commit 492dc29

Browse files
donald-pinckneydonald-pinckney
committed
add local model parameterization
1 parent 151e0d5 commit 492dc29

11 files changed

Lines changed: 81 additions & 51 deletions

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,6 @@ Some examples require extra dependencies. See each sample's directory for specif
9090
To run the tests:
9191

9292
uv run poe test
93+
94+
Note that this will skip running `openai_agents` tests against real OpenAI API calls if an API key is not found, and use only mocked models.
95+
To run with real model calls, set `OPENAI_API_KEY` in your environment.

‎tests/openai_agents/basic/test_agent_lifecycle_workflow.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -19,11 +21,13 @@ def agent_lifecycle_test_model():
1921
[ResponseBuilders.output_message('{"number": 10}')]
2022
)
2123

22-
23-
async def test_execute_workflow(client: Client):
24+
@pytest.mark.parametrize("mock_model", [True, False])
25+
async def test_execute_workflow(client: Client, mock_model: bool):
2426
task_queue_name = str(uuid.uuid4())
27+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
28+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2529

26-
async with AgentEnvironment(model=agent_lifecycle_test_model()) as agent_env:
30+
async with AgentEnvironment(model=agent_lifecycle_test_model() if mock_model else None) as agent_env:
2731
client = agent_env.applied_on_client(client)
2832
async with Worker(
2933
client,

‎tests/openai_agents/basic/test_dynamic_system_prompt_workflow.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -24,10 +26,13 @@ def dynamic_system_prompt_test_model():
2426
)
2527

2628

27-
async def test_execute_workflow_with_random_style(client: Client):
29+
@pytest.mark.parametrize("mock_model", [True, False])
30+
async def test_execute_workflow_with_random_style(client: Client, mock_model: bool):
2831
task_queue_name = str(uuid.uuid4())
32+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
33+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2934

30-
async with AgentEnvironment(model=dynamic_system_prompt_test_model()) as agent_env:
35+
async with AgentEnvironment(model=dynamic_system_prompt_test_model() if mock_model else None) as agent_env:
3136
client = agent_env.applied_on_client(client)
3237
async with Worker(
3338
client,
@@ -48,10 +53,13 @@ async def test_execute_workflow_with_random_style(client: Client):
4853
assert any(style in result for style in ["haiku", "pirate", "robot"])
4954

5055

51-
async def test_execute_workflow_with_specific_style(client: Client):
56+
@pytest.mark.parametrize("mock_model", [True, False])
57+
async def test_execute_workflow_with_specific_style(client: Client, mock_model: bool):
5258
task_queue_name = str(uuid.uuid4())
59+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
60+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
5361

54-
async with AgentEnvironment(model=dynamic_system_prompt_test_model()) as agent_env:
62+
async with AgentEnvironment(model=dynamic_system_prompt_test_model() if mock_model else None) as agent_env:
5563
client = agent_env.applied_on_client(client)
5664
async with Worker(
5765
client,

‎tests/openai_agents/basic/test_hello_world_workflow.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -18,10 +20,13 @@ def hello_world_test_model():
1820
)
1921

2022

21-
async def test_execute_workflow(client: Client):
23+
@pytest.mark.parametrize("mock_model", [True, False])
24+
async def test_execute_workflow(client: Client, mock_model: bool):
2225
task_queue_name = str(uuid.uuid4())
26+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
27+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2328

24-
async with AgentEnvironment(model=hello_world_test_model()) as agent_env:
29+
async with AgentEnvironment(model=hello_world_test_model() if mock_model else None) as agent_env:
2530
client = agent_env.applied_on_client(client)
2631
async with Worker(
2732
client,

‎tests/openai_agents/basic/test_lifecycle_workflow.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -18,10 +20,13 @@ def lifecycle_test_model():
1820
)
1921

2022

21-
async def test_execute_workflow(client: Client):
23+
@pytest.mark.parametrize("mock_model", [True, False])
24+
async def test_execute_workflow(client: Client, mock_model: bool):
2225
task_queue_name = str(uuid.uuid4())
26+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
27+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2328

24-
async with AgentEnvironment(model=lifecycle_test_model()) as agent_env:
29+
async with AgentEnvironment(model=lifecycle_test_model() if mock_model else None) as agent_env:
2530
client = agent_env.applied_on_client(client)
2631
async with Worker(
2732
client,

‎tests/openai_agents/basic/test_local_image_workflow.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -19,10 +21,13 @@ def local_image_test_model():
1921
)
2022

2123

22-
async def test_execute_workflow_default_question(client: Client):
24+
@pytest.mark.parametrize("mock_model", [True, False])
25+
async def test_execute_workflow_default_question(client: Client, mock_model: bool):
2326
task_queue_name = str(uuid.uuid4())
27+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
28+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2429

25-
async with AgentEnvironment(model=local_image_test_model()) as agent_env:
30+
async with AgentEnvironment(model=local_image_test_model() if mock_model else None) as agent_env:
2631
client = agent_env.applied_on_client(client)
2732
async with Worker(
2833
client,
@@ -43,10 +48,13 @@ async def test_execute_workflow_default_question(client: Client):
4348
assert len(result) > 0
4449

4550

46-
async def test_execute_workflow_custom_question(client: Client):
51+
@pytest.mark.parametrize("mock_model", [True, False])
52+
async def test_execute_workflow_custom_question(client: Client, mock_model: bool):
4753
task_queue_name = str(uuid.uuid4())
54+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
55+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
4856

49-
async with AgentEnvironment(model=local_image_test_model()) as agent_env:
57+
async with AgentEnvironment(model=local_image_test_model() if mock_model else None) as agent_env:
5058
client = agent_env.applied_on_client(client)
5159
async with Worker(
5260
client,

‎tests/openai_agents/basic/test_non_strict_output_workflow.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -26,10 +28,13 @@ def non_strict_output_test_model():
2628
)
2729

2830

29-
async def test_execute_workflow(client: Client):
31+
@pytest.mark.parametrize("mock_model", [True, False])
32+
async def test_execute_workflow(client: Client, mock_model: bool):
3033
task_queue_name = str(uuid.uuid4())
34+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
35+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
3136

32-
async with AgentEnvironment(model=non_strict_output_test_model()) as agent_env:
37+
async with AgentEnvironment(model=non_strict_output_test_model() if mock_model else None) as agent_env:
3338
client = agent_env.applied_on_client(client)
3439
async with Worker(
3540
client,

‎tests/openai_agents/basic/test_previous_response_id_workflow.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -25,10 +27,13 @@ def previous_response_id_test_model():
2527
)
2628

2729

28-
async def test_execute_workflow(client: Client):
30+
@pytest.mark.parametrize("mock_model", [True, False])
31+
async def test_execute_workflow(client: Client, mock_model: bool):
2932
task_queue_name = str(uuid.uuid4())
33+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
34+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
3035

31-
async with AgentEnvironment(model=previous_response_id_test_model()) as agent_env:
36+
async with AgentEnvironment(model=previous_response_id_test_model() if mock_model else None) as agent_env:
3237
client = agent_env.applied_on_client(client)
3338
async with Worker(
3439
client,

‎tests/openai_agents/basic/test_remote_image_workflow.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -22,10 +24,13 @@ def remote_image_test_model():
2224
)
2325

2426

25-
async def test_execute_workflow_default_question(client: Client):
27+
@pytest.mark.parametrize("mock_model", [True, False])
28+
async def test_execute_workflow_default_question(client: Client, mock_model: bool):
2629
task_queue_name = str(uuid.uuid4())
30+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
31+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2732

28-
async with AgentEnvironment(model=remote_image_test_model()) as agent_env:
33+
async with AgentEnvironment(model=remote_image_test_model() if mock_model else None) as agent_env:
2934
client = agent_env.applied_on_client(client)
3035
async with Worker(
3136
client,
@@ -47,10 +52,13 @@ async def test_execute_workflow_default_question(client: Client):
4752
assert len(result) > 0
4853

4954

50-
async def test_execute_workflow_custom_question(client: Client):
55+
@pytest.mark.parametrize("mock_model", [True, False])
56+
async def test_execute_workflow_custom_question(client: Client, mock_model: bool):
5157
task_queue_name = str(uuid.uuid4())
58+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
59+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
5260

53-
async with AgentEnvironment(model=remote_image_test_model()) as agent_env:
61+
async with AgentEnvironment(model=remote_image_test_model() if mock_model else None) as agent_env:
5462
client = agent_env.applied_on_client(client)
5563
async with Worker(
5664
client,

‎tests/openai_agents/basic/test_tools_workflow.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import uuid
2+
import pytest
3+
import os
24
from concurrent.futures import ThreadPoolExecutor
35

46
from temporalio.client import Client
@@ -24,10 +26,13 @@ def tools_test_model():
2426
)
2527

2628

27-
async def test_execute_workflow(client: Client):
29+
@pytest.mark.parametrize("mock_model", [True, False])
30+
async def test_execute_workflow(client: Client, mock_model: bool):
2831
task_queue_name = str(uuid.uuid4())
32+
if not mock_model and not os.environ.get("OPENAI_API_KEY"):
33+
pytest.skip(f"Skipping test (mock_model={mock_model}), because OPENAI_API_KEY is not set")
2934

30-
async with AgentEnvironment(model=tools_test_model()) as agent_env:
35+
async with AgentEnvironment(model=tools_test_model() if mock_model else None) as agent_env:
3136
client = agent_env.applied_on_client(client)
3237
async with Worker(
3338
client,

0 commit comments

Comments
 (0)