Skip to content

Commit b04d7e0

Browse files
maxisbeymaxisbey
committed
test: add interaction-model e2e suite with requirements manifest
New tests/interaction/ suite asserting client<->server round trips through the public API only. Tests are organised around a requirements manifest (_requirements.py) mapping each test to the spec or SDK behaviour it exercises, with known divergences from the spec recorded on the requirement; test_coverage.py enforces that every non-deferred requirement is exercised by at least one test. Covers tools, prompts, resources, and ping against the low-level Server, plus MCPServer tool-call behaviours. Removes two 'pragma: no cover' comments on the ping send/answer paths now that they are covered.
1 parent e8e6484 commit b04d7e0

13 files changed

Lines changed: 937 additions & 2 deletions

File tree

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ strict-no-cover = { git = "https://github.com/pydantic/strict-no-cover" }
193193
[tool.pytest.ini_options]
194194
log_cli = true
195195
xfail_strict = true
196+
markers = [
197+
"requirement(id): links a test to the entry in tests/interaction/_requirements.py it exercises",
198+
]
196199
addopts = """
197200
--color=yes
198201
--capture=fd

‎src/mcp/client/session.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ async def _received_request(self, responder: RequestResponder[types.ServerReques
449449
client_response = ClientResponse.validate_python(response)
450450
await responder.respond(client_response)
451451

452-
case types.PingRequest(): # pragma: no cover
452+
case types.PingRequest():
453453
with responder:
454454
return await responder.respond(types.EmptyResult())
455455

‎src/mcp/server/session.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -447,7 +447,7 @@ async def elicit_url(
447447
metadata=ServerMessageMetadata(related_request_id=related_request_id),
448448
)
449449

450-
async def send_ping(self) -> types.EmptyResult: # pragma: no cover
450+
async def send_ping(self) -> types.EmptyResult:
451451
"""Send a ping request."""
452452
return await self.send_request(
453453
types.PingRequest(),

‎tests/interaction/__init__.py‎

Whitespace-only changes.

‎tests/interaction/_requirements.py‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
"""Requirements manifest for the interaction-model test suite.
2+
3+
Every user-facing behaviour the SDK must satisfy, keyed by a stable `<area>:<feature>[:<variant>]`
4+
ID. Each entry owns the tests that exercise it: tests declare `@requirement("<id>")` and
5+
`test_coverage.py` enforces that every non-deferred requirement is exercised by at least one test.
6+
7+
Sources:
8+
spec URL -- externally mandated by the MCP specification (deep link to the section)
9+
`sdk` -- a behavioural guarantee the SDK chose; not spec-mandated
10+
`issue:#n` -- regression lock-in for a previously fixed bug
11+
12+
The `behavior` sentence describes what the suite *asserts* -- which is always the SDK's current
13+
behaviour. Where that differs from what `source` mandates, the gap is recorded in `divergence`
14+
and the tests still pin current behaviour: this suite is the parity bar for the receive-path
15+
rewrite, so a test that fails today proves nothing about equivalence.
16+
"""
17+
18+
from collections.abc import Callable
19+
from dataclasses import dataclass
20+
from typing import TypeVar
21+
22+
import pytest
23+
24+
SPEC_REVISION = "2025-11-25"
25+
SPEC_BASE_URL = f"https://modelcontextprotocol.io/specification/{SPEC_REVISION}"
26+
27+
_TestFn = TypeVar("_TestFn", bound=Callable[..., object])
28+
29+
30+
@dataclass(frozen=True, kw_only=True)
31+
class Divergence:
32+
"""A documented gap between the SDK behaviour this suite pins and what `source` mandates."""
33+
34+
note: str
35+
issue: str | None = None
36+
37+
38+
@dataclass(frozen=True, kw_only=True)
39+
class Requirement:
40+
"""A single testable behaviour and the provenance of why it must hold."""
41+
42+
source: str
43+
behavior: str
44+
divergence: Divergence | None = None
45+
deferred: str | None = None
46+
47+
48+
REQUIREMENTS: dict[str, Requirement] = {
49+
# ═══════════════════════════════════════════════════════════════════════════
50+
# Protocol primitives
51+
# ═══════════════════════════════════════════════════════════════════════════
52+
"protocol:error:internal-error": Requirement(
53+
source=f"{SPEC_BASE_URL}/basic#responses",
54+
behavior="An unhandled exception in a request handler is returned to the caller as a JSON-RPC error.",
55+
divergence=Divergence(
56+
note=(
57+
"The spec reserves -32603 Internal error for this; the low-level Server returns code 0 "
58+
"(not a defined JSON-RPC code) and leaks str(exc) as the error message."
59+
),
60+
),
61+
),
62+
# ═══════════════════════════════════════════════════════════════════════════
63+
# Ping
64+
# ═══════════════════════════════════════════════════════════════════════════
65+
"ping:client-to-server": Requirement(
66+
source=f"{SPEC_BASE_URL}/basic/utilities/ping#behavior-requirements",
67+
behavior="A client-initiated ping receives an empty result from the server.",
68+
),
69+
"ping:server-to-client": Requirement(
70+
source=f"{SPEC_BASE_URL}/basic/utilities/ping#behavior-requirements",
71+
behavior="A server-initiated ping receives an empty result from the client.",
72+
),
73+
# ═══════════════════════════════════════════════════════════════════════════
74+
# Tools
75+
# ═══════════════════════════════════════════════════════════════════════════
76+
"tools:list:basic": Requirement(
77+
source=f"{SPEC_BASE_URL}/server/tools#listing-tools",
78+
behavior="tools/list returns the registered tools with name, description, and inputSchema.",
79+
),
80+
"tools:list:optional-fields": Requirement(
81+
source=f"{SPEC_BASE_URL}/server/tools#tool",
82+
behavior=(
83+
"Optional Tool fields supplied by the server (title, annotations, outputSchema, icons, _meta) "
84+
"are delivered to the client unchanged."
85+
),
86+
),
87+
"tools:call:content:text": Requirement(
88+
source=f"{SPEC_BASE_URL}/server/tools#text-content",
89+
behavior="tools/call delivers arguments to the tool handler and returns its text content to the caller.",
90+
),
91+
"tools:call:content:image": Requirement(
92+
source=f"{SPEC_BASE_URL}/server/tools#image-content",
93+
behavior="A tool result can carry image content: base64 data with a mimeType.",
94+
),
95+
"tools:call:content:audio": Requirement(
96+
source=f"{SPEC_BASE_URL}/server/tools#audio-content",
97+
behavior="A tool result can carry audio content: base64 data with a mimeType.",
98+
),
99+
"tools:call:content:resource-link": Requirement(
100+
source=f"{SPEC_BASE_URL}/server/tools#resource-links",
101+
behavior="A tool result can carry a resource_link content block referencing a resource by URI.",
102+
),
103+
"tools:call:content:embedded-resource": Requirement(
104+
source=f"{SPEC_BASE_URL}/server/tools#embedded-resources",
105+
behavior="A tool result can carry an embedded resource with full text or blob contents.",
106+
),
107+
"tools:call:content:multiple": Requirement(
108+
source=f"{SPEC_BASE_URL}/server/tools#calling-tools",
109+
behavior="A tool result can carry multiple content blocks of different types; order is preserved.",
110+
),
111+
"tools:call:structured-content": Requirement(
112+
source=f"{SPEC_BASE_URL}/server/tools#structured-content",
113+
behavior="A tool result can carry structuredContent alongside content; the client receives both.",
114+
),
115+
"tools:call:is-error": Requirement(
116+
source=f"{SPEC_BASE_URL}/server/tools#error-handling",
117+
behavior=(
118+
"A tool execution failure is returned as a result with isError true and the failure described "
119+
"in content, not as a JSON-RPC error."
120+
),
121+
),
122+
"tools:call:unknown-name": Requirement(
123+
source=f"{SPEC_BASE_URL}/server/tools#error-handling",
124+
behavior="tools/call for a name the server does not recognise returns a JSON-RPC error.",
125+
),
126+
# ═══════════════════════════════════════════════════════════════════════════
127+
# Resources
128+
# ═══════════════════════════════════════════════════════════════════════════
129+
"resources:list:basic": Requirement(
130+
source=f"{SPEC_BASE_URL}/server/resources#listing-resources",
131+
behavior=(
132+
"resources/list returns the registered resources with uri, name, and the optional descriptive "
133+
"fields supplied by the server."
134+
),
135+
),
136+
"resources:read:text": Requirement(
137+
source=f"{SPEC_BASE_URL}/server/resources#reading-resources",
138+
behavior="resources/read returns text contents carrying uri, mimeType, and the text.",
139+
),
140+
"resources:read:binary": Requirement(
141+
source=f"{SPEC_BASE_URL}/server/resources#reading-resources",
142+
behavior="resources/read returns binary contents base64-encoded in blob.",
143+
),
144+
"resources:read:not-found": Requirement(
145+
source=f"{SPEC_BASE_URL}/server/resources#error-handling",
146+
behavior="resources/read for an unknown URI returns a JSON-RPC error; the spec reserves -32002 for it.",
147+
),
148+
# ═══════════════════════════════════════════════════════════════════════════
149+
# Prompts
150+
# ═══════════════════════════════════════════════════════════════════════════
151+
"prompts:list:basic": Requirement(
152+
source=f"{SPEC_BASE_URL}/server/prompts#listing-prompts",
153+
behavior="prompts/list returns the registered prompts with name, description, and argument declarations.",
154+
),
155+
"prompts:get:arguments": Requirement(
156+
source=f"{SPEC_BASE_URL}/server/prompts#getting-a-prompt",
157+
behavior="prompts/get delivers the supplied arguments to the prompt handler and returns its messages.",
158+
),
159+
"prompts:get:multi-message": Requirement(
160+
source=f"{SPEC_BASE_URL}/server/prompts#getting-a-prompt",
161+
behavior="A prompt can return multiple messages mixing user and assistant roles; order is preserved.",
162+
),
163+
"prompts:get:unknown-name": Requirement(
164+
source=f"{SPEC_BASE_URL}/server/prompts#error-handling",
165+
behavior="prompts/get for an unknown prompt name returns a JSON-RPC error.",
166+
),
167+
# ═══════════════════════════════════════════════════════════════════════════
168+
# MCPServer behavioural guarantees (not spec-mandated)
169+
# ═══════════════════════════════════════════════════════════════════════════
170+
"mcpserver:tools:handler-exception": Requirement(
171+
source="sdk",
172+
behavior=(
173+
"An exception raised by a tool function (ToolError or otherwise) is caught and returned as a "
174+
"tool result with isError true and the failure text in content; it does not become a JSON-RPC error."
175+
),
176+
),
177+
"mcpserver:tools:unknown-name": Requirement(
178+
source="sdk",
179+
behavior="Calling a tool name that was never registered returns a tool result with isError true.",
180+
divergence=Divergence(
181+
note=(
182+
"The spec classifies unknown tools as a protocol error (its example uses -32602 Invalid "
183+
"params); MCPServer reports a tool execution error instead. The low-level path follows the "
184+
"spec example (see tools:call:unknown-name)."
185+
),
186+
),
187+
),
188+
}
189+
190+
191+
def requirement(requirement_id: str) -> Callable[[_TestFn], _TestFn]:
192+
"""Mark a test as exercising a requirement from :data:`REQUIREMENTS`.
193+
194+
Applies the `requirement` pytest marker and records the coverage link checked by
195+
`test_coverage.py`. Unknown IDs fail at import time so a typo surfaces as a collection
196+
error on the offending test, not as a missing-coverage report later.
197+
"""
198+
if requirement_id not in REQUIREMENTS:
199+
raise KeyError(f"Unknown requirement id {requirement_id!r}: add it to REQUIREMENTS in {__name__}")
200+
201+
def apply(test_fn: _TestFn) -> _TestFn:
202+
covered_by(requirement_id).append(f"{test_fn.__module__}.{test_fn.__qualname__}")
203+
return pytest.mark.requirement(requirement_id)(test_fn)
204+
205+
return apply
206+
207+
208+
_COVERAGE: dict[str, list[str]] = {}
209+
210+
211+
def covered_by(requirement_id: str) -> list[str]:
212+
"""Return the (mutable) list of test names recorded as exercising `requirement_id`."""
213+
return _COVERAGE.setdefault(requirement_id, [])

‎tests/interaction/lowlevel/__init__.py‎

Whitespace-only changes.

‎tests/interaction/lowlevel/test_ping.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
"""Ping interactions against the low-level Server, driven through the public Client API."""
2+
3+
import pytest
4+
from inline_snapshot import snapshot
5+
6+
from mcp import types
7+
from mcp.client.client import Client
8+
from mcp.server import Server, ServerRequestContext
9+
from mcp.types import CallToolResult, EmptyResult, TextContent
10+
from tests.interaction._requirements import requirement
11+
12+
pytestmark = pytest.mark.anyio
13+
14+
15+
@requirement("ping:client-to-server")
16+
async def test_client_ping_returns_empty_result() -> None:
17+
"""A client ping is answered with an empty result, even by a server with no handlers."""
18+
server = Server("silent")
19+
20+
async with Client(server) as client:
21+
result = await client.send_ping()
22+
23+
assert result == snapshot(EmptyResult())
24+
25+
26+
@requirement("ping:server-to-client")
27+
async def test_server_ping_returns_empty_result() -> None:
28+
"""A server-initiated ping sent while a request is in flight is answered by the client.
29+
30+
The tool returns the type of the ping response, proving the round trip completed inside
31+
the handler before the tool result was produced.
32+
"""
33+
34+
async def list_tools(
35+
ctx: ServerRequestContext, params: types.PaginatedRequestParams | None
36+
) -> types.ListToolsResult:
37+
return types.ListToolsResult(
38+
tools=[types.Tool(name="ping_back", description="Ping the client.", input_schema={"type": "object"})]
39+
)
40+
41+
async def call_tool(ctx: ServerRequestContext, params: types.CallToolRequestParams) -> CallToolResult:
42+
assert params.name == "ping_back"
43+
pong = await ctx.session.send_ping()
44+
return CallToolResult(content=[TextContent(text=type(pong).__name__)])
45+
46+
server = Server("pinger", on_list_tools=list_tools, on_call_tool=call_tool)
47+
48+
async with Client(server) as client:
49+
result = await client.call_tool("ping_back", {})
50+
51+
assert result == snapshot(CallToolResult(content=[TextContent(text="EmptyResult")]))

0 commit comments

Comments
 (0)