Skip to content

Commit 52f200b

Browse files
committed
Conformance client fixture: drive Client(mode='auto') for the modern leg
- client.py reads MCP_CONFORMANCE_PROTOCOL_VERSION and passes mode='auto' (modern) or 'legacy' (handshake-era) to the high-level Client; auth flows wrap the OAuth-authed httpx client in streamable_http_client and hand that as a Transport - New fixture handlers for request-metadata and http-standard-headers - json-schema-ref-no-deref pinned to legacy (its mock only speaks the handshake-era lifecycle; the check is lifecycle-agnostic) - Baselines: request-metadata + auth/authorization-server-migration removed from expected-failures.yml; tools_call + auth/scope-step-up + auth/scope-retry-limit + the two above removed from expected-failures.2026-07-28.yml. http-custom-headers / http-invalid-tool-headers (Mcp-Param-* headers) and sep-2322-client-request-state (multi-round-trip) stay waived.
1 parent a97459d commit 52f200b

3 files changed

Lines changed: 134 additions & 106 deletions

File tree

.github/actions/conformance/client.py

Lines changed: 132 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
initialize - Connect, initialize, list tools, close
1818
tools_call - Connect, call add_numbers(a=5, b=3), close
1919
sse-retry - Connect, call test_reconnection, close
20+
json-schema-ref-no-deref - Connect, list tools (no $ref deref)
21+
request-metadata - Connect with all callbacks; client stamps _meta
22+
http-standard-headers - Connect, call a tool (Mcp-* headers checked)
2023
elicitation-sep1034-client-defaults - Elicitation with default accept callback
2124
auth/client-credentials-jwt - Client credentials with private_key_jwt
2225
auth/client-credentials-basic - Client credentials with client_secret_basic
@@ -35,16 +38,18 @@
3538
import httpx
3639
from pydantic import AnyUrl
3740

38-
from mcp import ClientSession, types
41+
from mcp import types
3942
from mcp.client.auth import OAuthClientProvider, TokenStorage
4043
from mcp.client.auth.extensions.client_credentials import (
4144
ClientCredentialsOAuthProvider,
4245
PrivateKeyJWTOAuthProvider,
4346
SignedJWTParameters,
4447
)
48+
from mcp.client.client import Client
4549
from mcp.client.context import ClientRequestContext
4650
from mcp.client.streamable_http import streamable_http_client
4751
from mcp.shared.auth import AuthorizationCodeResult, OAuthClientInformationFull, OAuthClientMetadata, OAuthToken
52+
from mcp.shared.version import MODERN_PROTOCOL_VERSIONS
4853

4954
# Set up logging to stderr (stdout is for conformance test output)
5055
logging.basicConfig(
@@ -58,10 +63,24 @@
5863
#: "2026-07-28"). The harness always sets this (when --spec-version is omitted
5964
#: it picks per-scenario: LATEST_SPEC_VERSION for active scenarios,
6065
#: DRAFT_PROTOCOL_VERSION for draft-only ones), so None means we were invoked
61-
#: outside the harness. Handlers that need to take the stateless 2026 path will
62-
#: branch on this once the SDK has one; today it is logged only.
66+
#: outside the harness.
6367
PROTOCOL_VERSION: str | None = os.environ.get("MCP_CONFORMANCE_PROTOCOL_VERSION")
6468

69+
70+
def client_mode() -> str:
71+
"""Pick the Client(mode=) for the harness leg.
72+
73+
On a modern leg (2026-07-28+) -> 'auto' so Client.discover() runs and the
74+
_meta envelope + MCP-Protocol-Version header are stamped on every request.
75+
On a handshake-era leg -> 'legacy' so the initialize handshake runs exactly
76+
as before (no server/discover probe is sent against a mock that would 400 it).
77+
Outside the harness -> 'auto' (probe + fallback).
78+
"""
79+
if PROTOCOL_VERSION is None or PROTOCOL_VERSION in MODERN_PROTOCOL_VERSIONS:
80+
return "auto"
81+
return "legacy"
82+
83+
6584
# Type for async scenario handler functions
6685
ScenarioHandler = Callable[[str], Coroutine[Any, None, None]]
6786

@@ -165,52 +184,22 @@ async def handle_callback(self) -> AuthorizationCodeResult:
165184
return result
166185

167186

168-
# --- Scenario Handlers ---
169-
170-
171-
@register("initialize")
172-
async def run_initialize(server_url: str) -> None:
173-
"""Connect, initialize, list tools, close."""
174-
async with streamable_http_client(url=server_url) as (read_stream, write_stream):
175-
async with ClientSession(read_stream, write_stream) as session:
176-
await session.initialize()
177-
logger.debug("Initialized successfully")
178-
await session.list_tools()
179-
logger.debug("Listed tools successfully")
180-
181-
182-
@register("json-schema-ref-no-deref")
183-
async def run_json_schema_ref_no_deref(server_url: str) -> None:
184-
"""Initialize and list tools; the scenario fails only if the client fetches a network $ref.
185-
186-
ClientSession never walks inputSchema or resolves $refs, so listing is enough (SEP-2106).
187-
"""
188-
async with streamable_http_client(url=server_url) as (read_stream, write_stream):
189-
async with ClientSession(read_stream, write_stream) as session:
190-
await session.initialize()
191-
await session.list_tools()
187+
# --- Stub callbacks (declare capabilities in _meta without doing real work) ---
192188

193189

194-
@register("tools_call")
195-
async def run_tools_call(server_url: str) -> None:
196-
"""Connect, initialize, list tools, call add_numbers(a=5, b=3), close."""
197-
async with streamable_http_client(url=server_url) as (read_stream, write_stream):
198-
async with ClientSession(read_stream, write_stream) as session:
199-
await session.initialize()
200-
await session.list_tools()
201-
result = await session.call_tool("add_numbers", {"a": 5, "b": 3})
202-
logger.debug(f"add_numbers result: {result}")
190+
async def stub_sampling_callback(
191+
context: ClientRequestContext,
192+
params: types.CreateMessageRequestParams,
193+
) -> types.CreateMessageResult | types.ErrorData:
194+
return types.CreateMessageResult(
195+
role="assistant",
196+
content=types.TextContent(type="text", text=""),
197+
model="conformance-stub",
198+
)
203199

204200

205-
@register("sse-retry")
206-
async def run_sse_retry(server_url: str) -> None:
207-
"""Connect, initialize, list tools, call test_reconnection, close."""
208-
async with streamable_http_client(url=server_url) as (read_stream, write_stream):
209-
async with ClientSession(read_stream, write_stream) as session:
210-
await session.initialize()
211-
await session.list_tools()
212-
result = await session.call_tool("test_reconnection", {})
213-
logger.debug(f"test_reconnection result: {result}")
201+
async def stub_list_roots_callback(context: ClientRequestContext) -> types.ListRootsResult | types.ErrorData:
202+
return types.ListRootsResult(roots=[])
214203

215204

216205
async def default_elicitation_callback(
@@ -233,17 +222,87 @@ async def default_elicitation_callback(
233222
return types.ElicitResult(action="accept", content=content)
234223

235224

225+
# --- Scenario Handlers ---
226+
227+
228+
@register("initialize")
229+
async def run_initialize(server_url: str) -> None:
230+
"""Connect, initialize, list tools, close."""
231+
async with Client(server_url, mode=client_mode()) as client:
232+
logger.debug("Initialized successfully")
233+
await client.list_tools()
234+
logger.debug("Listed tools successfully")
235+
236+
237+
@register("json-schema-ref-no-deref")
238+
async def run_json_schema_ref_no_deref(server_url: str) -> None:
239+
"""Initialize and list tools; the scenario fails only if the client fetches a network $ref.
240+
241+
The client never walks inputSchema or resolves $refs, so listing is enough (SEP-2106).
242+
Pinned to mode='legacy': the harness reports PROTOCOL_VERSION=2026-07-28 for this
243+
scenario but its mock server only speaks the handshake-era lifecycle and 400s a
244+
modern-stamped tools/list. The check is lifecycle-agnostic so this is harmless.
245+
"""
246+
async with Client(server_url, mode="legacy") as client:
247+
await client.list_tools()
248+
249+
250+
@register("tools_call")
251+
async def run_tools_call(server_url: str) -> None:
252+
"""Connect, list tools, call add_numbers(a=5, b=3), close."""
253+
async with Client(server_url, mode=client_mode()) as client:
254+
await client.list_tools()
255+
result = await client.call_tool("add_numbers", {"a": 5, "b": 3})
256+
logger.debug(f"add_numbers result: {result}")
257+
258+
259+
@register("sse-retry")
260+
async def run_sse_retry(server_url: str) -> None:
261+
"""Connect, list tools, call test_reconnection, close."""
262+
async with Client(server_url, mode=client_mode()) as client:
263+
await client.list_tools()
264+
result = await client.call_tool("test_reconnection", {})
265+
logger.debug(f"test_reconnection result: {result}")
266+
267+
268+
@register("request-metadata")
269+
async def run_request_metadata(server_url: str) -> None:
270+
"""Connect on the modern path with every client capability declared.
271+
272+
The scenario inspects every request's `_meta` envelope (SEP-2575) for
273+
protocolVersion / clientInfo / clientCapabilities, and the matching
274+
MCP-Protocol-Version header. mode='auto' makes the SDK send
275+
server/discover (covering the unsupported-version retry check), then adopt
276+
and stamp the envelope on the follow-up requests.
277+
"""
278+
async with Client(
279+
server_url,
280+
mode=client_mode(),
281+
sampling_callback=stub_sampling_callback,
282+
list_roots_callback=stub_list_roots_callback,
283+
elicitation_callback=default_elicitation_callback,
284+
) as client:
285+
await client.list_tools()
286+
result = await client.call_tool("add_numbers", {"a": 5, "b": 3})
287+
logger.debug(f"add_numbers result: {result}")
288+
289+
290+
@register("http-standard-headers")
291+
async def run_http_standard_headers(server_url: str) -> None:
292+
"""Connect on the modern path so Mcp-Method / Mcp-Name / MCP-Protocol-Version are sent (SEP-2243)."""
293+
async with Client(server_url, mode=client_mode()) as client:
294+
await client.list_tools()
295+
result = await client.call_tool("add_numbers", {"a": 5, "b": 3})
296+
logger.debug(f"add_numbers result: {result}")
297+
298+
236299
@register("elicitation-sep1034-client-defaults")
237300
async def run_elicitation_defaults(server_url: str) -> None:
238301
"""Connect with elicitation callback that applies schema defaults."""
239-
async with streamable_http_client(url=server_url) as (read_stream, write_stream):
240-
async with ClientSession(
241-
read_stream, write_stream, elicitation_callback=default_elicitation_callback
242-
) as session:
243-
await session.initialize()
244-
await session.list_tools()
245-
result = await session.call_tool("test_client_elicitation_defaults", {})
246-
logger.debug(f"test_client_elicitation_defaults result: {result}")
302+
async with Client(server_url, mode=client_mode(), elicitation_callback=default_elicitation_callback) as client:
303+
await client.list_tools()
304+
result = await client.call_tool("test_client_elicitation_defaults", {})
305+
logger.debug(f"test_client_elicitation_defaults result: {result}")
247306

248307

249308
@register("auth/client-credentials-jwt")
@@ -343,25 +402,22 @@ async def run_auth_code_client(server_url: str) -> None:
343402

344403
async def _run_auth_session(server_url: str, oauth_auth: OAuthClientProvider) -> None:
345404
"""Common session logic for all OAuth flows."""
346-
client = httpx.AsyncClient(auth=oauth_auth, timeout=30.0)
347-
async with streamable_http_client(url=server_url, http_client=client) as (read_stream, write_stream):
348-
async with ClientSession(
349-
read_stream, write_stream, elicitation_callback=default_elicitation_callback
350-
) as session:
351-
await session.initialize()
352-
logger.debug("Initialized successfully")
353-
354-
tools_result = await session.list_tools()
355-
logger.debug(f"Listed tools: {[t.name for t in tools_result.tools]}")
356-
357-
# Call the first available tool (different tests have different tools)
358-
if tools_result.tools:
359-
tool_name = tools_result.tools[0].name
360-
try:
361-
result = await session.call_tool(tool_name, {})
362-
logger.debug(f"Called {tool_name}, result: {result}")
363-
except Exception as e:
364-
logger.debug(f"Tool call result/error: {e}")
405+
http_client = httpx.AsyncClient(auth=oauth_auth, timeout=30.0)
406+
transport = streamable_http_client(url=server_url, http_client=http_client)
407+
async with Client(transport, mode=client_mode(), elicitation_callback=default_elicitation_callback) as client:
408+
logger.debug("Initialized successfully")
409+
410+
tools_result = await client.list_tools()
411+
logger.debug(f"Listed tools: {[t.name for t in tools_result.tools]}")
412+
413+
# Call the first available tool (different tests have different tools)
414+
if tools_result.tools:
415+
tool_name = tools_result.tools[0].name
416+
try:
417+
result = await client.call_tool(tool_name, {})
418+
logger.debug(f"Called {tool_name}, result: {result}")
419+
except Exception as e:
420+
logger.debug(f"Tool call result/error: {e}")
365421

366422
logger.debug("Connection closed successfully")
367423

@@ -374,7 +430,7 @@ def main() -> None:
374430

375431
server_url = sys.argv[1]
376432
scenario = os.environ.get("MCP_CONFORMANCE_SCENARIO")
377-
logger.debug(f"Conformance protocol version: {PROTOCOL_VERSION!r}")
433+
logger.debug(f"Conformance protocol version: {PROTOCOL_VERSION!r} -> mode={client_mode()!r}")
378434

379435
if scenario:
380436
logger.debug(f"Running explicit scenario '{scenario}' against {server_url}")
@@ -384,6 +440,9 @@ def main() -> None:
384440
elif scenario.startswith("auth/"):
385441
asyncio.run(run_auth_code_client(server_url))
386442
else:
443+
# Unhandled scenarios:
444+
# - sep-2322-client-request-state (SEP-2322 / S6: MRTR client loop)
445+
# - http-custom-headers, http-invalid-tool-headers (SEP-2243 / S8: Mcp-Param-* headers)
387446
print(f"Unknown scenario: {scenario}", file=sys.stderr)
388447
sys.exit(1)
389448
else:

.github/actions/conformance/expected-failures.2026-07-28.yml

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -21,36 +21,13 @@
2121
# milestone.
2222

2323
client:
24-
# --- No stateless client path on main yet ---
25-
# client.py drives the 2025 stateful lifecycle (initialize handshake +
26-
# session). The 2026-mode mock server is stateless, so the call sequence
27-
# never reaches the assertion. Unblocks when client.py's is_modern_protocol()
28-
# branch takes the per-request _meta path.
29-
- tools_call
30-
31-
# --- Auth scenarios cut short by the 2026 connection lifecycle ---
32-
# The auth fixture flow drives the 2025 stateful lifecycle; the 2026-mode
33-
# mock rejects the MCP POST before the scope-escalation behaviour these
34-
# scenarios measure, so no authorization requests are observed. Unblocks
35-
# when client.py's auth flow speaks the 2026 per-request lifecycle.
36-
- auth/scope-step-up
37-
- auth/scope-retry-limit
38-
3924
# --- Same gaps as the 2025 baseline (fail identically when forced to 2026-07-28) ---
40-
# SEP-2575 (request metadata / _meta envelope): client does not populate the
41-
# _meta envelope or the MCP-Protocol-Version header semantics yet.
42-
- request-metadata
4325
# SEP-2322 (multi-round-trip requests): client does not echo requestState /
4426
# handle IncompleteResult yet.
4527
- sep-2322-client-request-state
46-
# SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
28+
# SEP-2243 (HTTP standardization): no fixture handler / client Mcp-Param-* support yet.
4729
- http-custom-headers
4830
- http-invalid-tool-headers
49-
# SEP-2352 (authorization server migration): the client re-registers and does not reuse the old
50-
# AS credentials, but the 2026-mode mock rejects the MCP POST before the migration 401 fires
51-
# (client.py drives the 2025 stateful lifecycle), so the re-register check is never reached.
52-
# Unblocks with the 2026 stateless client lifecycle.
53-
- auth/authorization-server-migration
5431
# auth/enterprise-managed-authorization (SEP-990) is in the 2025 baseline but
5532
# NOT here: the harness skips it as inapplicable at --spec-version 2026-07-28
5633
# (it is an extension scenario not carried into the 2026 wire), so it is

.github/actions/conformance/expected-failures.yml

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,20 +12,12 @@
1212

1313
client:
1414
# --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) ---
15-
# SEP-2575 (request metadata / _meta envelope): client does not populate the
16-
# _meta envelope or the MCP-Protocol-Version header semantics yet.
17-
- request-metadata
1815
# SEP-2322 (multi-round-trip requests): client does not echo requestState /
1916
# handle IncompleteResult yet.
2017
- sep-2322-client-request-state
21-
# SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
18+
# SEP-2243 (HTTP standardization): no fixture handler / client Mcp-Param-* support yet.
2219
- http-custom-headers
2320
- http-invalid-tool-headers
24-
# SEP-2352 (authorization server migration): the client re-registers and does not reuse the old
25-
# AS credentials, but this 2026-introduced scenario runs at 2026-07-28, where client.py's 2025
26-
# stateful lifecycle is rejected (400 on initialize) before the migration 401 fires, so the
27-
# re-register check is never reached. Unblocks with the 2026 stateless client lifecycle.
28-
- auth/authorization-server-migration
2921

3022
# --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 ---
3123
# SEP-990 (enterprise-managed authorization extension): no fixture handler /

0 commit comments

Comments
 (0)