From d76c5f9ca4d2670cf0149551141e74f18a6df0a7 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 24 Jun 2026 17:08:38 +0000 Subject: [PATCH] =?UTF-8?q?test(conformance):=20fixture=20sweep=20?= =?UTF-8?q?=E2=80=94=20surface=20silently-skipped=20scenarios;=20public-im?= =?UTF-8?q?port=20lint;=20tasks=20extension=20baseline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/conformance.yml | 1 + package.json | 1 + test/conformance/eslint.config.mjs | 30 ++++++- test/conformance/expected-failures.yaml | 25 +++++- test/conformance/package.json | 1 + test/conformance/src/authTestServer.ts | 25 ++---- test/conformance/src/everythingClient.ts | 67 ++++++++++++--- test/conformance/src/everythingServer.ts | 83 +++++++++---------- .../src/helpers/conformanceOAuthProvider.ts | 1 - .../conformance/src/helpers/withOAuthRetry.ts | 11 ++- 10 files changed, 166 insertions(+), 79 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index dd01a74c10..719e41f697 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -49,4 +49,5 @@ jobs: - run: pnpm run build:all - run: pnpm run test:conformance:server - run: pnpm run test:conformance:server:draft + - run: pnpm run test:conformance:server:extensions - run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:2026 diff --git a/package.json b/package.json index ab5510cfa1..d247408de1 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "test:conformance:client:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:run", "test:conformance:server": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server", "test:conformance:server:draft": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft", + "test:conformance:server:extensions": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:extensions", "test:conformance:server:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:all", "test:conformance:server:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:run", "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all" diff --git a/test/conformance/eslint.config.mjs b/test/conformance/eslint.config.mjs index 951c9f3a91..9f247cf600 100644 --- a/test/conformance/eslint.config.mjs +++ b/test/conformance/eslint.config.mjs @@ -2,4 +2,32 @@ import baseConfig from '@modelcontextprotocol/eslint-config'; -export default baseConfig; +export default [ + ...baseConfig, + { + files: ['**/*.{ts,tsx,js,jsx,mts,cts}'], + rules: { + // Conformance fixtures MUST use only what a consumer would `npm install` and import: + // public package entry points. Anything reaching into core or package internals is banned. + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + group: ['@modelcontextprotocol/core', '@modelcontextprotocol/core/*'], + message: 'Conformance fixtures must import from @modelcontextprotocol/{server,client}, not core.' + }, + { + group: ['@modelcontextprotocol/*/src/*'], + message: 'Conformance fixtures must import only public package entry points.' + }, + { + group: ['@modelcontextprotocol/*/dist/*'], + message: 'Conformance fixtures must import only public package entry points.' + } + ] + } + ] + } + } +]; diff --git a/test/conformance/expected-failures.yaml b/test/conformance/expected-failures.yaml index 3fe3da4b92..6f31a0844a 100644 --- a/test/conformance/expected-failures.yaml +++ b/test/conformance/expected-failures.yaml @@ -22,6 +22,25 @@ client: [] # (empty: SEP-2468/2352/2350/837/2207/990 burned by the auth bundle; the # last referee-side gap — conformance#361 callback-iss — closed at alpha.6) -# No server entries — the spec#2907 renumber-mismatch cells burned at the -# alpha.5 referee. -server: [] +server: + # --- SEP-2663 (io.modelcontextprotocol/tasks) — server SDK does not implement the tasks extension --- + # Extension-tagged scenarios; selected only by `--suite all` (the alpha.6 referee + # has no server-side `--suite extensions`). The active/draft/2026 legs never select + # them, so they cannot flag these entries as stale. `tasks-status-notifications` is + # intentionally absent: the referee SKIPs it unconditionally (harness rewrite pending + # against the SEP-2575 subscriptions/listen channel), so a baseline entry would be + # flagged stale. + - tasks-lifecycle + - tasks-capability-negotiation + - tasks-wire-fields + - tasks-request-state-removal + - tasks-mrtr-input + - tasks-request-headers + - tasks-dispatch-and-envelope + - tasks-required-task-error + - tasks-mrtr-composition + # --- conformance#256 server-sse-polling — referee-side pending scenario --- + # SHOULD-level checks emit WARNING (priming event + retry field) which the baseline + # checker treats as failure. Selected only by `--suite all`; on hold pending + # conformance#366 (referee sends 2025-03-26 while testing a 2025-11-25 feature; SDK correctly version-gates priming). + - server-sse-polling diff --git a/test/conformance/package.json b/test/conformance/package.json index f4ba012982..4a17607e57 100644 --- a/test/conformance/package.json +++ b/test/conformance/package.json @@ -34,6 +34,7 @@ "test:conformance:client:run": "node --import tsx ./src/everythingClient.ts", "test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml", "test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml", + "test:conformance:server:extensions": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml", "test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml", "test:conformance:server:2026": "scripts/run-server-conformance.sh --suite all --spec-version 2026-07-28 --expected-failures ./expected-failures.2026-07-28.yaml", "test:conformance:server:run": "node --import tsx ./src/everythingServer.ts", diff --git a/test/conformance/src/authTestServer.ts b/test/conformance/src/authTestServer.ts index 5fbd5785d6..25fc790481 100644 --- a/test/conformance/src/authTestServer.ts +++ b/test/conformance/src/authTestServer.ts @@ -51,17 +51,10 @@ const ADMIN_SCOPE = 'admin'; // Function to create a new MCP server instance (one per session) function createMcpServer(): McpServer { - const mcpServer = new McpServer( - { - name: 'mcp-auth-test-server', - version: '1.0.0' - }, - { - capabilities: { - tools: {} - } - } - ); + const mcpServer = new McpServer({ + name: 'mcp-auth-test-server', + version: '1.0.0' + }); // Simple echo tool for testing authenticated calls mcpServer.registerTool( @@ -72,8 +65,7 @@ function createMcpServer(): McpServer { message: z.string().optional().describe('The message to echo back') }) }, - async (args: { message?: string }) => { - const message = args.message || 'No message provided'; + async ({ message = 'No message provided' }) => { return { content: [{ type: 'text', text: `Echo: ${message}` }] }; @@ -102,8 +94,7 @@ function createMcpServer(): McpServer { action: z.string().optional().describe('The admin action to perform') }) }, - async (args: { action?: string }) => { - const action = args.action || 'default-admin-action'; + async ({ action = 'default-admin-action' }) => { return { content: [{ type: 'text', text: `Admin action performed: ${action}` }] }; @@ -274,8 +265,8 @@ async function startServer() { app.use( cors({ origin: '*', - exposedHeaders: ['Mcp-Session-Id'], - allowedHeaders: ['Content-Type', 'mcp-session-id', 'last-event-id', 'Authorization'] + exposedHeaders: ['Mcp-Session-Id', 'WWW-Authenticate'], + allowedHeaders: ['Content-Type', 'mcp-session-id', 'last-event-id', 'Authorization', 'mcp-protocol-version'] }) ); diff --git a/test/conformance/src/everythingClient.ts b/test/conformance/src/everythingClient.ts index 9dc7e1a614..78a1bded3b 100644 --- a/test/conformance/src/everythingClient.ts +++ b/test/conformance/src/everythingClient.ts @@ -160,8 +160,7 @@ async function runToolsCallClient(serverUrl: string): Promise { logger.debug('Successfully listed tools'); // Call the add_numbers tool - const addTool = tools.tools.find(t => t.name === 'add_numbers'); - if (addTool) { + if (tools.tools.some(t => t.name === 'add_numbers')) { const result = await client.callTool({ name: 'add_numbers', arguments: { a: 5, b: 3 } @@ -188,8 +187,7 @@ async function runToolsCallModernClient(serverUrl: string): Promise { logger.debug('Successfully listed tools'); // Call the add_numbers tool - const addTool = tools.tools.find(t => t.name === 'add_numbers'); - if (addTool) { + if (tools.tools.some(t => t.name === 'add_numbers')) { const result = await client.callTool({ name: 'add_numbers', arguments: { a: 5, b: 3 } @@ -226,6 +224,51 @@ registerScenario('initialize', runBasicClient); registerScenario('tools_call', runToolsCallClient); registerScenario('request-metadata', runRequestMetadataClient); +// ============================================================================ +// SEP-2243 standard-header client scenario (Mcp-Method / Mcp-Name) +// ============================================================================ + +// http-standard-headers: the referee mock answers initialize, tools/list, +// tools/call, resources/list, resources/read, prompts/list, prompts/get and +// asserts that each POST carried the correct Mcp-Method header (and Mcp-Name +// for the call/read/get verbs). The SDK emits both headers on the modern +// streamableHttp path, so the fixture just needs to drive each method once. +// The mock has no server/discover handler and its 2025-shaped initialize +// response doesn't satisfy the v2 client — same connect-time gap as the other +// SEP-2243 mocks — so connect via the withLocalDiscoverResponse shim. The +// initialize / notifications/initialized checks are intentionally left +// SKIPPED; the legacy initialize path's missing Mcp-Method is tracked as a +// baseline bug. The mock advertises its own surface (test_headers / +// file:///path/to/file%20name.txt / test_prompt) — the fixture lists first +// and uses whatever the mock returned so it stays referee-version-agnostic. +async function runHttpStandardHeadersClient(serverUrl: string): Promise { + const client = await connectModernHeaderClient(serverUrl); + logger.debug('Successfully connected to MCP server'); + + const { tools } = await client.listTools(); + const tool = tools[0]; + if (tool) { + await client.callTool({ name: tool.name, arguments: {} }); + } + + const { resources } = await client.listResources(); + const resource = resources[0]; + if (resource) { + await client.readResource({ uri: resource.uri }); + } + + const { prompts } = await client.listPrompts(); + const prompt = prompts[0]; + if (prompt) { + await client.getPrompt({ name: prompt.name, arguments: {} }); + } + + await client.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('http-standard-headers', runHttpStandardHeadersClient); + // ============================================================================ // SEP-2243 custom-header client scenarios (protocol revision 2026-07-28) // ============================================================================ @@ -319,7 +362,10 @@ function withLocalDiscoverResponse(serverInfo: { name: string; version: string } id: message.id, result: { supportedVersions: ['2026-07-28'], - capabilities: { tools: { listChanged: true } }, + // Advertise the full read surface so capability-gated + // list/read/get calls reach the real mock; callers that + // only use tools are unaffected by the extra entries. + capabilities: { tools: { listChanged: true }, resources: {}, prompts: {} }, serverInfo } }, @@ -425,6 +471,10 @@ registerScenarios( 'auth/metadata-var3', 'auth/2025-03-26-oauth-metadata-backcompat', 'auth/2025-03-26-oauth-endpoint-fallback', + // RFC 8707 resource-indicator binding: the referee serves a PRM whose + // `resource` does not match the MCP server URL; the SDK's discovery path + // must reject before token exchange (the referee sets `allowClientError`). + 'auth/resource-mismatch', 'auth/scope-from-www-authenticate', 'auth/scope-from-scopes-supported', 'auth/scope-omitted-when-undefined', @@ -789,9 +839,4 @@ async function main(): Promise { } } -try { - await main(); -} catch (error) { - logger.error('Error:', error); - process.exit(1); -} +await main(); diff --git a/test/conformance/src/everythingServer.ts b/test/conformance/src/everythingServer.ts index 7e11a83e67..0df9f18a0a 100644 --- a/test/conformance/src/everythingServer.ts +++ b/test/conformance/src/everythingServer.ts @@ -55,7 +55,9 @@ const eventStoreData = new Map { - const eventId = `${streamId}::${Date.now()}_${randomUUID()}`; + // Fixed-width timestamp so the lexicographic sort in + // replayEventsAfter is robustly chronological. + const eventId = `${streamId}::${String(Date.now()).padStart(15, '0')}_${randomUUID()}`; eventStoreData.set(eventId, { eventId, message, streamId }); return eventId; }, @@ -125,6 +127,11 @@ function createMcpServer() { prompts: { listChanged: true }, + // `logging` is deprecated as of protocol version 2026-07-28 + // (SEP-2577). Intentionally retained so the 2025-era + // logging/setLevel conformance leg still negotiates the + // capability; the 2026-07-28 path uses the per-request + // envelope and ignores this field. logging: {}, completions: {} }, @@ -140,7 +147,7 @@ function createMcpServer() { function sendLog( level: 'debug' | 'info' | 'notice' | 'warning' | 'error' | 'critical' | 'alert' | 'emergency', message: string, - _data?: unknown + data?: unknown ) { mcpServer.server .notification({ @@ -148,7 +155,7 @@ function createMcpServer() { params: { level, logger: 'conformance-test-server', - data: _data || message + data: data ?? message } }) .catch(() => { @@ -311,42 +318,28 @@ function createMcpServer() { inputSchema: z.object({}) }, async (_args, ctx): Promise => { - const progressToken = ctx.mcpReq._meta?.progressToken ?? 0; - console.log('Progress token:', progressToken); - await ctx.mcpReq.notify({ - method: 'notifications/progress', - params: { - progressToken, - progress: 0, - total: 100, - message: `Completed step ${0} of ${100}` - } - }); - await new Promise(resolve => setTimeout(resolve, 50)); - - await ctx.mcpReq.notify({ - method: 'notifications/progress', - params: { - progressToken, - progress: 50, - total: 100, - message: `Completed step ${50} of ${100}` - } - }); - await new Promise(resolve => setTimeout(resolve, 50)); - - await ctx.mcpReq.notify({ - method: 'notifications/progress', - params: { - progressToken, - progress: 100, - total: 100, - message: `Completed step ${100} of ${100}` + const progressToken = ctx.mcpReq._meta?.progressToken; + // Per spec, servers MUST NOT emit notifications/progress without a + // client-supplied token — only report progress when one was sent. + if (progressToken !== undefined) { + for (const progress of [0, 50, 100]) { + await ctx.mcpReq.notify({ + method: 'notifications/progress', + params: { + progressToken, + progress, + total: 100, + message: `Completed step ${progress} of ${100}` + } + }); + if (progress < 100) { + await new Promise(resolve => setTimeout(resolve, 50)); + } } - }); + } return { - content: [{ type: 'text', text: String(progressToken) }] + content: [{ type: 'text', text: String(progressToken ?? 'no-progress-token') }] }; } ); @@ -408,7 +401,7 @@ function createMcpServer() { prompt: z.string().describe('The prompt to send to the LLM') }) }, - async (args: { prompt: string }, ctx): Promise => { + async (args, ctx): Promise => { try { // Request sampling from client const result = (await ctx.mcpReq.send({ @@ -459,7 +452,7 @@ function createMcpServer() { message: z.string().describe('The message to show the user') }) }, - async (args: { message: string }, ctx): Promise => { + async (args, ctx): Promise => { try { // Request user input from client const result = await ctx.mcpReq.send({ @@ -967,6 +960,12 @@ function createMcpServer() { if (ctx.mcpReq.inputResponses !== undefined) { return { content: [{ type: 'text', text: 'Capability-aware input requests fulfilled' }] }; } + // `sampling` and `roots` on ClientCapabilities are @deprecated as + // of protocol version 2026-07-28 (SEP-2577). This fixture reads + // them intentionally: the conformance scenario asserts that the + // server only emits input-request kinds the client declared, and + // the per-request envelope carries the declared capabilities in + // the (deprecated) wire vocabulary. const declared = ctx.mcpReq.envelope?.[CLIENT_CAPABILITIES_META_KEY]; const inputRequests: InputRequests = {}; if (declared?.elicitation !== undefined) { @@ -1111,7 +1110,7 @@ function createMcpServer() { 'test://watched-resource', { title: 'Watched Resource', - description: 'A resource that auto-updates every 3 seconds', + description: 'Static resource registered for subscribe/unsubscribe testing', mimeType: 'text/plain' }, async (): Promise => { @@ -1177,7 +1176,7 @@ function createMcpServer() { arg2: z.string().describe('Second test argument') }) }, - async (args: { arg1: string; arg2: string }): Promise => { + async (args): Promise => { return { messages: [ { @@ -1202,7 +1201,7 @@ function createMcpServer() { resourceUri: z.string().describe('URI of the resource to embed') }) }, - async (args: { resourceUri: string }): Promise => { + async (args): Promise => { return { messages: [ { @@ -1352,7 +1351,7 @@ app.use( cors({ origin: '*', exposedHeaders: ['Mcp-Session-Id'], - allowedHeaders: ['Content-Type', 'mcp-session-id', 'last-event-id'] + allowedHeaders: ['Content-Type', 'mcp-session-id', 'last-event-id', 'mcp-protocol-version', 'mcp-method'] }) ); diff --git a/test/conformance/src/helpers/conformanceOAuthProvider.ts b/test/conformance/src/helpers/conformanceOAuthProvider.ts index 457ed58a58..b35e370407 100644 --- a/test/conformance/src/helpers/conformanceOAuthProvider.ts +++ b/test/conformance/src/helpers/conformanceOAuthProvider.ts @@ -16,7 +16,6 @@ export class ConformanceOAuthProvider implements OAuthClientProvider { private _codeVerifier?: string; private _authCode?: string; private _iss?: string; - private _authCodePromise?: Promise; private _discoveryState?: OAuthDiscoveryState; constructor( diff --git a/test/conformance/src/helpers/withOAuthRetry.ts b/test/conformance/src/helpers/withOAuthRetry.ts index 1b26883769..a156dbdff9 100644 --- a/test/conformance/src/helpers/withOAuthRetry.ts +++ b/test/conformance/src/helpers/withOAuthRetry.ts @@ -70,8 +70,11 @@ export const handle401 = async ( * - Does not throw UnauthorizedError on redirect, but instead retries * - Calls next() instead of throwing for redirect-based auth * - * @param provider - OAuth client provider for authentication - * @param baseUrl - Base URL for OAuth server discovery (defaults to request URL domain) + * @param clientName - `client_name` for the auto-created ConformanceOAuthProvider (ignored when `existingProvider` is supplied) + * @param baseUrl - Base URL for OAuth server discovery (defaults to request URL origin) + * @param handle401Fn - Challenge handler invoked on 401/403 (defaults to {@link handle401}) + * @param clientMetadataUrl - CIMD URL for the auto-created provider (ignored when `existingProvider` is supplied) + * @param existingProvider - Pre-populated provider; when set, `clientName`/`clientMetadataUrl` are unused * @returns A fetch middleware function */ export const withOAuthRetry = ( @@ -107,7 +110,7 @@ export const withOAuthRetry = ( let response = await makeRequest(); - // Handle 401 responses by attempting re-authentication + // Handle 401/403 responses by attempting re-authentication if (response.status === 401 || response.status === 403) { const serverUrl = baseUrl || (typeof input === 'string' ? new URL(input).origin : input.origin); await handle401Fn(response, provider, next, serverUrl); @@ -115,7 +118,7 @@ export const withOAuthRetry = ( response = await makeRequest(); } - // If we still have a 401 after re-auth attempt, throw an error + // If we still have a 401/403 after re-auth attempt, throw an error if (response.status === 401 || response.status === 403) { const url = typeof input === 'string' ? input : input.toString(); throw new UnauthorizedError(`Authentication failed for ${url}`);