From 734b35612c072738c306286a9127551f076b3b56 Mon Sep 17 00:00:00 2001 From: JF Date: Mon, 9 Nov 2026 21:45:12 -0500 Subject: [PATCH 1/6] feat(http): serve both protocol eras on /mcp and add the 2026-07-28 conformance lane (#1025) The http endpoint now speaks MCP protocol revision 2026-07-28 beside 2025-11-25 and earlier. handleMcpRequest reads the session-id header as before, then asks the SDK's isLegacyRequest which era the request speaks: a request carrying the 2026 per-request envelope (or a modern MCP-Protocol-Version header) goes to createMcpHandler(..., { legacy: 'reject' }) behind toNodeHandler, one deliver: 'external' facade per request over the shared core, sessionless; everything 2025-shaped lands on the sessionful app verbatim, 400/404 bodies included. NotificationHub gains handlerSink(modern.notify) so the coalesced resources/updated pings and resources/list_changed reach open subscriptions/listen streams; a listen stream counts the process as attended for the idle policy, like a legacy SSE stream. /health reports listenStreams and the requests served per era. CORS allows the 2026 headers and echoes a preflight's requested headers. Shutdown closes the modern handler, then the legacy transports, then the core. The same 10 MiB body limit reaches every SDK entry the request passes through. resources/templates/list is answered (an empty list) in both eras: the new lane found that a server advertising the resources capability must answer it, and we answered "Method not found". The 2026-07-28 conformance lane: `scripts/mcp-conformance.mjs --era 2026-07-28` runs the suite's 0.2 line (0.2.0-alpha.12, the one the SDKs run) with --requirements 2026-07-28 against a local http server and judges it under scripts/conformance/expected-failures-2026-07-28.yaml, the scenarios and checks that drive the suite's own reference fixtures. The verdict is read from the CLI's per-scenario summary lines and the checks.json it writes per scenario (server--/), never its exit code; an unlisted failure fails the lane and so does a listed entry that passes. CI job mcp-conformance-2026, required in test-summary. Measured: 37 scored scenarios conform under the baseline; caching 7/7 (6/7 before the templates fix); the 0.1.16 lane unchanged at 7/7. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 36 ++- CLAUDE.md | 2 +- README.md | 8 +- changelog.d/1025-conformance-2026.added.md | 1 + changelog.d/1025-protocol-2026-http.added.md | 1 + docs/architecture/system-overview.md | 2 +- docs/architecture/testing-architecture.md | 6 +- docs/getting-started.md | 13 +- .../expected-failures-2026-07-28.yaml | 39 +++ scripts/mcp-conformance.d.mts | 32 ++ scripts/mcp-conformance.mjs | 248 ++++++++++++-- src/cli/http-command.ts | 99 +++++- src/server/notification-hub.ts | 19 ++ src/server/output-resources.ts | 5 + src/server/protocol-facade.ts | 2 +- .../core/unit/server/notification-hub.test.ts | 15 + .../core/unit/server/protocol-facade.test.ts | 5 +- .../unit/server/server-initialization.test.ts | 5 +- .../core/unit/server/server-resources.test.ts | 10 + tests/core/unit/server/server-test-helpers.ts | 4 +- tests/e2e/mcp-server-smoke-http.test.ts | 31 ++ .../cli/http-era-negotiation.test.ts | 146 +++++++++ .../cli/http-host-allowlist.test.ts | 1 + .../cli/http-spec-contract-2026.test.ts | 302 ++++++++++++++++++ tests/unit/cli/http-command.test.ts | 188 ++++++++++- tests/unit/scripts/mcp-conformance.test.ts | 133 +++++++- 26 files changed, 1299 insertions(+), 54 deletions(-) create mode 100644 changelog.d/1025-conformance-2026.added.md create mode 100644 changelog.d/1025-protocol-2026-http.added.md create mode 100644 scripts/conformance/expected-failures-2026-07-28.yaml create mode 100644 tests/integration/cli/http-era-negotiation.test.ts create mode 100644 tests/integration/cli/http-spec-contract-2026.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0280d433c..11065cc40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -645,9 +645,39 @@ jobs: - name: Run the applicable conformance scenarios run: node scripts/mcp-conformance.mjs --verbose + mcp-conformance-2026: + # The same suite's 0.2 line (the one the SDKs run) against the 2026-07-28 revision this + # server now speaks beside 2025-11-25 (issue #1025, step 2): `--requirements 2026-07-28`, + # everything the revision requires of a server, judged under + # scripts/conformance/expected-failures-2026-07-28.yaml β€” the scenarios that drive the + # suite's own reference fixtures (tools, resources, prompts this server does not have). + # An unlisted failure fails the job, and so does a listed entry that passes, so the + # baseline can only shrink. Linux only, for the same libuv reason as the lane above. + name: MCP Conformance (2026-07-28) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22.x' + cache: 'pnpm' + - name: Install dependencies + run: pnpm install --frozen-lockfile + env: + SKIP_ADAPTER_VENDOR: 'true' + - name: Build project + run: pnpm run build:ci + - name: Run the 2026-07-28 requirements under the expected-failures baseline + run: node scripts/mcp-conformance.mjs --era 2026-07-28 --verbose + test-summary: name: Test Summary - needs: [build-and-test, windows-python-integration, lint, container-tests, cobol-host, dart-host, flutter-host, go-current, mcp-conformance] + needs: [build-and-test, windows-python-integration, lint, container-tests, cobol-host, dart-host, flutter-host, go-current, mcp-conformance, mcp-conformance-2026] runs-on: ubuntu-latest if: always() @@ -663,6 +693,7 @@ jobs: echo "flutter-host: ${{ needs.flutter-host.result }}" echo "go-current: ${{ needs.go-current.result }}" echo "mcp-conformance: ${{ needs.mcp-conformance.result }}" + echo "mcp-conformance-2026: ${{ needs.mcp-conformance-2026.result }}" if [ "${{ needs.build-and-test.result }}" != "success" ] \ || [ "${{ needs.windows-python-integration.result }}" != "success" ] \ || [ "${{ needs.lint.result }}" != "success" ] \ @@ -671,7 +702,8 @@ jobs: || [ "${{ needs.dart-host.result }}" != "success" ] \ || [ "${{ needs.flutter-host.result }}" != "success" ] \ || [ "${{ needs.go-current.result }}" != "success" ] \ - || [ "${{ needs.mcp-conformance.result }}" != "success" ]; then + || [ "${{ needs.mcp-conformance.result }}" != "success" ] \ + || [ "${{ needs.mcp-conformance-2026.result }}" != "success" ]; then echo "❌ One or more required jobs did not succeed" exit 1 fi diff --git a/CLAUDE.md b/CLAUDE.md index 6d510b29d..2525d1dcb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -222,7 +222,7 @@ The codebase follows a **layered architecture with dependency injection** and ** - Entry point for MCP protocol communication - `src/server.ts` is the composition root and the shared core (`DebugMcpServer`: dependencies, lifecycle, session validation, breakpoint gating, the public facade methods, the `NotificationHub`); it implements `ToolContext` and passes itself to the handlers. It owns no SDK `Server`: `src/server/protocol-facade.ts` (`createProtocolServer(core, { era, deliver })`) builds one cheap SDK `Server` per connection or request over the core, and `src/server/notification-hub.ts` fans the coalesced `resources/updated` pings and `resources/list_changed` out to every facade's sink (issue #1025) - `src/server/` holds the tool schemas (`tool-schemas.ts`), argument coercion (`tool-arguments.ts`), the tools/call dispatch wrapper (`tool-dispatch.ts` over `handlers/index.ts`), argument validation (`tool-validation.ts`), result envelopes and the session-error catch helpers (`tool-result.ts`), one handler module per tool family (`handlers/*.ts`) over their shared helpers (`handlers/shared.ts`), and the resource (`output-resources.ts`) and prompt (`prompts.ts`) handlers - - Supports STDIO and Streamable HTTP transport modes. `stdio` is served through the SDK's `serveStdio(factory, { transport })` and speaks both protocol eras (2026-07-28 for a client that probes with `server/discover`, 2025-11-25 and earlier for one that opens with `initialize`); `http` speaks the 2025 era (the 2026 era follows in #1025 step 2, PR 4). One core per process in both: on `http` every MCP session is a facade over the same debug sessions (the id is the capability; one process per agent is the isolation), a debug session outlives the MCP session that created it, and an abandoned session tree is closed by the per-debug-session idle policy (`MCP_DEBUG_SESSION_IDLE_MS`, 30 min on `http`, off on `stdio`, `0` disables; sweep `MCP_DEBUG_SESSION_IDLE_SWEEP_MS`) unless a client is attending the process (an open SSE stream). The HTTP reapers (`MCP_HTTP_STALE_SESSION_MS`, `MCP_HTTP_STREAM_LOST_SESSION_MS`) reap abandoned MCP sessions only (issue #1025) + - Supports STDIO and Streamable HTTP transport modes, both in both protocol eras (issue #1025): `stdio` is served through the SDK's `serveStdio(factory, { transport })`, which pins a connection to the era its opening message chose (2026-07-28 for a `server/discover` probe, 2025-11-25 and earlier for `initialize`); `http` routes each request with the SDK's `isLegacyRequest` β€” a POST carrying the 2026 per-request envelope goes to a strict `createMcpHandler(factory, { legacy: 'reject' })` behind `toNodeHandler` (one facade per request, sessionless), everything 2025-shaped to the sessionful app as before. One core per process in both: on `http` every MCP session is a facade over the same debug sessions (the id is the capability; one process per agent is the isolation), a debug session outlives the MCP session that created it, and an abandoned session tree is closed by the per-debug-session idle policy (`MCP_DEBUG_SESSION_IDLE_MS`, 30 min on `http`, off on `stdio`, `0` disables; sweep `MCP_DEBUG_SESSION_IDLE_SWEEP_MS`) unless a client is attending the process (an open SSE stream). The HTTP reapers (`MCP_HTTP_STALE_SESSION_MS`, `MCP_HTTP_STREAM_LOST_SESSION_MS`) reap abandoned MCP sessions only (issue #1025) - Dynamically discovers available language adapters 2. **Adapter System** (NEW) diff --git a/README.md b/README.md index 366a63670..f94eaa55b 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ If your agent runs in a terminal, a pipeline, or a cloud sandbox β€” or needs to - πŸ’₯ **Crash-state debugging by default** – Launch sessions pause on uncaught exceptions with stack and locals live (`breakOnExceptions`; exception class/message surfaced via `lastStop`) - πŸͺž **Read-only IDE mirror** – `expose_session` opens a loopback, token-gated DAP endpoint so a human's IDE can inspect the agent's live session without taking control - ☸️ **Kubernetes debugging** – port-forward attach for interpreted runtimes, `kubectl debug --target` + attach-by-PID for native processes ([recipe](docs/kubernetes.md), [turnkey manifests](examples/kubernetes/)) -- πŸ”Œ **STDIO and Streamable HTTP transports** – Works with any MCP client. `stdio` speaks both protocol eras β€” the current 2026-07-28 revision (a probing client such as Claude Code's v2 runtime lands on it) and 2025-11-25 and earlier (a client that opens with `initialize`); `http` speaks 2025-11-25 and earlier, with 2026-07-28 next +- πŸ”Œ **STDIO and Streamable HTTP transports** – Works with any MCP client. Both speak both protocol eras: the current 2026-07-28 revision (a probing client such as Claude Code's v2 runtime lands on it; sessionless on `http`) and 2025-11-25 and earlier (a client that opens with `initialize`) - πŸ“¦ **Zero-runtime dependencies** – Self-contained bundles via esbuild + tsup - ⚑ **npx ready** – Run directly with `npx @debugmcp/mcp-debugger` - no installation needed - 🐳 **Docker and npm packages** – Deploy anywhere @@ -224,6 +224,12 @@ and point the client at it: } ``` +The endpoint speaks both eras of the protocol: a client that probes with `server/discover` (Claude +Code's v2 runtime does) gets the current 2026-07-28 revision, sessionless β€” every request stands +alone, resource notifications flow over `subscriptions/listen` streams β€” and a client that opens +with `initialize` gets the 2025 sessionful era, `Mcp-Session-Id` and all, exactly as before. +`GET /health` counts the requests served per era under `requests`. + One `http` process is one debugger: every MCP client of it sees the same debug sessions, and a session is addressable by whoever holds its id (the handle is the capability β€” run one process per agent if agents must not see each other's work). A debug session outlives the MCP session diff --git a/changelog.d/1025-conformance-2026.added.md b/changelog.d/1025-conformance-2026.added.md new file mode 100644 index 000000000..7e0defdca --- /dev/null +++ b/changelog.d/1025-conformance-2026.added.md @@ -0,0 +1 @@ +**A second MCP conformance lane for the 2026-07-28 revision.** `node scripts/mcp-conformance.mjs --era 2026-07-28` runs the protocol org's conformance suite on its 0.2 line (`@modelcontextprotocol/conformance@0.2.0-alpha.12`, the line the SDKs run) with `--requirements 2026-07-28` β€” everything the revision requires of a server β€” against a local `http` server, and judges the run under `scripts/conformance/expected-failures-2026-07-28.yaml`: the scenarios, or single checks, that drive the suite's own reference fixtures (the "everything server" tools, resources and prompts this server does not have), each with the fixture it needs named beside it. An unlisted failure fails the lane; so does a listed entry that passes, so the baseline only shrinks. CI runs it as `mcp-conformance-2026`, required in the test summary beside the 2025-11-25 lane, which is unchanged. Measuring the lane also found that `resources/templates/list` answered "Method not found": a server advertising the resources capability must answer it, so both eras now answer an empty list. diff --git a/changelog.d/1025-protocol-2026-http.added.md b/changelog.d/1025-protocol-2026-http.added.md new file mode 100644 index 000000000..18f057bac --- /dev/null +++ b/changelog.d/1025-protocol-2026-http.added.md @@ -0,0 +1 @@ +**`http` speaks the current MCP protocol revision, 2026-07-28, beside 2025-11-25 and earlier β€” on the same `/mcp` endpoint.** A request carrying the 2026 per-request envelope (`_meta` with `io.modelcontextprotocol/protocolVersion`) is served by the SDK's sessionless handler: `server/discover` answers the supported versions and capabilities, every request stands alone with no `Mcp-Session-Id`, results carry `resultType` and the cache fields, resource notifications flow over `subscriptions/listen` streams, and the 2026 header rules apply (`MCP-Protocol-Version`, `Mcp-Method`, `Mcp-Name` required on a POST; a missing one is refused with `-32020`, an unsupported claimed revision with `-32022` naming the supported ones). A 2025-shaped request β€” `initialize`, a request carrying `Mcp-Session-Id`, a GET or DELETE, a claim-less POST β€” lands on the sessionful app exactly as before, 400/404 bodies included, so every 2025 client keeps working. A probing client such as Claude Code's v2 runtime therefore lands on 2026-07-28 against `http` too. An open `subscriptions/listen` stream counts the process as attended for the idle policy, the same as a 2025 session's SSE stream; `GET /health` reports `listenStreams` and the requests served per era (`requests.legacy` / `requests.modern`). Browser clients: CORS now allows `Mcp-Method`, `Mcp-Name` and echoes a preflight's requested headers. Step 2 of #1025 completes the protocol move on both transports. diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index 405f69608..513e497bf 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -55,7 +55,7 @@ graph TB - **Purpose**: Entry point for MCP protocol communication - **Key Files**: - `src/server.ts` - `DebugMcpServer`, the composition root and the shared core: dependencies, lifecycle, session validation, breakpoint gating, the public facade methods and the `NotificationHub`; implements `ToolContext`. It owns no SDK `Server` - - `src/server/protocol-facade.ts` - `createProtocolServer(core, { era, deliver })`: one cheap SDK `Server` per connection or request over the core (the eight MCP handlers, the error and initialized hooks, one notification sink that leaves with the facade); `src/server/notification-hub.ts` - the process-wide fan-out of coalesced `resources/updated` pings and `resources/list_changed` to every facade's sink (issue #1025) + - `src/server/protocol-facade.ts` - `createProtocolServer(core, { era, deliver })`: one cheap SDK `Server` per connection or request over the core (the nine MCP handlers, the error and initialized hooks, one notification sink that leaves with the facade); `src/server/notification-hub.ts` - the process-wide fan-out of coalesced `resources/updated` pings and `resources/list_changed` to every facade's sink (issue #1025) - `src/server/` - tool schemas (`tool-schemas.ts`), argument coercion (`tool-arguments.ts`), the tools/call dispatch wrapper (`tool-dispatch.ts` over `handlers/index.ts`), argument validation (`tool-validation.ts`), result envelopes and the session-error catch helpers (`tool-result.ts`), one handler module per tool family (`handlers/*.ts`) over their shared helpers (`handlers/shared.ts`), and the resource (`output-resources.ts`) and prompt (`prompts.ts`) handlers - `src/index.ts` - the `dist/index.js` entry: the prologue (`src/cli/entry-prologue.ts`, console silencing and argv cleanup, evaluated before anything else), then `main()` from `src/main.ts` β€” the side-effect-free module the npm bundle's `cli-entry.ts` imports directly (#732); the four subcommands are wired in `src/cli/setup.ts`: `stdio` (default), `http` (Streamable HTTP, recommended, `-p/--port`, default 3001), `doctor` (toolchain self-check), `check-rust-binary` - `src/server/output-resources.ts` - the MCP resource handlers: `debug://sessions/{id}/output` (subscribable plain-text transcript of captured debuggee output, coalesced `resources/updated` notifications) and `debug://sessions/{id}/proxy-log` (a sanitized, bounded tail of the session's proxy log β€” 64 KiB / 80 lines, on-demand snapshot, not subscribable, and listed only once the session has a log directory) diff --git a/docs/architecture/testing-architecture.md b/docs/architecture/testing-architecture.md index 4c30995c0..f1adb7c10 100644 --- a/docs/architecture/testing-architecture.md +++ b/docs/architecture/testing-architecture.md @@ -270,11 +270,11 @@ Protocol-level correctness checks. `breakpoint-messages/` contains Python script ### MCP Spec Contract and Conformance -**Location:** `tests/integration/cli/http-spec-contract.test.ts`, `scripts/mcp-conformance.mjs` (issue #1023) +**Location:** `tests/integration/cli/http-spec-contract.test.ts` (2025-11-25), `tests/integration/cli/http-spec-contract-2026.test.ts` (2026-07-28), `tests/integration/cli/http-era-negotiation.test.ts` and `stdio-era-negotiation.test.ts`, `scripts/mcp-conformance.mjs` (issues #1023, #1025) -Two tiers, split along the seam between this server's own code and the SDK transport. The clauses of the Streamable HTTP transport (spec 2025-11-25, the revision the pinned SDK speaks) that `createHttpApp` decides before it hands a request to `transport.handleRequest` β€” session routing, the Host/Origin allowlist, the stale-session reaper β€” are pinned one case per clause against the real server over raw HTTP: a missing `Mcp-Session-Id` is 400; an unknown, DELETEd or reaped one is 404 `Session not found`; an invalid `Origin` is 403; an unsupported `MCP-Protocol-Version` is 400. Raw HTTP because the SDK client folds every non-2xx answer into a thrown error, and a case about a status code has to read it. The 404 cases are the ones #1017 fixed; nothing in the suite could have caught the 400 they replaced. +Two tiers, split along the seam between this server's own code and the SDK, and two eras, served side by side on the same endpoint (issue #1025, step 2). The 2025-11-25 clauses of the Streamable HTTP transport that `createHttpApp` decides before it hands a request to `transport.handleRequest` β€” session routing, the Host/Origin allowlist, the stale-session reaper β€” are pinned one case per clause against the real server over raw HTTP: a missing `Mcp-Session-Id` is 400; an unknown, DELETEd or reaped one is 404 `Session not found`; an invalid `Origin` is 403; an unsupported `MCP-Protocol-Version` is 400. Raw HTTP because the SDK client folds every non-2xx answer into a thrown error, and a case about a status code has to read it. The 404 cases are the ones #1017 fixed; nothing in the suite could have caught the 400 they replaced. The 2026-07-28 file pins the same way what the SDK's sessionless handler answers once our router (`isLegacyRequest`) has sent a modern request to it: `server/discover` with no session minted, `tools/list` with its cache fields and a stable order, a stale `Mcp-Session-Id` ignored, the `-32602` / `-32020` / `-32022` refusals, a 2025 `initialize` still opening a session on the same endpoint, a claim-less POST still landing on the sessionful app's 400, and an open `subscriptions/listen` stream counting the process as attended. The two negotiation files drive the v2 SDK client in `auto`, pinned and `legacy` mode through a full mock debug session on each transport, so what a real client experiences on either era is pinned too. -Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance`) runs the protocol org's `@modelcontextprotocol/conformance` suite for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection β€” against a local `http` server, as the `mcp-conformance` CI lane on Linux. The suite has no session-lifecycle scenario, which is why the first tier exists; its CLI is pinned, and on Windows its exit code is unusable (it aborts at exit with a libuv assertion even after a pass), so the script reads the printed summary instead. The 2026-07-28 revision removes protocol-level sessions altogether; the legacy era this file pins stays supported under the spec's feature lifecycle policy. +Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance`) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” under an expected-failures baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) that lists each scenario needing a fixture this server does not have (the `tools-call-*`, `prompts-get-*`, `resources-read-*`, `completion` and most `input-required-result-*` scenarios drive the suite's reference tools) with the fixture it needs; the lane fails on an unlisted failure and on a listed scenario that starts passing, so the baseline can only shrink honestly. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned, and on Windows their exit codes are unusable (they abort at exit with a libuv assertion even after a pass), so the script reads the printed summaries instead. ## Coverage Strategy diff --git a/docs/getting-started.md b/docs/getting-started.md index 21c0ef963..42cad5ec7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -99,12 +99,13 @@ Because `stdio` is the default subcommand, bare `mcp-debugger` starts in stdio m explicitly in a config file is still worth doing: it documents the intent. Console output is silenced in every mode so nothing can corrupt the JSON-RPC stream. -Both eras of the MCP protocol are served on `stdio`: a client that probes with `server/discover` -(Claude Code's v2 runtime does, on every connect) gets the current 2026-07-28 revision, and a -client that opens with `initialize` gets 2025-11-25 or earlier, as before. Claude Code may keep a -stdio server it once saw answer as legacy pinned that way for up to seven days; restarting Claude -Code after upgrading the server, or waiting it out, lets it probe again. `http` speaks 2025-11-25 -and earlier today; the 2026-07-28 era follows there. +Both transports serve both eras of the MCP protocol: a client that probes with `server/discover` +(Claude Code's v2 runtime does, unless it has a cached verdict for that server) gets the current +2026-07-28 revision β€” sessionless on `http`, every request standing alone β€” and a client that +opens with `initialize` gets 2025-11-25 or earlier, as before. Claude Code may keep a stdio server +it once saw answer as legacy pinned that way for up to seven days (its own per-server MCP log +records the era it chose); restarting Claude Code after upgrading the server, or waiting it out, +lets it probe again. ### Claude Desktop, or any client with a JSON config diff --git a/scripts/conformance/expected-failures-2026-07-28.yaml b/scripts/conformance/expected-failures-2026-07-28.yaml new file mode 100644 index 000000000..018ae57ca --- /dev/null +++ b/scripts/conformance/expected-failures-2026-07-28.yaml @@ -0,0 +1,39 @@ +# Expected failures for `@modelcontextprotocol/conformance server --requirements 2026-07-28` +# against mcp-debugger (scripts/mcp-conformance.mjs --era 2026-07-28; issue #1025). +# +# Every entry is a scenario, or a `scenario:check` pair, that drives one of the conformance +# suite's own reference fixtures β€” the "everything server" tools (`test_*`), resources +# (`test://…`) and prompts β€” which this server does not have. None is a protocol divergence. +# The lane fails on a failure that is NOT listed here, and on a listed entry that starts +# passing (a stale baseline), so this file can only shrink honestly. Measured 2026-11-10 +# against 0.2.0-alpha.12: 37 scored scenarios, of which these need a fixture. +server: + - completion-complete # completion/complete on a reference prompt argument + - tools-call-simple-text # tools/call test_simple_text + - tools-call-image # tools/call test_image + - tools-call-audio # tools/call test_audio + - tools-call-embedded-resource # tools/call test_embedded_resource + - tools-call-mixed-content # tools/call test_mixed_content + - tools-call-error # tools/call test_error + - tools-call-with-progress # tools/call test_progress (progress notifications) + - resources-read-text # resources/read test://static-text + - resources-read-binary # resources/read test://static-binary + - resources-templates-read # resources/read of a reference template URI + - prompts-get-simple # prompts/get test_simple_prompt + - prompts-get-with-args # prompts/get test_prompt_with_args + - prompts-get-embedded-resource # prompts/get test_prompt_embedded_resource + - prompts-get-with-image # prompts/get test_prompt_with_image + - input-required-result-basic-elicitation # a reference tool that elicits (MRTR) + - input-required-result-basic-sampling # a reference tool that samples (MRTR) + - input-required-result-basic-list-roots # a reference tool that lists roots (MRTR) + - input-required-result-request-state # the reference MRTR tool's requestState + - input-required-result-multiple-input-requests # the reference MRTR tool, several inputs + - input-required-result-multi-round # the reference MRTR tool, several rounds + - input-required-result-non-tool-request # a reference prompt that elicits (MRTR) + - input-required-result-result-type # the reference MRTR tool's resultType + - input-required-result-tampered-state # the reference MRTR tool, tampered state + - input-required-result-capability-check # the reference MRTR tool vs. client capabilities + - server-stateless:ServerRejectsUndeclaredCapability # tools/call test_missing_capability + - server-stateless:MissingCapabilityHttp400 # tools/call test_missing_capability (HTTP status) + - server-stateless:HttpServerNoIndependentRequestsOnStream # tools/call test_streaming_elicitation + - server-stateless:ServerNoLogWithoutLogLevel # tools/call test_logging_tool diff --git a/scripts/mcp-conformance.d.mts b/scripts/mcp-conformance.d.mts index aad7f25b9..d44551172 100644 --- a/scripts/mcp-conformance.d.mts +++ b/scripts/mcp-conformance.d.mts @@ -19,3 +19,35 @@ export function runConformance(options?: { verbose?: boolean; log?: (line: string) => void; }): Promise; + +// The 2026-07-28 requirements lane (issue #1025, step 2). +export const CONFORMANCE_CLI_2026: string; +export const REQUIREMENTS_2026: string; +export interface RequirementsScenario { + name: string; + passed: number; + failed: number; +} +export interface RequirementsSummary { + scenarios: RequirementsScenario[]; + /** Scenarios the revision lists as not scored (extensions and pending ones). */ + notScored: Set; +} +/** A whole scenario expected to fail (`'all'`), or the names of its checks that are. */ +export type ExpectedFailures = Map>; +export function parseRequirementsSummary(output: string): RequirementsSummary | undefined; +export function parseBaseline(text: string): ExpectedFailures; +export function failingChecks(checksJson: unknown): string[]; +export function judgeRequirements( + summary: RequirementsSummary, + baseline: ExpectedFailures, + failingChecksOf: (scenario: string) => string[] +): string[]; +export function collectChecks(dir: string): Map; +export function runRequirements(options?: { + root?: string; + revision?: string; + baselinePath?: string; + verbose?: boolean; + log?: (line: string) => void; +}): Promise<{ problems: string[]; summary: RequirementsSummary }>; diff --git a/scripts/mcp-conformance.mjs b/scripts/mcp-conformance.mjs index 3bbeffe23..7531dd14f 100644 --- a/scripts/mcp-conformance.mjs +++ b/scripts/mcp-conformance.mjs @@ -13,16 +13,30 @@ * The CLI's exit code is not trusted: on Windows it aborts at exit with a libuv assertion even * when every check passed, so the printed `Passed: n/m, k failed` summary is what counts. * + * The 2026-07-28 era (issue #1025, step 2) is a second lane on the suite's 0.2 line, which the + * SDKs themselves run: `--era 2026-07-28` runs `--requirements 2026-07-28` β€” everything the + * revision requires of a server β€” under an expected-failures baseline + * (`scripts/conformance/expected-failures-2026-07-28.yaml`) that names each scenario, or + * scenario:check, that needs one of the suite's reference fixtures. An unlisted failure fails the + * lane; so does a listed entry that passes (a stale baseline), so the baseline only shrinks + * honestly. The verdict is read from the CLI's own per-scenario summary lines and the + * `checks.json` it writes per scenario (`--output-dir`), never from its exit code. + * * Usage: node scripts/mcp-conformance.mjs [scenario ...] (default: the list below) + * node scripts/mcp-conformance.mjs --era 2026-07-28 [--verbose] */ import { spawn } from 'node:child_process'; -import { existsSync } from 'node:fs'; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs'; import { createServer } from 'node:net'; +import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { isMain } from './lib/is-main.mjs'; export const CONFORMANCE_CLI = '@modelcontextprotocol/conformance@0.1.16'; +/** The 0.2 line, where the 2026-07-28 scenarios live; pinned to an exact prerelease. */ +export const CONFORMANCE_CLI_2026 = '@modelcontextprotocol/conformance@0.2.0-alpha.12'; +export const REQUIREMENTS_2026 = '2026-07-28'; /** The scenarios that apply to a server without the reference fixtures (measured 2026-10-27: all pass). */ export const APPLICABLE_SCENARIOS = [ @@ -51,6 +65,129 @@ export function judgeScenario(name, output) { return { name, ok: true, summary }; } +/** + * The per-scenario verdict lines of a `--requirements` run (`βœ“ name: n passed, m failed`), and + * the scenarios the revision lists as not scored (the CLI prints them after the total; they do + * not affect conformance). Undefined when the output has no summary. + */ +export function parseRequirementsSummary(output) { + const scenarios = []; + const line = /^[βœ“βœ—]\s+([A-Za-z0-9-]+):\s+(\d+)\s+passed,\s+(\d+)\s+failed\s*$/gm; + for (const match of output.matchAll(line)) { + scenarios.push({ name: match[1], passed: Number(match[2]), failed: Number(match[3]) }); + } + if (scenarios.length === 0) return undefined; + const notScored = new Set(); + const notScoredBlock = output.split(/^Not scored for [^\n]*$/m)[1]; + if (notScoredBlock) { + for (const match of notScoredBlock.matchAll(/^\s+[βœ“βœ—]\s+([A-Za-z0-9-]+)\s+\(/gm)) { + notScored.add(match[1]); + } + } + return { scenarios, notScored }; +} + +/** + * The expected-failures file: `server:` entries of `` (the whole scenario is expected to + * fail) or `:` (that check is). Comments and blank lines are ignored. Deliberately + * a tiny reader for this one shape rather than a YAML dependency. + */ +export function parseBaseline(text) { + const expected = new Map(); + let inServer = false; + for (const raw of text.split(/\r?\n/)) { + const line = raw.replace(/\s+#.*$/, '').replace(/^#.*$/, '').trimEnd(); + if (!line.trim()) continue; + if (/^[A-Za-z]+:\s*$/.test(line)) { + inServer = line.trim() === 'server:'; + continue; + } + const entry = /^\s*-\s*['"]?([A-Za-z0-9:_-]+)['"]?\s*$/.exec(line); + if (!entry || !inServer) continue; + const [scenario, check] = entry[1].split(':'); + const current = expected.get(scenario); + if (check === undefined) { + expected.set(scenario, 'all'); + } else if (current !== 'all') { + expected.set(scenario, new Set([...(current ?? []), check])); + } + } + return expected; +} + +/** The names of the failing checks in one scenario's `checks.json` (an array, or an object holding `checks`). */ +export function failingChecks(checksJson) { + const checks = Array.isArray(checksJson) ? checksJson : Array.isArray(checksJson?.checks) ? checksJson.checks : []; + return checks.filter((check) => check?.status === 'FAILURE').map((check) => String(check.name)); +} + +/** + * The lane's verdict for a requirements run: every scored scenario either passes, or fails only + * in ways the baseline expects β€” and every baseline entry still fails (otherwise it is stale). + * `failingChecksOf(name)` answers the failing check names of a scenario, read from its checks.json. + */ +export function judgeRequirements(summary, baseline, failingChecksOf) { + const problems = []; + const seen = new Set(); + for (const { name, failed } of summary.scenarios) { + if (summary.notScored.has(name)) continue; + seen.add(name); + const expected = baseline.get(name); + if (failed === 0) { + if (expected !== undefined) problems.push(`${name}: listed as an expected failure but it passed β€” remove it from the baseline`); + continue; + } + if (expected === 'all') continue; + const failing = failingChecksOf(name); + if (failing.length === 0) { + // The summary says it failed but no checks.json named the checks: nothing to compare the + // per-check entries against, and nothing to call passed. + problems.push(expected === undefined + ? `${name}: ${failed} failed check(s) not in the baseline` + : `${name}: ${failed} failed check(s) but no checks.json was written for it β€” the per-check baseline entries cannot be judged`); + continue; + } + const unexpected = failing.filter((check) => !(expected?.has(check))); + if (unexpected.length > 0) { + problems.push(`${name}: ${failed} failed check(s) not in the baseline (${unexpected.join(', ')})`); + } + for (const check of expected ?? []) { + if (!failing.includes(check)) problems.push(`${name}:${check}: listed as an expected failure but it passed β€” remove it from the baseline`); + } + } + for (const name of baseline.keys()) { + if (!seen.has(name) && !summary.notScored.has(name)) { + problems.push(`${name}: in the baseline but the requirements run did not report it`); + } + } + return problems; +} + +/** + * Every `checks.json` the CLI wrote under `dir`, keyed by scenario. The CLI names the folder + * `server--` with the timestamp's colons and dots turned into dashes + * (`server-server-stateless-2026-11-10T02-36-40-817Z`, measured against 0.2.0-alpha.12). + */ +export function collectChecks(dir) { + const byScenario = new Map(); + const walk = (current) => { + for (const entry of readdirSync(current)) { + const full = path.join(current, entry); + if (statSync(full).isDirectory()) { + walk(full); + } else if (entry === 'checks.json') { + const folder = path.basename(path.dirname(full)); + const match = /^(?:server-)?(.+)-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.exec(folder); + if (match) { + byScenario.set(match[1], failingChecks(JSON.parse(readFileSync(full, 'utf8')))); + } + } + } + }; + if (existsSync(dir)) walk(dir); + return byScenario; +} + function freePort() { return new Promise((resolve, reject) => { const probe = createServer(); @@ -76,12 +213,12 @@ async function waitForHealth(port, timeoutMs) { return false; } -/** Every argument the CLI is given is one of ours (a package spec, a loopback URL, a scenario name). */ -const SAFE_ARGUMENT = /^[A-Za-z0-9@./:_-]+$/; +/** Every argument the CLI is given is one of ours (a package spec, a loopback URL, a scenario name, a path). */ +const SAFE_ARGUMENT = /^[A-Za-z0-9@./:_\\-]+$/; -function runCli(args, cwd) { +function runCli(args, cwd, cli = CONFORMANCE_CLI) { return new Promise((resolve) => { - const argv = ['--yes', CONFORMANCE_CLI, ...args]; + const argv = ['--yes', cli, ...args]; const unsafe = argv.find((arg) => !SAFE_ARGUMENT.test(arg)); if (unsafe !== undefined) throw new Error(`refusing to pass "${unsafe}" to the conformance CLI`); // Windows: `npx` is a .cmd shim, which Node spawns only through a shell. The argv is checked @@ -134,21 +271,94 @@ export async function runConformance(options = {}) { } } +/** + * The 2026-07-28 lane: one `--requirements` run of the 0.2 CLI against a local `http` server, + * judged against the baseline. Answers `{ problems, summary }`; an empty `problems` is a pass. + */ +export async function runRequirements(options = {}) { + const root = options.root ?? fileURLToPath(new URL('../', import.meta.url)); + const entry = path.join(root, 'dist', 'index.js'); + if (!existsSync(entry)) throw new Error(`${entry} not found: run the build first`); + const revision = options.revision ?? REQUIREMENTS_2026; + const baselinePath = options.baselinePath ?? path.join(root, 'scripts', 'conformance', `expected-failures-${revision}.yaml`); + const baseline = parseBaseline(readFileSync(baselinePath, 'utf8')); + const log = options.log ?? console.log; + + const port = await freePort(); + const server = spawn(process.execPath, [entry, 'http', '-p', String(port)], { + cwd: root, + env: { ...process.env, MCP_SKIP_ORPHAN_REAPERS: '1' }, + windowsHide: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let serverOutput = ''; + server.stdout.on('data', (chunk) => { serverOutput += chunk.toString(); }); + server.stderr.on('data', (chunk) => { serverOutput += chunk.toString(); }); + const outputDir = mkdtempSync(path.join(os.tmpdir(), 'mcp-conformance-2026-')); + + try { + if (!(await waitForHealth(port, 20_000))) { + throw new Error(`the server never answered /health on port ${port}:\n${serverOutput}`); + } + const output = await runCli( + ['server', '--url', `http://127.0.0.1:${port}/mcp`, '--requirements', revision, '--output-dir', outputDir], + root, + CONFORMANCE_CLI_2026 + ); + const summary = parseRequirementsSummary(output); + if (!summary) { + throw new Error(`no per-scenario summary in the conformance CLI output:\n${output}`); + } + const checks = collectChecks(outputDir); + const problems = judgeRequirements(summary, baseline, (name) => checks.get(name) ?? []); + for (const { name, passed, failed } of summary.scenarios) { + const scored = !summary.notScored.has(name); + const expected = baseline.get(name); + const tag = !scored ? 'skip' : failed === 0 ? 'ok ' : expected ? 'base' : 'FAIL'; + log(`${tag} ${name.padEnd(48)} ${passed}/${passed + failed}${!scored ? ' (not scored)' : expected ? ' (expected: fixture)' : ''}`); + } + if (problems.length && options.verbose) log(output); + return { problems, summary }; + } finally { + server.kill(); + rmSync(outputDir, { recursive: true, force: true }); + } +} + if (isMain(import.meta.url)) { const verbose = process.argv.includes('--verbose'); - const scenarios = process.argv.slice(2).filter((arg) => !arg.startsWith('--')); - runConformance({ scenarios, verbose }) - .then((verdicts) => { - const failed = verdicts.filter((verdict) => !verdict.ok); - if (failed.length) { - console.error(`MCP conformance: ${failed.length} of ${verdicts.length} scenario(s) failed`); + const eraIndex = process.argv.indexOf('--era'); + if (eraIndex !== -1) { + const revision = process.argv[eraIndex + 1]; + runRequirements({ revision, verbose }) + .then(({ problems, summary }) => { + const scored = summary.scenarios.filter((scenario) => !summary.notScored.has(scenario.name)); + if (problems.length) { + for (const problem of problems) console.error(`MCP conformance ${revision}: ${problem}`); + process.exitCode = 1; + } else { + console.log(`MCP conformance ${revision}: ${scored.length} scored scenario(s) conform under the baseline against ${CONFORMANCE_CLI_2026}`); + } + }) + .catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; - } else { - console.log(`MCP conformance: ${verdicts.length} scenario(s) passed against ${CONFORMANCE_CLI}`); - } - }) - .catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; - }); + }); + } else { + const scenarios = process.argv.slice(2).filter((arg) => !arg.startsWith('--')); + runConformance({ scenarios, verbose }) + .then((verdicts) => { + const failed = verdicts.filter((verdict) => !verdict.ok); + if (failed.length) { + console.error(`MCP conformance: ${failed.length} of ${verdicts.length} scenario(s) failed`); + process.exitCode = 1; + } else { + console.log(`MCP conformance: ${verdicts.length} scenario(s) passed against ${CONFORMANCE_CLI}`); + } + }) + .catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); + } } diff --git a/src/cli/http-command.ts b/src/cli/http-command.ts index 8e47eb356..59772940a 100644 --- a/src/cli/http-command.ts +++ b/src/cli/http-command.ts @@ -3,10 +3,11 @@ import type { Express, Request, Response, NextFunction } from 'express'; import express from 'express'; import { randomUUID } from 'crypto'; import { IncomingMessage, ServerResponse } from 'http'; -import { isInitializeRequest, ProtocolErrorCode } from '@modelcontextprotocol/server'; -import { NodeStreamableHTTPServerTransport } from '@modelcontextprotocol/node'; +import { createMcpHandler, isInitializeRequest, isLegacyRequest, ProtocolErrorCode } from '@modelcontextprotocol/server'; +import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from '@modelcontextprotocol/node'; import { DebugMcpServer, type DebugMcpServerOptions } from '../server.js'; import { createProtocolServer } from '../server/protocol-facade.js'; +import { handlerSink } from '../server/notification-hub.js'; import { attachSharedFileTransport } from '../utils/logger.js'; import type { HttpOptions } from './setup.js'; import { @@ -67,6 +68,13 @@ interface SessionData { /** Largest JSON-RPC request body accepted; also named in the 413 the parser answers with (issue #670). */ const JSON_BODY_LIMIT = '10mb'; +/** The same limit in bytes, for the SDK's classifier and modern handler (their default is 4 MiB). */ +const JSON_BODY_LIMIT_BYTES = 10 * 1024 * 1024; + +/** A modern-era `subscriptions/listen` request: its response is a long-lived stream, so the process is attended while it is open. */ +function isListenRequest(body: unknown): boolean { + return typeof body === 'object' && body !== null && (body as { method?: unknown }).method === 'subscriptions/listen'; +} /** Idle window before a streamless HTTP session is reaped (issue #337). */ const DEFAULT_STALE_SESSION_MS = 30 * 60 * 1000; @@ -210,10 +218,41 @@ export function createHttpApp( logFile: options.logFile, debugSessionIdle: { defaultMs: DEFAULT_DEBUG_SESSION_IDLE_MS }, }); - // While any client holds an open SSE stream the process is attended and the - // idle policy closes nothing; the modern era's listen streams join this - // count when they are served. One predicate for the supplier and /health. + // The 2026-07-28 leg (issue #1025, step 2): the SDK's sessionless handler, + // strict β€” a 2025-era request that reaches it is refused with the supported + // versions β€” behind the request classifier in handleMcpRequest, which sends + // 2025-shaped traffic to the sessionful app below. One facade per request, + // delivering nothing itself: the handler's notifier is this leg's hub sink, + // routing to the open subscriptions/listen streams. Host/Origin stay ours + // (the allowlist middleware runs first); the body limit matches the parser's. + const modern = createMcpHandler( + ({ era }) => protocolServerFactory(core, { era, deliver: 'external' }), + { + legacy: 'reject', + maxRequestBodySize: JSON_BODY_LIMIT_BYTES, + onerror: (error: Error) => { + logger.error('Modern-era MCP handler error', { error }); + }, + } + ); + core.notifications.addSink(handlerSink(modern.notify)); + const modernNode = toNodeHandler(modern, { + onerror: (error: Error) => { + logger.error('Modern-era MCP request failed', { error }); + }, + }); + /** Requests served per era since start, for /health and the #1025 telemetry. */ + const requests = { legacy: 0, modern: 0 }; + /** Open modern subscriptions/listen streams (their responses are long-lived). */ + let modernListenStreams = 0; + + // While any client holds an open SSE stream β€” a 2025 session's GET stream or + // a 2026 subscriptions/listen stream β€” the process is attended and the idle + // policy closes nothing. One predicate for the supplier and /health. const attended = (): boolean => { + if (modernListenStreams > 0) { + return true; + } for (const session of httpSessions.values()) { if (session.openStreams > 0) { return true; @@ -271,7 +310,10 @@ export function createHttpApp( (app as any).staleSweepTimer = staleSweepTimer; // CORS β€” Mcp-Session-Id and last-event-id must be exposed for the MCP Inspector - // and for clients to read the session ID from the Initialize response. + // and for clients to read the session ID from the Initialize response. The + // 2026-07-28 revision requires Mcp-Method and Mcp-Name on every POST and + // mirrors tool parameters into Mcp-Param-* headers, so a browser preflight's + // own requested headers are echoed back as allowed. app.use((req: Request, res: Response, next: NextFunction) => { // A request carrying an Origin has already passed the allowlist above: // echo that origin (and Vary on it) instead of granting every origin. @@ -281,9 +323,12 @@ export function createHttpApp( res.header('Vary', 'Origin'); } res.header('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS'); + const requestedHeaders = req.headers['access-control-request-headers']; + const requested = Array.isArray(requestedHeaders) ? requestedHeaders.join(', ') : requestedHeaders; res.header( 'Access-Control-Allow-Headers', - 'Content-Type, Mcp-Session-Id, Mcp-Protocol-Version, Last-Event-Id' + 'Content-Type, Accept, Mcp-Session-Id, Mcp-Protocol-Version, Mcp-Method, Mcp-Name, Last-Event-Id' + + (requested ? `, ${requested}` : '') ); res.header( 'Access-Control-Expose-Headers', @@ -307,6 +352,27 @@ export function createHttpApp( const sessionIdHeader = req.headers['mcp-session-id']; const sessionId = Array.isArray(sessionIdHeader) ? sessionIdHeader[0] : sessionIdHeader; + // Which era is this request? The SDK's own classifier decides (issue + // #1025, step 2): a POST carrying the 2026 per-request envelope β€” or a + // modern protocol header the strict handler must refuse β€” is the modern + // leg's; anything 2025-shaped (initialize, a session id, GET, DELETE, a + // claim-less POST) stays with the sessionful app below, unchanged. The + // body Express already parsed is handed on so nothing re-reads the stream. + const body = req.method === 'POST' ? req.body : undefined; + const probe = await toWebRequest(req, body, { maxRequestBodySize: JSON_BODY_LIMIT_BYTES }); + if (!(await isLegacyRequest(probe, body, { maxRequestBodySize: JSON_BODY_LIMIT_BYTES }))) { + requests.modern++; + if (isListenRequest(body)) { + modernListenStreams++; + res.once('close', () => { + modernListenStreams = Math.max(0, modernListenStreams - 1); + }); + } + await modernNode(req, res, body); + return; + } + requests.legacy++; + let transport: NodeStreamableHTTPServerTransport; if (sessionId && httpSessions.has(sessionId)) { @@ -435,7 +501,8 @@ export function createHttpApp( listening: { ...listening }, connections: httpSessions.size, attended: attended(), - listenStreams: 0, + listenStreams: modernListenStreams, + requests: { ...requests }, sessions: Array.from(httpSessions.keys()), details: sessions, debugSessions: core.sessionManager.getAllSessions().map((debugSession) => { @@ -475,6 +542,8 @@ export function createHttpApp( // eslint-disable-next-line @typescript-eslint/no-explicit-any (app as any).core = core; // eslint-disable-next-line @typescript-eslint/no-explicit-any + (app as any).modernHandler = modern; + // eslint-disable-next-line @typescript-eslint/no-explicit-any (app as any).bindAddress = bind; // eslint-disable-next-line @typescript-eslint/no-explicit-any (app as any).listening = listening; @@ -532,6 +601,8 @@ export async function handleHttpCommand( const httpSessions = (app as any).httpSessions as Map; // eslint-disable-next-line @typescript-eslint/no-explicit-any const core = (app as any).core as DebugMcpServer; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const modernHandler = (app as any).modernHandler as { close: () => Promise }; // The one core (issue #1025, step 2): started once, before anything can // reach it; its idle sweep is armed here. await core.start(); @@ -588,10 +659,16 @@ export async function handleHttpCommand( }, 15000); hardExit.unref?.(); - // Close every active transport, then stop the one core (every debug - // session with it), bounded so one wedged proxy cannot stall the whole - // shutdown. Transports first: no request lands on a core that is closing. + // Close the modern handler (its listen streams end gracefully), then every + // active transport, then the one core (every debug session with it), + // bounded so one wedged proxy cannot stall the whole shutdown. Transports + // before the core: no request lands on a core that is closing. const stopWork = (async () => { + try { + await modernHandler.close(); + } catch (err) { + logger.error('Error closing the modern-era handler during shutdown', { error: err }); + } for (const { transport } of httpSessions.values()) { try { await transport.close(); diff --git a/src/server/notification-hub.ts b/src/server/notification-hub.ts index 7bd30e9eb..b1b4f3d2d 100644 --- a/src/server/notification-hub.ts +++ b/src/server/notification-hub.ts @@ -118,6 +118,25 @@ export class NotificationHub { } } +/** The slice of the SDK's `ServerNotifier` (an HTTP handler's `notify`) the hub feeds. */ +export interface HandlerNotifier { + resourceUpdated(uri: string): void | Promise; + resourcesChanged(): void | Promise; +} + +/** + * The sink for the 2026-07-28 HTTP leg: its facades serve one request each and + * cannot send anything afterwards, so the hub feeds the handler's own notifier, + * which routes to the `subscriptions/listen` streams and filters by what each + * listen named. Nothing per URI to forget. + */ +export function handlerSink(notify: HandlerNotifier): NotificationSink { + return { + resourceUpdated: (uri: string) => notify.resourceUpdated(uri), + resourcesChanged: () => notify.resourcesChanged() + }; +} + /** * The sink a protocol facade attaches for its own SDK Server. With a filter * (a legacy-era facade) only the URIs that facade subscribed to through diff --git a/src/server/output-resources.ts b/src/server/output-resources.ts index 44a789387..959bfdd04 100644 --- a/src/server/output-resources.ts +++ b/src/server/output-resources.ts @@ -98,6 +98,11 @@ export function registerResourceHandlers( }; }); + // This server exposes no resource templates, but a server advertising the + // resources capability must answer the method β€” the 2026-07-28 conformance + // suite's caching scenario asks it (issue #1025); the SDK stamps the cache fields. + server.setRequestHandler('resources/templates/list', async () => ({ resourceTemplates: [] })); + server.setRequestHandler('resources/read', async (request, callCtx) => { const uri = request.params.uri; const outputSessionId = parseOutputResourceUri(uri); diff --git a/src/server/protocol-facade.ts b/src/server/protocol-facade.ts index a3195d1b2..280ed653c 100644 --- a/src/server/protocol-facade.ts +++ b/src/server/protocol-facade.ts @@ -4,7 +4,7 @@ * * The 2026-07-28 serving entries build a Server per HTTP request * (`createMcpHandler`) or per stdio connection plus a discarded probe instance - * (`serveStdio`), so a facade has to be cheap: the eight MCP handlers closing + * (`serveStdio`), so a facade has to be cheap: the nine MCP handlers closing * over the core, the error and initialized hooks, and one notification sink * that lives exactly as long as the facade does. No dependency container, no * SessionManager, no logger of its own. diff --git a/tests/core/unit/server/notification-hub.test.ts b/tests/core/unit/server/notification-hub.test.ts index 62e367728..77766a3d2 100644 --- a/tests/core/unit/server/notification-hub.test.ts +++ b/tests/core/unit/server/notification-hub.test.ts @@ -11,6 +11,7 @@ import { NotificationHub, OUTPUT_UPDATE_DEBOUNCE_MS, facadeSink, + handlerSink, type NotificationSink } from '../../../../src/server/notification-hub.js'; import { FacadeSubscriptions } from '../../../../src/server/output-resources.js'; @@ -179,6 +180,20 @@ describe('NotificationHub forget guard', () => { }); }); +describe('handlerSink', () => { + it('forwards every ping and list change to the HTTP handler notifier, which filters by its listen streams', async () => { + const notify = { resourceUpdated: vi.fn(), resourcesChanged: vi.fn(), toolsChanged: vi.fn(), promptsChanged: vi.fn() }; + const sink = handlerSink(notify); + + await sink.resourceUpdated(URI); + await sink.resourcesChanged(); + + expect(notify.resourceUpdated).toHaveBeenCalledWith(URI); + expect(notify.resourcesChanged).toHaveBeenCalledTimes(1); + expect(sink.forget).toBeUndefined(); + }); +}); + describe('facadeSink', () => { const server = () => ({ sendResourceUpdated: vi.fn().mockResolvedValue(undefined), diff --git a/tests/core/unit/server/protocol-facade.test.ts b/tests/core/unit/server/protocol-facade.test.ts index c610924f0..76c03eabc 100644 --- a/tests/core/unit/server/protocol-facade.test.ts +++ b/tests/core/unit/server/protocol-facade.test.ts @@ -1,7 +1,7 @@ /** * The protocol facade (issue #1025, step 2): one cheap SDK `Server` per * connection or request over the one shared DebugMcpServer core. It carries - * the eight MCP handlers, the error and initialized hooks, and one + * the nine MCP handlers, the error and initialized hooks, and one * notification sink that lives exactly as long as the facade does. The core * never constructs a Server itself any more. */ @@ -119,7 +119,7 @@ describe('createProtocolServer', () => { expect(b.sendResourceUpdated).toHaveBeenCalledWith({ uri: 'debug://sessions/sess-2/output' }); }); - it('registers exactly the eight MCP handlers', () => { + it('registers exactly the nine MCP handlers', () => { legacy(); const [server] = mockServers; @@ -130,6 +130,7 @@ describe('createProtocolServer', () => { 'resources/list', 'resources/read', 'resources/subscribe', + 'resources/templates/list', 'resources/unsubscribe', 'tools/call', 'tools/list' diff --git a/tests/core/unit/server/server-initialization.test.ts b/tests/core/unit/server/server-initialization.test.ts index 0487fc172..8964e0ff4 100644 --- a/tests/core/unit/server/server-initialization.test.ts +++ b/tests/core/unit/server/server-initialization.test.ts @@ -97,9 +97,10 @@ describe('Server Initialization Tests', () => { it('a facade over the core registers the tool, resource and prompt handlers', () => { debugServer = createDebugServerWithFacade(); - // ListTools + CallTool, ListResources/ReadResource/Subscribe/Unsubscribe (issue #218), + // ListTools + CallTool, ListResources/ReadResource/Subscribe/Unsubscribe (issue #218) + // + ListResourceTemplates (empty; the resources capability must answer it, issue #1025), // plus ListPrompts/GetPrompt for the debugging-workflow prompt - expect(mockServer.setRequestHandler).toHaveBeenCalledTimes(8); + expect(mockServer.setRequestHandler).toHaveBeenCalledTimes(9); }); it('a facade routes SDK errors to the core logger', () => { diff --git a/tests/core/unit/server/server-resources.test.ts b/tests/core/unit/server/server-resources.test.ts index 263ef4d9b..58312379f 100644 --- a/tests/core/unit/server/server-resources.test.ts +++ b/tests/core/unit/server/server-resources.test.ts @@ -185,6 +185,16 @@ describe('Server Output Resources Tests', () => { }); }); + describe('resources/templates/list', () => { + it('answers an empty list: this server exposes no resource templates, and a resources capability must answer the method (issue #1025)', async () => { + const { listResourceTemplatesHandler } = getResourceHandlers(mockServer); + + const result = await listResourceTemplatesHandler({ method: 'resources/templates/list', params: {} }); + + expect(result).toEqual({ resourceTemplates: [] }); + }); + }); + describe('resources/read', () => { it('returns the verbatim transcript', async () => { const session = mockSession(); diff --git a/tests/core/unit/server/server-test-helpers.ts b/tests/core/unit/server/server-test-helpers.ts index 46c0571b4..3a0c87cde 100644 --- a/tests/core/unit/server/server-test-helpers.ts +++ b/tests/core/unit/server/server-test-helpers.ts @@ -275,7 +275,9 @@ export function getResourceHandlers(mockServer: MockServer) { subscribeHandler: findHandler(mockServer, 'resources/subscribe') as (request: TestRequest) => Promise, unsubscribeHandler: findHandler(mockServer, 'resources/unsubscribe') as - (request: TestRequest) => Promise + (request: TestRequest) => Promise, + listResourceTemplatesHandler: findHandler(mockServer, 'resources/templates/list') as + (request?: TestRequest) => Promise<{ resourceTemplates: unknown[] }> }; } diff --git a/tests/e2e/mcp-server-smoke-http.test.ts b/tests/e2e/mcp-server-smoke-http.test.ts index 8c51e5ad2..85e1dab19 100644 --- a/tests/e2e/mcp-server-smoke-http.test.ts +++ b/tests/e2e/mcp-server-smoke-http.test.ts @@ -318,6 +318,37 @@ describe('MCP Server E2E HTTP Smoke Test', () => { } }, TEST_TIMEOUT); + it('debugs fibonacci.py over the 2026-07-28 era too: a client pinned to it runs sessionless (issue #1025)', async () => { + let debugSessionId: string | undefined; + + try { + serverPort = await startHttpServer(); + expect(await waitForHealthEndpoint(serverPort, TEST_TIMEOUT)).toBe(true); + + // The pin never falls back: this connection is modern or the test fails. + mcpSdkClient = new Client( + { name: 'e2e-http-smoke-test-client-2026', version: '0.1.0' }, + { versionNegotiation: { mode: { pin: '2026-07-28' } } } + ); + await mcpSdkClient.connect(new StreamableHTTPClientTransport(new URL(`http://127.0.0.1:${serverPort}/mcp`))); + expect(mcpSdkClient.getProtocolEra()).toBe('modern'); + + const fibonacciPath = path.join(projectRoot, 'examples', 'python', 'fibonacci.py'); + const result = await executeDebugSequence(mcpSdkClient, fibonacciPath, 'E2E HTTP Smoke Test Session (2026-07-28)'); + expect(result.success).toBe(true); + debugSessionId = result.sessionId; + + // No MCP session exists for this client; the debug session is the process's. + const health = await (await fetch(`http://127.0.0.1:${serverPort}/health`)).json() as { sessions: string[]; requests: { modern: number } }; + expect(health.sessions).toEqual([]); + expect(health.requests.modern).toBeGreaterThan(0); + } finally { + if (debugSessionId && mcpSdkClient) { + await mcpSdkClient.callTool({ name: 'close_debug_session', arguments: { sessionId: debugSessionId } }).catch(() => undefined); + } + } + }, TEST_TIMEOUT); + // Test spawning the server from a different working directory it('should work when the HTTP server is spawned from a different working directory', async () => { const tempDir = os.tmpdir(); diff --git a/tests/integration/cli/http-era-negotiation.test.ts b/tests/integration/cli/http-era-negotiation.test.ts new file mode 100644 index 000000000..bf4c7b97d --- /dev/null +++ b/tests/integration/cli/http-era-negotiation.test.ts @@ -0,0 +1,146 @@ +/** + * The http transport speaks both protocol eras (issue #1025, step 2): a + * probing v2 client lands on 2026-07-28 and drives a full debug session with + * no MCP session at all; a client that opens with initialize gets the 2025 + * sessionful era as before. Real server (`dist/index.js http`), the v2 SDK + * client in each negotiation mode, the mock adapter β€” no toolchain. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { spawn, type ChildProcess } from 'node:child_process'; +import net, { type AddressInfo } from 'node:net'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'; +import { serverEnvironment } from '../../test-utils/helpers/mcp-stdio-client.js'; +import { waitForHealthEndpoint } from '../../e2e/smoke-test-utils.js'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const entry = path.join(root, 'dist/index.js'); +const MOCK_SCRIPT = path.join(root, 'examples/python/simple_test.py'); + +type Mode = 'auto' | 'legacy' | { pin: string }; + +interface Answer { + success: boolean; + sessionId?: string; + state?: string; + message?: string; + error?: string; +} + +let server: ChildProcess | undefined; +let serverOutput = ''; +const clients: Client[] = []; + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const probe = net.createServer(); + probe.on('error', reject); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address() as AddressInfo; + probe.close(() => resolve(port)); + }); + }); +} + +async function startServer(): Promise { + const port = await freePort(); + server = spawn(process.execPath, [entry, 'http', '-p', String(port)], { + cwd: root, + env: serverEnvironment({ scrub: ['MCP_HTTP_BIND'], exitOnStdinClose: false }), + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + server.stdout!.on('data', (chunk: Buffer) => { serverOutput += chunk.toString(); }); + server.stderr!.on('data', (chunk: Buffer) => { serverOutput += chunk.toString(); }); + expect(await waitForHealthEndpoint(port, 20_000), serverOutput).toBe(true); + return port; +} + +async function connect(port: number, mode: Mode, name: string): Promise { + const client = new Client({ name, version: '1.0.0' }, { versionNegotiation: { mode } }); + await client.connect(new StreamableHTTPClientTransport(new URL(`http://127.0.0.1:${port}/mcp`))); + clients.push(client); + return client; +} + +async function call(client: Client, name: string, args: Record = {}): Promise { + const raw = await client.callTool({ name, arguments: args }, { timeout: 30_000 }); + const text = (raw.content as Array<{ type: string; text?: string }>).find((item) => item.type === 'text')?.text; + expect(text, JSON.stringify(raw)).toBeDefined(); + return JSON.parse(text!) as Answer; +} + +async function health(port: number): Promise<{ sessions: string[]; debugSessions: Array<{ id: string }>; requests: { legacy: number; modern: number } }> { + const response = await fetch(`http://127.0.0.1:${port}/health`); + return (await response.json()) as { sessions: string[]; debugSessions: Array<{ id: string }>; requests: { legacy: number; modern: number } }; +} + +afterEach(async () => { + for (const client of clients.splice(0)) { + await client.close().catch(() => {}); + } + if (server && server.exitCode === null) { + const exited = new Promise((resolve) => server!.once('exit', () => resolve())); + server.kill(); + await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 5_000))]); + } + server = undefined; + serverOutput = ''; +}); + +describe('http serves both protocol eras (issue #1025, step 2)', () => { + it('a probing client lands on 2026-07-28 with no MCP session, and the server counts it as modern traffic', async () => { + const port = await startServer(); + const client = await connect(port, 'auto', 'era-auto'); + + expect(client.getProtocolEra()).toBe('modern'); + expect(client.getNegotiatedProtocolVersion()).toBe('2026-07-28'); + expect(client.getServerVersion()?.name).toBe('debug-mcp-server'); + expect((await client.listTools()).tools.length).toBe(29); + const h = await health(port); + expect(h.sessions).toEqual([]); + expect(h.requests.modern).toBeGreaterThan(0); + }, 60_000); + + it('a client pinned to 2026-07-28 drives a full debug session sessionlessly and hears list changes', async () => { + const port = await startServer(); + const client = await connect(port, { pin: '2026-07-28' }, 'era-pin'); + expect(client.getProtocolEra()).toBe('modern'); + + const changes: string[] = []; + client.setNotificationHandler('notifications/resources/list_changed', () => { changes.push('list_changed'); }); + const subscription = await client.listen({ resourcesListChanged: true }); + expect(subscription.honoredFilter.resourcesListChanged).toBe(true); + + const created = await call(client, 'create_debug_session', { language: 'mock', name: 'era-pin' }); + expect(created.success, JSON.stringify(created)).toBe(true); + const sessionId = created.sessionId!; + await expect.poll(() => changes.length, { timeout: 10_000 }).toBeGreaterThanOrEqual(1); + expect((await health(port)).debugSessions.map((s) => s.id)).toContain(sessionId); + + const bp = await call(client, 'set_breakpoint', { sessionId, file: MOCK_SCRIPT, line: 32 }); + expect(bp.success, JSON.stringify(bp)).toBe(true); + const started = await call(client, 'start_debugging', { sessionId, scriptPath: MOCK_SCRIPT, dapLaunchArgs: { stopOnEntry: true } }); + expect(started.success, JSON.stringify(started)).toBe(true); + const waited = await call(client, 'wait_for_stop', { sessionId, timeout: 15_000 }); + expect(waited.success, JSON.stringify(waited)).toBe(true); + expect(waited.state).toBe('paused'); + expect((await call(client, 'close_debug_session', { sessionId })).success).toBe(true); + + await subscription.close(); + }, 90_000); + + it('a client that opens with initialize gets the 2025-11-25 era and an MCP session, as before', async () => { + const port = await startServer(); + const client = await connect(port, 'legacy', 'era-legacy'); + + expect(client.getProtocolEra()).toBe('legacy'); + expect(client.getNegotiatedProtocolVersion()).toBe('2025-11-25'); + expect((await health(port)).sessions).toHaveLength(1); + const created = await call(client, 'create_debug_session', { language: 'mock', name: 'era-legacy' }); + expect(created.success).toBe(true); + await client.subscribeResource({ uri: `debug://sessions/${created.sessionId}/output` }); + expect((await call(client, 'close_debug_session', { sessionId: created.sessionId! })).success).toBe(true); + }, 60_000); +}); diff --git a/tests/integration/cli/http-host-allowlist.test.ts b/tests/integration/cli/http-host-allowlist.test.ts index 1a1221081..2a7b480b9 100644 --- a/tests/integration/cli/http-host-allowlist.test.ts +++ b/tests/integration/cli/http-host-allowlist.test.ts @@ -36,6 +36,7 @@ async function listen(options: { allowedHost?: string[] } = {}, env: Record () => undefined }, idlePolicy: { idleMs: 0, sweepMs: 60_000 }, sessionManager: { getAllSessions: () => [], getSession: () => undefined, idleMsOf: () => undefined } }; diff --git a/tests/integration/cli/http-spec-contract-2026.test.ts b/tests/integration/cli/http-spec-contract-2026.test.ts new file mode 100644 index 000000000..1c65c8982 --- /dev/null +++ b/tests/integration/cli/http-spec-contract-2026.test.ts @@ -0,0 +1,302 @@ +/** + * Streamable HTTP contract for the 2026-07-28 revision (issue #1025, step 2): + * the modern, sessionless era served beside the 2025 sessionful app on the + * same /mcp endpoint. One case per clause the SDK's modern handler answers on + * our behalf β€” a raw request per case, because the SDK client hides status + * codes and error bodies β€” plus the two routing facts this server decides + * itself: a 2025-shaped request still lands on the sessionful app, and the + * modern listen stream counts the process as attended. The 2025 clauses stay + * pinned by http-spec-contract.test.ts; both eras are served until the legacy + * one is retired. + * + * Spec: https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { spawn, type ChildProcess } from 'node:child_process'; +import * as http from 'node:http'; +import net, { type AddressInfo } from 'node:net'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { serverEnvironment } from '../../test-utils/helpers/mcp-stdio-client.js'; +import { waitForHealthEndpoint } from '../../e2e/smoke-test-utils.js'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const entry = path.join(root, 'dist/index.js'); + +const VERSION = '2026-07-28'; +const META = { + 'io.modelcontextprotocol/protocolVersion': VERSION, + 'io.modelcontextprotocol/clientInfo': { name: 'spec-contract-2026', version: '1.0.0' }, + 'io.modelcontextprotocol/clientCapabilities': {} +}; + +interface Reply { + status: number; + headers: http.IncomingHttpHeaders; + body: string; +} + +interface RunningServer { + port: number; + child: ChildProcess; + output: () => string; +} + +interface Health { + attended: boolean; + listenStreams: number; + sessions: string[]; + requests?: { legacy: number; modern: number }; +} + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const probe = net.createServer(); + probe.on('error', reject); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address() as AddressInfo; + probe.close(() => resolve(port)); + }); + }); +} + +async function startServer(): Promise { + const port = await freePort(); + let output = ''; + const child = spawn(process.execPath, [entry, 'http', '-p', String(port)], { + cwd: root, + env: serverEnvironment({ scrub: ['MCP_HTTP_BIND', 'MCP_HTTP_ALLOWED_HOSTS'], exitOnStdinClose: false }), + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + child.stdout!.on('data', (chunk: Buffer) => { output += chunk.toString(); }); + child.stderr!.on('data', (chunk: Buffer) => { output += chunk.toString(); }); + expect(await waitForHealthEndpoint(port, 20_000), output).toBe(true); + return { port, child, output: () => output }; +} + +async function stopServer(server: RunningServer | undefined): Promise { + if (!server || server.child.exitCode !== null) return; + const exited = new Promise((resolve) => server.child.once('exit', () => resolve())); + server.child.kill(); + await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 5_000))]); +} + +/** A modern-era request body: the method's params plus the per-request `_meta` envelope. */ +function envelope(id: number, method: string, params: Record = {}, meta: Record = META) { + return { jsonrpc: '2.0', id, method, params: { ...params, _meta: meta } }; +} + +/** The headers the 2026-07-28 revision requires on a POST, derived from the body unless overridden. */ +function modernHeaders( + body: { method: string; params?: { name?: string; uri?: string; [key: string]: unknown } }, + overrides: Record = {} +): Record { + const headers: Record = { + 'MCP-Protocol-Version': VERSION, + 'Mcp-Method': body.method, + ...(body.params?.name !== undefined ? { 'Mcp-Name': body.params.name } : {}), + ...(body.params?.uri !== undefined ? { 'Mcp-Name': body.params.uri } : {}), + ...overrides, + }; + return Object.fromEntries(Object.entries(headers).filter((entry): entry is [string, string] => entry[1] !== undefined)); +} + +function request( + port: number, + options: { method?: string; body?: unknown; headers?: Record; path?: string } = {} +): Promise { + const method = options.method ?? 'POST'; + const payload = options.body === undefined ? undefined : JSON.stringify(options.body); + const headers: Record = { + Accept: method === 'GET' ? 'text/event-stream' : 'application/json, text/event-stream', + ...(payload !== undefined ? { 'Content-Type': 'application/json', 'Content-Length': String(Buffer.byteLength(payload)) } : {}), + ...(options.headers ?? {}), + }; + return new Promise((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port, path: options.path ?? '/mcp', method, headers }, (res) => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => { body += chunk; }); + res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body })); + }); + req.on('error', reject); + if (payload !== undefined) req.write(payload); + req.end(); + }); +} + +/** The JSON-RPC message with `id` out of a JSON or SSE-framed reply. */ +function messageOf(reply: Reply, id: number): Record { + const contentType = String(reply.headers['content-type'] ?? ''); + if (contentType.startsWith('text/event-stream')) { + for (const line of reply.body.split('\n')) { + if (!line.startsWith('data:')) continue; + const parsed = JSON.parse(line.slice(5).trim()) as Record; + if (parsed.id === id) return parsed; + } + throw new Error(`no message with id ${id} in the SSE reply:\n${reply.body}`); + } + const parsed = JSON.parse(reply.body) as Record; + if (Array.isArray(parsed)) { + const found = parsed.find((message: { id?: number }) => message.id === id); + if (found) return found; + throw new Error(`no message with id ${id} in the batch reply:\n${reply.body}`); + } + return parsed; +} + +async function health(port: number): Promise { + const reply = await request(port, { method: 'GET', path: '/health', headers: { Accept: 'application/json' } }); + return JSON.parse(reply.body) as Health; +} + +async function healthUntil(port: number, predicate: (h: Health) => boolean, deadlineMs = 10_000): Promise { + const deadline = Date.now() + deadlineMs; + let last = await health(port); + while (!predicate(last) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)); + last = await health(port); + } + return last; +} + +describe('Streamable HTTP contract (spec 2026-07-28, served beside 2025-11-25)', () => { + let server: RunningServer | undefined; + + beforeAll(async () => { + server = await startServer(); + }, 30_000); + + afterAll(async () => { + await stopServer(server); + }); + + it('server/discover answers the supported versions, the capabilities and the server identity, with no session minted', async () => { + const body = envelope(1, 'server/discover'); + const reply = await request(server!.port, { body, headers: modernHeaders(body) }); + + expect(reply.status, reply.body).toBe(200); + expect(reply.headers['mcp-session-id']).toBeUndefined(); + const message = messageOf(reply, 1); + expect(message.result.supportedVersions).toContain(VERSION); + expect(message.result.resultType).toBe('complete'); + expect(message.result.capabilities).toMatchObject({ tools: {}, resources: { subscribe: true, listChanged: true }, prompts: {} }); + expect(message.result._meta?.['io.modelcontextprotocol/serverInfo']?.name).toBe('debug-mcp-server'); + }); + + it('tools/list carries the cache fields and a deterministic order', async () => { + const body = envelope(2, 'tools/list'); + const first = messageOf(await request(server!.port, { body, headers: modernHeaders(body) }), 2); + const second = messageOf(await request(server!.port, { body, headers: modernHeaders(body) }), 2); + + expect(first.result.tools).toHaveLength(29); + expect(typeof first.result.ttlMs).toBe('number'); + expect(['public', 'private']).toContain(first.result.cacheScope); + expect(first.result.resultType).toBe('complete'); + expect(second.result.tools.map((tool: { name: string }) => tool.name)).toEqual(first.result.tools.map((tool: { name: string }) => tool.name)); + }); + + it('a modern request carrying a stale Mcp-Session-Id is served statelessly, the header ignored', async () => { + const body = envelope(3, 'tools/list'); + const reply = await request(server!.port, { body, headers: { ...modernHeaders(body), 'Mcp-Session-Id': 'stale-from-another-life' } }); + + expect(reply.status, reply.body).toBe(200); + expect(messageOf(reply, 3).result.tools).toHaveLength(29); + expect(reply.headers['mcp-session-id']).toBeUndefined(); + }); + + it('a modern protocol header without the per-request envelope is refused with -32602', async () => { + const body = { jsonrpc: '2.0', id: 4, method: 'tools/list', params: {} }; + const reply = await request(server!.port, { body, headers: modernHeaders(body) }); + + expect(reply.status, reply.body).toBe(400); + expect(JSON.parse(reply.body).error.code).toBe(-32602); + }); + + it('a modern request missing the Mcp-Method header is refused with -32020 (HeaderMismatch)', async () => { + const body = envelope(5, 'tools/list'); + const reply = await request(server!.port, { body, headers: modernHeaders(body, { 'Mcp-Method': undefined }) }); + + expect(reply.status, reply.body).toBe(400); + expect(JSON.parse(reply.body).error.code).toBe(-32020); + }); + + it('an unsupported claimed revision is refused with -32022 naming the supported ones', async () => { + const body = envelope(6, 'tools/list', {}, { ...META, 'io.modelcontextprotocol/protocolVersion': '2099-01-01' }); + const reply = await request(server!.port, { body, headers: modernHeaders(body, { 'MCP-Protocol-Version': '2099-01-01' }) }); + + expect(reply.status, reply.body).toBe(400); + const error = JSON.parse(reply.body).error; + expect(error.code).toBe(-32022); + expect(error.data?.supported).toContain(VERSION); + }); + + it('a 2025 initialize on the same endpoint still opens a sessionful MCP session (both eras served)', async () => { + const reply = await request(server!.port, { + body: { + jsonrpc: '2.0', + id: 7, + method: 'initialize', + params: { protocolVersion: '2025-11-25', capabilities: {}, clientInfo: { name: 'spec-contract-2026', version: '1.0.0' } }, + }, + }); + + expect(reply.status, reply.body).toBe(200); + const sessionId = reply.headers['mcp-session-id']; + expect(typeof sessionId).toBe('string'); + expect((await health(server!.port)).sessions).toContain(sessionId); + await request(server!.port, { method: 'DELETE', headers: { 'Mcp-Session-Id': sessionId as string } }); + }); + + it('a session-less 2025-shaped request still lands on the sessionful app and its 400 (unchanged)', async () => { + const reply = await request(server!.port, { body: { jsonrpc: '2.0', id: 8, method: 'tools/list' } }); + + expect(reply.status, reply.body).toBe(400); + expect(JSON.parse(reply.body).error.code).toBe(-32600); + const get = await request(server!.port, { method: 'GET' }); + expect(get.status).toBe(400); + }); + + it('an open subscriptions/listen stream counts the process as attended, and its close as unattended', async () => { + const before = await health(server!.port); + expect(before.listenStreams).toBe(0); + + const body = envelope(9, 'subscriptions/listen', { notifications: { resourcesListChanged: true } }); + const payload = JSON.stringify(body); + const headers = { + Accept: 'application/json, text/event-stream', + 'Content-Type': 'application/json', + 'Content-Length': String(Buffer.byteLength(payload)), + ...modernHeaders(body), + }; + const firstEvent = new Promise((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port: server!.port, path: '/mcp', method: 'POST', headers }, (res) => { + expect(res.statusCode, 'listen must open a stream').toBe(200); + res.setEncoding('utf8'); + let buffer = ''; + res.on('data', (chunk: string) => { + buffer += chunk; + if (buffer.includes('\n\n')) { + resolve(buffer); + // Hold the stream open a moment longer, then drop it: cancellation. + setTimeout(() => req.destroy(), 1_500); + } + }); + }); + req.on('error', reject); + req.write(payload); + req.end(); + }); + + const opened = await firstEvent; + expect(opened).toContain('notifications/subscriptions/acknowledged'); + const during = await healthUntil(server!.port, (h) => h.listenStreams === 1); + expect(during.listenStreams, JSON.stringify(during)).toBe(1); + expect(during.attended).toBe(true); + + const after = await healthUntil(server!.port, (h) => h.listenStreams === 0); + expect(after.listenStreams, JSON.stringify(after)).toBe(0); + expect(after.attended).toBe(false); + }, 30_000); +}); diff --git a/tests/unit/cli/http-command.test.ts b/tests/unit/cli/http-command.test.ts index ce369ab91..6b778e289 100644 --- a/tests/unit/cli/http-command.test.ts +++ b/tests/unit/cli/http-command.test.ts @@ -11,6 +11,14 @@ import type { createProtocolServer } from '../../../src/server/protocol-facade.j vi.mock('../../../src/server.js'); vi.mock('@modelcontextprotocol/node'); +// The 2026-07-28 leg (issue #1025, step 2): the SDK's modern handler and its +// classifier are doubles here; the handler's own behaviour is pinned over the +// wire by tests/integration/cli/http-spec-contract-2026.test.ts. +vi.mock('@modelcontextprotocol/server', async (importOriginal) => ({ + ...(await importOriginal()), + createMcpHandler: vi.fn(), + isLegacyRequest: vi.fn(), +})); // The app is built with express() directly (issues #667/#670): the default // export hands back the per-test mockApp, and express.json() returns one @@ -24,9 +32,14 @@ const expressMock = vi.hoisted(() => { }); vi.mock('express', () => ({ default: expressMock.express })); -import { NodeStreamableHTTPServerTransport } from '@modelcontextprotocol/node'; +import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from '@modelcontextprotocol/node'; +import { createMcpHandler, isLegacyRequest } from '@modelcontextprotocol/server'; const MockedStreamableHTTPServerTransport = vi.mocked(NodeStreamableHTTPServerTransport); +const mockCreateMcpHandler = vi.mocked(createMcpHandler); +const mockIsLegacyRequest = vi.mocked(isLegacyRequest); +const mockToNodeHandler = vi.mocked(toNodeHandler); +const mockToWebRequest = vi.mocked(toWebRequest); /** Drive a Host-checking middleware with one request; returns what it did. */ function runHost(middleware: (req: any, res: any, next: any) => void, host: string) { @@ -41,6 +54,8 @@ describe('HTTP Command Handler', () => { let mockTransport: any; let mockFacade: { connect: Mock<() => Promise> }; let mockProtocolServerFactory: Mock; + let mockModernHandler: { fetch: Mock; close: Mock<() => Promise>; notify: { resourceUpdated: Mock; resourcesChanged: Mock } }; + let mockModernNode: Mock<(req: unknown, res: unknown, body?: unknown) => Promise>; let fakeProc: FakeCurrentProcess; let mockApp: any; @@ -67,6 +82,7 @@ describe('HTTP Command Handler', () => { start: vi.fn().mockResolvedValue(undefined), stop: vi.fn().mockResolvedValue(undefined), setAttendedSupplier: vi.fn(), + notifications: { addSink: vi.fn(() => () => undefined) }, idlePolicy: { idleMs: 1_800_000, sweepMs: 60_000 }, sessionManager: { getAllSessions: vi.fn().mockReturnValue([]), @@ -76,6 +92,22 @@ describe('HTTP Command Handler', () => { } as any; mockFacade = { connect: vi.fn<() => Promise>().mockResolvedValue(undefined) }; mockProtocolServerFactory = vi.fn(() => mockFacade as any); + // The modern leg's doubles. Every request classifies as legacy unless a + // test says otherwise, so the sessionful cases below run exactly as before. + mockModernHandler = { + fetch: vi.fn(), + close: vi.fn<() => Promise>().mockResolvedValue(undefined), + notify: { resourceUpdated: vi.fn(), resourcesChanged: vi.fn() }, + }; + mockModernNode = vi.fn<(req: unknown, res: unknown, body?: unknown) => Promise>().mockResolvedValue(undefined); + mockCreateMcpHandler.mockReset(); + mockCreateMcpHandler.mockReturnValue(mockModernHandler as any); + mockToNodeHandler.mockReset(); + mockToNodeHandler.mockReturnValue(mockModernNode as any); + mockToWebRequest.mockReset(); + mockToWebRequest.mockResolvedValue({} as Request); + mockIsLegacyRequest.mockReset(); + mockIsLegacyRequest.mockResolvedValue(true); mockServerFactory = vi.fn<(options: ServerFactoryOptions) => DebugMcpServer>().mockReturnValue(mockServer); mockExitProcess = vi.fn<(code: number) => void>(); @@ -576,6 +608,154 @@ describe('HTTP Command Handler', () => { }); }); + describe('the 2026-07-28 leg beside the sessionful app (issue #1025, step 2)', () => { + // The modern handler is the SDK's; this suite pins only the router split + // and the bookkeeping around it. The handler's own behaviour is pinned over + // the wire by tests/integration/cli/http-spec-contract-2026.test.ts. + const MODERN_BODY = { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: { 'io.modelcontextprotocol/protocolVersion': '2026-07-28' } }, + }; + const INIT_BODY = { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { protocolVersion: '2025-11-25', capabilities: {}, clientInfo: { name: 'c', version: '1' } }, + }; + + function makeStreamRes() { + const res = new EventEmitter() as any; + res.status = vi.fn().mockReturnThis(); + res.json = vi.fn(); + res.end = vi.fn(); + res.headersSent = false; + return res; + } + + function makeReq(overrides: Partial<{ method: string; headers: any; body: any }> = {}) { + return { method: 'POST', headers: {}, body: undefined, ...overrides }; + } + + function makeRes() { + return { status: vi.fn().mockReturnThis(), json: vi.fn(), end: vi.fn(), headersSent: false }; + } + + function appAndHandler() { + const app = createHttpApp( + { port: '3001' }, + { logger: mockLogger, serverFactory: mockServerFactory, protocolServerFactory: mockProtocolServerFactory, proc: fakeProc } + ); + const postCall = mockApp.post.mock.calls.find((c: any) => c[0] === '/mcp'); + const healthCall = mockApp.get.mock.calls.find((c: any) => c[0] === '/health'); + return { app, handler: postCall![1] as (req: any, res: any) => Promise, healthHandler: healthCall![1] as (req: any, res: any) => void }; + } + + function healthBody(healthHandler: (req: any, res: any) => void) { + const res = { json: vi.fn() }; + healthHandler({}, res); + return res.json.mock.calls[0][0]; + } + + it('builds one strict modern handler per app over per-request facades, and feeds its notifier from the hub', () => { + appAndHandler(); + + expect(mockCreateMcpHandler).toHaveBeenCalledTimes(1); + const [factory, options] = mockCreateMcpHandler.mock.calls[0]; + expect(options).toMatchObject({ legacy: 'reject', maxRequestBodySize: 10 * 1024 * 1024 }); + expect(typeof options?.onerror).toBe('function'); + expect(factory({ era: 'modern' })).toBe(mockFacade); + expect(mockProtocolServerFactory).toHaveBeenLastCalledWith(mockServer, { era: 'modern', deliver: 'external' }); + expect(mockToNodeHandler).toHaveBeenCalledWith(mockModernHandler, expect.objectContaining({ onerror: expect.any(Function) })); + + // The hub's sink for this leg is the handler's notifier. + expect(mockServer.notifications.addSink).toHaveBeenCalledTimes(1); + const sink = vi.mocked(mockServer.notifications.addSink).mock.calls[0][0]; + sink.resourceUpdated('debug://sessions/s/output'); + sink.resourcesChanged(); + expect(mockModernHandler.notify.resourceUpdated).toHaveBeenCalledWith('debug://sessions/s/output'); + expect(mockModernHandler.notify.resourcesChanged).toHaveBeenCalledTimes(1); + }); + + it('routes a request the classifier calls modern to the SDK handler, never to the session branch', async () => { + const { app, handler, healthHandler } = appAndHandler(); + mockIsLegacyRequest.mockResolvedValue(false); + const req = makeReq({ body: MODERN_BODY }); + const res = makeRes(); + + await handler(req, res); + + expect(mockToWebRequest).toHaveBeenCalledWith(req, MODERN_BODY, { maxRequestBodySize: 10 * 1024 * 1024 }); + expect(mockIsLegacyRequest).toHaveBeenCalledWith(expect.anything(), MODERN_BODY, { maxRequestBodySize: 10 * 1024 * 1024 }); + expect(mockModernNode).toHaveBeenCalledWith(req, res, MODERN_BODY); + expect(MockedStreamableHTTPServerTransport).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect((app as any).httpSessions.size).toBe(0); + expect(healthBody(healthHandler).requests).toEqual({ legacy: 0, modern: 1 }); + }); + + it('routes a request the classifier calls legacy to the sessionful app, untouched', async () => { + const { app, handler, healthHandler } = appAndHandler(); + mockIsLegacyRequest.mockResolvedValue(true); + + await handler(makeReq({ body: INIT_BODY }), makeRes()); + mockTransport.triggerSessionInit(); + + expect(mockModernNode).not.toHaveBeenCalled(); + expect(MockedStreamableHTTPServerTransport).toHaveBeenCalledTimes(1); + expect((app as any).httpSessions.size).toBe(1); + expect(healthBody(healthHandler).requests).toEqual({ legacy: 1, modern: 0 }); + }); + + it('classifies a GET or DELETE without a body', async () => { + const { handler } = appAndHandler(); + mockIsLegacyRequest.mockResolvedValue(true); + const req = makeReq({ method: 'GET', headers: { 'mcp-session-id': 'nope' }, body: undefined }); + + await handler(req, makeRes()); + + expect(mockToWebRequest).toHaveBeenCalledWith(req, undefined, { maxRequestBodySize: 10 * 1024 * 1024 }); + expect(mockIsLegacyRequest).toHaveBeenCalledWith(expect.anything(), undefined, { maxRequestBodySize: 10 * 1024 * 1024 }); + }); + + it('counts an open modern subscriptions/listen stream as attendance until its response closes', async () => { + const { handler, healthHandler } = appAndHandler(); + mockIsLegacyRequest.mockResolvedValue(false); + const attended = vi.mocked(mockServer.setAttendedSupplier).mock.calls[0][0] as () => boolean; + let release: () => void = () => undefined; + mockModernNode.mockImplementation(() => new Promise((resolve) => { release = resolve; })); + const res = makeStreamRes(); + const listen = { ...MODERN_BODY, method: 'subscriptions/listen', params: { ...MODERN_BODY.params, notifications: { resourcesListChanged: true } } }; + + const pending = handler(makeReq({ body: listen }), res); + // The classifier is asynchronous: the count lands once it has answered. + await vi.waitFor(() => expect(attended()).toBe(true)); + expect(healthBody(healthHandler)).toMatchObject({ attended: true, listenStreams: 1 }); + + res.emit('close'); + release(); + await pending; + expect(attended()).toBe(false); + expect(healthBody(healthHandler)).toMatchObject({ attended: false, listenStreams: 0 }); + }); + + it('allows the 2026 request headers through CORS and echoes a preflight\'s requested headers', () => { + appAndHandler(); + const corsMiddleware = mockApp.use.mock.calls[1][0]; + const headers = new Map(); + const res = { header: vi.fn((name: string, value: string) => headers.set(name.toLowerCase(), value)), sendStatus: vi.fn() }; + + corsMiddleware({ method: 'OPTIONS', headers: { 'access-control-request-headers': 'Mcp-Param-Region, X-Custom' } }, res, vi.fn()); + + const allowed = headers.get('access-control-allow-headers')?.toLowerCase() ?? ''; + for (const name of ['mcp-method', 'mcp-name', 'mcp-protocol-version', 'mcp-session-id', 'accept', 'content-type', 'mcp-param-region', 'x-custom']) { + expect(allowed, name).toContain(name); + } + expect(res.sendStatus).toHaveBeenCalledWith(200); + }); + }); + describe('/health endpoint', () => { it('reports the address and port the server listens on (issue #680)', () => { createHttpApp( @@ -605,6 +785,7 @@ describe('HTTP Command Handler', () => { connections: 0, attended: false, listenStreams: 0, + requests: { legacy: 0, modern: 0 }, sessions: [], details: [], debugSessions: [], @@ -1286,7 +1467,10 @@ describe('HTTP Command Handler', () => { expect(t1.close).toHaveBeenCalled(); expect(t2.close).toHaveBeenCalled(); expect(mockServer.stop).toHaveBeenCalledTimes(1); - // Transports first, so no request lands on a core that is closing its sessions. + // The modern handler first (its listen streams end gracefully), then the + // transports, so no request lands on a core that is closing its sessions. + expect(mockModernHandler.close).toHaveBeenCalledTimes(1); + expect(mockModernHandler.close.mock.invocationCallOrder[0]).toBeLessThan(t1.close.mock.invocationCallOrder[0]); expect(vi.mocked(mockServer.stop).mock.invocationCallOrder[0]).toBeGreaterThan(t2.close.mock.invocationCallOrder[0]); expect(mockHttpServer.close).toHaveBeenCalled(); expect(mockExitProcess).toHaveBeenCalledWith(0); diff --git a/tests/unit/scripts/mcp-conformance.test.ts b/tests/unit/scripts/mcp-conformance.test.ts index 7143c9130..e1e1df6bb 100644 --- a/tests/unit/scripts/mcp-conformance.test.ts +++ b/tests/unit/scripts/mcp-conformance.test.ts @@ -1,5 +1,40 @@ -import { describe, expect, it } from 'vitest'; -import { APPLICABLE_SCENARIOS, judgeScenario, parseSummary } from '../../../scripts/mcp-conformance.mjs'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + APPLICABLE_SCENARIOS, + collectChecks, + failingChecks, + judgeRequirements, + judgeScenario, + parseBaseline, + parseRequirementsSummary, + parseSummary +} from '../../../scripts/mcp-conformance.mjs'; + +const REQUIREMENTS_OUTPUT = [ + 'Running requirements 2026-07-28 (50 scenarios) against http://127.0.0.1:3997/mcp', + '=== SUMMARY ===', + 'βœ— server-stateless: 24 passed, 4 failed', + 'βœ“ tools-list: 4 passed, 0 failed', + 'βœ— tools-call-simple-text: 1 passed, 1 failed', + 'βœ“ caching: 7 passed, 0 failed', + 'βœ— tasks-lifecycle: 1 passed, 8 failed', + '', + 'Total: 37 passed, 13 failed', + '', + 'Not scored for 2026-07-28: 1 scenario(s) run, 1 failing. These do not affect conformance.', + ' βœ— tasks-lifecycle (extension)', +].join('\r\n'); + +const BASELINE = [ + '# fixtures this server does not have', + 'server:', + ' - tools-call-simple-text # tools/call test_simple_text', + ' - server-stateless:ServerRejectsUndeclaredCapability # test_missing_capability', + ' - server-stateless:ServerNoLogWithoutLogLevel', +].join('\n'); const PASSING = [ "Running client scenario 'ping' against server: http://127.0.0.1:3919/mcp", @@ -36,3 +71,97 @@ describe('mcp-conformance runner: reading the CLI summary', () => { expect(APPLICABLE_SCENARIOS).toContain('dns-rebinding-protection'); }); }); + +describe('mcp-conformance runner: the 2026-07-28 requirements lane (issue #1025)', () => { + it('reads the per-scenario verdict lines and the not-scored list, CRLF included', () => { + const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; + expect(summary.scenarios).toEqual([ + { name: 'server-stateless', passed: 24, failed: 4 }, + { name: 'tools-list', passed: 4, failed: 0 }, + { name: 'tools-call-simple-text', passed: 1, failed: 1 }, + { name: 'caching', passed: 7, failed: 0 }, + { name: 'tasks-lifecycle', passed: 1, failed: 8 }, + ]); + expect([...summary.notScored]).toEqual(['tasks-lifecycle']); + expect(parseRequirementsSummary('Running requirements ...\nSegmentation fault')).toBeUndefined(); + }); + + it('reads whole-scenario and per-check entries out of the baseline, comments stripped', () => { + const baseline = parseBaseline(BASELINE); + expect(baseline.get('tools-call-simple-text')).toBe('all'); + expect(baseline.get('server-stateless')).toEqual(new Set(['ServerRejectsUndeclaredCapability', 'ServerNoLogWithoutLogLevel'])); + expect(baseline.has('fixtures')).toBe(false); + }); + + it('names the failing checks of a checks.json, whichever shape the CLI wrote; a SKIPPED check is not a failure', () => { + const checks = [{ name: 'A', status: 'SUCCESS' }, { name: 'B', status: 'FAILURE' }, { name: 'C', status: 'SKIPPED' }]; + expect(failingChecks(checks)).toEqual(['B']); + expect(failingChecks({ checks })).toEqual(['B']); + expect(failingChecks(undefined)).toEqual([]); + }); + + describe('collecting the per-scenario checks.json files', () => { + const dirs: string[] = []; + afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + it('keys each checks.json by its scenario, read out of the CLI folder name server--', () => { + // Measured 2026-11-10 against 0.2.0-alpha.12: `--output-dir` holds one folder per scenario, + // `server-server-stateless-2026-11-10T02-36-40-817Z/checks.json` (colons and dots of the + // ISO timestamp replaced by dashes); a scenario name itself carries dashes. + const dir = mkdtempSync(path.join(os.tmpdir(), 'mcp-conformance-test-')); + dirs.push(dir); + const folder = path.join(dir, 'server-server-stateless-2026-11-10T02-36-40-817Z'); + mkdirSync(folder); + writeFileSync(path.join(folder, 'checks.json'), JSON.stringify([ + { id: 'sep-2575-a', name: 'ServerRejectsUndeclaredCapability', status: 'FAILURE' }, + { id: 'sep-2575-b', name: 'ServerSendsPromptsListChangedOnSubscription', status: 'SKIPPED' }, + { id: 'sep-2575-c', name: 'StatelessNoSessionId', status: 'SUCCESS' }, + ])); + const other = path.join(dir, 'server-tools-list-2026-11-10T02-36-41-000Z'); + mkdirSync(other); + writeFileSync(path.join(other, 'checks.json'), JSON.stringify([{ name: 'ToolsListOk', status: 'SUCCESS' }])); + + const byScenario = collectChecks(dir); + expect(byScenario.get('server-stateless')).toEqual(['ServerRejectsUndeclaredCapability']); + expect(byScenario.get('tools-list')).toEqual([]); + expect(collectChecks(path.join(dir, 'does-not-exist')).size).toBe(0); + }); + }); + + it('cannot judge a per-check entry without the scenario\'s checks.json, and says so instead of calling it passed', () => { + const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; + const problems = judgeRequirements(summary, parseBaseline(BASELINE), () => []); + expect(problems).toEqual([ + expect.stringContaining('server-stateless: 4 failed check(s) but no checks.json was written for it'), + ]); + expect(problems.join('\n')).not.toContain('listed as an expected failure but it passed'); + }); + + it('passes when every failure is expected and every expectation still fails', () => { + const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; + const baseline = parseBaseline(BASELINE); + const checks = new Map([['server-stateless', ['ServerRejectsUndeclaredCapability', 'ServerNoLogWithoutLogLevel']]]); + // Two of the four server-stateless failures are listed by check; the other two are not. + expect(judgeRequirements(summary, baseline, (name) => checks.get(name) ?? [])).toEqual([]); + }); + + it('fails on an unlisted failure, a listed check that does not fail, and a stale whole-scenario entry', () => { + const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; + const checks = new Map([['server-stateless', ['ServerRejectsUndeclaredCapability', 'SomethingNew']]]); + const problems = judgeRequirements( + summary, + parseBaseline(['server:', ' - caching', ' - server-stateless:ServerRejectsUndeclaredCapability', ' - server-stateless:ServerNoLogWithoutLogLevel'].join('\n')), + (name) => checks.get(name) ?? [] + ); + expect(problems).toEqual(expect.arrayContaining([ + expect.stringContaining('server-stateless: 4 failed check(s) not in the baseline (SomethingNew)'), + expect.stringContaining('server-stateless:ServerNoLogWithoutLogLevel: listed as an expected failure but it passed'), + expect.stringContaining('tools-call-simple-text: 1 failed check(s) not in the baseline'), + expect.stringContaining('caching: listed as an expected failure but it passed'), + ])); + // A not-scored scenario never counts, listed or not. + expect(problems.join('\n')).not.toContain('tasks-lifecycle'); + }); +}); From 6b2b82db36a739af3f65e6507354dd274d4711ac Mon Sep 17 00:00:00 2001 From: JF Date: Mon, 9 Nov 2026 22:31:08 -0500 Subject: [PATCH 2/6] fix(conformance): reconcile the 2026 lane's summary with checks.json, read every verdict line, and state the modern-era attendance rule (#1025) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the review of PR #1117. The judge never reconciled the CLI's per-scenario summary with the checks.json rows: a scenario counted at four failures whose checks.json named two, both in the baseline, passed the lane β€” and the unit fixture pinned that as desired. A scenario's failed count must now equal the FAILURE rows its checks.json names, or the lane fails naming both. The summary parser matched scenario names against [A-Za-z0-9-] and dropped anything else silently, so a scenario the suite adds with an underscore or a dot would fail on every run and never be judged. The name is now any non-blank, non-colon run; a verdict line in the summary block the parser cannot read is itself a failure; the header's announced scenario count must match the lines reported; a baseline entry naming a scenario the revision does not score is flagged stale. The CLI runs with NO_COLOR=1/FORCE_COLOR=0 so a coloured summary cannot break the parser. The README said nothing expires while any client holds its SSE stream; a 2026-07-28 client attends only while it holds a subscriptions/listen stream, and a 2026 client that only calls tools keeps its sessions alive by tool activity alone. README and the changelog fragment now say so. Also: toNodeHandler receives the same 10 MiB maxRequestBodySize as the other SDK entries (pinned); the 2026 contract test kills its server child when the health wait fails; both new integration files take the tool count from TOOL_NAMES; `pnpm test:conformance:2026` runs the new lane; the script header carries the measured note on the alpha's tools-list; the fragment's header-rule wording and the /health.requests description are corrected. Co-Authored-By: Claude Fable 5.1 --- README.md | 7 ++- changelog.d/1025-protocol-2026-http.added.md | 2 +- docs/architecture/testing-architecture.md | 2 +- package.json | 1 + scripts/mcp-conformance.d.mts | 4 ++ scripts/mcp-conformance.mjs | 50 ++++++++++++--- src/cli/http-command.ts | 3 + .../cli/http-era-negotiation.test.ts | 3 +- .../cli/http-spec-contract-2026.test.ts | 10 ++- tests/unit/cli/http-command.test.ts | 7 ++- tests/unit/scripts/mcp-conformance.test.ts | 62 ++++++++++++++++++- 11 files changed, 130 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f94eaa55b..2192f57b6 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,7 @@ The endpoint speaks both eras of the protocol: a client that probes with `server Code's v2 runtime does) gets the current 2026-07-28 revision, sessionless β€” every request stands alone, resource notifications flow over `subscriptions/listen` streams β€” and a client that opens with `initialize` gets the 2025 sessionful era, `Mcp-Session-Id` and all, exactly as before. -`GET /health` counts the requests served per era under `requests`. +`GET /health` counts the requests classified per era under `requests` (refused ones included). One `http` process is one debugger: every MCP client of it sees the same debug sessions, and a session is addressable by whoever holds its id (the handle is the capability β€” run one process @@ -236,7 +236,10 @@ per agent if agents must not see each other's work). A debug session outlives th that created it, so a client that reconnects can list and resume or close what it left behind. What ends an abandoned one is the idle policy: a session tree that no tool call has named for 30 minutes (`MCP_DEBUG_SESSION_IDLE_MS`; `0` disables) is closed, unless a client is attending -the process β€” while any client holds its SSE stream open, nothing expires. Abandoned MCP +the process β€” while any client holds a stream open (a 2025 session's SSE stream, a 2026-07-28 +client's `subscriptions/listen` stream), nothing expires. A 2026 client that only calls tools is +not attending: its sessions live by tool activity (and an in-flight `wait_for_stop`) alone, so +raise or disable the window if such a client pauses for long. Abandoned MCP sessions themselves are reaped separately (`MCP_HTTP_STREAM_LOST_SESSION_MS`, `MCP_HTTP_STALE_SESSION_MS`). `GET /health` lists the MCP sessions, the debug sessions with their idle time, and whether anyone is attending. diff --git a/changelog.d/1025-protocol-2026-http.added.md b/changelog.d/1025-protocol-2026-http.added.md index 18f057bac..d7c70da3c 100644 --- a/changelog.d/1025-protocol-2026-http.added.md +++ b/changelog.d/1025-protocol-2026-http.added.md @@ -1 +1 @@ -**`http` speaks the current MCP protocol revision, 2026-07-28, beside 2025-11-25 and earlier β€” on the same `/mcp` endpoint.** A request carrying the 2026 per-request envelope (`_meta` with `io.modelcontextprotocol/protocolVersion`) is served by the SDK's sessionless handler: `server/discover` answers the supported versions and capabilities, every request stands alone with no `Mcp-Session-Id`, results carry `resultType` and the cache fields, resource notifications flow over `subscriptions/listen` streams, and the 2026 header rules apply (`MCP-Protocol-Version`, `Mcp-Method`, `Mcp-Name` required on a POST; a missing one is refused with `-32020`, an unsupported claimed revision with `-32022` naming the supported ones). A 2025-shaped request β€” `initialize`, a request carrying `Mcp-Session-Id`, a GET or DELETE, a claim-less POST β€” lands on the sessionful app exactly as before, 400/404 bodies included, so every 2025 client keeps working. A probing client such as Claude Code's v2 runtime therefore lands on 2026-07-28 against `http` too. An open `subscriptions/listen` stream counts the process as attended for the idle policy, the same as a 2025 session's SSE stream; `GET /health` reports `listenStreams` and the requests served per era (`requests.legacy` / `requests.modern`). Browser clients: CORS now allows `Mcp-Method`, `Mcp-Name` and echoes a preflight's requested headers. Step 2 of #1025 completes the protocol move on both transports. +**`http` speaks the current MCP protocol revision, 2026-07-28, beside 2025-11-25 and earlier β€” on the same `/mcp` endpoint.** A request carrying the 2026 per-request envelope (`_meta` with `io.modelcontextprotocol/protocolVersion`) is served by the SDK's sessionless handler: `server/discover` answers the supported versions and capabilities, every request stands alone with no `Mcp-Session-Id`, results carry `resultType` and the cache fields, resource notifications flow over `subscriptions/listen` streams, and the 2026 header rules apply (`MCP-Protocol-Version` and `Mcp-Method` on every POST, `Mcp-Name` on the methods that name a tool, prompt or resource; a missing or mismatched one is refused with `-32020`, an unsupported claimed revision with `-32022` naming the supported ones). A 2025-shaped request β€” `initialize`, a request carrying `Mcp-Session-Id` and no envelope, a GET or DELETE, a claim-less POST β€” lands on the sessionful app exactly as before, status codes included, so every 2025 client keeps working. An open `subscriptions/listen` stream counts the process as attended for the idle policy, like a 2025 session's SSE stream; a 2026 client that only calls tools is not attending, and its sessions live by tool activity alone. A probing client such as Claude Code's v2 runtime therefore lands on 2026-07-28 against `http` too. `GET /health` reports `listenStreams` and the requests classified per era (`requests.legacy` / `requests.modern`). Browser clients: CORS now allows `Mcp-Method`, `Mcp-Name` and echoes a preflight's requested headers. Step 2 of #1025 completes the protocol move on both transports. diff --git a/docs/architecture/testing-architecture.md b/docs/architecture/testing-architecture.md index f1adb7c10..75410e149 100644 --- a/docs/architecture/testing-architecture.md +++ b/docs/architecture/testing-architecture.md @@ -274,7 +274,7 @@ Protocol-level correctness checks. `breakpoint-messages/` contains Python script Two tiers, split along the seam between this server's own code and the SDK, and two eras, served side by side on the same endpoint (issue #1025, step 2). The 2025-11-25 clauses of the Streamable HTTP transport that `createHttpApp` decides before it hands a request to `transport.handleRequest` β€” session routing, the Host/Origin allowlist, the stale-session reaper β€” are pinned one case per clause against the real server over raw HTTP: a missing `Mcp-Session-Id` is 400; an unknown, DELETEd or reaped one is 404 `Session not found`; an invalid `Origin` is 403; an unsupported `MCP-Protocol-Version` is 400. Raw HTTP because the SDK client folds every non-2xx answer into a thrown error, and a case about a status code has to read it. The 404 cases are the ones #1017 fixed; nothing in the suite could have caught the 400 they replaced. The 2026-07-28 file pins the same way what the SDK's sessionless handler answers once our router (`isLegacyRequest`) has sent a modern request to it: `server/discover` with no session minted, `tools/list` with its cache fields and a stable order, a stale `Mcp-Session-Id` ignored, the `-32602` / `-32020` / `-32022` refusals, a 2025 `initialize` still opening a session on the same endpoint, a claim-less POST still landing on the sessionful app's 400, and an open `subscriptions/listen` stream counting the process as attended. The two negotiation files drive the v2 SDK client in `auto`, pinned and `legacy` mode through a full mock debug session on each transport, so what a real client experiences on either era is pinned too. -Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance`) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” under an expected-failures baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) that lists each scenario needing a fixture this server does not have (the `tools-call-*`, `prompts-get-*`, `resources-read-*`, `completion` and most `input-required-result-*` scenarios drive the suite's reference tools) with the fixture it needs; the lane fails on an unlisted failure and on a listed scenario that starts passing, so the baseline can only shrink honestly. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned, and on Windows their exit codes are unusable (they abort at exit with a libuv assertion even after a pass), so the script reads the printed summaries instead. +Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance` for the 2025-11-25 lane, `pnpm test:conformance:2026` for the 2026-07-28 one) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” under an expected-failures baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) that lists each scenario needing a fixture this server does not have (the `tools-call-*`, `prompts-get-*`, `resources-read-*`, `completion` and most `input-required-result-*` scenarios drive the suite's reference tools) with the fixture it needs; the lane fails on an unlisted failure and on a listed scenario that starts passing, so the baseline can only shrink honestly. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned, and on Windows their exit codes are unusable (they abort at exit with a libuv assertion even after a pass), so the script reads the printed summaries instead. ## Coverage Strategy diff --git a/package.json b/package.json index fe52e9a6a..4f2caec23 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "bench:mem:docker": "node scripts/mem-bench.mjs --target docker --scenario all", "test:integration": "vitest run --project integration", "test:conformance": "node scripts/mcp-conformance.mjs", + "test:conformance:2026": "node scripts/mcp-conformance.mjs --era 2026-07-28", "posttest:integration": "node scripts/cleanup-test-processes.js", "test:e2e": "pnpm run pretest:docker && vitest run --project e2e", "pretest:e2e": "pnpm run build", diff --git a/scripts/mcp-conformance.d.mts b/scripts/mcp-conformance.d.mts index d44551172..863907ec6 100644 --- a/scripts/mcp-conformance.d.mts +++ b/scripts/mcp-conformance.d.mts @@ -32,6 +32,10 @@ export interface RequirementsSummary { scenarios: RequirementsScenario[]; /** Scenarios the revision lists as not scored (extensions and pending ones). */ notScored: Set; + /** The scenario count the run's header announced, when it printed one. */ + announced?: number; + /** Verdict lines in the summary block the parser could not read; the judge fails on each. */ + unparsed: string[]; } /** A whole scenario expected to fail (`'all'`), or the names of its checks that are. */ export type ExpectedFailures = Map>; diff --git a/scripts/mcp-conformance.mjs b/scripts/mcp-conformance.mjs index 7531dd14f..dbeb50048 100644 --- a/scripts/mcp-conformance.mjs +++ b/scripts/mcp-conformance.mjs @@ -20,7 +20,11 @@ * scenario:check, that needs one of the suite's reference fixtures. An unlisted failure fails the * lane; so does a listed entry that passes (a stale baseline), so the baseline only shrinks * honestly. The verdict is read from the CLI's own per-scenario summary lines and the - * `checks.json` it writes per scenario (`--output-dir`), never from its exit code. + * `checks.json` it writes per scenario (`--output-dir`), never from its exit code, and the two + * are reconciled: a scenario's failed count must equal the FAILURE rows its checks.json names, a + * summary line the parser cannot read is a failure, and the header's scenario count must match + * the lines reported. Measured 2026-11-10 against 0.2.0-alpha.12: the alpha's `tools-list` + * (on the v1 SDK) runs its four checks against this server's 2026-07-28 leg and passes them. * * Usage: node scripts/mcp-conformance.mjs [scenario ...] (default: the list below) * node scripts/mcp-conformance.mjs --era 2026-07-28 [--verbose] @@ -72,11 +76,23 @@ export function judgeScenario(name, output) { */ export function parseRequirementsSummary(output) { const scenarios = []; - const line = /^[βœ“βœ—]\s+([A-Za-z0-9-]+):\s+(\d+)\s+passed,\s+(\d+)\s+failed\s*$/gm; - for (const match of output.matchAll(line)) { - scenarios.push({ name: match[1], passed: Number(match[2]), failed: Number(match[3]) }); + // A verdict line the parser cannot read is kept, not dropped: the judge fails on it. + const unparsed = []; + const verdict = /^[βœ“βœ—]\s+([^\s:]+):\s+(\d+)\s+passed,\s+(\d+)\s+failed\s*$/; + // The summary block runs from the banner to the Total line; the not-scored list after it is indented. + const lines = output.split(/\r?\n/); + const banner = lines.findIndex((text) => /^=== SUMMARY ===/.test(text)); + for (const raw of lines.slice(banner === -1 ? 0 : banner + 1)) { + if (/^Total:/.test(raw)) break; + if (!/^[βœ“βœ—]/.test(raw)) continue; + const match = verdict.exec(raw); + if (match) scenarios.push({ name: match[1], passed: Number(match[2]), failed: Number(match[3]) }); + else unparsed.push(raw.trimEnd()); } - if (scenarios.length === 0) return undefined; + if (scenarios.length === 0 && unparsed.length === 0) return undefined; + // `Running requirements 2026-07-28 (50 scenarios) against …`: what the run set out to report. + const announcedMatch = /^Running requirements \S+ \((\d+) scenarios?\)/m.exec(output); + const announced = announcedMatch ? Number(announcedMatch[1]) : undefined; const notScored = new Set(); const notScoredBlock = output.split(/^Not scored for [^\n]*$/m)[1]; if (notScoredBlock) { @@ -84,7 +100,7 @@ export function parseRequirementsSummary(output) { notScored.add(match[1]); } } - return { scenarios, notScored }; + return { scenarios, notScored, announced, unparsed }; } /** @@ -129,6 +145,12 @@ export function failingChecks(checksJson) { export function judgeRequirements(summary, baseline, failingChecksOf) { const problems = []; const seen = new Set(); + // The lane's one promise β€” an unlisted failure fails it β€” holds only if every scenario the run + // set out to report was read: a line the parser could not read, or a count short of the header's. + for (const line of summary.unparsed ?? []) problems.push(`could not read a summary line: ${line}`); + if (summary.announced !== undefined && summary.announced !== summary.scenarios.length) { + problems.push(`the run announced ${summary.announced} scenario(s) but the summary reported ${summary.scenarios.length}`); + } for (const { name, failed } of summary.scenarios) { if (summary.notScored.has(name)) continue; seen.add(name); @@ -147,6 +169,12 @@ export function judgeRequirements(summary, baseline, failingChecksOf) { : `${name}: ${failed} failed check(s) but no checks.json was written for it β€” the per-check baseline entries cannot be judged`); continue; } + // Nothing may go unaccounted: every failure the summary counts must be a FAILURE row that + // checks.json names (a scenario-level failure with no row, or a truncated file, would otherwise + // pass under a baseline that lists the rows it does have). + if (failing.length !== failed) { + problems.push(`${name}: the summary counts ${failed} failed check(s) but checks.json names ${failing.length} (${failing.join(', ')})`); + } const unexpected = failing.filter((check) => !(expected?.has(check))); if (unexpected.length > 0) { problems.push(`${name}: ${failed} failed check(s) not in the baseline (${unexpected.join(', ')})`); @@ -156,7 +184,9 @@ export function judgeRequirements(summary, baseline, failingChecksOf) { } } for (const name of baseline.keys()) { - if (!seen.has(name) && !summary.notScored.has(name)) { + if (summary.notScored.has(name)) { + problems.push(`${name}: listed in the baseline but not scored for this revision β€” remove it`); + } else if (!seen.has(name)) { problems.push(`${name}: in the baseline but the requirements run did not report it`); } } @@ -223,9 +253,11 @@ function runCli(args, cwd, cli = CONFORMANCE_CLI) { if (unsafe !== undefined) throw new Error(`refusing to pass "${unsafe}" to the conformance CLI`); // Windows: `npx` is a .cmd shim, which Node spawns only through a shell. The argv is checked // above, so one command string (no args array) keeps the shell out of DEP0190's warning. + // No colour: the per-scenario summary lines are parsed, and FORCE_COLOR would wrap them in escapes. + const env = { ...process.env, NO_COLOR: '1', FORCE_COLOR: '0' }; const child = process.platform === 'win32' - ? spawn(['npx', ...argv].join(' '), { cwd, shell: true, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }) - : spawn('npx', argv, { cwd, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); + ? spawn(['npx', ...argv].join(' '), { cwd, env, shell: true, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }) + : spawn('npx', argv, { cwd, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); let output = ''; child.stdout.on('data', (chunk) => { output += chunk.toString(); }); child.stderr.on('data', (chunk) => { output += chunk.toString(); }); diff --git a/src/cli/http-command.ts b/src/cli/http-command.ts index 59772940a..798e35c3a 100644 --- a/src/cli/http-command.ts +++ b/src/cli/http-command.ts @@ -237,6 +237,9 @@ export function createHttpApp( ); core.notifications.addSink(handlerSink(modern.notify)); const modernNode = toNodeHandler(modern, { + // The same limit as express.json, the classifier and the handler: a body between two + // different limits would pass one entry and die in the next. + maxRequestBodySize: JSON_BODY_LIMIT_BYTES, onerror: (error: Error) => { logger.error('Modern-era MCP request failed', { error }); }, diff --git a/tests/integration/cli/http-era-negotiation.test.ts b/tests/integration/cli/http-era-negotiation.test.ts index bf4c7b97d..3ab6b53bb 100644 --- a/tests/integration/cli/http-era-negotiation.test.ts +++ b/tests/integration/cli/http-era-negotiation.test.ts @@ -13,6 +13,7 @@ import { fileURLToPath } from 'node:url'; import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'; import { serverEnvironment } from '../../test-utils/helpers/mcp-stdio-client.js'; import { waitForHealthEndpoint } from '../../e2e/smoke-test-utils.js'; +import { TOOL_NAMES } from '../../../src/server/tool-schemas.js'; const root = fileURLToPath(new URL('../../../', import.meta.url)); const entry = path.join(root, 'dist/index.js'); @@ -97,7 +98,7 @@ describe('http serves both protocol eras (issue #1025, step 2)', () => { expect(client.getProtocolEra()).toBe('modern'); expect(client.getNegotiatedProtocolVersion()).toBe('2026-07-28'); expect(client.getServerVersion()?.name).toBe('debug-mcp-server'); - expect((await client.listTools()).tools.length).toBe(29); + expect((await client.listTools()).tools.length).toBe(TOOL_NAMES.length); const h = await health(port); expect(h.sessions).toEqual([]); expect(h.requests.modern).toBeGreaterThan(0); diff --git a/tests/integration/cli/http-spec-contract-2026.test.ts b/tests/integration/cli/http-spec-contract-2026.test.ts index 1c65c8982..fb029070a 100644 --- a/tests/integration/cli/http-spec-contract-2026.test.ts +++ b/tests/integration/cli/http-spec-contract-2026.test.ts @@ -19,6 +19,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { serverEnvironment } from '../../test-utils/helpers/mcp-stdio-client.js'; import { waitForHealthEndpoint } from '../../e2e/smoke-test-utils.js'; +import { TOOL_NAMES } from '../../../src/server/tool-schemas.js'; const root = fileURLToPath(new URL('../../../', import.meta.url)); const entry = path.join(root, 'dist/index.js'); @@ -71,7 +72,10 @@ async function startServer(): Promise { }); child.stdout!.on('data', (chunk: Buffer) => { output += chunk.toString(); }); child.stderr!.on('data', (chunk: Buffer) => { output += chunk.toString(); }); - expect(await waitForHealthEndpoint(port, 20_000), output).toBe(true); + // A server that never answers /health must not outlive the failed assertion. + const healthy = await waitForHealthEndpoint(port, 20_000); + if (!healthy) child.kill(); + expect(healthy, output).toBe(true); return { port, child, output: () => output }; } @@ -190,7 +194,7 @@ describe('Streamable HTTP contract (spec 2026-07-28, served beside 2025-11-25)', const first = messageOf(await request(server!.port, { body, headers: modernHeaders(body) }), 2); const second = messageOf(await request(server!.port, { body, headers: modernHeaders(body) }), 2); - expect(first.result.tools).toHaveLength(29); + expect(first.result.tools).toHaveLength(TOOL_NAMES.length); expect(typeof first.result.ttlMs).toBe('number'); expect(['public', 'private']).toContain(first.result.cacheScope); expect(first.result.resultType).toBe('complete'); @@ -202,7 +206,7 @@ describe('Streamable HTTP contract (spec 2026-07-28, served beside 2025-11-25)', const reply = await request(server!.port, { body, headers: { ...modernHeaders(body), 'Mcp-Session-Id': 'stale-from-another-life' } }); expect(reply.status, reply.body).toBe(200); - expect(messageOf(reply, 3).result.tools).toHaveLength(29); + expect(messageOf(reply, 3).result.tools).toHaveLength(TOOL_NAMES.length); expect(reply.headers['mcp-session-id']).toBeUndefined(); }); diff --git a/tests/unit/cli/http-command.test.ts b/tests/unit/cli/http-command.test.ts index 6b778e289..7ab793098 100644 --- a/tests/unit/cli/http-command.test.ts +++ b/tests/unit/cli/http-command.test.ts @@ -667,7 +667,12 @@ describe('HTTP Command Handler', () => { expect(typeof options?.onerror).toBe('function'); expect(factory({ era: 'modern' })).toBe(mockFacade); expect(mockProtocolServerFactory).toHaveBeenLastCalledWith(mockServer, { era: 'modern', deliver: 'external' }); - expect(mockToNodeHandler).toHaveBeenCalledWith(mockModernHandler, expect.objectContaining({ onerror: expect.any(Function) })); + // The one 10 MiB limit reaches every SDK entry the request passes through β€” the + // Node adapter included, or a non-JSON POST between the two limits dies in it. + expect(mockToNodeHandler).toHaveBeenCalledWith(mockModernHandler, expect.objectContaining({ + onerror: expect.any(Function), + maxRequestBodySize: 10 * 1024 * 1024, + })); // The hub's sink for this leg is the handler's notifier. expect(mockServer.notifications.addSink).toHaveBeenCalledTimes(1); diff --git a/tests/unit/scripts/mcp-conformance.test.ts b/tests/unit/scripts/mcp-conformance.test.ts index e1e1df6bb..026c0fbff 100644 --- a/tests/unit/scripts/mcp-conformance.test.ts +++ b/tests/unit/scripts/mcp-conformance.test.ts @@ -14,7 +14,7 @@ import { } from '../../../scripts/mcp-conformance.mjs'; const REQUIREMENTS_OUTPUT = [ - 'Running requirements 2026-07-28 (50 scenarios) against http://127.0.0.1:3997/mcp', + 'Running requirements 2026-07-28 (5 scenarios) against http://127.0.0.1:3997/mcp', '=== SUMMARY ===', 'βœ— server-stateless: 24 passed, 4 failed', 'βœ“ tools-list: 4 passed, 0 failed', @@ -28,6 +28,17 @@ const REQUIREMENTS_OUTPUT = [ ' βœ— tasks-lifecycle (extension)', ].join('\r\n'); +// A scenario name outside [A-Za-z0-9-] and a summary line in a shape the parser does not know. +const ODD_OUTPUT = [ + 'Running requirements 2026-07-28 (3 scenarios) against http://127.0.0.1:1/mcp', + '=== SUMMARY ===', + 'βœ“ sep-1234_foo.bar: 1 passed, 0 failed', + 'βœ— tools-list: 4 passed, 1 failed', + 'βœ— a line the CLI printed in a shape this parser does not know', + '', + 'Total: 5 passed, 1 failed', +].join('\n'); + const BASELINE = [ '# fixtures this server does not have', 'server:', @@ -83,9 +94,26 @@ describe('mcp-conformance runner: the 2026-07-28 requirements lane (issue #1025) { name: 'tasks-lifecycle', passed: 1, failed: 8 }, ]); expect([...summary.notScored]).toEqual(['tasks-lifecycle']); + expect(summary.announced).toBe(5); + expect(summary.unparsed).toEqual([]); expect(parseRequirementsSummary('Running requirements ...\nSegmentation fault')).toBeUndefined(); }); + it('keeps a scenario whose name carries an underscore or a dot, and surfaces a summary line it cannot read instead of dropping it', () => { + const summary = parseRequirementsSummary(ODD_OUTPUT)!; + expect(summary.scenarios.map((s) => s.name)).toEqual(['sep-1234_foo.bar', 'tools-list']); + expect(summary.unparsed).toEqual(['βœ— a line the CLI printed in a shape this parser does not know']); + expect(summary.announced).toBe(3); + }); + + it('fails when the header announces more scenarios than the summary reported, or a summary line could not be read', () => { + const problems = judgeRequirements(parseRequirementsSummary(ODD_OUTPUT)!, parseBaseline('server:\n'), () => []); + expect(problems).toEqual(expect.arrayContaining([ + expect.stringContaining('the run announced 3 scenario(s) but the summary reported 2'), + expect.stringContaining('could not read a summary line: βœ— a line the CLI printed'), + ])); + }); + it('reads whole-scenario and per-check entries out of the baseline, comments stripped', () => { const baseline = parseBaseline(BASELINE); expect(baseline.get('tools-call-simple-text')).toBe('all'); @@ -140,11 +168,39 @@ describe('mcp-conformance runner: the 2026-07-28 requirements lane (issue #1025) }); it('passes when every failure is expected and every expectation still fails', () => { + const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; + const baseline = parseBaseline([ + 'server:', + ' - tools-call-simple-text', + ' - server-stateless:ServerRejectsUndeclaredCapability', + ' - server-stateless:MissingCapabilityHttp400', + ' - server-stateless:HttpServerNoIndependentRequestsOnStream', + ' - server-stateless:ServerNoLogWithoutLogLevel', + ].join('\n')); + // All four server-stateless failures are named by checks.json and listed by check. + const checks = new Map([['server-stateless', [ + 'ServerRejectsUndeclaredCapability', 'MissingCapabilityHttp400', + 'HttpServerNoIndependentRequestsOnStream', 'ServerNoLogWithoutLogLevel', + ]]]); + expect(judgeRequirements(summary, baseline, (name) => checks.get(name) ?? [])).toEqual([]); + }); + + it('fails when the summary counts more failures than checks.json names: nothing may go unaccounted', () => { const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; const baseline = parseBaseline(BASELINE); + // Both named checks are listed, but the summary says four failed. const checks = new Map([['server-stateless', ['ServerRejectsUndeclaredCapability', 'ServerNoLogWithoutLogLevel']]]); - // Two of the four server-stateless failures are listed by check; the other two are not. - expect(judgeRequirements(summary, baseline, (name) => checks.get(name) ?? [])).toEqual([]); + expect(judgeRequirements(summary, baseline, (name) => checks.get(name) ?? [])).toEqual(expect.arrayContaining([ + expect.stringContaining('server-stateless: the summary counts 4 failed check(s) but checks.json names 2'), + ])); + }); + + it('flags a baseline entry naming a scenario the revision does not score', () => { + const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; + const problems = judgeRequirements(summary, parseBaseline('server:\n - tasks-lifecycle\n'), () => []); + expect(problems).toEqual(expect.arrayContaining([ + expect.stringContaining('tasks-lifecycle: listed in the baseline but not scored for this revision'), + ])); }); it('fails on an unlisted failure, a listed check that does not fail, and a stale whole-scenario entry', () => { From 996baf1e1acd1598e49b477f4eec686f7bdaa729 Mon Sep 17 00:00:00 2001 From: JF Date: Tue, 10 Nov 2026 11:57:23 -0500 Subject: [PATCH 3/6] fix(conformance): let the suite judge its own baseline, fix the divergence it found, and answer refused bodies honestly (#1025) A second look at two rulings made while building step 2, on JF's review. The 2026-07-28 lane judged the run itself: a summary parser, a hand-written YAML reader, a checks.json reader and a reconciliation between them, because "the CLI's exit code is unreliable on Windows". The exit abort is a Node bug (nodejs/node#56645, fixed in 24.20.0 and 26.7.0), not the suite's, and CI runs Linux. The suite already does this job: --expected-failures reads the baseline with a real parser, takes per-check entries, and fails on an unlisted failure or warning and on a stale entry. The lane now hands it the baseline and takes its exit code; on a Windows Node that aborts at exit, and only then, it reads the suite's own closing line. The custom judge and its tests are deleted (the script drops from 396 to 274 lines). The hand-built judge was also weaker than the suite's in two ways, which the first suite-judged run showed. It matched check names where the suite matches check ids, and it counted only failures where the suite's baseline also counts warnings. Three warning-only scenarios had read as passes: - sep-2164-resource-not-found was a real SHOULD-level divergence of ours: the error for an unknown resource carried no data. It now names the requested uri. Fixed, not baselined. - two input-required scenarios cannot run without the reference tools; they join the baseline. The baseline is rewritten as what it is: 31 scenarios and checks that cannot be tested against a server without the suite's reference fixtures (18) or without an optional feature this server does not declare (13), each with the fixture or feature beside it, every name read from the suite's source (seven of the old comments named a fixture wrongly). None is a divergence; by the suite's policy a missing prerequisite is a failure, not a skip. Measured on alpha.12: "Baseline check passed: all failures are expected", exit 0; sep-2164-resource-not-found 4 passed, no warning. Also on the http leg: - A body too large to classify answers 413, as the JSON parser already does for an oversized JSON body, where it answered 500 "Internal error" and wrote an error log. - A request the SDK's modern handler refuses is logged as a warning; only a fault of the handler is an error. A scanner posting {} wrote error lines. - The modern handler's notification sink is detached from the hub at shutdown, before the handler closes. - A test pins that an http client is named once in the log however many per-request facades serve it. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/ci.yml | 18 +- changelog.d/1025-conformance-2026.added.md | 2 +- changelog.d/1025-protocol-2026-http.added.md | 2 +- docs/architecture/testing-architecture.md | 2 +- docs/tool-reference.md | 2 +- .../expected-failures-2026-07-28.yaml | 93 +++--- scripts/mcp-conformance.d.mts | 39 +-- scripts/mcp-conformance.mjs | 268 +++++------------ src/cli/http-command.ts | 45 ++- src/server/output-resources.ts | 16 +- .../core/unit/server/mcp-wire-errors.test.ts | 3 + .../server/protocol-facade-transport.test.ts | 23 +- tests/unit/cli/http-command.test.ts | 66 +++++ tests/unit/scripts/mcp-conformance.test.ts | 274 ++++++++---------- 14 files changed, 414 insertions(+), 439 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11065cc40..7548b6d9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -619,8 +619,9 @@ jobs: mcp-conformance: # The protocol org's own conformance suite (`@modelcontextprotocol/conformance`, pinned in # the script) against a local `http` server: the scenarios that apply to a server without - # the suite's reference fixtures (issue #1023). Linux only β€” on Windows the CLI aborts at - # exit with a libuv assertion even after every check passed. What the suite does not cover, + # the suite's reference fixtures (issue #1023). Linux only β€” on Windows an affected Node build + # aborts inside process.exit after every check passed (nodejs/node#56645, fixed in 24.20.0 + # and 26.7.0), which is a Node bug and not the suite's. What the suite does not cover, # the session-lifecycle clauses this server answers itself, is pinned by # tests/integration/cli/http-spec-contract.test.ts in build-and-test. name: MCP Conformance (official suite) @@ -648,11 +649,12 @@ jobs: mcp-conformance-2026: # The same suite's 0.2 line (the one the SDKs run) against the 2026-07-28 revision this # server now speaks beside 2025-11-25 (issue #1025, step 2): `--requirements 2026-07-28`, - # everything the revision requires of a server, judged under - # scripts/conformance/expected-failures-2026-07-28.yaml β€” the scenarios that drive the - # suite's own reference fixtures (tools, resources, prompts this server does not have). - # An unlisted failure fails the job, and so does a listed entry that passes, so the - # baseline can only shrink. Linux only, for the same libuv reason as the lane above. + # everything the revision requires of a server, judged by the suite itself against + # scripts/conformance/expected-failures-2026-07-28.yaml (`--expected-failures`) β€” what + # cannot be tested without the suite's reference fixtures or an optional feature this + # server does not declare. The suite fails on an unlisted failure or warning and on a + # listed entry that passes, so the baseline can only shrink; its exit code is the verdict. + # Linux, where that exit code is always trustworthy (see the script's header). name: MCP Conformance (2026-07-28) runs-on: ubuntu-latest timeout-minutes: 15 @@ -672,7 +674,7 @@ jobs: SKIP_ADAPTER_VENDOR: 'true' - name: Build project run: pnpm run build:ci - - name: Run the 2026-07-28 requirements under the expected-failures baseline + - name: Run the 2026-07-28 requirements against the baseline, judged by the suite run: node scripts/mcp-conformance.mjs --era 2026-07-28 --verbose test-summary: diff --git a/changelog.d/1025-conformance-2026.added.md b/changelog.d/1025-conformance-2026.added.md index 7e0defdca..4e4ba8960 100644 --- a/changelog.d/1025-conformance-2026.added.md +++ b/changelog.d/1025-conformance-2026.added.md @@ -1 +1 @@ -**A second MCP conformance lane for the 2026-07-28 revision.** `node scripts/mcp-conformance.mjs --era 2026-07-28` runs the protocol org's conformance suite on its 0.2 line (`@modelcontextprotocol/conformance@0.2.0-alpha.12`, the line the SDKs run) with `--requirements 2026-07-28` β€” everything the revision requires of a server β€” against a local `http` server, and judges the run under `scripts/conformance/expected-failures-2026-07-28.yaml`: the scenarios, or single checks, that drive the suite's own reference fixtures (the "everything server" tools, resources and prompts this server does not have), each with the fixture it needs named beside it. An unlisted failure fails the lane; so does a listed entry that passes, so the baseline only shrinks. CI runs it as `mcp-conformance-2026`, required in the test summary beside the 2025-11-25 lane, which is unchanged. Measuring the lane also found that `resources/templates/list` answered "Method not found": a server advertising the resources capability must answer it, so both eras now answer an empty list. +**A second MCP conformance lane for the 2026-07-28 revision, judged by the suite itself.** `node scripts/mcp-conformance.mjs --era 2026-07-28` runs the protocol org's conformance suite on its 0.2 line (`@modelcontextprotocol/conformance@0.2.0-alpha.12`, the line the SDKs run) with `--requirements 2026-07-28` β€” everything the revision requires of a server β€” against a local `http` server, and hands the suite its own baseline with `--expected-failures`: the suite fails the run on a failure or warning that is not listed and on a listed entry that passes, and its exit code is the lane's verdict. `scripts/conformance/expected-failures-2026-07-28.yaml` lists what cannot be tested against this server β€” 18 scenarios and checks that only drive the suite's reference fixtures (the "everything server" tools, resources and prompts), and 13 that need an optional feature this server does not declare (`completions`, input-required results) β€” each with the fixture or feature named beside it; none is a protocol divergence, and by the suite's own policy a missing prerequisite is a failure, not a skip (upstream discussion: modelcontextprotocol/conformance#516). CI runs it as `mcp-conformance-2026`, required in the test summary beside the 2025-11-25 lane, which is unchanged. Running the suite found two real gaps, both fixed in both eras: `resources/templates/list` answered "Method not found" (a server advertising the resources capability must answer it; it now answers an empty list), and the error for an unknown resource carried no `data` (it now names the requested `uri`, SEP-2164). diff --git a/changelog.d/1025-protocol-2026-http.added.md b/changelog.d/1025-protocol-2026-http.added.md index 198ff784d..a54d408fe 100644 --- a/changelog.d/1025-protocol-2026-http.added.md +++ b/changelog.d/1025-protocol-2026-http.added.md @@ -1 +1 @@ -**`http` speaks the current MCP protocol revision, 2026-07-28, beside 2025-11-25 and earlier β€” on the same `/mcp` endpoint.** A request carrying the 2026 per-request envelope (`_meta` with `io.modelcontextprotocol/protocolVersion`) is served by the SDK's sessionless handler: `server/discover` answers the supported versions and capabilities, every request stands alone with no `Mcp-Session-Id`, results carry `resultType` and the cache fields, resource notifications flow over `subscriptions/listen` streams, and the 2026 header rules apply (`MCP-Protocol-Version` and `Mcp-Method` on every POST, `Mcp-Name` on the methods that name a tool, prompt or resource; a missing or mismatched one is refused with `-32020`, an unsupported claimed revision with `-32022` naming the supported ones). A 2025-shaped request β€” `initialize`, a request carrying `Mcp-Session-Id` and no envelope, a GET or DELETE, a claim-less POST β€” lands on the sessionful app exactly as before, status codes included, so every 2025 client keeps working. An open `subscriptions/listen` stream counts the process as attended for the idle policy, like a 2025 session's SSE stream; a 2026 client that only calls tools is not attending, so with the idle policy turned on its sessions live by tool activity alone. A probing client such as Claude Code's v2 runtime therefore lands on 2026-07-28 against `http` too. `GET /health` reports `listenStreams` and the requests classified per era (`requests.legacy` / `requests.modern`). Browser clients: CORS now allows `Mcp-Method`, `Mcp-Name` and echoes a preflight's requested headers. Step 2 of #1025 completes the protocol move on both transports. +**`http` speaks the current MCP protocol revision, 2026-07-28, beside 2025-11-25 and earlier β€” on the same `/mcp` endpoint.** A request carrying the 2026 per-request envelope (`_meta` with `io.modelcontextprotocol/protocolVersion`) is served by the SDK's sessionless handler: `server/discover` answers the supported versions and capabilities, every request stands alone with no `Mcp-Session-Id`, results carry `resultType` and the cache fields, resource notifications flow over `subscriptions/listen` streams, and the 2026 header rules apply (`MCP-Protocol-Version` and `Mcp-Method` on every POST, `Mcp-Name` on the methods that name a tool, prompt or resource; a missing or mismatched one is refused with `-32020`, an unsupported claimed revision with `-32022` naming the supported ones). A 2025-shaped request β€” `initialize`, a request carrying `Mcp-Session-Id` and no envelope, a GET or DELETE, a claim-less POST β€” lands on the sessionful app exactly as before, status codes included, so every 2025 client keeps working. An open `subscriptions/listen` stream counts the process as attended for the idle policy, like a 2025 session's SSE stream; a 2026 client that only calls tools is not attending, so with the idle policy turned on its sessions live by tool activity alone. A probing client such as Claude Code's v2 runtime therefore lands on 2026-07-28 against `http` too. `GET /health` reports `listenStreams` and the requests classified per era (`requests.legacy` / `requests.modern`). Browser clients: CORS now allows `Mcp-Method`, `Mcp-Name` and echoes a preflight's requested headers. In the log, each 2026-07-28 client is named once (`[MCP Server] Client identified`), a request the handler refuses is a warning rather than an error, and a body too large to classify answers 413 like any other oversized body. Step 2 of #1025 completes the protocol move on both transports. diff --git a/docs/architecture/testing-architecture.md b/docs/architecture/testing-architecture.md index 75410e149..a3a150ad0 100644 --- a/docs/architecture/testing-architecture.md +++ b/docs/architecture/testing-architecture.md @@ -274,7 +274,7 @@ Protocol-level correctness checks. `breakpoint-messages/` contains Python script Two tiers, split along the seam between this server's own code and the SDK, and two eras, served side by side on the same endpoint (issue #1025, step 2). The 2025-11-25 clauses of the Streamable HTTP transport that `createHttpApp` decides before it hands a request to `transport.handleRequest` β€” session routing, the Host/Origin allowlist, the stale-session reaper β€” are pinned one case per clause against the real server over raw HTTP: a missing `Mcp-Session-Id` is 400; an unknown, DELETEd or reaped one is 404 `Session not found`; an invalid `Origin` is 403; an unsupported `MCP-Protocol-Version` is 400. Raw HTTP because the SDK client folds every non-2xx answer into a thrown error, and a case about a status code has to read it. The 404 cases are the ones #1017 fixed; nothing in the suite could have caught the 400 they replaced. The 2026-07-28 file pins the same way what the SDK's sessionless handler answers once our router (`isLegacyRequest`) has sent a modern request to it: `server/discover` with no session minted, `tools/list` with its cache fields and a stable order, a stale `Mcp-Session-Id` ignored, the `-32602` / `-32020` / `-32022` refusals, a 2025 `initialize` still opening a session on the same endpoint, a claim-less POST still landing on the sessionful app's 400, and an open `subscriptions/listen` stream counting the process as attended. The two negotiation files drive the v2 SDK client in `auto`, pinned and `legacy` mode through a full mock debug session on each transport, so what a real client experiences on either era is pinned too. -Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance` for the 2025-11-25 lane, `pnpm test:conformance:2026` for the 2026-07-28 one) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” under an expected-failures baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) that lists each scenario needing a fixture this server does not have (the `tools-call-*`, `prompts-get-*`, `resources-read-*`, `completion` and most `input-required-result-*` scenarios drive the suite's reference tools) with the fixture it needs; the lane fails on an unlisted failure and on a listed scenario that starts passing, so the baseline can only shrink honestly. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned, and on Windows their exit codes are unusable (they abort at exit with a libuv assertion even after a pass), so the script reads the printed summaries instead. +Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance` for the 2025-11-25 lane, `pnpm test:conformance:2026` for the 2026-07-28 one) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” and lets the suite judge it: the suite reads the baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) with its own `--expected-failures`, fails on a failure or warning that is not listed and on a listed entry that passes, and its exit code is the verdict; the script adds no judging of its own. The baseline lists what cannot be tested without the suite's reference fixtures (the `tools-call-*`, `prompts-get-*`, `resources-read-*` scenarios and four `server-stateless` checks) or without an optional feature this server does not declare (`completion-complete`, the `input-required-result-*` scenarios), each with its reason; by the suite's policy a missing prerequisite is a failure, not a skip, and a baseline is its named mechanism for that. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned. One caveat for local runs: Node 23 to 24.19 and 26.0 to 26.6 abort inside `process.exit` on Windows (nodejs/node#56645, fixed in 24.20.0 and 26.7.0) after the suite has printed everything, so there the legacy lane reads the printed summaries and the 2026 lane reads the suite's closing line when that abort is in the output; CI runs Linux and takes the exit code. ## Coverage Strategy diff --git a/docs/tool-reference.md b/docs/tool-reference.md index a1b21f59b..5429fbb5e 100644 --- a/docs/tool-reference.md +++ b/docs/tool-reference.md @@ -87,7 +87,7 @@ Creates a new debugging session. **Notes:** - Session IDs are UUIDs in the format `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` - Sessions start in `"created"` state -- **One debugger core per process; the id is the capability (issue #1025).** On Streamable HTTP every MCP client of a process sees the same debug sessions: `list_debug_sessions` is the whole process's list, and any client holding an id can drive or close that session β€” run one `http` process per agent where agents must not see each other's work. A debug session outlives the MCP session that created it, so a one-shot script that opens a fresh MCP session per call reaches the sessions its previous run created, and a client that reconnects can resume or close what it left behind. A session ends by `close_debug_session`, by its program ending, or with the server: nothing expires on a timer by default. An operator can turn on the **idle policy** (`MCP_DEBUG_SESSION_IDLE_MS`, off by default on every transport): a session tree that no tool call has named for that long is then closed through the normal close cascade (a launched program is ended, an attached one released), unless a client is attending the process β€” while any client holds a stream open (a 2025 session’s SSE stream, a 2026-07-28 client’s `subscriptions/listen` stream), nothing expires; an in-flight `wait_for_stop` holds its tree; children never expire on a clock of their own. `list_debug_sessions` shows each session's `idleMs` and, with a policy on, `expiresInMs`; a later call naming a session the policy closed answers what happened, with `closedBy: "idle-policy"`. The server is built to scope sessions per principal once it verifies callers itself (no authenticated `http` mode ships yet; an authenticating reverse proxy does not identify callers to the server), so today every caller is in the one shared scope +- **One debugger core per process; the id is the capability (issue #1025).** On Streamable HTTP every MCP client of a process sees the same debug sessions: `list_debug_sessions` is the whole process's list, and any client holding an id can drive or close that session β€” run one `http` process per agent where agents must not see each other's work. A debug session outlives the MCP session that created it, so a one-shot script that opens a fresh MCP session per call reaches the sessions its previous run created, and a client that reconnects can resume or close what it left behind. A session ends by `close_debug_session`, by its program ending, or with the server: nothing expires on a timer by default. An operator can turn on the **idle policy** (`MCP_DEBUG_SESSION_IDLE_MS`, off by default on every transport): a session tree that no tool call has named for that long is then closed through the normal close cascade (a launched program is ended, an attached one released), unless a client is attending the process β€” while any client holds a stream open (a 2025 session's SSE stream, a 2026-07-28 client's `subscriptions/listen` stream), nothing expires; an in-flight `wait_for_stop` holds its tree; children never expire on a clock of their own. `list_debug_sessions` shows each session's `idleMs` and, with a policy on, `expiresInMs`; a later call naming a session the policy closed answers what happened, with `closedBy: "idle-policy"`. The server is built to scope sessions per principal once it verifies callers itself (no authenticated `http` mode ships yet; an authenticating reverse proxy does not identify callers to the server), so today every caller is in the one shared scope - When `port`, `processId` or an `adapterConfig` URI/info-file key is provided in `create_debug_session`, the server performs an inline attach (creating the session and immediately attaching). The session exists before the addressing gate can refuse the form (a `port` on C/C++, say), so such a refusal answers `success: false` with a usable `sessionId`. The response then mirrors `attach_to_process`: alongside `sessionId` it carries `state`, the attach `data` payload, an optional `warning`, and β€” when a requested post-attach pause has not landed yet β€” `pending: true`, with `message` saying so (`…; post-attach pause pending β€” the target stops when it next executes code; call wait_for_stop to block until it does (pass stopOnEntry: false to attach without pausing)`) --- diff --git a/scripts/conformance/expected-failures-2026-07-28.yaml b/scripts/conformance/expected-failures-2026-07-28.yaml index 018ae57ca..d5194cbb9 100644 --- a/scripts/conformance/expected-failures-2026-07-28.yaml +++ b/scripts/conformance/expected-failures-2026-07-28.yaml @@ -1,39 +1,58 @@ -# Expected failures for `@modelcontextprotocol/conformance server --requirements 2026-07-28` -# against mcp-debugger (scripts/mcp-conformance.mjs --era 2026-07-28; issue #1025). +# Not testable against mcp-debugger: the scenarios and checks of +# `@modelcontextprotocol/conformance server --requirements 2026-07-28` that need something +# this server does not host. Nothing in this file is a protocol divergence of ours. # -# Every entry is a scenario, or a `scenario:check` pair, that drives one of the conformance -# suite's own reference fixtures β€” the "everything server" tools (`test_*`), resources -# (`test://…`) and prompts β€” which this server does not have. None is a protocol divergence. -# The lane fails on a failure that is NOT listed here, and on a listed entry that starts -# passing (a stale baseline), so this file can only shrink honestly. Measured 2026-11-10 -# against 0.2.0-alpha.12: 37 scored scenarios, of which these need a fixture. +# The suite is built to test SDKs against a reference "everything server", and by its own +# policy a missing prerequisite is a failure, not a skip (suite AGENTS.md; issues #248, #372); +# a baseline like this one is the mechanism it names for a server without the fixtures. +# The suite reads this file itself (`--expected-failures`, scripts/mcp-conformance.mjs +# --era 2026-07-28) and fails the lane on a failure or warning that is NOT listed, and on a +# listed entry that starts passing β€” so the file can only shrink honestly, and a real +# divergence can never hide in it. An entry is a scenario, or `scenario:check-id`. +# +# Measured 2026-11-10 against 0.2.0-alpha.12: 37 scored scenarios; the fixture names below +# are read from the suite's source. Upstream discussion of reporting these as not applicable: +# modelcontextprotocol/conformance#516. server: - - completion-complete # completion/complete on a reference prompt argument - - tools-call-simple-text # tools/call test_simple_text - - tools-call-image # tools/call test_image - - tools-call-audio # tools/call test_audio - - tools-call-embedded-resource # tools/call test_embedded_resource - - tools-call-mixed-content # tools/call test_mixed_content - - tools-call-error # tools/call test_error - - tools-call-with-progress # tools/call test_progress (progress notifications) - - resources-read-text # resources/read test://static-text - - resources-read-binary # resources/read test://static-binary - - resources-templates-read # resources/read of a reference template URI - - prompts-get-simple # prompts/get test_simple_prompt - - prompts-get-with-args # prompts/get test_prompt_with_args - - prompts-get-embedded-resource # prompts/get test_prompt_embedded_resource - - prompts-get-with-image # prompts/get test_prompt_with_image - - input-required-result-basic-elicitation # a reference tool that elicits (MRTR) - - input-required-result-basic-sampling # a reference tool that samples (MRTR) - - input-required-result-basic-list-roots # a reference tool that lists roots (MRTR) - - input-required-result-request-state # the reference MRTR tool's requestState - - input-required-result-multiple-input-requests # the reference MRTR tool, several inputs - - input-required-result-multi-round # the reference MRTR tool, several rounds - - input-required-result-non-tool-request # a reference prompt that elicits (MRTR) - - input-required-result-result-type # the reference MRTR tool's resultType - - input-required-result-tampered-state # the reference MRTR tool, tampered state - - input-required-result-capability-check # the reference MRTR tool vs. client capabilities - - server-stateless:ServerRejectsUndeclaredCapability # tools/call test_missing_capability - - server-stateless:MissingCapabilityHttp400 # tools/call test_missing_capability (HTTP status) - - server-stateless:HttpServerNoIndependentRequestsOnStream # tools/call test_streaming_elicitation - - server-stateless:ServerNoLogWithoutLogLevel # tools/call test_logging_tool + # --- Needs only a reference fixture this server does not have (18) --- + - tools-call-simple-text # tool test_simple_text + - tools-call-image # tool test_image_content + - tools-call-audio # tool test_audio_content + - tools-call-embedded-resource # tool test_embedded_resource + - tools-call-mixed-content # tool test_multiple_content_types + - tools-call-error # tool test_error_handling + - tools-call-with-progress # tool test_tool_with_progress + - resources-read-text # resource test://static-text + - resources-read-binary # resource test://static-binary + - resources-templates-read # resource template test://template/{id}/data + - prompts-get-simple # prompt test_simple_prompt + - prompts-get-with-args # prompt test_prompt_with_arguments + - prompts-get-embedded-resource # prompt test_prompt_with_embedded_resource + - prompts-get-with-image # prompt test_prompt_with_image + # Four checks of the core 2026 scenario report "Not testable: server does not list the + # diagnostic tool". The MUSTs behind them hold vacuously here: this server never requires a + # client capability and never sends a server-to-client request or a log notification. + - server-stateless:sep-2575-server-rejects-undeclared-capability # tool test_missing_capability + - server-stateless:sep-2575-missing-capability-http-400 # tool test_missing_capability + - server-stateless:sep-2575-http-server-no-independent-requests-on-stream # tool test_streaming_elicitation + - server-stateless:sep-2575-server-no-log-without-loglevel # tool test_logging_tool + + # --- Needs an optional feature this server does not declare (13) --- + # `completions` is not declared; the scenario also asks about the reference prompt. + - completion-complete # completion/complete on prompt test_prompt_with_arguments + # Input-required results (SEP-2322): a tool that answers by asking the client for + # elicitation, sampling or roots. Each scenario drives a reference tool or prompt. + - input-required-result-basic-elicitation # tool test_input_required_result_elicitation + - input-required-result-basic-sampling # tool test_input_required_result_sampling + - input-required-result-basic-list-roots # tool test_input_required_result_list_roots + - input-required-result-request-state # tool test_input_required_result_request_state + - input-required-result-multiple-input-requests # tool test_input_required_result_multiple_inputs + - input-required-result-multi-round # tool test_input_required_result_multi_round + - input-required-result-non-tool-request # prompt test_input_required_result_prompt + - input-required-result-result-type # tool test_input_required_result_elicitation + - input-required-result-tampered-state # tool test_input_required_result_tampered_state + - input-required-result-capability-check # tools test_input_required_result_capabilities, _elicitation + # These two end in a WARNING, not a failure (the unknown tool answers with an error where + # the suite recommends another input request); the suite's baseline counts a warning. + - input-required-result-missing-input-response # tool test_input_required_result_elicitation, prompt _prompt + - input-required-result-ignore-extra-params # tool test_input_required_result_elicitation diff --git a/scripts/mcp-conformance.d.mts b/scripts/mcp-conformance.d.mts index 863907ec6..13623928d 100644 --- a/scripts/mcp-conformance.d.mts +++ b/scripts/mcp-conformance.d.mts @@ -20,38 +20,23 @@ export function runConformance(options?: { log?: (line: string) => void; }): Promise; -// The 2026-07-28 requirements lane (issue #1025, step 2). +// The 2026-07-28 requirements lane (issue #1025, step 2): judged by the suite itself. export const CONFORMANCE_CLI_2026: string; export const REQUIREMENTS_2026: string; -export interface RequirementsScenario { - name: string; - passed: number; - failed: number; -} -export interface RequirementsSummary { - scenarios: RequirementsScenario[]; - /** Scenarios the revision lists as not scored (extensions and pending ones). */ - notScored: Set; - /** The scenario count the run's header announced, when it printed one. */ - announced?: number; - /** Verdict lines in the summary block the parser could not read; the judge fails on each. */ - unparsed: string[]; +/** The suite's verdict on a `--requirements … --expected-failures` run. */ +export interface BaselineVerdict { + ok: boolean; + /** Node's Windows exit abort hid the exit code; the suite's closing line was read instead. */ + abortedAtExit: boolean; + /** What the suite said was wrong, colour stripped; empty on a pass. */ + report: string; } -/** A whole scenario expected to fail (`'all'`), or the names of its checks that are. */ -export type ExpectedFailures = Map>; -export function parseRequirementsSummary(output: string): RequirementsSummary | undefined; -export function parseBaseline(text: string): ExpectedFailures; -export function failingChecks(checksJson: unknown): string[]; -export function judgeRequirements( - summary: RequirementsSummary, - baseline: ExpectedFailures, - failingChecksOf: (scenario: string) => string[] -): string[]; -export function collectChecks(dir: string): Map; +export function requirementsArgs(revision: string, port: number): string[]; +export function baselineReport(output: string): string; +export function judgeBaselineRun(run: { code: number | null; output: string }): BaselineVerdict; export function runRequirements(options?: { root?: string; revision?: string; - baselinePath?: string; verbose?: boolean; log?: (line: string) => void; -}): Promise<{ problems: string[]; summary: RequirementsSummary }>; +}): Promise; diff --git a/scripts/mcp-conformance.mjs b/scripts/mcp-conformance.mjs index dbeb50048..a692e0ed6 100644 --- a/scripts/mcp-conformance.mjs +++ b/scripts/mcp-conformance.mjs @@ -3,36 +3,39 @@ * mcp-debugger `http` server and fail on any failed check. * * The suite (`@modelcontextprotocol/conformance`, pinned) is the protocol org's own client-side - * check of a server; each scenario cites the spec clause it covers. Most of its scenarios drive - * the reference "everything" server's fixtures (`test_simple_text`, `test://static-text`, - * `logging/setLevel`, …) that this server does not have, so they are not run: the list below is - * the transport- and listing-level set that applies to any server. What the suite does NOT - * cover β€” the session-lifecycle clauses this server answers with its own code β€” is pinned by - * `tests/integration/cli/http-spec-contract.test.ts` instead. + * check of a server; each scenario cites the spec clause it covers. It is built to test SDKs + * against a reference "everything" server, so most of its scenarios drive that server's + * fixtures (`test_simple_text`, `test://static-text`, `logging/setLevel`, …), which this server + * does not have. * - * The CLI's exit code is not trusted: on Windows it aborts at exit with a libuv assertion even - * when every check passed, so the printed `Passed: n/m, k failed` summary is what counts. + * Two lanes: * - * The 2026-07-28 era (issue #1025, step 2) is a second lane on the suite's 0.2 line, which the - * SDKs themselves run: `--era 2026-07-28` runs `--requirements 2026-07-28` β€” everything the - * revision requires of a server β€” under an expected-failures baseline - * (`scripts/conformance/expected-failures-2026-07-28.yaml`) that names each scenario, or - * scenario:check, that needs one of the suite's reference fixtures. An unlisted failure fails the - * lane; so does a listed entry that passes (a stale baseline), so the baseline only shrinks - * honestly. The verdict is read from the CLI's own per-scenario summary lines and the - * `checks.json` it writes per scenario (`--output-dir`), never from its exit code, and the two - * are reconciled: a scenario's failed count must equal the FAILURE rows its checks.json names, a - * summary line the parser cannot read is a failure, and the header's scenario count must match - * the lines reported. Measured 2026-11-10 against 0.2.0-alpha.12: the alpha's `tools-list` - * (on the v1 SDK) runs its four checks against this server's 2026-07-28 leg and passes them. + * - The 2025-11-25 lane (the default) runs the list below β€” the transport- and listing-level + * scenarios that apply to any server β€” one `--scenario` at a time on the 0.1 line, and reads + * each scenario's printed `Passed: n/m, k failed` summary. What the suite does NOT cover β€” the + * session-lifecycle clauses this server answers with its own code β€” is pinned by + * `tests/integration/cli/http-spec-contract.test.ts` instead. + * - The 2026-07-28 lane (`--era 2026-07-28`; issue #1025, step 2) runs everything the revision + * requires of a server (`--requirements 2026-07-28`) on the suite's 0.2 line, which the SDKs + * themselves run, and lets the suite judge it: the suite reads the baseline + * (`scripts/conformance/expected-failures-2026-07-28.yaml`, the scenarios and checks that + * cannot be tested without its reference fixtures) with `--expected-failures`, fails on a + * failure or warning that is not listed and on a listed entry that passes, and its exit code + * is the verdict. This script adds no judging of its own. + * + * One exception to trusting an exit code, on both lanes: Node 23 to 24.19 and 26.0 to 26.6 abort + * inside `process.exit` on Windows with a libuv assertion (nodejs/node#56645, fixed in 24.20.0 + * and 26.7.0), after the suite has printed everything. CI runs Linux and never sees it. On an + * affected Windows Node the 2025 lane's printed summary already is the verdict, and the 2026 + * lane reads the suite's own closing line ("Baseline check passed: …") β€” only when that + * assertion is in the output. * * Usage: node scripts/mcp-conformance.mjs [scenario ...] (default: the list below) * node scripts/mcp-conformance.mjs --era 2026-07-28 [--verbose] */ import { spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import { createServer } from 'node:net'; -import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { isMain } from './lib/is-main.mjs'; @@ -69,153 +72,42 @@ export function judgeScenario(name, output) { return { name, ok: true, summary }; } -/** - * The per-scenario verdict lines of a `--requirements` run (`βœ“ name: n passed, m failed`), and - * the scenarios the revision lists as not scored (the CLI prints them after the total; they do - * not affect conformance). Undefined when the output has no summary. - */ -export function parseRequirementsSummary(output) { - const scenarios = []; - // A verdict line the parser cannot read is kept, not dropped: the judge fails on it. - const unparsed = []; - const verdict = /^[βœ“βœ—]\s+([^\s:]+):\s+(\d+)\s+passed,\s+(\d+)\s+failed\s*$/; - // The summary block runs from the banner to the Total line; the not-scored list after it is indented. - const lines = output.split(/\r?\n/); - const banner = lines.findIndex((text) => /^=== SUMMARY ===/.test(text)); - for (const raw of lines.slice(banner === -1 ? 0 : banner + 1)) { - if (/^Total:/.test(raw)) break; - if (!/^[βœ“βœ—]/.test(raw)) continue; - const match = verdict.exec(raw); - if (match) scenarios.push({ name: match[1], passed: Number(match[2]), failed: Number(match[3]) }); - else unparsed.push(raw.trimEnd()); - } - if (scenarios.length === 0 && unparsed.length === 0) return undefined; - // `Running requirements 2026-07-28 (50 scenarios) against …`: what the run set out to report. - const announcedMatch = /^Running requirements \S+ \((\d+) scenarios?\)/m.exec(output); - const announced = announcedMatch ? Number(announcedMatch[1]) : undefined; - const notScored = new Set(); - const notScoredBlock = output.split(/^Not scored for [^\n]*$/m)[1]; - if (notScoredBlock) { - for (const match of notScoredBlock.matchAll(/^\s+[βœ“βœ—]\s+([A-Za-z0-9-]+)\s+\(/gm)) { - notScored.add(match[1]); - } - } - return { scenarios, notScored, announced, unparsed }; +/** The arguments of a `--requirements` run judged against the baseline; the path is relative to the repository root. */ +export function requirementsArgs(revision, port) { + return [ + 'server', + '--url', `http://127.0.0.1:${port}/mcp`, + '--requirements', revision, + '--expected-failures', `scripts/conformance/expected-failures-${revision}.yaml`, + ]; } -/** - * The expected-failures file: `server:` entries of `` (the whole scenario is expected to - * fail) or `:` (that check is). Comments and blank lines are ignored. Deliberately - * a tiny reader for this one shape rather than a YAML dependency. - */ -export function parseBaseline(text) { - const expected = new Map(); - let inServer = false; - for (const raw of text.split(/\r?\n/)) { - const line = raw.replace(/\s+#.*$/, '').replace(/^#.*$/, '').trimEnd(); - if (!line.trim()) continue; - if (/^[A-Za-z]+:\s*$/.test(line)) { - inServer = line.trim() === 'server:'; - continue; - } - const entry = /^\s*-\s*['"]?([A-Za-z0-9:_-]+)['"]?\s*$/.exec(line); - if (!entry || !inServer) continue; - const [scenario, check] = entry[1].split(':'); - const current = expected.get(scenario); - if (check === undefined) { - expected.set(scenario, 'all'); - } else if (current !== 'all') { - expected.set(scenario, new Set([...(current ?? []), check])); - } - } - return expected; -} - -/** The names of the failing checks in one scenario's `checks.json` (an array, or an object holding `checks`). */ -export function failingChecks(checksJson) { - const checks = Array.isArray(checksJson) ? checksJson : Array.isArray(checksJson?.checks) ? checksJson.checks : []; - return checks.filter((check) => check?.status === 'FAILURE').map((check) => String(check.name)); -} +/** The suite's closing line when every failure and warning is in the baseline and no entry is stale. */ +const BASELINE_PASSED = 'Baseline check passed: all failures are expected.'; +/** Node's abort inside process.exit on Windows (nodejs/node#56645; fixed in 24.20.0 and 26.7.0). */ +const EXIT_ABORT = /Assertion failed: !\(handle->flags & UV_HANDLE_CLOSING\)/; /** - * The lane's verdict for a requirements run: every scored scenario either passes, or fails only - * in ways the baseline expects β€” and every baseline entry still fails (otherwise it is stale). - * `failingChecksOf(name)` answers the failing check names of a scenario, read from its checks.json. + * What the suite said was wrong, in its own words: from its first finding list (stale entries, + * unexpected failures) to the end, colour stripped; the tail of the output when it never got + * that far. */ -export function judgeRequirements(summary, baseline, failingChecksOf) { - const problems = []; - const seen = new Set(); - // The lane's one promise β€” an unlisted failure fails it β€” holds only if every scenario the run - // set out to report was read: a line the parser could not read, or a count short of the header's. - for (const line of summary.unparsed ?? []) problems.push(`could not read a summary line: ${line}`); - if (summary.announced !== undefined && summary.announced !== summary.scenarios.length) { - problems.push(`the run announced ${summary.announced} scenario(s) but the summary reported ${summary.scenarios.length}`); - } - for (const { name, failed } of summary.scenarios) { - if (summary.notScored.has(name)) continue; - seen.add(name); - const expected = baseline.get(name); - if (failed === 0) { - if (expected !== undefined) problems.push(`${name}: listed as an expected failure but it passed β€” remove it from the baseline`); - continue; - } - if (expected === 'all') continue; - const failing = failingChecksOf(name); - if (failing.length === 0) { - // The summary says it failed but no checks.json named the checks: nothing to compare the - // per-check entries against, and nothing to call passed. - problems.push(expected === undefined - ? `${name}: ${failed} failed check(s) not in the baseline` - : `${name}: ${failed} failed check(s) but no checks.json was written for it β€” the per-check baseline entries cannot be judged`); - continue; - } - // Nothing may go unaccounted: every failure the summary counts must be a FAILURE row that - // checks.json names (a scenario-level failure with no row, or a truncated file, would otherwise - // pass under a baseline that lists the rows it does have). - if (failing.length !== failed) { - problems.push(`${name}: the summary counts ${failed} failed check(s) but checks.json names ${failing.length} (${failing.join(', ')})`); - } - const unexpected = failing.filter((check) => !(expected?.has(check))); - if (unexpected.length > 0) { - problems.push(`${name}: ${failed} failed check(s) not in the baseline (${unexpected.join(', ')})`); - } - for (const check of expected ?? []) { - if (!failing.includes(check)) problems.push(`${name}:${check}: listed as an expected failure but it passed β€” remove it from the baseline`); - } - } - for (const name of baseline.keys()) { - if (summary.notScored.has(name)) { - problems.push(`${name}: listed in the baseline but not scored for this revision β€” remove it`); - } else if (!seen.has(name)) { - problems.push(`${name}: in the baseline but the requirements run did not report it`); - } - } - return problems; +export function baselineReport(output) { + // eslint-disable-next-line no-control-regex -- stripping the CLI's ANSI colour codes + const text = output.replace(/\u001b\[[0-9;]*m/g, ''); + const at = text.search(/^(Stale baseline entries|Unexpected failures \(not in baseline\))/m); + return (at >= 0 ? text.slice(at) : text.split(/\r?\n/).slice(-40).join('\n')).trim(); } /** - * Every `checks.json` the CLI wrote under `dir`, keyed by scenario. The CLI names the folder - * `server--` with the timestamp's colons and dots turned into dashes - * (`server-server-stateless-2026-11-10T02-36-40-817Z`, measured against 0.2.0-alpha.12). + * The verdict of a `--requirements … --expected-failures` run: the suite's exit code. Only when + * the process died of Node's Windows exit abort β€” after the suite printed everything β€” is the + * suite's own closing line read instead. */ -export function collectChecks(dir) { - const byScenario = new Map(); - const walk = (current) => { - for (const entry of readdirSync(current)) { - const full = path.join(current, entry); - if (statSync(full).isDirectory()) { - walk(full); - } else if (entry === 'checks.json') { - const folder = path.basename(path.dirname(full)); - const match = /^(?:server-)?(.+)-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.exec(folder); - if (match) { - byScenario.set(match[1], failingChecks(JSON.parse(readFileSync(full, 'utf8')))); - } - } - } - }; - if (existsSync(dir)) walk(dir); - return byScenario; +export function judgeBaselineRun({ code, output }) { + const abortedAtExit = code !== 0 && EXIT_ABORT.test(output); + const ok = code === 0 || (abortedAtExit && output.includes(BASELINE_PASSED)); + return { ok, abortedAtExit, report: ok ? '' : baselineReport(output) }; } function freePort() { @@ -243,7 +135,7 @@ async function waitForHealth(port, timeoutMs) { return false; } -/** Every argument the CLI is given is one of ours (a package spec, a loopback URL, a scenario name, a path). */ +/** Every argument the CLI is given is one of ours (a package spec, a loopback URL, a scenario name, a repository-relative path). */ const SAFE_ARGUMENT = /^[A-Za-z0-9@./:_\\-]+$/; function runCli(args, cwd, cli = CONFORMANCE_CLI) { @@ -253,7 +145,7 @@ function runCli(args, cwd, cli = CONFORMANCE_CLI) { if (unsafe !== undefined) throw new Error(`refusing to pass "${unsafe}" to the conformance CLI`); // Windows: `npx` is a .cmd shim, which Node spawns only through a shell. The argv is checked // above, so one command string (no args array) keeps the shell out of DEP0190's warning. - // No colour: the per-scenario summary lines are parsed, and FORCE_COLOR would wrap them in escapes. + // No colour: the summary line is parsed, and FORCE_COLOR would wrap it in escapes. const env = { ...process.env, NO_COLOR: '1', FORCE_COLOR: '0' }; const child = process.platform === 'win32' ? spawn(['npx', ...argv].join(' '), { cwd, env, shell: true, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }) @@ -261,8 +153,8 @@ function runCli(args, cwd, cli = CONFORMANCE_CLI) { let output = ''; child.stdout.on('data', (chunk) => { output += chunk.toString(); }); child.stderr.on('data', (chunk) => { output += chunk.toString(); }); - child.on('close', () => resolve(output)); - child.on('error', (error) => resolve(`${output}\n${error.message}`)); + child.on('close', (code) => resolve({ output, code })); + child.on('error', (error) => resolve({ output: `${output}\n${error.message}`, code: null })); }); } @@ -290,7 +182,7 @@ export async function runConformance(options = {}) { } const verdicts = []; for (const name of scenarios) { - const output = await runCli(['server', '--url', `http://127.0.0.1:${port}/mcp`, '--scenario', name], root); + const { output } = await runCli(['server', '--url', `http://127.0.0.1:${port}/mcp`, '--scenario', name], root); const verdict = judgeScenario(name, output); verdicts.push(verdict); const counts = verdict.summary ? `${verdict.summary.passed}/${verdict.summary.total}` : '-'; @@ -305,15 +197,13 @@ export async function runConformance(options = {}) { /** * The 2026-07-28 lane: one `--requirements` run of the 0.2 CLI against a local `http` server, - * judged against the baseline. Answers `{ problems, summary }`; an empty `problems` is a pass. + * judged by the suite against the baseline. Answers `{ ok, abortedAtExit, report, output }`. */ export async function runRequirements(options = {}) { const root = options.root ?? fileURLToPath(new URL('../', import.meta.url)); const entry = path.join(root, 'dist', 'index.js'); if (!existsSync(entry)) throw new Error(`${entry} not found: run the build first`); const revision = options.revision ?? REQUIREMENTS_2026; - const baselinePath = options.baselinePath ?? path.join(root, 'scripts', 'conformance', `expected-failures-${revision}.yaml`); - const baseline = parseBaseline(readFileSync(baselinePath, 'utf8')); const log = options.log ?? console.log; const port = await freePort(); @@ -326,34 +216,20 @@ export async function runRequirements(options = {}) { let serverOutput = ''; server.stdout.on('data', (chunk) => { serverOutput += chunk.toString(); }); server.stderr.on('data', (chunk) => { serverOutput += chunk.toString(); }); - const outputDir = mkdtempSync(path.join(os.tmpdir(), 'mcp-conformance-2026-')); try { if (!(await waitForHealth(port, 20_000))) { throw new Error(`the server never answered /health on port ${port}:\n${serverOutput}`); } - const output = await runCli( - ['server', '--url', `http://127.0.0.1:${port}/mcp`, '--requirements', revision, '--output-dir', outputDir], - root, - CONFORMANCE_CLI_2026 - ); - const summary = parseRequirementsSummary(output); - if (!summary) { - throw new Error(`no per-scenario summary in the conformance CLI output:\n${output}`); - } - const checks = collectChecks(outputDir); - const problems = judgeRequirements(summary, baseline, (name) => checks.get(name) ?? []); - for (const { name, passed, failed } of summary.scenarios) { - const scored = !summary.notScored.has(name); - const expected = baseline.get(name); - const tag = !scored ? 'skip' : failed === 0 ? 'ok ' : expected ? 'base' : 'FAIL'; - log(`${tag} ${name.padEnd(48)} ${passed}/${passed + failed}${!scored ? ' (not scored)' : expected ? ' (expected: fixture)' : ''}`); - } - if (problems.length && options.verbose) log(output); - return { problems, summary }; + // cwd is the repository root, so the baseline's relative path resolves and no temp or + // home path β€” which may hold a space β€” ever reaches the shell. + const { output, code } = await runCli(requirementsArgs(revision, port), root, CONFORMANCE_CLI_2026); + const verdict = judgeBaselineRun({ code, output }); + if (options.verbose) log(output); + else if (!verdict.ok) log(verdict.report); + return { ...verdict, output }; } finally { server.kill(); - rmSync(outputDir, { recursive: true, force: true }); } } @@ -363,13 +239,15 @@ if (isMain(import.meta.url)) { if (eraIndex !== -1) { const revision = process.argv[eraIndex + 1]; runRequirements({ revision, verbose }) - .then(({ problems, summary }) => { - const scored = summary.scenarios.filter((scenario) => !summary.notScored.has(scenario.name)); - if (problems.length) { - for (const problem of problems) console.error(`MCP conformance ${revision}: ${problem}`); - process.exitCode = 1; + .then(({ ok, abortedAtExit }) => { + const how = abortedAtExit + ? ' (read from the suite\'s closing line: this Node build aborts at exit on Windows β€” nodejs/node#56645, fixed in 24.20.0 and 26.7.0)' + : ''; + if (ok) { + console.log(`MCP conformance ${revision}: the suite's baseline check passed against ${CONFORMANCE_CLI_2026}${how}`); } else { - console.log(`MCP conformance ${revision}: ${scored.length} scored scenario(s) conform under the baseline against ${CONFORMANCE_CLI_2026}`); + console.error(`MCP conformance ${revision}: the suite's baseline check failed against ${CONFORMANCE_CLI_2026}${how}`); + process.exitCode = 1; } }) .catch((error) => { diff --git a/src/cli/http-command.ts b/src/cli/http-command.ts index cec9dc28f..22f4820d8 100644 --- a/src/cli/http-command.ts +++ b/src/cli/http-command.ts @@ -68,6 +68,19 @@ interface SessionData { /** Largest JSON-RPC request body accepted; also named in the 413 the parser answers with (issue #670). */ const JSON_BODY_LIMIT = '10mb'; + +/** + * What the SDK's modern handler says when it refuses a request it was sent (as + * opposed to failing on its own): the messages it builds at its rejection + * points. Matched by their openings because the SDK reports them as plain + * Errors; a wording it changes falls back to the error level, never the reverse. + */ +const SDK_REFUSAL = /^(Rejected |Unsupported Media Type)/; + +/** The Node adapter's body-limit error (its class is not exported): the client sent too much. */ +function isBodyTooLarge(error: unknown): error is Error & { status: 413 } { + return error instanceof Error && (error as { status?: unknown }).status === 413; +} /** The same limit in bytes, for the SDK's classifier and modern handler (their default is 4 MiB). */ const JSON_BODY_LIMIT_BYTES = 10 * 1024 * 1024; @@ -224,11 +237,19 @@ export function createHttpApp( legacy: 'reject', maxRequestBodySize: JSON_BODY_LIMIT_BYTES, onerror: (error: Error) => { - logger.error('Modern-era MCP handler error', { error }); + // The SDK reports every request it refuses through the same hook as its + // own faults, as a plain Error. A refusal is the caller's doing β€” a junk + // body, a missing header, a wrong content type β€” and anyone who can reach + // the port can cause one, so it is a warning; only a real fault is an error. + if (SDK_REFUSAL.test(error.message)) { + logger.warn('Modern-era MCP request refused', { reason: error.message }); + } else { + logger.error('Modern-era MCP handler error', { error }); + } }, } ); - core.notifications.addSink(handlerSink(modern.notify)); + const detachModernSink = core.notifications.addSink(handlerSink(modern.notify)); const modernNode = toNodeHandler(modern, { // The same limit as express.json, the classifier and the handler: a body between two // different limits would pass one entry and die in the next. @@ -456,6 +477,17 @@ export function createHttpApp( await transport.handleRequest(req as IncomingMessage, res as ServerResponse, req.body); } catch (error) { + if (isBodyTooLarge(error)) { + // Classifying reads a body Express did not parse (a non-JSON content + // type). Over the limit is the caller's error, answered as the JSON + // parser answers it β€” not an "Internal error" and not an error log. + if (!res.headersSent) { + res.status(413).json( + jsonRpcErrorBody(ProtocolErrorCode.InvalidRequest, `Request body too large (limit ${JSON_BODY_LIMIT})`, error.message) + ); + } + return; + } logger.error('Error handling MCP request', { error }); if (!res.headersSent) { res.status(500).json( @@ -538,7 +570,14 @@ export function createHttpApp( // eslint-disable-next-line @typescript-eslint/no-explicit-any (app as any).core = core; // eslint-disable-next-line @typescript-eslint/no-explicit-any - (app as any).modernHandler = modern; + (app as any).modernHandler = { + close: async (): Promise => { + // The sink first: the core closes its sessions after this, and each close + // fires a resource change that a closed handler has no stream for. + detachModernSink(); + await modern.close(); + }, + }; // eslint-disable-next-line @typescript-eslint/no-explicit-any (app as any).bindAddress = bind; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/server/output-resources.ts b/src/server/output-resources.ts index 7b1524f8b..3f28f5175 100644 --- a/src/server/output-resources.ts +++ b/src/server/output-resources.ts @@ -64,6 +64,16 @@ export class FacadeSubscriptions implements ResourceSubscriptions { } } +/** + * The one refusal for a resource this caller cannot have: unknown, gone, or + * outside its scope β€” all the same answer. The error's data names the URI + * that was asked for (SEP-2164's SHOULD), so a client can tell which of its + * requests this was without parsing the message. + */ +function unknownResource(uri: string): WireMcpError { + return new WireMcpError(ProtocolErrorCode.InvalidParams, `Unknown resource: ${uri}`, { uri }); +} + /** * Registers the MCP resource handlers. Only debuggee output is subscribable; * the bounded proxy-log tail is an on-demand diagnostic snapshot. @@ -113,11 +123,11 @@ export function registerResourceHandlers( ? sessionManager.getSession(sessionId) : undefined; if (!session) { - throw new WireMcpError(ProtocolErrorCode.InvalidParams, `Unknown resource: ${uri}`); + throw unknownResource(uri); } if (proxyLogSessionId) { if (!session.logDir) { - throw new WireMcpError(ProtocolErrorCode.InvalidParams, `Unknown resource: ${uri}`); + throw unknownResource(uri); } const text = await readProxyLogTail( fileSystem, @@ -147,7 +157,7 @@ export function registerResourceHandlers( const uri = request.params.uri; const sessionId = parseOutputResourceUri(uri); if (!sessionId || !sessionManager.isSessionVisibleTo(sessionId, ownerKeyOf(callCtx))) { - throw new WireMcpError(ProtocolErrorCode.InvalidParams, `Unknown resource: ${uri}`); + throw unknownResource(uri); } subscriptions.subscribe(uri); return {}; diff --git a/tests/core/unit/server/mcp-wire-errors.test.ts b/tests/core/unit/server/mcp-wire-errors.test.ts index be4b438f0..fa8c43ce4 100644 --- a/tests/core/unit/server/mcp-wire-errors.test.ts +++ b/tests/core/unit/server/mcp-wire-errors.test.ts @@ -132,6 +132,9 @@ describe('protocol errors reach SDK clients as the bare reason, with the code be const error = await caught(request); expect(error.code).toBe(ProtocolErrorCode.InvalidParams); expect(error.message).toBe(`Unknown resource: ${uri}`); + // SEP-2164: the error names the URI that was asked for, in its data + // (a SHOULD; the conformance suite's sep-2164-data-uri check). + expect(error.data).toEqual({ uri }); } }); }); diff --git a/tests/core/unit/server/protocol-facade-transport.test.ts b/tests/core/unit/server/protocol-facade-transport.test.ts index 30aea0859..64db5130d 100644 --- a/tests/core/unit/server/protocol-facade-transport.test.ts +++ b/tests/core/unit/server/protocol-facade-transport.test.ts @@ -8,9 +8,9 @@ * connection's Server be collected. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { InMemoryTransport } from '@modelcontextprotocol/server'; +import { createMcpHandler, InMemoryTransport } from '@modelcontextprotocol/server'; import { serveStdio } from '@modelcontextprotocol/server/stdio'; -import { Client } from '@modelcontextprotocol/client'; +import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'; import { DebugMcpServer } from '../../../../src/server.js'; import { createProtocolServer } from '../../../../src/server/protocol-facade.js'; import { SessionManager } from '../../../../src/session/session-manager.js'; @@ -137,6 +137,25 @@ describe('a 2026-07-28 client is identified in the log', () => { expect(identified().map(([, meta]) => (meta as { client: string }).client)).toEqual(['era-client 1.2.3', 'other-client 1.2.3']); }); + it('logs an http client once, however many per-request facades serve it', async () => { + // The http leg as the command builds it: a strict modern handler over a + // facade per request, each of which delivers nothing itself. + const handler = createMcpHandler(({ era }) => createProtocolServer(core, { era, deliver: 'external' }), { legacy: 'reject' }); + const client = new Client({ name: 'http-client', version: '2.0.0' }, { versionNegotiation: { mode: { pin: '2026-07-28' } } }); + await client.connect(new StreamableHTTPClientTransport(new URL('http://mcp.test/mcp'), { + fetch: (input, init) => handler.fetch(new Request(input, init)) + })); + closers.push(() => client.close(), () => handler.close()); + + await client.listPrompts(); + await client.listPrompts(); + await client.listResources(); + + expect(identified()).toEqual([ + ['[MCP Server] Client identified', { era: 'modern', protocolVersion: '2026-07-28', client: 'http-client 2.0.0' }] + ]); + }); + it('leaves a 2025 client to its Client initialized line', async () => { await (await connect('legacy-client', 'legacy')).listPrompts(); diff --git a/tests/unit/cli/http-command.test.ts b/tests/unit/cli/http-command.test.ts index 05345de60..55ea1e5d6 100644 --- a/tests/unit/cli/http-command.test.ts +++ b/tests/unit/cli/http-command.test.ts @@ -699,6 +699,66 @@ describe('HTTP Command Handler', () => { expect(healthBody(healthHandler).requests).toEqual({ legacy: 0, modern: 1 }); }); + // Classifying a request reads its body when Express did not parse it (a non-JSON + // content type). A body over the limit is the client's error and is answered as one β€” + // the same 413 the JSON parser gives β€” not as an "Internal error" with an error log. + it('answers 413 when the body is too large to classify, and logs no server error', async () => { + const { handler, healthHandler } = appAndHandler(); + mockToWebRequest.mockRejectedValueOnce( + Object.assign(new Error('Payload Too Large: Request body must not exceed 10485760 bytes'), { + name: 'RequestBodyTooLargeError', + status: 413, + }) + ); + const res = makeRes(); + + await handler(makeReq({ body: undefined, headers: { 'content-type': 'text/plain' } }), res); + + expect(res.status).toHaveBeenCalledWith(413); + expect(res.json.mock.calls[0][0]).toMatchObject({ + jsonrpc: '2.0', + error: { code: -32600, message: 'Request body too large (limit 10mb)' }, + id: null, + }); + expect(mockLogger.error).not.toHaveBeenCalledWith('Error handling MCP request', expect.anything()); + expect(mockModernNode).not.toHaveBeenCalled(); + expect(healthBody(healthHandler).requests).toEqual({ legacy: 0, modern: 0 }); + }); + + it('still answers 500 for a fault of its own while classifying', async () => { + const { handler } = appAndHandler(); + mockToWebRequest.mockRejectedValueOnce(new Error('stream exploded')); + const res = makeRes(); + + await handler(makeReq({ body: undefined }), res); + + expect(res.status).toHaveBeenCalledWith(500); + expect(mockLogger.error).toHaveBeenCalledWith('Error handling MCP request', expect.anything()); + }); + + // The SDK reports every request it refuses β€” a junk body, a missing header, a wrong + // content type β€” through the same hook as its own faults. A refusal is the caller's + // doing: a scanner posting `{}` must not write error lines. + it('logs a request the SDK refused as a warning, and a fault of the handler as an error', () => { + appAndHandler(); + const [, options] = mockCreateMcpHandler.mock.calls[0]; + + options!.onerror!(new Error('Rejected inbound request (jsonrpc-shape): Bad Request: the request body is not a valid JSON-RPC message')); + options!.onerror!(new Error('Rejected 2025-era request on a modern-only endpoint (era): Unsupported protocol version')); + options!.onerror!(new Error('Unsupported Media Type: Content-Type must be application/json')); + + expect(mockLogger.warn).toHaveBeenCalledTimes(3); + expect(mockLogger.warn).toHaveBeenCalledWith('Modern-era MCP request refused', { + reason: 'Rejected inbound request (jsonrpc-shape): Bad Request: the request body is not a valid JSON-RPC message', + }); + expect(mockLogger.error).not.toHaveBeenCalledWith('Modern-era MCP handler error', expect.anything()); + + const fault = new Error('factory exploded'); + options!.onerror!(fault); + + expect(mockLogger.error).toHaveBeenCalledWith('Modern-era MCP handler error', { error: fault }); + }); + it('routes a request the classifier calls legacy to the sessionful app, untouched', async () => { const { app, handler, healthHandler } = appAndHandler(); mockIsLegacyRequest.mockResolvedValue(true); @@ -1450,6 +1510,8 @@ describe('HTTP Command Handler', () => { it('handles SIGINT by closing all transports, stopping the one core, then exiting', async () => { const listen = vi.fn(() => mockHttpServer); mockApp.listen = listen; + const detachModernSink = vi.fn(); + vi.mocked(mockServer.notifications.addSink).mockReturnValue(detachModernSink); await handleHttpCommand( { port: '3001' }, @@ -1475,6 +1537,10 @@ describe('HTTP Command Handler', () => { // transports, so no request lands on a core that is closing its sessions. expect(mockModernHandler.close).toHaveBeenCalledTimes(1); expect(mockModernHandler.close.mock.invocationCallOrder[0]).toBeLessThan(t1.close.mock.invocationCallOrder[0]); + // Its sink leaves the hub before the handler closes: the core's own + // teardown fires resource changes a closed handler has no stream for. + expect(detachModernSink).toHaveBeenCalledTimes(1); + expect(detachModernSink.mock.invocationCallOrder[0]).toBeLessThan(mockModernHandler.close.mock.invocationCallOrder[0]); expect(vi.mocked(mockServer.stop).mock.invocationCallOrder[0]).toBeGreaterThan(t2.close.mock.invocationCallOrder[0]); expect(mockHttpServer.close).toHaveBeenCalled(); expect(mockExitProcess).toHaveBeenCalledWith(0); diff --git a/tests/unit/scripts/mcp-conformance.test.ts b/tests/unit/scripts/mcp-conformance.test.ts index 026c0fbff..d2c32dc23 100644 --- a/tests/unit/scripts/mcp-conformance.test.ts +++ b/tests/unit/scripts/mcp-conformance.test.ts @@ -1,52 +1,15 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; import { APPLICABLE_SCENARIOS, - collectChecks, - failingChecks, - judgeRequirements, + baselineReport, + judgeBaselineRun, judgeScenario, - parseBaseline, - parseRequirementsSummary, - parseSummary + parseSummary, + requirementsArgs } from '../../../scripts/mcp-conformance.mjs'; -const REQUIREMENTS_OUTPUT = [ - 'Running requirements 2026-07-28 (5 scenarios) against http://127.0.0.1:3997/mcp', - '=== SUMMARY ===', - 'βœ— server-stateless: 24 passed, 4 failed', - 'βœ“ tools-list: 4 passed, 0 failed', - 'βœ— tools-call-simple-text: 1 passed, 1 failed', - 'βœ“ caching: 7 passed, 0 failed', - 'βœ— tasks-lifecycle: 1 passed, 8 failed', - '', - 'Total: 37 passed, 13 failed', - '', - 'Not scored for 2026-07-28: 1 scenario(s) run, 1 failing. These do not affect conformance.', - ' βœ— tasks-lifecycle (extension)', -].join('\r\n'); - -// A scenario name outside [A-Za-z0-9-] and a summary line in a shape the parser does not know. -const ODD_OUTPUT = [ - 'Running requirements 2026-07-28 (3 scenarios) against http://127.0.0.1:1/mcp', - '=== SUMMARY ===', - 'βœ“ sep-1234_foo.bar: 1 passed, 0 failed', - 'βœ— tools-list: 4 passed, 1 failed', - 'βœ— a line the CLI printed in a shape this parser does not know', - '', - 'Total: 5 passed, 1 failed', -].join('\n'); - -const BASELINE = [ - '# fixtures this server does not have', - 'server:', - ' - tools-call-simple-text # tools/call test_simple_text', - ' - server-stateless:ServerRejectsUndeclaredCapability # test_missing_capability', - ' - server-stateless:ServerNoLogWithoutLogLevel', -].join('\n'); - const PASSING = [ "Running client scenario 'ping' against server: http://127.0.0.1:3919/mcp", 'Checks:', @@ -83,141 +46,132 @@ describe('mcp-conformance runner: reading the CLI summary', () => { }); }); +// The tail of a `--requirements … --expected-failures` run, as 0.2.0-alpha.12 prints it +// (colour codes included: the CLI colours these lines whatever NO_COLOR says). +const BASELINE_PASS = [ + 'Running requirements 2026-07-28 (50 scenarios) against http://127.0.0.1:3997/mcp', + 'βœ— server-stateless: 24 passed, 4 failed', + 'βœ“ tools-list: 4 passed, 0 failed', + '', + 'Total: 104 passed, 66 failed', + '', + '\u001b[33mExpected failures (in baseline):\u001b[0m', + ' ~ server-stateless:sep-2575-server-no-log-without-loglevel', + ' ~ tools-call-simple-text', + '', + '\u001b[32mBaseline check passed: all failures are expected.\u001b[0m', +].join('\r\n'); + +const BASELINE_FAIL = [ + 'Running requirements 2026-07-28 (50 scenarios) against http://127.0.0.1:3997/mcp', + 'βœ“ sep-2164-resource-not-found: 3 passed, 0 failed', + '', + 'Total: 104 passed, 66 failed', + '', + '\u001b[33mExpected failures (in baseline):\u001b[0m', + ' ~ tools-call-simple-text', + '', + '\u001b[31mStale baseline entries (now passing - remove from baseline):\u001b[0m', + ' βœ“ tools-call-image', + '', + '\u001b[31mUnexpected failures (not in baseline):\u001b[0m', + ' βœ— sep-2164-resource-not-found', + '', + '\u001b[31mBaseline is stale: update your expected-failures file to remove passing entries.\u001b[0m', + '', + '\u001b[31mUnexpected failures detected: these are not in your expected-failures baseline.\u001b[0m', +].join('\n'); + +// Node 23 to 24.19 and 26.0 to 26.6 abort inside process.exit on Windows (nodejs/node#56645). +const EXIT_ABORT = 'Assertion failed: !(handle->flags & UV_HANDLE_CLOSING), file src\\win\\async.c, line 76'; +const ABORT_STATUS = 3221226505; + describe('mcp-conformance runner: the 2026-07-28 requirements lane (issue #1025)', () => { - it('reads the per-scenario verdict lines and the not-scored list, CRLF included', () => { - const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; - expect(summary.scenarios).toEqual([ - { name: 'server-stateless', passed: 24, failed: 4 }, - { name: 'tools-list', passed: 4, failed: 0 }, - { name: 'tools-call-simple-text', passed: 1, failed: 1 }, - { name: 'caching', passed: 7, failed: 0 }, - { name: 'tasks-lifecycle', passed: 1, failed: 8 }, + // The suite judges its own baseline: it reads the file, fails on a failure or + // warning that is not listed and on a listed entry that passes, and says so in + // its exit code. This runner adds no judging of its own. + it('hands the suite its own baseline by a path relative to the repository root', () => { + expect(requirementsArgs('2026-07-28', 43210)).toEqual([ + 'server', + '--url', 'http://127.0.0.1:43210/mcp', + '--requirements', '2026-07-28', + '--expected-failures', 'scripts/conformance/expected-failures-2026-07-28.yaml' ]); - expect([...summary.notScored]).toEqual(['tasks-lifecycle']); - expect(summary.announced).toBe(5); - expect(summary.unparsed).toEqual([]); - expect(parseRequirementsSummary('Running requirements ...\nSegmentation fault')).toBeUndefined(); }); - it('keeps a scenario whose name carries an underscore or a dot, and surfaces a summary line it cannot read instead of dropping it', () => { - const summary = parseRequirementsSummary(ODD_OUTPUT)!; - expect(summary.scenarios.map((s) => s.name)).toEqual(['sep-1234_foo.bar', 'tools-list']); - expect(summary.unparsed).toEqual(['βœ— a line the CLI printed in a shape this parser does not know']); - expect(summary.announced).toBe(3); + it('takes the exit code as the verdict', () => { + expect(judgeBaselineRun({ code: 0, output: BASELINE_PASS })).toEqual({ ok: true, abortedAtExit: false, report: '' }); + expect(judgeBaselineRun({ code: 1, output: BASELINE_FAIL })).toMatchObject({ ok: false, abortedAtExit: false }); }); - it('fails when the header announces more scenarios than the summary reported, or a summary line could not be read', () => { - const problems = judgeRequirements(parseRequirementsSummary(ODD_OUTPUT)!, parseBaseline('server:\n'), () => []); - expect(problems).toEqual(expect.arrayContaining([ - expect.stringContaining('the run announced 3 scenario(s) but the summary reported 2'), - expect.stringContaining('could not read a summary line: βœ— a line the CLI printed'), - ])); + it('never calls a run a pass on its printed text when the exit code says it failed', () => { + // A pass line in the output of a run that exited 1 for another reason proves nothing. + expect(judgeBaselineRun({ code: 1, output: BASELINE_PASS }).ok).toBe(false); }); - it('reads whole-scenario and per-check entries out of the baseline, comments stripped', () => { - const baseline = parseBaseline(BASELINE); - expect(baseline.get('tools-call-simple-text')).toBe('all'); - expect(baseline.get('server-stateless')).toEqual(new Set(['ServerRejectsUndeclaredCapability', 'ServerNoLogWithoutLogLevel'])); - expect(baseline.has('fixtures')).toBe(false); - }); + it('reports a failed run in the words of the suite: its stale and unexpected lists, colour stripped', () => { + const { report } = judgeBaselineRun({ code: 1, output: BASELINE_FAIL }); - it('names the failing checks of a checks.json, whichever shape the CLI wrote; a SKIPPED check is not a failure', () => { - const checks = [{ name: 'A', status: 'SUCCESS' }, { name: 'B', status: 'FAILURE' }, { name: 'C', status: 'SKIPPED' }]; - expect(failingChecks(checks)).toEqual(['B']); - expect(failingChecks({ checks })).toEqual(['B']); - expect(failingChecks(undefined)).toEqual([]); + expect(report).toMatch(/^Stale baseline entries \(now passing - remove from baseline\):/); + expect(report).toContain('βœ“ tools-call-image'); + expect(report).toContain('βœ— sep-2164-resource-not-found'); + expect(report).not.toContain('\u001b'); + expect(report).not.toContain('Expected failures (in baseline)'); }); - describe('collecting the per-scenario checks.json files', () => { - const dirs: string[] = []; - afterEach(() => { - for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); - }); + it('reports the tail of the output when the suite never reached its verdict', () => { + const output = ['npm error code E404', 'npm error 404 Not Found'].join('\n'); - it('keys each checks.json by its scenario, read out of the CLI folder name server--', () => { - // Measured 2026-11-10 against 0.2.0-alpha.12: `--output-dir` holds one folder per scenario, - // `server-server-stateless-2026-11-10T02-36-40-817Z/checks.json` (colons and dots of the - // ISO timestamp replaced by dashes); a scenario name itself carries dashes. - const dir = mkdtempSync(path.join(os.tmpdir(), 'mcp-conformance-test-')); - dirs.push(dir); - const folder = path.join(dir, 'server-server-stateless-2026-11-10T02-36-40-817Z'); - mkdirSync(folder); - writeFileSync(path.join(folder, 'checks.json'), JSON.stringify([ - { id: 'sep-2575-a', name: 'ServerRejectsUndeclaredCapability', status: 'FAILURE' }, - { id: 'sep-2575-b', name: 'ServerSendsPromptsListChangedOnSubscription', status: 'SKIPPED' }, - { id: 'sep-2575-c', name: 'StatelessNoSessionId', status: 'SUCCESS' }, - ])); - const other = path.join(dir, 'server-tools-list-2026-11-10T02-36-41-000Z'); - mkdirSync(other); - writeFileSync(path.join(other, 'checks.json'), JSON.stringify([{ name: 'ToolsListOk', status: 'SUCCESS' }])); - - const byScenario = collectChecks(dir); - expect(byScenario.get('server-stateless')).toEqual(['ServerRejectsUndeclaredCapability']); - expect(byScenario.get('tools-list')).toEqual([]); - expect(collectChecks(path.join(dir, 'does-not-exist')).size).toBe(0); - }); + expect(baselineReport(output)).toBe(output); + expect(judgeBaselineRun({ code: 1, output })).toMatchObject({ ok: false, report: output }); }); - it('cannot judge a per-check entry without the scenario\'s checks.json, and says so instead of calling it passed', () => { - const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; - const problems = judgeRequirements(summary, parseBaseline(BASELINE), () => []); - expect(problems).toEqual([ - expect.stringContaining('server-stateless: 4 failed check(s) but no checks.json was written for it'), - ]); - expect(problems.join('\n')).not.toContain('listed as an expected failure but it passed'); - }); + describe('a Node build that aborts at exit on Windows', () => { + // The process dies inside process.exit, after the suite printed its verdict, so + // the exit status says nothing. Only then is the suite's closing line the verdict. + it('reads the closing line: a pass is a pass, and the answer says how it was read', () => { + const output = `${BASELINE_PASS}\r\n${EXIT_ABORT}`; - it('passes when every failure is expected and every expectation still fails', () => { - const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; - const baseline = parseBaseline([ - 'server:', - ' - tools-call-simple-text', - ' - server-stateless:ServerRejectsUndeclaredCapability', - ' - server-stateless:MissingCapabilityHttp400', - ' - server-stateless:HttpServerNoIndependentRequestsOnStream', - ' - server-stateless:ServerNoLogWithoutLogLevel', - ].join('\n')); - // All four server-stateless failures are named by checks.json and listed by check. - const checks = new Map([['server-stateless', [ - 'ServerRejectsUndeclaredCapability', 'MissingCapabilityHttp400', - 'HttpServerNoIndependentRequestsOnStream', 'ServerNoLogWithoutLogLevel', - ]]]); - expect(judgeRequirements(summary, baseline, (name) => checks.get(name) ?? [])).toEqual([]); - }); + expect(judgeBaselineRun({ code: ABORT_STATUS, output })).toEqual({ ok: true, abortedAtExit: true, report: '' }); + }); - it('fails when the summary counts more failures than checks.json names: nothing may go unaccounted', () => { - const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; - const baseline = parseBaseline(BASELINE); - // Both named checks are listed, but the summary says four failed. - const checks = new Map([['server-stateless', ['ServerRejectsUndeclaredCapability', 'ServerNoLogWithoutLogLevel']]]); - expect(judgeRequirements(summary, baseline, (name) => checks.get(name) ?? [])).toEqual(expect.arrayContaining([ - expect.stringContaining('server-stateless: the summary counts 4 failed check(s) but checks.json names 2'), - ])); - }); + it('still fails a run whose closing line is not the pass line', () => { + const output = `${BASELINE_FAIL}\n${EXIT_ABORT}`; - it('flags a baseline entry naming a scenario the revision does not score', () => { - const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; - const problems = judgeRequirements(summary, parseBaseline('server:\n - tasks-lifecycle\n'), () => []); - expect(problems).toEqual(expect.arrayContaining([ - expect.stringContaining('tasks-lifecycle: listed in the baseline but not scored for this revision'), - ])); + const verdict = judgeBaselineRun({ code: ABORT_STATUS, output }); + expect(verdict).toMatchObject({ ok: false, abortedAtExit: true }); + expect(verdict.report).toContain('βœ— sep-2164-resource-not-found'); + }); }); - it('fails on an unlisted failure, a listed check that does not fail, and a stale whole-scenario entry', () => { - const summary = parseRequirementsSummary(REQUIREMENTS_OUTPUT)!; - const checks = new Map([['server-stateless', ['ServerRejectsUndeclaredCapability', 'SomethingNew']]]); - const problems = judgeRequirements( - summary, - parseBaseline(['server:', ' - caching', ' - server-stateless:ServerRejectsUndeclaredCapability', ' - server-stateless:ServerNoLogWithoutLogLevel'].join('\n')), - (name) => checks.get(name) ?? [] + describe('the baseline file', () => { + const text = readFileSync( + fileURLToPath(new URL('../../../scripts/conformance/expected-failures-2026-07-28.yaml', import.meta.url)), + 'utf8' ); - expect(problems).toEqual(expect.arrayContaining([ - expect.stringContaining('server-stateless: 4 failed check(s) not in the baseline (SomethingNew)'), - expect.stringContaining('server-stateless:ServerNoLogWithoutLogLevel: listed as an expected failure but it passed'), - expect.stringContaining('tools-call-simple-text: 1 failed check(s) not in the baseline'), - expect.stringContaining('caching: listed as an expected failure but it passed'), - ])); - // A not-scored scenario never counts, listed or not. - expect(problems.join('\n')).not.toContain('tasks-lifecycle'); + const entries = text.split(/\r?\n/).filter((line) => /^\s+- /.test(line)); + + it('gives every entry the fixture or feature that makes it untestable here', () => { + for (const line of entries) { + expect(line, line).toMatch(/^\s+- [a-z0-9-]+(:[a-z0-9-]+)?\s+# \S/); + } + expect(entries.length).toBeGreaterThan(0); + }); + + it('lists each entry once, and no scenario both whole and by check (the suite refuses that)', () => { + const names = entries.map((line) => /- (\S+)/.exec(line)![1]); + expect(new Set(names).size).toBe(names.length); + const whole = new Set(names.filter((name) => !name.includes(':'))); + for (const name of names.filter((entry) => entry.includes(':'))) { + expect(whole.has(name.split(':')[0]), name).toBe(false); + } + }); + + it('names checks by the suite\'s check ids, which is what the suite matches', () => { + const checks = entries.map((line) => /- (\S+)/.exec(line)![1]).filter((name) => name.includes(':')); + for (const name of checks) { + expect(name, name).toMatch(/^[a-z0-9-]+:sep-\d+-[a-z0-9-]+$/); + } + }); }); }); From 9e0d009cd0020c372237286b424e03a798459464 Mon Sep 17 00:00:00 2001 From: JF Date: Tue, 10 Nov 2026 13:26:54 -0500 Subject: [PATCH 4/6] fix(conformance): excuse one check per baseline entry, never a whole scenario; log the SDK's typed refusals as warnings (#1025) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the second-look pass (fresh context, range bdf402ee..84d1bad8). The baseline listed 27 whole scenarios. A whole-scenario entry excuses every check in the scenario, so the suite's wire-schema-valid check β€” every message this server sent, its error answers included, is valid per the spec's JSON schema β€” was excused in all of them, and the file's claim that a real divergence could not hide in it was stronger than the mechanism. Every entry is now scenario:check-id, the one check that cannot run without the fixture or feature, with the ids read from a measured run's checks.json. Same 31 entries; the other checks of those scenarios are enforced again. Measured on alpha.12: "Baseline check passed", exit 0. The header says what the mechanism does not cover (an entry for a scenario the suite does not run is ignored). The http leg logged two classes of refusal at error level after all: an unsupported protocol revision (-32022) and a missing client capability reach the hook as typed protocol errors, with no "Rejected" in front for the prefix match to find. A protocol error other than an internal one is the caller's doing; it is a warning now, and an internal error stays an error. Co-Authored-By: Claude Fable 5.2 --- changelog.d/1025-conformance-2026.added.md | 2 +- docs/architecture/testing-architecture.md | 2 +- .../expected-failures-2026-07-28.yaml | 66 ++++++++++--------- src/cli/http-command.ts | 18 ++++- tests/unit/cli/http-command.test.ts | 23 ++++++- tests/unit/scripts/mcp-conformance.test.ts | 20 ++++-- 6 files changed, 91 insertions(+), 40 deletions(-) diff --git a/changelog.d/1025-conformance-2026.added.md b/changelog.d/1025-conformance-2026.added.md index 4e4ba8960..641070ddb 100644 --- a/changelog.d/1025-conformance-2026.added.md +++ b/changelog.d/1025-conformance-2026.added.md @@ -1 +1 @@ -**A second MCP conformance lane for the 2026-07-28 revision, judged by the suite itself.** `node scripts/mcp-conformance.mjs --era 2026-07-28` runs the protocol org's conformance suite on its 0.2 line (`@modelcontextprotocol/conformance@0.2.0-alpha.12`, the line the SDKs run) with `--requirements 2026-07-28` β€” everything the revision requires of a server β€” against a local `http` server, and hands the suite its own baseline with `--expected-failures`: the suite fails the run on a failure or warning that is not listed and on a listed entry that passes, and its exit code is the lane's verdict. `scripts/conformance/expected-failures-2026-07-28.yaml` lists what cannot be tested against this server β€” 18 scenarios and checks that only drive the suite's reference fixtures (the "everything server" tools, resources and prompts), and 13 that need an optional feature this server does not declare (`completions`, input-required results) β€” each with the fixture or feature named beside it; none is a protocol divergence, and by the suite's own policy a missing prerequisite is a failure, not a skip (upstream discussion: modelcontextprotocol/conformance#516). CI runs it as `mcp-conformance-2026`, required in the test summary beside the 2025-11-25 lane, which is unchanged. Running the suite found two real gaps, both fixed in both eras: `resources/templates/list` answered "Method not found" (a server advertising the resources capability must answer it; it now answers an empty list), and the error for an unknown resource carried no `data` (it now names the requested `uri`, SEP-2164). +**A second MCP conformance lane for the 2026-07-28 revision, judged by the suite itself.** `node scripts/mcp-conformance.mjs --era 2026-07-28` runs the protocol org's conformance suite on its 0.2 line (`@modelcontextprotocol/conformance@0.2.0-alpha.12`, the line the SDKs run) with `--requirements 2026-07-28` β€” everything the revision requires of a server β€” against a local `http` server, and hands the suite its own baseline with `--expected-failures`: the suite fails the run on a failure or warning that is not listed and on a listed entry that passes, and its exit code is the lane's verdict. `scripts/conformance/expected-failures-2026-07-28.yaml` lists what cannot be tested against this server β€” 18 checks that only drive the suite's reference fixtures (the "everything server" tools, resources and prompts), and 13 that need an optional feature this server does not declare (`completions`, input-required results) β€” each a single check with the fixture or feature named beside it, never a whole scenario, so every scenario's other checks (the schema validation of each message this server sends) stay enforced; none is a protocol divergence, and by the suite's own policy a missing prerequisite is a failure, not a skip (upstream discussion: modelcontextprotocol/conformance#516). CI runs it as `mcp-conformance-2026`, required in the test summary beside the 2025-11-25 lane, which is unchanged. Running the suite found two real gaps, both fixed in both eras: `resources/templates/list` answered "Method not found" (a server advertising the resources capability must answer it; it now answers an empty list), and the error for an unknown resource carried no `data` (it now names the requested `uri`, SEP-2164). diff --git a/docs/architecture/testing-architecture.md b/docs/architecture/testing-architecture.md index a3a150ad0..a1747f4e1 100644 --- a/docs/architecture/testing-architecture.md +++ b/docs/architecture/testing-architecture.md @@ -274,7 +274,7 @@ Protocol-level correctness checks. `breakpoint-messages/` contains Python script Two tiers, split along the seam between this server's own code and the SDK, and two eras, served side by side on the same endpoint (issue #1025, step 2). The 2025-11-25 clauses of the Streamable HTTP transport that `createHttpApp` decides before it hands a request to `transport.handleRequest` β€” session routing, the Host/Origin allowlist, the stale-session reaper β€” are pinned one case per clause against the real server over raw HTTP: a missing `Mcp-Session-Id` is 400; an unknown, DELETEd or reaped one is 404 `Session not found`; an invalid `Origin` is 403; an unsupported `MCP-Protocol-Version` is 400. Raw HTTP because the SDK client folds every non-2xx answer into a thrown error, and a case about a status code has to read it. The 404 cases are the ones #1017 fixed; nothing in the suite could have caught the 400 they replaced. The 2026-07-28 file pins the same way what the SDK's sessionless handler answers once our router (`isLegacyRequest`) has sent a modern request to it: `server/discover` with no session minted, `tools/list` with its cache fields and a stable order, a stale `Mcp-Session-Id` ignored, the `-32602` / `-32020` / `-32022` refusals, a 2025 `initialize` still opening a session on the same endpoint, a claim-less POST still landing on the sessionful app's 400, and an open `subscriptions/listen` stream counting the process as attended. The two negotiation files drive the v2 SDK client in `auto`, pinned and `legacy` mode through a full mock debug session on each transport, so what a real client experiences on either era is pinned too. -Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance` for the 2025-11-25 lane, `pnpm test:conformance:2026` for the 2026-07-28 one) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” and lets the suite judge it: the suite reads the baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) with its own `--expected-failures`, fails on a failure or warning that is not listed and on a listed entry that passes, and its exit code is the verdict; the script adds no judging of its own. The baseline lists what cannot be tested without the suite's reference fixtures (the `tools-call-*`, `prompts-get-*`, `resources-read-*` scenarios and four `server-stateless` checks) or without an optional feature this server does not declare (`completion-complete`, the `input-required-result-*` scenarios), each with its reason; by the suite's policy a missing prerequisite is a failure, not a skip, and a baseline is its named mechanism for that. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned. One caveat for local runs: Node 23 to 24.19 and 26.0 to 26.6 abort inside `process.exit` on Windows (nodejs/node#56645, fixed in 24.20.0 and 26.7.0) after the suite has printed everything, so there the legacy lane reads the printed summaries and the 2026 lane reads the suite's closing line when that abort is in the output; CI runs Linux and takes the exit code. +Everything after the hand-off is the SDK's conformance, and `scripts/mcp-conformance.mjs` (`pnpm test:conformance` for the 2025-11-25 lane, `pnpm test:conformance:2026` for the 2026-07-28 one) runs the protocol org's `@modelcontextprotocol/conformance` suite against a local `http` server in two lanes on Linux. The legacy lane runs the pinned 0.1.16 CLI for the scenarios that apply to a server without its reference fixtures β€” initialize, ping, the three listings, multiple SSE streams, DNS-rebinding protection. The 2026-07-28 lane (`--era 2026-07-28`, CI job `mcp-conformance-2026`) runs the pinned 0.2 alpha's `--requirements 2026-07-28` set β€” everything the revision requires of a server β€” and lets the suite judge it: the suite reads the baseline (`scripts/conformance/expected-failures-2026-07-28.yaml`) with its own `--expected-failures`, fails on a failure or warning that is not listed and on a listed entry that passes, and its exit code is the verdict; the script adds no judging of its own. The baseline lists the checks that cannot be tested without the suite's reference fixtures (one in each `tools-call-*`, `prompts-get-*` and `resources-read-*` scenario, four in `server-stateless`) or without an optional feature this server does not declare (`completion-complete`, the `input-required-result-*` scenarios), each a single `scenario:check-id` with its reason β€” never a whole scenario, so the other checks of those scenarios, the schema validation of every message this server sends among them, stay enforced; by the suite's policy a missing prerequisite is a failure, not a skip, and a baseline is its named mechanism for that. The suite has no session-lifecycle scenario, which is why the first tier exists; both CLIs are pinned. One caveat for local runs: Node 23 to 24.19 and 26.0 to 26.6 abort inside `process.exit` on Windows (nodejs/node#56645, fixed in 24.20.0 and 26.7.0) after the suite has printed everything, so there the legacy lane reads the printed summaries and the 2026 lane reads the suite's closing line when that abort is in the output; CI runs Linux and takes the exit code. ## Coverage Strategy diff --git a/scripts/conformance/expected-failures-2026-07-28.yaml b/scripts/conformance/expected-failures-2026-07-28.yaml index d5194cbb9..650759c41 100644 --- a/scripts/conformance/expected-failures-2026-07-28.yaml +++ b/scripts/conformance/expected-failures-2026-07-28.yaml @@ -1,4 +1,4 @@ -# Not testable against mcp-debugger: the scenarios and checks of +# Not testable against mcp-debugger: the checks of # `@modelcontextprotocol/conformance server --requirements 2026-07-28` that need something # this server does not host. Nothing in this file is a protocol divergence of ours. # @@ -7,28 +7,34 @@ # a baseline like this one is the mechanism it names for a server without the fixtures. # The suite reads this file itself (`--expected-failures`, scripts/mcp-conformance.mjs # --era 2026-07-28) and fails the lane on a failure or warning that is NOT listed, and on a -# listed entry that starts passing β€” so the file can only shrink honestly, and a real -# divergence can never hide in it. An entry is a scenario, or `scenario:check-id`. +# listed check that starts passing, so the file can only shrink honestly. +# +# Every entry is `scenario:check-id` β€” one check, never a whole scenario. A whole-scenario +# entry would excuse every check in it; this way each scenario's other checks stay enforced, +# above all `wire-schema-valid` (every message this server sent, its error answers included, +# is valid per the spec's JSON schema). One limit of the mechanism: the suite ignores an entry +# for a scenario it does not run or score, so a mistyped entry is silent β€” the failure it +# was meant to cover then shows as unexpected, which is what fails the lane. # # Measured 2026-11-10 against 0.2.0-alpha.12: 37 scored scenarios; the fixture names below # are read from the suite's source. Upstream discussion of reporting these as not applicable: # modelcontextprotocol/conformance#516. server: # --- Needs only a reference fixture this server does not have (18) --- - - tools-call-simple-text # tool test_simple_text - - tools-call-image # tool test_image_content - - tools-call-audio # tool test_audio_content - - tools-call-embedded-resource # tool test_embedded_resource - - tools-call-mixed-content # tool test_multiple_content_types - - tools-call-error # tool test_error_handling - - tools-call-with-progress # tool test_tool_with_progress - - resources-read-text # resource test://static-text - - resources-read-binary # resource test://static-binary - - resources-templates-read # resource template test://template/{id}/data - - prompts-get-simple # prompt test_simple_prompt - - prompts-get-with-args # prompt test_prompt_with_arguments - - prompts-get-embedded-resource # prompt test_prompt_with_embedded_resource - - prompts-get-with-image # prompt test_prompt_with_image + - tools-call-simple-text:tools-call-simple-text # tool test_simple_text + - tools-call-image:tools-call-image # tool test_image_content + - tools-call-audio:tools-call-audio # tool test_audio_content + - tools-call-embedded-resource:tools-call-embedded-resource # tool test_embedded_resource + - tools-call-mixed-content:tools-call-mixed-content # tool test_multiple_content_types + - tools-call-error:tools-call-error # tool test_error_handling + - tools-call-with-progress:tools-call-with-progress # tool test_tool_with_progress + - resources-read-text:resources-read-text # resource test://static-text + - resources-read-binary:resources-read-binary # resource test://static-binary + - resources-templates-read:resources-templates-read # resource template test://template/{id}/data + - prompts-get-simple:prompts-get-simple # prompt test_simple_prompt + - prompts-get-with-args:prompts-get-with-args # prompt test_prompt_with_arguments + - prompts-get-embedded-resource:prompts-get-embedded-resource # prompt test_prompt_with_embedded_resource + - prompts-get-with-image:prompts-get-with-image # prompt test_prompt_with_image # Four checks of the core 2026 scenario report "Not testable: server does not list the # diagnostic tool". The MUSTs behind them hold vacuously here: this server never requires a # client capability and never sends a server-to-client request or a log notification. @@ -39,20 +45,20 @@ server: # --- Needs an optional feature this server does not declare (13) --- # `completions` is not declared; the scenario also asks about the reference prompt. - - completion-complete # completion/complete on prompt test_prompt_with_arguments + - completion-complete:completion-complete # completion/complete on prompt test_prompt_with_arguments # Input-required results (SEP-2322): a tool that answers by asking the client for # elicitation, sampling or roots. Each scenario drives a reference tool or prompt. - - input-required-result-basic-elicitation # tool test_input_required_result_elicitation - - input-required-result-basic-sampling # tool test_input_required_result_sampling - - input-required-result-basic-list-roots # tool test_input_required_result_list_roots - - input-required-result-request-state # tool test_input_required_result_request_state - - input-required-result-multiple-input-requests # tool test_input_required_result_multiple_inputs - - input-required-result-multi-round # tool test_input_required_result_multi_round - - input-required-result-non-tool-request # prompt test_input_required_result_prompt - - input-required-result-result-type # tool test_input_required_result_elicitation - - input-required-result-tampered-state # tool test_input_required_result_tampered_state - - input-required-result-capability-check # tools test_input_required_result_capabilities, _elicitation + - input-required-result-basic-elicitation:sep-2322-elicitation-incomplete # tool test_input_required_result_elicitation + - input-required-result-basic-sampling:sep-2322-sampling-incomplete # tool test_input_required_result_sampling + - input-required-result-basic-list-roots:sep-2322-list-roots-incomplete # tool test_input_required_result_list_roots + - input-required-result-request-state:sep-2322-request-state-incomplete # tool test_input_required_result_request_state + - input-required-result-multiple-input-requests:sep-2322-multiple-inputs-incomplete # tool test_input_required_result_multiple_inputs + - input-required-result-multi-round:sep-2322-multi-round-r1 # tool test_input_required_result_multi_round + - input-required-result-non-tool-request:sep-2322-non-tool-incomplete # prompt test_input_required_result_prompt + - input-required-result-result-type:sep-2322-result-type-included # tool test_input_required_result_elicitation + - input-required-result-tampered-state:sep-2322-reject-tampered-state # tool test_input_required_result_tampered_state + - input-required-result-capability-check:sep-2322-respect-client-capabilities # tools test_input_required_result_capabilities, _elicitation # These two end in a WARNING, not a failure (the unknown tool answers with an error where # the suite recommends another input request); the suite's baseline counts a warning. - - input-required-result-missing-input-response # tool test_input_required_result_elicitation, prompt _prompt - - input-required-result-ignore-extra-params # tool test_input_required_result_elicitation + - input-required-result-missing-input-response:sep-2322-missing-response-rerequests # tool test_input_required_result_elicitation, prompt _prompt + - input-required-result-ignore-extra-params:sep-2322-ignore-unexpected-params # tool test_input_required_result_elicitation diff --git a/src/cli/http-command.ts b/src/cli/http-command.ts index 3c3fc5abf..e5e262123 100644 --- a/src/cli/http-command.ts +++ b/src/cli/http-command.ts @@ -3,7 +3,7 @@ import type { Express, Request, Response, NextFunction } from 'express'; import express from 'express'; import { randomUUID } from 'crypto'; import { IncomingMessage, ServerResponse } from 'http'; -import { createMcpHandler, isInitializeRequest, isLegacyRequest, ProtocolErrorCode } from '@modelcontextprotocol/server'; +import { createMcpHandler, isInitializeRequest, isLegacyRequest, ProtocolErrorCode, ProtocolError } from '@modelcontextprotocol/server'; import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from '@modelcontextprotocol/node'; import { DebugMcpServer, type DebugMcpServerOptions } from '../server.js'; import { createProtocolServer } from '../server/protocol-facade.js'; @@ -77,6 +77,20 @@ const JSON_BODY_LIMIT = '10mb'; */ const SDK_REFUSAL = /^(Rejected |Unsupported Media Type)/; +/** + * Whether an error the modern handler reported is a refusal of the caller's + * request rather than a fault. Besides the plain-Error refusals above, the SDK + * reports some as typed protocol errors β€” an unsupported protocol revision + * (-32022), a missing client capability β€” which are the caller's doing by + * definition; only an internal error is the server's. + */ +function isRefusedRequest(error: Error): boolean { + if (error instanceof ProtocolError) { + return error.code !== ProtocolErrorCode.InternalError; + } + return SDK_REFUSAL.test(error.message); +} + /** The Node adapter's body-limit error (its class is not exported): the client sent too much. */ function isBodyTooLarge(error: unknown): error is Error & { status: 413 } { return error instanceof Error && (error as { status?: unknown }).status === 413; @@ -241,7 +255,7 @@ export function createHttpApp( // own faults, as a plain Error. A refusal is the caller's doing β€” a junk // body, a missing header, a wrong content type β€” and anyone who can reach // the port can cause one, so it is a warning; only a real fault is an error. - if (SDK_REFUSAL.test(error.message)) { + if (isRefusedRequest(error)) { logger.warn('Modern-era MCP request refused', { reason: error.message }); } else { logger.error('Modern-era MCP handler error', { error }); diff --git a/tests/unit/cli/http-command.test.ts b/tests/unit/cli/http-command.test.ts index 55ea1e5d6..fece4b270 100644 --- a/tests/unit/cli/http-command.test.ts +++ b/tests/unit/cli/http-command.test.ts @@ -33,7 +33,7 @@ const expressMock = vi.hoisted(() => { vi.mock('express', () => ({ default: expressMock.express })); import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from '@modelcontextprotocol/node'; -import { createMcpHandler, isLegacyRequest } from '@modelcontextprotocol/server'; +import { createMcpHandler, isLegacyRequest, ProtocolError, ProtocolErrorCode } from '@modelcontextprotocol/server'; const MockedStreamableHTTPServerTransport = vi.mocked(NodeStreamableHTTPServerTransport); const mockCreateMcpHandler = vi.mocked(createMcpHandler); @@ -759,6 +759,27 @@ describe('HTTP Command Handler', () => { expect(mockLogger.error).toHaveBeenCalledWith('Modern-era MCP handler error', { error: fault }); }); + // Some refusals reach the hook as the SDK's typed protocol errors, with no + // "Rejected" in front: an unsupported protocol revision (-32022), a missing + // client capability. They are refusals all the same β€” a client whose + // preferred revision is newer than ours would write one on every connect. + it('logs a typed protocol refusal as a warning too, but an internal error as an error', () => { + appAndHandler(); + const [, options] = mockCreateMcpHandler.mock.calls[0]; + + options!.onerror!(new ProtocolError(-32022, 'Unsupported protocol version: 2099-01-01')); + + expect(mockLogger.warn).toHaveBeenCalledWith('Modern-era MCP request refused', { + reason: 'Unsupported protocol version: 2099-01-01', + }); + expect(mockLogger.error).not.toHaveBeenCalledWith('Modern-era MCP handler error', expect.anything()); + + const fault = new ProtocolError(ProtocolErrorCode.InternalError, 'the factory threw'); + options!.onerror!(fault); + + expect(mockLogger.error).toHaveBeenCalledWith('Modern-era MCP handler error', { error: fault }); + }); + it('routes a request the classifier calls legacy to the sessionful app, untouched', async () => { const { app, handler, healthHandler } = appAndHandler(); mockIsLegacyRequest.mockResolvedValue(true); diff --git a/tests/unit/scripts/mcp-conformance.test.ts b/tests/unit/scripts/mcp-conformance.test.ts index d2c32dc23..0cbcc00ca 100644 --- a/tests/unit/scripts/mcp-conformance.test.ts +++ b/tests/unit/scripts/mcp-conformance.test.ts @@ -153,11 +153,21 @@ describe('mcp-conformance runner: the 2026-07-28 requirements lane (issue #1025) it('gives every entry the fixture or feature that makes it untestable here', () => { for (const line of entries) { - expect(line, line).toMatch(/^\s+- [a-z0-9-]+(:[a-z0-9-]+)?\s+# \S/); + expect(line, line).toMatch(/^\s+- [a-z0-9-]+:[a-z0-9-]+\s+# \S/); } expect(entries.length).toBeGreaterThan(0); }); + // A whole-scenario entry excuses every check in the scenario. Naming one + // check keeps the others enforced β€” above all wire-schema-valid, the suite's + // check that every message this server sent is valid per the spec's schema. + it('excuses one check per entry, never a whole scenario, and never the schema check', () => { + for (const name of entries.map((line) => /- (\S+)/.exec(line)![1])) { + expect(name, name).toMatch(/^[a-z0-9-]+:[a-z0-9-]+$/); + expect(name, name).not.toMatch(/:wire-schema-valid$/); + } + }); + it('lists each entry once, and no scenario both whole and by check (the suite refuses that)', () => { const names = entries.map((line) => /- (\S+)/.exec(line)![1]); expect(new Set(names).size).toBe(names.length); @@ -167,10 +177,10 @@ describe('mcp-conformance runner: the 2026-07-28 requirements lane (issue #1025) } }); - it('names checks by the suite\'s check ids, which is what the suite matches', () => { - const checks = entries.map((line) => /- (\S+)/.exec(line)![1]).filter((name) => name.includes(':')); - for (const name of checks) { - expect(name, name).toMatch(/^[a-z0-9-]+:sep-\d+-[a-z0-9-]+$/); + it('names checks by the suite\'s check ids, which is what the suite matches (ids are kebab-case; names are CamelCase)', () => { + const checks = entries.map((line) => /- (\S+)/.exec(line)![1].split(':')[1]); + for (const id of checks) { + expect(id, id).toMatch(/^[a-z0-9]+(-[a-z0-9]+)+$/); } }); }); From 901db66f93bb70d9eb6012bda2c9523db0756ae1 Mon Sep 17 00:00:00 2001 From: debugmcpdev Date: Tue, 10 Nov 2026 22:43:33 -0500 Subject: [PATCH 5/6] ci: use the reviewed pnpm action in the new conformance job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7548b6d9c..cd4d5efda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -662,7 +662,7 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup pnpm - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1 - name: Setup Node.js uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: From 7f1c02d7b1797cde63131d40e179b448a38c6d85 Mon Sep 17 00:00:00 2001 From: debugmcpdev Date: Wed, 11 Nov 2026 06:29:29 -0500 Subject: [PATCH 6/6] test(http): verify cleanup after modern handler shutdown failure Assert that sink detachment, legacy transport closure, core shutdown and listener cleanup still complete when the modern handler rejects close. --- tests/unit/cli/http-command.test.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/unit/cli/http-command.test.ts b/tests/unit/cli/http-command.test.ts index fece4b270..5ee8be4cd 100644 --- a/tests/unit/cli/http-command.test.ts +++ b/tests/unit/cli/http-command.test.ts @@ -1567,6 +1567,34 @@ describe('HTTP Command Handler', () => { expect(mockExitProcess).toHaveBeenCalledWith(0); }); + it('completes shutdown after the modern handler fails to close', async () => { + mockApp.listen = vi.fn(() => mockHttpServer); + const error = new Error('modern close failed'); + mockModernHandler.close.mockRejectedValueOnce(error); + const detachModernSink = vi.fn(); + vi.mocked(mockServer.notifications.addSink).mockReturnValue(detachModernSink); + + await handleHttpCommand( + { port: '3001' }, + { logger: mockLogger, serverFactory: mockServerFactory, protocolServerFactory: mockProtocolServerFactory, exitProcess: mockExitProcess, proc: fakeProc } + ); + const transport = { close: vi.fn().mockResolvedValue(undefined) }; + const sessions = mockApp.httpSessions as Map; + sessions.set('legacy', { transport }); + + await fakeProc.lastListener('SIGINT')(); + + expect(mockLogger.error).toHaveBeenCalledWith('Error closing the modern-era handler during shutdown', { error }); + expect(detachModernSink).toHaveBeenCalledTimes(1); + expect(detachModernSink.mock.invocationCallOrder[0]).toBeLessThan(mockModernHandler.close.mock.invocationCallOrder[0]); + expect(transport.close).toHaveBeenCalledTimes(1); + expect(mockServer.stop).toHaveBeenCalledTimes(1); + expect(vi.mocked(mockServer.stop).mock.invocationCallOrder[0]).toBeGreaterThan(transport.close.mock.invocationCallOrder[0]); + expect(sessions.size).toBe(0); + expect(mockHttpServer.close).toHaveBeenCalledTimes(1); + expect(mockExitProcess).toHaveBeenCalledExactlyOnceWith(0); + }); + it('proceeds past a hung session stop after the guard and still exits 0 (issue #337)', async () => { const listen = vi.fn(() => mockHttpServer); mockApp.listen = listen;