diff --git a/src/fetch/README.md b/src/fetch/README.md index 09b4e8b5d8..9c48bbea78 100644 --- a/src/fetch/README.md +++ b/src/fetch/README.md @@ -39,6 +39,10 @@ The fetch tool will truncate the response, but by using the `start_index` argume The server is built on the MCP Python SDK's high-level `MCPServer`: the tool and the prompt are decorated functions, and their type hints and field descriptions are the input schema and the prompt's arguments. +### Content negotiation + +The tool and the prompt ask for markdown first, with the request header `Accept: text/markdown, text/html;q=0.9, */*;q=0.8`. A site that serves markdown through content negotiation sends it directly, and a `text/markdown` response is returned as-is: it is not simplified and carries no raw-content notice. A site that ignores the header sends what it always did, and HTML is simplified to markdown as before. A `raw` fetch keeps the default `Accept: */*`, so it gets whatever the site serves by default (for a web page, its HTML), unsimplified; a markdown response it receives anyway is returned as-is too. + ### Protocol versions The server speaks both eras of the Model Context Protocol over stdio, with nothing to configure: a **2026-07-28** client (discovered with `server/discover`, no session) and a **2025-11-25** client (the `initialize` handshake). The client's first request decides which. Tools and prompts behave the same in both. Methods the 2026-07-28 specification removes, such as `ping`, are answered only on a 2025-11-25 connection. diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index a36c5a2547..535e2cc805 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -507,6 +507,17 @@ async def check_may_autonomously_fetch_url( ) +# The Accept header of a non-raw fetch (#5198). A site that negotiates content +# can then serve native markdown, which is returned as-is, and a site that does +# not still serves HTML (preferred over everything else) to be simplified. +MARKDOWN_ACCEPT = "text/markdown, text/html;q=0.9, */*;q=0.8" + + +def _is_markdown(content_type: str) -> bool: + """Whether a Content-Type header declares markdown (``text/markdown``, RFC 7763).""" + return content_type.split(";", 1)[0].strip().lower() == "text/markdown" + + def _body_looks_like_html(page_raw: str) -> bool: """Whether a body served with no Content-Type is an HTML document. @@ -557,6 +568,10 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None: hooks.append(_check_redirect_robots_txt) if hooks: client.event_hooks = {"request": hooks} + # A raw fetch keeps httpx's default Accept: */*, so it gets the site's + # default representation (a page's HTML), as before this negotiation. + if not force_raw: + client.headers["Accept"] = MARKDOWN_ACCEPT try: response = await client.get( url, @@ -574,6 +589,10 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None: page_raw = decode_page(response) content_type = response.headers.get("content-type", "") + # Markdown is already what this tool returns, so it is not simplified, and + # it carries no raw-content prefix even under force_raw: it is unchanged. + if _is_markdown(content_type): + return page_raw, "" is_page_html = ( "Test Page +
+

Hello World

+

This is a link in a test paragraph +with enough words to count as the main content of the page.

+
+""" + + +def markdown(content_type: str = "text/markdown; charset=utf-8") -> httpx.Response: + return httpx.Response( + 200, content=MARKDOWN.encode(), headers={"content-type": content_type} + ) + + +async def call_fetch(web: FakeWeb, arguments: dict[str, object]) -> str: + async with connect() as (session, _): + result = await session.call_tool("fetch", {"url": PAGE, **arguments}) + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) + assert data["isError"] is False + return data["content"][0]["text"] + + +def page_requests(web: FakeWeb) -> list[httpx.Request]: + return [r for r in web.requests if str(r.url) == PAGE] + + +async def test_a_fetch_prefers_markdown_then_html(web: FakeWeb) -> None: + web.add(ROBOTS, httpx.Response(404)) + web.add(PAGE, markdown()) + await call_fetch(web, {}) + [request] = page_requests(web) + assert request.headers["accept"] == "text/markdown, text/html;q=0.9, */*;q=0.8" + assert MARKDOWN_ACCEPT == request.headers["accept"] + # The robots.txt lookup is not a page fetch and asks for nothing new. + [robots] = [r for r in web.requests if str(r.url) == ROBOTS] + assert robots.headers["accept"] == "*/*" + + +@pytest.mark.parametrize( + "content_type", ["text/markdown", "text/markdown; charset=utf-8", "Text/Markdown"] +) +async def test_markdown_is_returned_as_is_without_the_raw_prefix( + web: FakeWeb, content_type: str +) -> None: + web.add(ROBOTS, httpx.Response(404)) + web.add(PAGE, markdown(content_type)) + text = await call_fetch(web, {}) + assert text == f"Contents of {PAGE}:\n{MARKDOWN}" + + +async def test_a_raw_fetch_keeps_the_default_accept(web: FakeWeb) -> None: + web.add(ROBOTS, httpx.Response(404)) + web.add( + PAGE, httpx.Response(200, text=ARTICLE, headers={"content-type": "text/html"}) + ) + text = await call_fetch(web, {"raw": True}) + [request] = page_requests(web) + assert request.headers["accept"] == "*/*" + assert text.endswith(f"Contents of {PAGE}:\n{ARTICLE}") + + +async def test_markdown_under_raw_is_unchanged_and_unprefixed(web: FakeWeb) -> None: + # Markdown served unasked (a .md file, say) is the same either way. + web.add(ROBOTS, httpx.Response(404)) + web.add(PAGE, markdown()) + text = await call_fetch(web, {"raw": True}) + assert text == f"Contents of {PAGE}:\n{MARKDOWN}" + + +@pytest.mark.usefixtures("python_readability") +async def test_html_from_a_site_that_ignores_accept_is_still_simplified( + web: FakeWeb, +) -> None: + web.add(ROBOTS, httpx.Response(404)) + web.add( + PAGE, httpx.Response(200, text=ARTICLE, headers={"content-type": "text/html"}) + ) + text = await call_fetch(web, {}) + assert "cannot be simplified" not in text + assert "
" not in text + assert "Hello World" in text + + +async def test_the_prompt_returns_markdown_as_is(web: FakeWeb) -> None: + web.add(PAGE, markdown()) + async with connect() as (session, _): + result = await session.get_prompt("fetch", {"url": PAGE}) + message = result.model_dump(by_alias=True, mode="json")["messages"][0] + assert message["content"]["text"].endswith(MARKDOWN) + assert "cannot be simplified" not in message["content"]["text"] + [request] = page_requests(web) + assert request.headers["accept"] == MARKDOWN_ACCEPT + + +@pytest.mark.parametrize( + ("content_type", "expected"), + [ + ("text/markdown", True), + (" text/markdown ; variant=GFM", True), + ("text/markdownish", False), + ("text/x-markdown", False), + ("text/plain", False), + ("", False), + ], +) +def test_is_markdown(content_type: str, expected: bool) -> None: + assert _is_markdown(content_type) is expected