diff --git a/src/fetch/README.md b/src/fetch/README.md index 7f90ee4a7c..6c7a078920 100644 --- a/src/fetch/README.md +++ b/src/fetch/README.md @@ -15,6 +15,8 @@ Requires MCP Python SDK 2.x (`mcp>=2.3,<3`). The fetch tool will truncate the response, but by using the `start_index` argument, you can specify where to start the content extraction. This lets models read a webpage in chunks, until they find the information they need. +The markdown of an HTML page opens with the page's title as a `#` heading (its `og:title`, else its `

` when the `` holds it, else its `<title>` or `<h1>`), because the extraction drops a heading that repeats the title. It is not added when the markdown already opens with it, to a raw result, or to a page that failed to simplify. + ### Available Tools - `fetch` - Fetches a URL from the internet and extracts its contents as markdown. diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index 7315a65ce5..2135436c32 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -41,6 +41,8 @@ from protego import Protego from pydantic import AnyUrl, Field +from .title import with_page_title + # The version this server reports in serverInfo, read from the installed # distribution's metadata (pyproject.toml) so it cannot drift from the # published version (#360). Without it the SDK reports its own `mcp` version. @@ -601,7 +603,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None: content = await asyncio.to_thread( extract_content_from_html, page_raw, use_readability_js ) - return content, "" + return await asyncio.to_thread(with_page_title, page_raw, content), "" # A response with no Content-Type would otherwise leave an empty slot here. declared_type = content_type or "(not declared)" diff --git a/src/fetch/src/mcp_server_fetch/title.py b/src/fetch/src/mcp_server_fetch/title.py new file mode 100644 index 0000000000..074387211f --- /dev/null +++ b/src/fetch/src/mcp_server_fetch/title.py @@ -0,0 +1,150 @@ +# The page title for the fetch server's simplified (Markdown) result (#2472). +# +# Readability.js removes an article heading that repeats the document title, +# and neither extractor returns the `<title>` itself in the content the server +# converts, so a page such as a Sphinx release note came back without the line +# that says what it is. This module reads the title from the raw HTML and puts +# it at the top of the Markdown as a level-one heading, unless the Markdown +# already opens with it. +# +# It works from the raw page rather than from readabilipy's result, so it is +# independent of how the extractor is invoked and is applied at the one call +# site in `fetch_url` that simplifies HTML. The raw result (`raw: true`) and +# non-HTML bodies never pass through it. + +import re + +from bs4 import BeautifulSoup, Tag + +# The marker `extract_content_from_html` returns when a page simplifies to +# nothing. A title on top of it would make the failure read as content. +_FAILURE_PREFIX = "<error>" + +# Titles are compared word by word: case, Markdown heading markers, emphasis, +# escapes, Sphinx's "¶" permalink and other punctuation are ignored, but word +# boundaries are kept, so "Rust" does not match "Trust". +_WORD = re.compile(r"[^\W_]+") +# Characters that Markdown or inline HTML would interpret in the emitted +# heading ("<T>" read as a tag, "__init__" as emphasis, "<" as an entity), +# each escaped with a backslash so the heading shows the title as written. +_MARKDOWN_SPECIAL = re.compile(r"([\\`*_\[\]<>#&|~])") + + +def _normalized(text: str) -> str: + """``text`` as its casefolded words, one space apart, padded with spaces. + + The padding makes ``in`` a whole-word, whole-phrase containment test. + """ + words = _WORD.findall(text.casefold()) + return f" {' '.join(words)} " if words else "" + + +def _starts_tag(char: str) -> bool: + return char.isascii() and (char.isalpha() or char in "/!?") + + +def _visible(line: str) -> str: + """The text a reader sees on one Markdown line. + + A link's destination (`](...)`) and an HTML tag (an autolink's URL + included) are dropped; a code span's contents are kept, tag-like text + and all. As in CommonMark, a `<` starts a tag only before a letter, `/`, + `!` or `?` (so "x < 5 and y > 3" is text), and a backslash-escaped + character is text. One pass, so a long line with an unterminated `](`, `<` or + backtick costs linear time: each closer is searched for forward from + where it is needed, and once a search for one fails it is never repeated. + """ + out: list[str] = [] + missing: set[str] = set() + i, end = 0, len(line) + while i < end: + if line[i] == "\\" and i + 1 < end: + out.append(line[i + 1]) + i += 2 + continue + for opener, closer, keep in ( + ("`", "`", True), + ("<", ">", False), + ("](", ")", False), + ): + if closer in missing or not line.startswith(opener, i): + continue + if opener == "<" and not _starts_tag(line[i + 1 : i + 2]): + continue + j = line.find(closer, i + len(opener)) + if j == -1: + missing.add(closer) + continue + out.append(line[i + len(opener) : j] if keep else " ") + i = j + 1 + break + else: + out.append(line[i]) + i += 1 + return "".join(out) + + +def _collapsed(text: str) -> str: + return " ".join(text.split()) + + +def _text(candidate: str) -> str: + # A trailing "¶" is a permalink that is not marked up as one. + return _collapsed(candidate).rstrip("¶").strip() + + +def page_title(html: str) -> str | None: + """The page's title, or None when the page names none. + + In order: `og:title`; the first `<h1>` when `<title>` holds it, since + `<title>` then only adds the site's name ("What's new — pandas docs"); + `<title>`; the first `<h1>`. + """ + soup = BeautifulSoup(html, "html.parser") + og = soup.find("meta", attrs={"property": "og:title"}) + og_title = _text(str(og.get("content", ""))) if isinstance(og, Tag) else "" + title = _text(soup.title.get_text(" ")) if soup.title else "" + h1 = soup.find("h1") + heading = "" + if isinstance(h1, Tag): + # Sphinx appends a permalink anchor ("¶", or "#" in some themes) to + # each heading; it is not part of the heading's text. + for link in h1.find_all("a", class_="headerlink"): + link.decompose() + heading = _text(h1.get_text(" ")) + heading_in_title = bool(_normalized(heading)) and _normalized( + heading + ) in _normalized(title) + if og_title: + return og_title + if heading_in_title: + return heading + return title or heading or None + + +def _opens_with_title(content: str, title: str) -> bool: + """Whether the Markdown's first non-empty line already carries the title. + + A first line holding the whole title counts (the pure-Python extractor + opens with the title in a paragraph), and so does a first heading that the + title holds: `<title>` often adds the site name ("What's new — pandas + docs") to the page's own heading ("What's new"). + """ + first = next((line for line in content.splitlines() if line.strip()), "") + line, wanted = _normalized(_visible(first)), _normalized(title) + if not line or not wanted: + return False + if wanted in line: + return True + return first.lstrip().startswith("#") and line in wanted + + +def with_page_title(html: str, content: str) -> str: + """``content`` with the page's title as a heading on top, if it lacks one.""" + if content.startswith(_FAILURE_PREFIX): + return content + title = page_title(html) + if title is None or _opens_with_title(content, title): + return content + heading = _MARKDOWN_SPECIAL.sub(r"\\\1", title) + return f"# {heading}\n\n{content.lstrip()}" diff --git a/src/fetch/tests/test_page_title.py b/src/fetch/tests/test_page_title.py new file mode 100644 index 0000000000..be4978a31b --- /dev/null +++ b/src/fetch/tests/test_page_title.py @@ -0,0 +1,225 @@ +"""Tests for the page title on the fetch tool's simplified result (#2472). + +Readability.js drops an article heading that repeats the document title, so a +page such as pandas' "What's new in 2.1.0" release note came back without the +line naming it. ``with_page_title`` now puts the title on top as a heading +unless the Markdown already opens with it. The protocol-level tests drive the +real server through an MCP client with HTTP served by ``harness.FakeWeb``, on +both extractor paths, and pin that the raw result is untouched; the unit tests +below pin each rule for choosing the title and spotting a duplicate. +""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest +from mcp import ClientSession +from mcp.types import CallToolResult + +from mcp_server_fetch.title import page_title, with_page_title + +from .harness import FakeWeb, connect + +PAGE = "https://example.com/page" +ROBOTS = "https://example.com/robots.txt" +PREFIX = f"Contents of {PAGE}:\n" + +# The shape of a Sphinx (pydata theme) page: `<title>` adds the site name to +# the `<h1>`, which carries a "#" permalink. +BODY = "<p>These are the changes in pandas 2.1.0, with a long enough body. " * 8 +SPHINX_PAGE = f"""<!DOCTYPE html> +<html><head><title>What’s new in 2.1.0 (Aug 30, 2023) — pandas documentation +
+

What’s new in 2.1.0 (Aug 30, 2023)#

+{BODY}

+
""" + + +def text_of(result: CallToolResult) -> str: + assert not result.is_error, result + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) + assert len(data["content"]) == 1 + return data["content"][0]["text"] + + +async def fetch(web: FakeWeb, html: str, **arguments: Any) -> str: + web.add(ROBOTS, httpx.Response(404)) + web.add( + PAGE, + httpx.Response(200, text=html, headers={"content-type": "text/html"}), + ) + async with connect() as (session, _): + assert isinstance(session, ClientSession) + result = await session.call_tool("fetch", {"url": PAGE, **arguments}) + return text_of(result) + + +@pytest.mark.usefixtures("node_readability", "private_tempdir") +async def test_readability_js_result_gains_the_dropped_title(web: FakeWeb) -> None: + text = await fetch(web, SPHINX_PAGE) + assert text.startswith(f"{PREFIX}# What’s new in 2.1.0 (Aug 30, 2023)\n\n") + assert text.count("What’s new in 2.1.0") == 1 + assert "These are the changes in pandas 2.1.0" in text + + +@pytest.mark.usefixtures("python_readability") +async def test_python_result_is_not_given_a_second_title(web: FakeWeb) -> None: + # The pure-Python extractor keeps the `

` (and opens with the + # ``), so the title is already there and nothing is added. + text = await fetch(web, SPHINX_PAGE) + assert not text.startswith(f"{PREFIX}# ") + assert "What’s new in 2.1.0" in text + assert "These are the changes in pandas 2.1.0" in text + + +async def test_raw_result_is_untouched(web: FakeWeb) -> None: + text = await fetch(web, SPHINX_PAGE, raw=True) + assert text.endswith(SPHINX_PAGE) + assert "\n# What’s new" not in text + + +@pytest.mark.usefixtures("node_readability", "private_tempdir") +async def test_failed_simplification_is_not_titled(web: FakeWeb) -> None: + text = await fetch( + web, "<html><head><title>Shell" + ) + assert text == f"{PREFIX}Page failed to be simplified from HTML" + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + # og:title wins over everything else. + ( + 'T

T

', + "Social title", + ), + # The `

` wins when `` holds it. + ("<title>Heading | Site

Heading

", "Heading"), + # Otherwise ``. + ("<title>Document title

Something else

", "Document title"), + # A trailing "¶" permalink that is plain text is dropped. + ("Doc ¶", "Doc"), + # No ``: the `<h1>`. + ("<body><h1>Only heading</h1></body>", "Only heading"), + # An empty og:title and title fall through to the `<h1>`. + ( + '<meta property="og:title" content=""><title>

H

', + "H", + ), + # Nothing names the page. + ("

No title

", None), + ("

#

", None), + ], +) +def test_page_title(html: str, expected: str | None) -> None: + assert page_title(html) == expected + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + # No heading: the title goes on top. + ("Body text.\n", "# Title Here\n\nBody text.\n"), + # Leading blank lines are folded into the separator. + ("\n\nBody text.\n", "# Title Here\n\nBody text.\n"), + # Already opens with the title as a heading, at any level. + ("# Title Here\n\nBody.", "# Title Here\n\nBody."), + ("\n## title here ¶\n\nBody.", "\n## title here ¶\n\nBody."), + # A first line that holds the whole title, heading or not. + ("

Title Here — Site

Skip\n", "

Title Here — Site

Skip\n"), + # A first heading the title holds (the title adds a site name). + ("# Title\n\nBody.", "# Title\n\nBody."), + # A first paragraph the title holds is not a heading: title added. + ("Title\n\nBody.", "# Title Here\n\nTitle\n\nBody."), + # A first line of punctuation only cannot match. + ("---\n\nBody.", "# Title Here\n\n---\n\nBody."), + # The title only in a link destination or tag is not shown: added. + ( + "[Source](https://example.com/title-here)\n", + "# Title Here\n\n[Source](https://example.com/title-here)\n", + ), + ( + "x\n", + "# Title Here\n\nx\n", + ), + # The title as a link's visible text is shown: not added. + ("[Title Here](https://example.com)\n", "[Title Here](https://example.com)\n"), + # Empty content still gains the title. + ("", "# Title Here\n\n"), + ], +) +def test_with_page_title(content: str, expected: str) -> None: + assert with_page_title("Title Here", content) == expected + + +def test_with_page_title_leaves_untitled_page_and_failure_alone() -> None: + assert with_page_title("

x

", "Body.") == "Body." + failure = "Page failed to be simplified from HTML" + assert with_page_title("T", failure) == failure + + +@pytest.mark.parametrize( + ("title", "content"), + [ + # Matching is by whole words, in both directions. + ("Rust", "Trust the compiler."), + ("Go", "Algorithms make this possible."), + ("Rust Guide", "# Rust Gui\n"), + ], +) +def test_partial_words_do_not_match(title: str, content: str) -> None: + html = f"{title}" + assert with_page_title(html, content) == f"# {title}\n\n{content}" + + +def test_code_span_contents_count_as_visible() -> None: + # A heading's inline HTML example is text, not a tag, inside a code span. + content = "# The `` element\n\nBody." + assert ( + with_page_title("The <input> element", content) == content + ) + + +@pytest.mark.parametrize( + ("title_html", "content"), + [ + # A "<" before a space is text, not a tag. + ("Comparing x < 5 and y > 3", "# Comparing x < 5 and y > 3\n\nBody."), + # Backslash-escaped brackets are text too. + ("Using <T>", "# Using \\\n\nBody."), + ], +) +def test_literal_angle_brackets_count_as_visible(title_html: str, content: str) -> None: + assert with_page_title(f"{title_html}", content) == content + + +@pytest.mark.parametrize("opener", ["](", " None: + # Linear in the line's length: a quadratic scan of 200,000 openers would + # not finish in the test's time. The title is not on the line, so it is added. + content = opener * 200_000 + result = with_page_title("Title Here", content) + assert result == f"# Title Here\n\n{content}" + + +@pytest.mark.parametrize( + ("title_html", "heading"), + [ + ("Using <T>", "# Using \\"), + ("__init__ and *args", "# \\_\\_init\\_\\_ and \\*args"), + ( + "[Draft] R&D #3 `x` | a~b \\ c", + "# \\[Draft\\] R\\&D \\#3 \\`x\\` \\| a\\~b \\\\ c", + ), + ], +) +def test_heading_escapes_markdown_and_html(title_html: str, heading: str) -> None: + result = with_page_title(f"{title_html}", "Body.") + assert result == f"{heading}\n\nBody." + + +def test_title_with_no_word_characters_is_not_matched() -> None: + assert with_page_title("—", "Body.") == "# —\n\nBody."