From efbeb274aac27f002c22ccbc9f4236de734b3ab2 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Tue, 27 Oct 2026 00:51:52 -0400 Subject: [PATCH 1/3] fix(fetch): recover server-rendered content hidden until hydration Progressively rendered pages stream a visible loading shell, then the real content in a container hidden until the client hydrates it. Readability.js drops hidden nodes, so fetch returned only the shell. When a hidden element that is not page furniture holds more text than the whole extraction, unhide it and run Readability.js again, keeping the second result only if it has more text. Pages whose hidden elements are smaller are extracted once, as before. Closes #3878 Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- src/fetch/README.md | 6 + src/fetch/src/mcp_server_fetch/server.py | 114 ++++++++++-- src/fetch/tests/test_hidden_ssr.py | 210 +++++++++++++++++++++++ 3 files changed, 316 insertions(+), 14 deletions(-) create mode 100644 src/fetch/tests/test_hidden_ssr.py diff --git a/src/fetch/README.md b/src/fetch/README.md index 8aaf265c8a..09b4e8b5d8 100644 --- a/src/fetch/README.md +++ b/src/fetch/README.md @@ -219,6 +219,12 @@ pure-Python extractor, which keeps headings and text but drops link targets. The timeout, so a Node.js that hangs, or fails in a way readabilipy does not detect, stalls the tool call until the client gives up. +Readability.js discards hidden elements. Pages rendered progressively on the server often send a visible loading +shell, then the real content in a container hidden (`display: none`, `visibility: hidden`, `hidden`) until scripts +run in the browser. When a hidden element other than navigation or similar page furniture holds more text than +Readability.js extracted, the server unhides it and runs Readability.js again, keeping that result only if it has more +text. Other pages are extracted once, exactly as before. + To always use the pure-Python extractor, add the argument `--no-readability-js` to the `args` list in the configuration. The server then never starts Node.js (or npm), whether or not it is installed. diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index a69a2316eb..8b4b8dffb1 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -31,6 +31,7 @@ import httpx import markdownify import readabilipy.simple_json +from bs4 import BeautifulSoup, Tag from mcp.server import MCPServer, ServerRequestContext from mcp.server.context import CallNext, HandlerResult, ServerMiddleware from mcp.server.mcpserver.exceptions import ToolError @@ -167,6 +168,98 @@ async def _refuse_private_destination(request: httpx.Request) -> None: ) +def _readability_content(html: str, use_readability_js: bool) -> str: + """readabilipy's simplified HTML for ``html``, or "" when it has none.""" + try: + ret = readabilipy.simple_json.simple_json_from_html_string( + html, use_readability=use_readability_js + ) + except IndexError: + # readabilipy's pure-Python extractor wraps its simplified tree with + # `Tag.append`, which beautifulsoup4 4.13 implements as + # `self.insert(...)[0]`. A page that simplifies to nothing (an SPA + # shell, an empty body) inserts nothing, so that index raises (#5158). + # Under beautifulsoup4 <= 4.12 the same page comes back as an empty + # tree, reported by the caller as this same failure. + return "" + return ret["content"] or "" + + +def _hiding_declaration(declaration: str) -> bool: + """Whether one inline-style declaration is one Readability.js treats as hidden.""" + prop, _, value = declaration.partition(":") + value = value.replace("!important", "").strip().lower() + return (prop.strip().lower(), value) in { + ("display", "none"), + ("visibility", "hidden"), + } + + +def _is_hidden(tag: Tag) -> bool: + """Whether Readability.js drops ``tag`` as invisible (its `_isProbablyVisible`).""" + if tag.has_attr("hidden") or tag.get("aria-hidden") == "true": + return True + return any(_hiding_declaration(d) for d in str(tag.get("style", "")).split(";")) + + +# Hidden page furniture that is expected to stay hidden (a collapsed menu, a +# closed dialog): never a hydration container, so never unhidden. +_PAGE_CHROME_TAGS = frozenset({"nav", "header", "footer", "aside", "menu", "dialog"}) +_PAGE_CHROME_ROLES = frozenset( + {"navigation", "banner", "contentinfo", "menu", "menubar", "dialog", "alertdialog"} +) + + +def _is_page_chrome(tag: Tag) -> bool: + return tag.name in _PAGE_CHROME_TAGS or tag.get("role") in _PAGE_CHROME_ROLES + + +def _text_length(html: Tag) -> int: + """The length of the text in ``html``, whitespace collapsed.""" + return len(" ".join(html.get_text(" ").split())) + + +def _recover_hidden_content(html: str, content: str) -> str: + """Re-extract a page whose content Readability.js dropped as hidden (#3878). + + Progressive server-side rendering streams a visible loading shell first, + then the real page in a container hidden (``visibility: hidden``, + ``display: none``, ``hidden``) until the client hydrates it. Readability.js + discards hidden nodes, so only the shell's text survived. When a hidden + element holds more text than the whole extraction, those elements are + unhidden and Readability.js runs again; the second result is kept only if + it has more text. A page whose hidden elements are smaller than its + extracted content, or are page furniture (navigation, menus, dialogs), is + returned exactly as before, from the one Readability.js run. + """ + extracted = _text_length(BeautifulSoup(content, "html.parser")) + soup = BeautifulSoup(html, "html.parser") + larger = [ + tag + for tag in soup.find_all(_is_hidden) + # Outermost only: unhiding a container exposes what it holds, and + # anything hidden inside it stays hidden. + if not any(_is_hidden(parent) for parent in tag.parents) + and not _is_page_chrome(tag) + and _text_length(tag) > extracted + ] + if not larger: + return content + for tag in larger: + del tag["hidden"] + del tag["aria-hidden"] + style = [ + d + for d in str(tag.get("style", "")).split(";") + if not _hiding_declaration(d) + ] + tag["style"] = ";".join(style) + recovered = _readability_content(str(soup), use_readability_js=True) + if _text_length(BeautifulSoup(recovered, "html.parser")) > extracted: + return recovered + return content + + def extract_content_from_html(html: str, use_readability_js: bool = True) -> str: """Extract and convert HTML content to Markdown format. @@ -180,22 +273,15 @@ def extract_content_from_html(html: str, use_readability_js: bool = True) -> str Simplified markdown version of the content """ failed = "Page failed to be simplified from HTML" - try: - ret = readabilipy.simple_json.simple_json_from_html_string( - html, use_readability=use_readability_js - ) - except IndexError: - # readabilipy's pure-Python extractor wraps its simplified tree with - # `Tag.append`, which beautifulsoup4 4.13 implements as - # `self.insert(...)[0]`. A page that simplifies to nothing (an SPA - # shell, an empty body) inserts nothing, so that index raises (#5158). - # Under beautifulsoup4 <= 4.12 the same page comes back as an empty - # tree, reported below as this same failure. - return failed - if not ret["content"]: + content_html = _readability_content(html, use_readability_js) + if use_readability_js: + # Only Readability.js drops hidden elements; the pure-Python extractor + # keeps them, so its result never needs recovering. + content_html = _recover_hidden_content(html, content_html) + if not content_html: return failed content = markdownify.markdownify( - ret["content"], + content_html, heading_style=markdownify.ATX, ) # Without Node, readabilipy's pure-Python extractor never returns empty diff --git a/src/fetch/tests/test_hidden_ssr.py b/src/fetch/tests/test_hidden_ssr.py new file mode 100644 index 0000000000..54da607d1c --- /dev/null +++ b/src/fetch/tests/test_hidden_ssr.py @@ -0,0 +1,210 @@ +"""Tests for #3878: content that server-side rendering hides until hydration. + +A progressively rendered page streams a visible loading shell, then the real +page in a container hidden until the client hydrates it. Readability.js drops +hidden nodes, so fetch returned only the shell. The server now re-runs +Readability.js with such a container unhidden, but only when a hidden element +(that is not navigation or other page furniture) holds more text than the whole +first extraction, and keeps the second result only when it has more text. + +What counts as hidden depends on the Readability.js that readabilipy runs: the +copy bundled with readabilipy 0.2 (the lockfile's) drops ``display: none`` and +``hidden``; the ``@mozilla/readability`` that readabilipy 0.3 installs also +drops ``visibility: hidden`` and ``aria-hidden="true"``. The protocol tests +that prove the second run happens use the two hidings both versions drop. They +count readabilipy calls, so "a normal page is unchanged" is asserted as "it +was extracted once, as before", not inferred from the Markdown. +""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest +import readabilipy.simple_json +from bs4 import BeautifulSoup +from mcp.types import CallToolResult + +from mcp_server_fetch.server import _is_hidden + +from .harness import FakeWeb, connect + +PAGE = "https://example.com/docs" + +ARTICLE = """

Developer Concepts

+

Everything you need to integrate the framework: guides, concepts and +references for developers and architects, rendered on the server.

+

Install

+

Install the package and initialize your first project. The command line +handles project setup and configuration automatically.

""" + + +def ssr_page(hiding: str) -> str: + # The shape from the issue: a visible shell, then the content, hidden. + return f"""Docs +

Unified framework for full-stack applications

+
{ARTICLE}
""" + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Record every HTML string readabilipy is asked to simplify.""" + seen: list[str] = [] + original = readabilipy.simple_json.simple_json_from_html_string + + def spy(html: str, **kwargs: Any) -> Any: + seen.append(html) + return original(html, **kwargs) + + monkeypatch.setattr(readabilipy.simple_json, "simple_json_from_html_string", spy) + return seen + + +def html(body: str) -> httpx.Response: + return httpx.Response( + 200, content=body.encode(), headers={"content-type": "text/html"} + ) + + +def text_of(result: CallToolResult) -> str: + data = result.model_dump( + by_alias=True, mode="json", exclude_none=True, exclude_unset=True + ) + assert data["isError"] is False + [content] = data["content"] + assert content["type"] == "text" + return content["text"] + + +async def fetch(web: FakeWeb, page: str, **serve_kwargs: Any) -> str: + web.add(PAGE, html(page)) + async with connect(ignore_robots_txt=True, **serve_kwargs) as (session, _): + return text_of(await session.call_tool("fetch", {"url": PAGE})) + + +@pytest.mark.usefixtures("node_readability") +@pytest.mark.parametrize( + "hiding", + [ + 'style="position:absolute;top:-9999px;display:none"', + "hidden", + 'style="display: NONE !important"', + ], +) +async def test_hidden_ssr_content_is_recovered( + web: FakeWeb, calls: list[str], hiding: str +) -> None: + text = await fetch(web, ssr_page(hiding)) + assert "## Install" in text + assert "rendered on the server" in text + assert "configuration automatically" in text + # Extracted once as is, then once with the container unhidden. + assert len(calls) == 2 + + +@pytest.mark.usefixtures("node_readability") +@pytest.mark.parametrize("hiding", ['style="visibility: hidden"', 'aria-hidden="true"']) +async def test_newer_readability_hidings_come_back_too( + web: FakeWeb, hiding: str +) -> None: + # Dropped by readabilipy 0.3's Readability.js, kept by 0.2's: either way + # the content reaches the model. + text = await fetch(web, ssr_page(hiding)) + assert "rendered on the server" in text + + +@pytest.mark.usefixtures("node_readability") +async def test_page_with_small_hidden_element_is_extracted_once( + web: FakeWeb, calls: list[str] +) -> None: + page = f"""{ARTICLE} +
Copied to clipboard
""" + text = await fetch(web, page) + assert "rendered on the server" in text + assert "Copied to clipboard" not in text + assert len(calls) == 1 + + +@pytest.mark.usefixtures("node_readability") +@pytest.mark.parametrize( + ("opening", "closing"), + [(""), ('")], +) +async def test_hidden_navigation_is_left_hidden( + web: FakeWeb, calls: list[str], opening: str, closing: str +) -> None: + # A collapsed menu larger than a short page is not a hydration container. + links = "".join( + f'
  • Navigation link number {i}
    • ' for i in range(30) + ) + page = f"""

    A short page with one visible paragraph.

    +{opening}
      {links}
    {closing}""" + text = await fetch(web, page) + assert "one visible paragraph" in text + assert "Navigation link" not in text + assert len(calls) == 1 + + +@pytest.mark.usefixtures("node_readability") +async def test_second_run_is_kept_only_when_it_gains_text( + web: FakeWeb, calls: list[str] +) -> None: + # Readability.js discards a {options}""" + text = await fetch(web, page) + assert "one visible paragraph" in text + assert "Option number" not in text + assert len(calls) == 2 + + +@pytest.mark.usefixtures("node_readability") +async def test_shell_with_nothing_hidden_still_fails_to_simplify( + web: FakeWeb, calls: list[str] +) -> None: + text = await fetch(web, "") + assert text == ( + f"Contents of {PAGE}:\nPage failed to be simplified from HTML" + ) + assert len(calls) == 1 + + +@pytest.mark.usefixtures("python_readability") +async def test_pure_python_fallback_keeps_hidden_content_in_one_run( + web: FakeWeb, calls: list[str] +) -> None: + # Without Node, readabilipy's own extractor ignores visibility, so the + # content is there the first time and there is nothing to recover. + text = await fetch(web, ssr_page("hidden")) + assert "rendered on the server" in text + assert len(calls) == 1 + + +async def test_no_readability_js_is_never_rerun(web: FakeWeb, calls: list[str]) -> None: + text = await fetch(web, ssr_page("hidden"), use_readability_js=False) + assert "rendered on the server" in text + assert len(calls) == 1 + + +@pytest.mark.parametrize( + ("element", "hidden"), + [ + ('
    ', True), + ('
    ', True), + ('
    ', True), + ("