fix(fetch): recover server-rendered content hidden until hydration

src/fetch/README.md

@@ -219,6 +219,12 @@ pure-Python extractor, which keeps headings and text but drops link targets. The
 timeout, so a Node.js that hangs, or fails in a way readabilipy does not detect, stalls the tool call until the client
 gives up.
 
+Readability.js discards hidden elements. Pages rendered progressively on the server often send a visible loading
+shell, then the real content in a container hidden (`display: none`, `visibility: hidden`, `hidden`) until scripts
+run in the browser. When a hidden element other than navigation or similar page furniture holds more text than
+Readability.js extracted, the server unhides it and runs Readability.js again, keeping that result only if it has more
+text. Other pages are extracted once, exactly as before.
+
 To always use the pure-Python extractor, add the argument `--no-readability-js` to the `args` list in the
 configuration. The server then never starts Node.js (or npm), whether or not it is installed.
 

src/fetch/src/mcp_server_fetch/server.py

@@ -31,6 +31,7 @@
 import httpx
 import markdownify
 import readabilipy.simple_json
+from bs4 import BeautifulSoup, Tag
 from mcp.server import MCPServer, ServerRequestContext
 from mcp.server.context import CallNext, HandlerResult, ServerMiddleware
 from mcp.server.mcpserver.exceptions import ToolError
@@ -167,6 +168,110 @@ async def _refuse_private_destination(request: httpx.Request) -> None:
             )
 
 
+def _readability_content(html: str, use_readability_js: bool) -> str:
+    """readabilipy's simplified HTML for ``html``, or "" when it has none."""
+    try:
+        ret = readabilipy.simple_json.simple_json_from_html_string(
+            html, use_readability=use_readability_js
+        )
+    except IndexError:
+        # readabilipy's pure-Python extractor wraps its simplified tree with
+        # `Tag.append`, which beautifulsoup4 4.13 implements as
+        # `self.insert(...)[0]`. A page that simplifies to nothing (an SPA
+        # shell, an empty body) inserts nothing, so that index raises (#5158).
+        # Under beautifulsoup4 <= 4.12 the same page comes back as an empty
+        # tree, reported by the caller as this same failure.
+        return ""
+    return ret["content"] or ""
+
+
+def _hiding_declaration(declaration: str) -> bool:
+    """Whether one inline-style declaration is one Readability.js treats as hidden."""
+    prop, _, value = declaration.partition(":")
+    # Lowercased first: CSS keywords, `!important` included, ignore case.
+    value = value.lower().replace("!important", "").strip()
+    return (prop.strip().lower(), value) in {
+        ("display", "none"),
+        ("visibility", "hidden"),
+    }
+
+
+def _is_hidden(tag: Tag) -> bool:
+    """Whether Readability.js drops ``tag`` as invisible (its `_isProbablyVisible`)."""
+    if tag.has_attr("hidden") or tag.get("aria-hidden") == "true":
+        return True
+    return any(_hiding_declaration(d) for d in str(tag.get("style", "")).split(";"))
+
+
+# Hidden page furniture that is expected to stay hidden (a collapsed menu, a
+# closed dialog): never a hydration container, so never unhidden.
+_PAGE_CHROME_TAGS = frozenset({"nav", "header", "footer", "aside", "menu", "dialog"})
+_PAGE_CHROME_ROLES = frozenset(
+    {
+        "navigation",
+        "banner",
+        "contentinfo",
+        "complementary",
+        "menu",
+        "menubar",
+        "dialog",
+        "alertdialog",
+    }
+)
+
+
+def _is_page_chrome(tag: Tag) -> bool:
+    return tag.name in _PAGE_CHROME_TAGS or tag.get("role") in _PAGE_CHROME_ROLES
+
+
+def _text_length(html: Tag) -> int:
+    """The length of the text in ``html``, whitespace collapsed."""
+    return len(" ".join(html.get_text(" ").split()))
+
+
+def _recover_hidden_content(html: str, content: str) -> str:
+    """Re-extract a page whose content Readability.js dropped as hidden (#3878).
+
+    Progressive server-side rendering streams a visible loading shell first,
+    then the real page in a container hidden (``visibility: hidden``,
+    ``display: none``, ``hidden``) until the client hydrates it. Readability.js
+    discards hidden nodes, so only the shell's text survived. When a hidden
+    element holds more text than the whole extraction, those elements are
+    unhidden and Readability.js runs again; the second result is kept only if
+    it has more text. A page whose hidden elements are smaller than its
+    extracted content, or are page furniture (navigation, menus, dialogs), is
+    returned exactly as before, from the one Readability.js run.
+    """
+    extracted = _text_length(BeautifulSoup(content, "html.parser"))
+    soup = BeautifulSoup(html, "html.parser")
+    larger = [
+        tag
+        for tag in soup.find_all(_is_hidden)
+        # Outermost only: unhiding a container exposes what it holds, and
+        # anything hidden inside it stays hidden.
+        if not any(_is_hidden(parent) for parent in tag.parents)
+        # Furniture, or anything inside it (`<nav><div hidden>`), stays hidden.
+        and not _is_page_chrome(tag)
+        and not any(_is_page_chrome(parent) for parent in tag.parents)
+        and _text_length(tag) > extracted
+    ]
+    if not larger:
+        return content
+    for tag in larger:
+        del tag["hidden"]
+        del tag["aria-hidden"]
+        style = [
+            d
+            for d in str(tag.get("style", "")).split(";")
+            if not _hiding_declaration(d)
+        ]
+        tag["style"] = ";".join(style)
+    recovered = _readability_content(str(soup), use_readability_js=True)
+    if _text_length(BeautifulSoup(recovered, "html.parser")) > extracted:
+        return recovered
+    return content
+
+
 def extract_content_from_html(html: str, use_readability_js: bool = True) -> str:
     """Extract and convert HTML content to Markdown format.
 
@@ -180,22 +285,15 @@ def extract_content_from_html(html: str, use_readability_js: bool = True) -> str
         Simplified markdown version of the content
     """
     failed = "<error>Page failed to be simplified from HTML</error>"
-    try:
-        ret = readabilipy.simple_json.simple_json_from_html_string(
-            html, use_readability=use_readability_js
-        )
-    except IndexError:
-        # readabilipy's pure-Python extractor wraps its simplified tree with
-        # `Tag.append`, which beautifulsoup4 4.13 implements as
-        # `self.insert(...)[0]`. A page that simplifies to nothing (an SPA
-        # shell, an empty body) inserts nothing, so that index raises (#5158).
-        # Under beautifulsoup4 <= 4.12 the same page comes back as an empty
-        # tree, reported below as this same failure.
-        return failed
-    if not ret["content"]:
+    content_html = _readability_content(html, use_readability_js)
+    if use_readability_js:
+        # Only Readability.js drops hidden elements; the pure-Python extractor
+        # keeps them, so its result never needs recovering.
+        content_html = _recover_hidden_content(html, content_html)
+    if not content_html:
         return failed
     content = markdownify.markdownify(
-        ret["content"],
+        content_html,
         heading_style=markdownify.ATX,
     )
     # Without Node, readabilipy's pure-Python extractor never returns empty

src/fetch/tests/test_hidden_ssr.py

@@ -0,0 +1,218 @@
+"""Tests for #3878: content that server-side rendering hides until hydration.
+
+A progressively rendered page streams a visible loading shell, then the real
+page in a container hidden until the client hydrates it. Readability.js drops
+hidden nodes, so fetch returned only the shell. The server now re-runs
+Readability.js with such a container unhidden, but only when a hidden element
+(that is not navigation or other page furniture) holds more text than the whole
+first extraction, and keeps the second result only when it has more text.
+
+What counts as hidden depends on the Readability.js that readabilipy runs: the
+copy bundled with readabilipy 0.2 (the lockfile's) drops ``display: none`` and
+``hidden``; the ``@mozilla/readability`` that readabilipy 0.3 installs also
+drops ``visibility: hidden`` and ``aria-hidden="true"``. The protocol tests
+that prove the second run happens use the two hidings both versions drop. They
+count readabilipy calls, so "a normal page is unchanged" is asserted as "it
+was extracted once, as before", not inferred from the Markdown.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+import pytest
+import readabilipy.simple_json
+from bs4 import BeautifulSoup
+from mcp.types import CallToolResult
+
+from mcp_server_fetch.server import _is_hidden
+
+from .harness import FakeWeb, connect
+
+PAGE = "https://example.com/docs"
+
+ARTICLE = """<article><h1>Developer Concepts</h1>
+<p>Everything you need to integrate the framework: guides, concepts and
+references for developers and architects, rendered on the server.</p>
+<h2>Install</h2>
+<p>Install the package and initialize your first project. The command line
+handles project setup and configuration automatically.</p></article>"""
+
+
+def ssr_page(hiding: str) -> str:
+    # The shape from the issue: a visible shell, then the content, hidden.
+    return f"""<html><head><title>Docs</title></head><body>
+<div class="shell"><p>Unified framework for full-stack applications</p></div>
+<div {hiding}>{ARTICLE}</div></body></html>"""
+
+
+@pytest.fixture
+def calls(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+    """Record every HTML string readabilipy is asked to simplify."""
+    seen: list[str] = []
+    original = readabilipy.simple_json.simple_json_from_html_string
+
+    def spy(html: str, **kwargs: Any) -> Any:
+        seen.append(html)
+        return original(html, **kwargs)
+
+    monkeypatch.setattr(readabilipy.simple_json, "simple_json_from_html_string", spy)
+    return seen
+
+
+def html(body: str) -> httpx.Response:
+    return httpx.Response(
+        200, content=body.encode(), headers={"content-type": "text/html"}
+    )
+
+
+def text_of(result: CallToolResult) -> str:
+    data = result.model_dump(
+        by_alias=True, mode="json", exclude_none=True, exclude_unset=True
+    )
+    assert data["isError"] is False
+    [content] = data["content"]
+    assert content["type"] == "text"
+    return content["text"]
+
+
+async def fetch(web: FakeWeb, page: str, **serve_kwargs: Any) -> str:
+    web.add(PAGE, html(page))
+    async with connect(ignore_robots_txt=True, **serve_kwargs) as (session, _):
+        return text_of(await session.call_tool("fetch", {"url": PAGE}))
+
+
+@pytest.mark.usefixtures("node_readability")
+@pytest.mark.parametrize(
+    "hiding",
+    [
+        'style="position:absolute;top:-9999px;display:none"',
+        "hidden",
+        'style="display: NONE !important"',
+        'style="display:none !IMPORTANT"',
+    ],
+)
+async def test_hidden_ssr_content_is_recovered(
+    web: FakeWeb, calls: list[str], hiding: str
+) -> None:
+    text = await fetch(web, ssr_page(hiding))
+    assert "## Install" in text
+    assert "rendered on the server" in text
+    assert "configuration automatically" in text
+    # Extracted once as is, then once with the container unhidden.
+    assert len(calls) == 2
+
+
+@pytest.mark.usefixtures("node_readability")
+@pytest.mark.parametrize("hiding", ['style="visibility: hidden"', 'aria-hidden="true"'])
+async def test_newer_readability_hidings_come_back_too(
+    web: FakeWeb, hiding: str
+) -> None:
+    # Dropped by readabilipy 0.3's Readability.js, kept by 0.2's: either way
+    # the content reaches the model.
+    text = await fetch(web, ssr_page(hiding))
+    assert "rendered on the server" in text
+
+
+@pytest.mark.usefixtures("node_readability")
+async def test_page_with_small_hidden_element_is_extracted_once(
+    web: FakeWeb, calls: list[str]
+) -> None:
+    page = f"""<html><body>{ARTICLE}
+<div style="display:none">Copied to clipboard</div></body></html>"""
+    text = await fetch(web, page)
+    assert "rendered on the server" in text
+    assert "Copied to clipboard" not in text
+    assert len(calls) == 1
+
+
+@pytest.mark.usefixtures("node_readability")
+@pytest.mark.parametrize(
+    ("opening", "closing"),
+    [
+        ("<nav hidden>", "</nav>"),
+        ('<div role="navigation" hidden>', "</div>"),
+        ("<nav><div hidden>", "</div></nav>"),
+        ('<header><div style="display:none">', "</div></header>"),
+        ('<div role="complementary"><div hidden>', "</div></div>"),
+    ],
+)
+async def test_hidden_navigation_is_left_hidden(
+    web: FakeWeb, calls: list[str], opening: str, closing: str
+) -> None:
+    # A collapsed menu larger than a short page is not a hydration container.
+    links = "".join(
+        f'<li><a href="/p{i}">Navigation link number {i}</a></li>' for i in range(30)
+    )
+    page = f"""<html><body><p>A short page with one visible paragraph.</p>
+{opening}<ul>{links}</ul>{closing}</body></html>"""
+    text = await fetch(web, page)
+    assert "one visible paragraph" in text
+    assert "Navigation link" not in text
+    assert len(calls) == 1
+
+
+@pytest.mark.usefixtures("node_readability")
+async def test_second_run_is_kept_only_when_it_gains_text(
+    web: FakeWeb, calls: list[str]
+) -> None:
+    # Readability.js discards a <select> even when it is visible, so unhiding
+    # it gains nothing and the first extraction is returned.
+    options = "".join(f"<option>Option number {i}</option>" for i in range(30))
+    page = f"""<html><body><p>A short page with one visible paragraph.</p>
+<div hidden><select>{options}</select></div></body></html>"""
+    text = await fetch(web, page)
+    assert "one visible paragraph" in text
+    assert "Option number" not in text
+    assert len(calls) == 2
+
+
+@pytest.mark.usefixtures("node_readability")
+async def test_shell_with_nothing_hidden_still_fails_to_simplify(
+    web: FakeWeb, calls: list[str]
+) -> None:
+    text = await fetch(web, "<html><body></body></html>")
+    assert text == (
+        f"Contents of {PAGE}:\n<error>Page failed to be simplified from HTML</error>"
+    )
+    assert len(calls) == 1
+
+
+@pytest.mark.usefixtures("python_readability")
+async def test_pure_python_fallback_keeps_hidden_content_in_one_run(
+    web: FakeWeb, calls: list[str]
+) -> None:
+    # Without Node, readabilipy's own extractor ignores visibility, so the
+    # content is there the first time and there is nothing to recover.
+    text = await fetch(web, ssr_page("hidden"))
+    assert "rendered on the server" in text
+    assert len(calls) == 1
+
+
+async def test_no_readability_js_is_never_rerun(web: FakeWeb, calls: list[str]) -> None:
+    text = await fetch(web, ssr_page("hidden"), use_readability_js=False)
+    assert "rendered on the server" in text
+    assert len(calls) == 1
+
+
+@pytest.mark.parametrize(
+    ("element", "hidden"),
+    [
+        ('<div style="display:none">', True),
+        ('<div style="color: red; Display : None ;">', True),
+        ('<div style="visibility:hidden !important">', True),
+        ('<div style="visibility:hidden !Important">', True),
+        ("<div hidden>", True),
+        ('<div aria-hidden="true">', True),
+        ("<div>", False),
+        ('<div style="display:block">', False),
+        ('<div style="visibility: visible">', False),
+        ('<div style="display:none-ish">', False),
+        ('<div aria-hidden="false">', False),
+    ],
+)
+def test_is_hidden_follows_readability(element: str, hidden: bool) -> None:
+    tag = BeautifulSoup(f"{element}text</div>", "html.parser").div
+    assert tag is not None
+    assert _is_hidden(tag) is hidden