From 9b1570af3c4e04edd61f47cba7f2164b6b297b3a Mon Sep 17 00:00:00 2001 From: cliffhall Date: Wed, 28 Oct 2026 18:18:03 -0400 Subject: [PATCH 1/4] feat(fetch): open the simplified result with the page title Readability.js drops an article heading that repeats the document title, so a page like pandas' 2.1.0 release note came back without the line naming it. A new title module reads the title from the raw HTML (og:title, the h1 when holds it, <title>, the h1) and puts it on top of the markdown unless the markdown already opens with it. Raw results and failed simplifications are untouched. Issue: #2472 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: cliffhall <cliff@futurescale.com> --- src/fetch/README.md | 2 + src/fetch/src/mcp_server_fetch/server.py | 4 +- src/fetch/src/mcp_server_fetch/title.py | 94 ++++++++++++++ src/fetch/tests/test_page_title.py | 154 +++++++++++++++++++++++ 4 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 src/fetch/src/mcp_server_fetch/title.py create mode 100644 src/fetch/tests/test_page_title.py diff --git a/src/fetch/README.md b/src/fetch/README.md index 7f90ee4a7c..6c7a078920 100644 --- a/src/fetch/README.md +++ b/src/fetch/README.md @@ -15,6 +15,8 @@ Requires MCP Python SDK 2.x (`mcp>=2.3,<3`). The fetch tool will truncate the response, but by using the `start_index` argument, you can specify where to start the content extraction. This lets models read a webpage in chunks, until they find the information they need. +The markdown of an HTML page opens with the page's title as a `#` heading (its `og:title`, else its `<h1>` when the `<title>` holds it, else its `<title>` or `<h1>`), because the extraction drops a heading that repeats the title. It is not added when the markdown already opens with it, to a raw result, or to a page that failed to simplify. + ### Available Tools - `fetch` - Fetches a URL from the internet and extracts its contents as markdown. diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index 7315a65ce5..21d00eb169 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -41,6 +41,8 @@ from protego import Protego from pydantic import AnyUrl, Field +from .title import with_page_title + # The version this server reports in serverInfo, read from the installed # distribution's metadata (pyproject.toml) so it cannot drift from the # published version (#360). Without it the SDK reports its own `mcp` version. @@ -601,7 +603,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None: content = await asyncio.to_thread( extract_content_from_html, page_raw, use_readability_js ) - return content, "" + return with_page_title(page_raw, content), "" # A response with no Content-Type would otherwise leave an empty slot here. declared_type = content_type or "(not declared)" diff --git a/src/fetch/src/mcp_server_fetch/title.py b/src/fetch/src/mcp_server_fetch/title.py new file mode 100644 index 0000000000..b380af91f6 --- /dev/null +++ b/src/fetch/src/mcp_server_fetch/title.py @@ -0,0 +1,94 @@ +# The page title for the fetch server's simplified (Markdown) result (#2472). +# +# Readability.js removes an article heading that repeats the document title, +# and neither extractor returns the `<title>` itself in the content the server +# converts, so a page such as a Sphinx release note came back without the line +# that says what it is. This module reads the title from the raw HTML and puts +# it at the top of the Markdown as a level-one heading, unless the Markdown +# already opens with it. +# +# It works from the raw page rather than from readabilipy's result, so it is +# independent of how the extractor is invoked and is applied at the one call +# site in `fetch_url` that simplifies HTML. The raw result (`raw: true`) and +# non-HTML bodies never pass through it. + +import re + +from bs4 import BeautifulSoup, Tag + +# The marker `extract_content_from_html` returns when a page simplifies to +# nothing. A title on top of it would make the failure read as content. +_FAILURE_PREFIX = "<error>" + +# Markdown heading markers, emphasis, escapes, Sphinx's "¶" permalink and any +# other punctuation are ignored when comparing a heading with the title. +_NOT_WORD = re.compile(r"[\W_]+") + + +def _normalized(text: str) -> str: + return _NOT_WORD.sub("", text).casefold() + + +def _collapsed(text: str) -> str: + return " ".join(text.split()) + + +def _text(candidate: str) -> str: + # A trailing "¶" is a permalink that is not marked up as one. + return _collapsed(candidate).rstrip("¶").strip() + + +def page_title(html: str) -> str | None: + """The page's title, or None when the page names none. + + In order: `og:title`; the first `<h1>` when `<title>` holds it, since + `<title>` then only adds the site's name ("What's new — pandas docs"); + `<title>`; the first `<h1>`. + """ + soup = BeautifulSoup(html, "html.parser") + og = soup.find("meta", attrs={"property": "og:title"}) + og_title = _text(str(og.get("content", ""))) if isinstance(og, Tag) else "" + title = _text(soup.title.get_text(" ")) if soup.title else "" + h1 = soup.find("h1") + heading = "" + if isinstance(h1, Tag): + # Sphinx appends a permalink anchor ("¶", or "#" in some themes) to + # each heading; it is not part of the heading's text. + for link in h1.find_all("a", class_="headerlink"): + link.decompose() + heading = _text(h1.get_text(" ")) + heading_in_title = bool(_normalized(heading)) and _normalized( + heading + ) in _normalized(title) + if og_title: + return og_title + if heading_in_title: + return heading + return title or heading or None + + +def _opens_with_title(content: str, title: str) -> bool: + """Whether the Markdown's first non-empty line already carries the title. + + A first line holding the whole title counts (the pure-Python extractor + opens with the title in a paragraph), and so does a first heading that the + title holds: `<title>` often adds the site name ("What's new — pandas + docs") to the page's own heading ("What's new"). + """ + first = next((line for line in content.splitlines() if line.strip()), "") + line, wanted = _normalized(first), _normalized(title) + if not line or not wanted: + return False + if wanted in line: + return True + return first.lstrip().startswith("#") and line in wanted + + +def with_page_title(html: str, content: str) -> str: + """``content`` with the page's title as a heading on top, if it lacks one.""" + if content.startswith(_FAILURE_PREFIX): + return content + title = page_title(html) + if title is None or _opens_with_title(content, title): + return content + return f"# {title}\n\n{content.lstrip()}" diff --git a/src/fetch/tests/test_page_title.py b/src/fetch/tests/test_page_title.py new file mode 100644 index 0000000000..9d8fa3fb8f --- /dev/null +++ b/src/fetch/tests/test_page_title.py @@ -0,0 +1,154 @@ +"""Tests for the page title on the fetch tool's simplified result (#2472). + +Readability.js drops an article heading that repeats the document title, so a +page such as pandas' "What's new in 2.1.0" release note came back without the +line naming it. ``with_page_title`` now puts the title on top as a heading +unless the Markdown already opens with it. The protocol-level tests drive the +real server through an MCP client with HTTP served by ``harness.FakeWeb``, on +both extractor paths, and pin that the raw result is untouched; the unit tests +below pin each rule for choosing the title and spotting a duplicate. +""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest +from mcp import ClientSession +from mcp.types import CallToolResult + +from mcp_server_fetch.title import page_title, with_page_title + +from .harness import FakeWeb, connect + +PAGE = "https://example.com/page" +ROBOTS = "https://example.com/robots.txt" +PREFIX = f"Contents of {PAGE}:\n" + +# The shape of a Sphinx (pydata theme) page: `<title>` adds the site name to +# the `<h1>`, which carries a "#" permalink. +BODY = "<p>These are the changes in pandas 2.1.0, with a long enough body. " * 8 +SPHINX_PAGE = f"""<!DOCTYPE html> +<html><head><title>What’s new in 2.1.0 (Aug 30, 2023) — pandas documentation +
+

What’s new in 2.1.0 (Aug 30, 2023)#

+{BODY}

+
""" + + +def text_of(result: CallToolResult) -> str: + assert not result.is_error, result + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) + assert len(data["content"]) == 1 + return data["content"][0]["text"] + + +async def fetch(web: FakeWeb, html: str, **arguments: Any) -> str: + web.add(ROBOTS, httpx.Response(404)) + web.add( + PAGE, + httpx.Response(200, text=html, headers={"content-type": "text/html"}), + ) + async with connect() as (session, _): + assert isinstance(session, ClientSession) + result = await session.call_tool("fetch", {"url": PAGE, **arguments}) + return text_of(result) + + +@pytest.mark.usefixtures("node_readability", "private_tempdir") +async def test_readability_js_result_gains_the_dropped_title(web: FakeWeb) -> None: + text = await fetch(web, SPHINX_PAGE) + assert text.startswith(f"{PREFIX}# What’s new in 2.1.0 (Aug 30, 2023)\n\n") + assert text.count("What’s new in 2.1.0") == 1 + assert "These are the changes in pandas 2.1.0" in text + + +@pytest.mark.usefixtures("python_readability") +async def test_python_result_is_not_given_a_second_title(web: FakeWeb) -> None: + # The pure-Python extractor keeps the `

` (and opens with the + # ``), so the title is already there and nothing is added. + text = await fetch(web, SPHINX_PAGE) + assert not text.startswith(f"{PREFIX}# ") + assert "What’s new in 2.1.0" in text + assert "These are the changes in pandas 2.1.0" in text + + +async def test_raw_result_is_untouched(web: FakeWeb) -> None: + text = await fetch(web, SPHINX_PAGE, raw=True) + assert text.endswith(SPHINX_PAGE) + assert "\n# What’s new" not in text + + +@pytest.mark.usefixtures("node_readability", "private_tempdir") +async def test_failed_simplification_is_not_titled(web: FakeWeb) -> None: + text = await fetch( + web, "<html><head><title>Shell" + ) + assert text == f"{PREFIX}Page failed to be simplified from HTML" + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + # og:title wins over everything else. + ( + 'T

T

', + "Social title", + ), + # The `

` wins when `` holds it. + ("<title>Heading | Site

Heading

", "Heading"), + # Otherwise ``. + ("<title>Document title

Something else

", "Document title"), + # A trailing "¶" permalink that is plain text is dropped. + ("Doc ¶", "Doc"), + # No ``: the `<h1>`. + ("<body><h1>Only heading</h1></body>", "Only heading"), + # An empty og:title and title fall through to the `<h1>`. + ( + '<meta property="og:title" content=""><title>

H

', + "H", + ), + # Nothing names the page. + ("

No title

", None), + ("

#

", None), + ], +) +def test_page_title(html: str, expected: str | None) -> None: + assert page_title(html) == expected + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + # No heading: the title goes on top. + ("Body text.\n", "# Title Here\n\nBody text.\n"), + # Leading blank lines are folded into the separator. + ("\n\nBody text.\n", "# Title Here\n\nBody text.\n"), + # Already opens with the title as a heading, at any level. + ("# Title Here\n\nBody.", "# Title Here\n\nBody."), + ("\n## title here ¶\n\nBody.", "\n## title here ¶\n\nBody."), + # A first line that holds the whole title, heading or not. + ("

Title Here — Site

Skip\n", "

Title Here — Site

Skip\n"), + # A first heading the title holds (the title adds a site name). + ("# Title\n\nBody.", "# Title\n\nBody."), + # A first paragraph the title holds is not a heading: title added. + ("Title\n\nBody.", "# Title Here\n\nTitle\n\nBody."), + # A first line of punctuation only cannot match. + ("---\n\nBody.", "# Title Here\n\n---\n\nBody."), + # Empty content still gains the title. + ("", "# Title Here\n\n"), + ], +) +def test_with_page_title(content: str, expected: str) -> None: + assert with_page_title("Title Here", content) == expected + + +def test_with_page_title_leaves_untitled_page_and_failure_alone() -> None: + assert with_page_title("

x

", "Body.") == "Body." + failure = "Page failed to be simplified from HTML" + assert with_page_title("T", failure) == failure + + +def test_title_with_no_word_characters_is_not_matched() -> None: + assert with_page_title("—", "Body.") == "# —\n\nBody." From 8b912e12f85eaec80832ef3d310b64e120a8919d Mon Sep 17 00:00:00 2001 From: cliffhall Date: Wed, 28 Oct 2026 20:32:39 -0400 Subject: [PATCH 2/4] fix(fetch): match the page title by visible whole words, off the loop Copilot round 1 on #5398: the duplicate check compared characters with word boundaries removed and read link destinations and tags, so 'Rust' matched 'Trust' and a title only in a URL suppressed the heading. Compare the first line's visible words instead. Title extraction parses the whole page, so run it in a worker thread like the extraction itself. Issue: #2472 Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- src/fetch/src/mcp_server_fetch/server.py | 2 +- src/fetch/src/mcp_server_fetch/title.py | 24 ++++++++++++++++++----- src/fetch/tests/test_page_title.py | 25 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index 21d00eb169..2135436c32 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -603,7 +603,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None: content = await asyncio.to_thread( extract_content_from_html, page_raw, use_readability_js ) - return with_page_title(page_raw, content), "" + return await asyncio.to_thread(with_page_title, page_raw, content), "" # A response with no Content-Type would otherwise leave an empty slot here. declared_type = content_type or "(not declared)" diff --git a/src/fetch/src/mcp_server_fetch/title.py b/src/fetch/src/mcp_server_fetch/title.py index b380af91f6..5d58599137 100644 --- a/src/fetch/src/mcp_server_fetch/title.py +++ b/src/fetch/src/mcp_server_fetch/title.py @@ -20,13 +20,27 @@ # nothing. A title on top of it would make the failure read as content. _FAILURE_PREFIX = "" -# Markdown heading markers, emphasis, escapes, Sphinx's "¶" permalink and any -# other punctuation are ignored when comparing a heading with the title. -_NOT_WORD = re.compile(r"[\W_]+") +# Titles are compared word by word: case, Markdown heading markers, emphasis, +# escapes, Sphinx's "¶" permalink and other punctuation are ignored, but word +# boundaries are kept, so "Rust" does not match "Trust". +_WORD = re.compile(r"[^\W_]+") +# What a reader does not see on a Markdown line: a link's destination and any +# HTML tag (an autolink's URL included). A title found only there is not shown. +_LINK_DESTINATION = re.compile(r"\]\([^)]*\)") +_HTML_TAG = re.compile(r"<[^>]*>") def _normalized(text: str) -> str: - return _NOT_WORD.sub("", text).casefold() + """``text`` as its casefolded words, one space apart, padded with spaces. + + The padding makes ``in`` a whole-word, whole-phrase containment test. + """ + words = _WORD.findall(text.casefold()) + return f" {' '.join(words)} " if words else "" + + +def _visible(line: str) -> str: + return _HTML_TAG.sub(" ", _LINK_DESTINATION.sub("]", line)) def _collapsed(text: str) -> str: @@ -76,7 +90,7 @@ def _opens_with_title(content: str, title: str) -> bool: docs") to the page's own heading ("What's new"). """ first = next((line for line in content.splitlines() if line.strip()), "") - line, wanted = _normalized(first), _normalized(title) + line, wanted = _normalized(_visible(first)), _normalized(title) if not line or not wanted: return False if wanted in line: diff --git a/src/fetch/tests/test_page_title.py b/src/fetch/tests/test_page_title.py index 9d8fa3fb8f..f0c6b383c1 100644 --- a/src/fetch/tests/test_page_title.py +++ b/src/fetch/tests/test_page_title.py @@ -136,6 +136,17 @@ def test_page_title(html: str, expected: str | None) -> None: ("Title\n\nBody.", "# Title Here\n\nTitle\n\nBody."), # A first line of punctuation only cannot match. ("---\n\nBody.", "# Title Here\n\n---\n\nBody."), + # The title only in a link destination or tag is not shown: added. + ( + "[Source](https://example.com/title-here)\n", + "# Title Here\n\n[Source](https://example.com/title-here)\n", + ), + ( + "x\n", + "# Title Here\n\nx\n", + ), + # The title as a link's visible text is shown: not added. + ("[Title Here](https://example.com)\n", "[Title Here](https://example.com)\n"), # Empty content still gains the title. ("", "# Title Here\n\n"), ], @@ -150,5 +161,19 @@ def test_with_page_title_leaves_untitled_page_and_failure_alone() -> None: assert with_page_title("T", failure) == failure +@pytest.mark.parametrize( + ("title", "content"), + [ + # Matching is by whole words, in both directions. + ("Rust", "Trust the compiler."), + ("Go", "Algorithms make this possible."), + ("Rust Guide", "# Rust Gui\n"), + ], +) +def test_partial_words_do_not_match(title: str, content: str) -> None: + html = f"{title}" + assert with_page_title(html, content) == f"# {title}\n\n{content}" + + def test_title_with_no_word_characters_is_not_matched() -> None: assert with_page_title("—", "Body.") == "# —\n\nBody." From eabad3f48fceb044006bed0bed2f10e5f0331a70 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Wed, 28 Oct 2026 20:59:17 -0400 Subject: [PATCH 3/4] fix(fetch): scan the first line in linear time and escape the heading Copilot round 2 on #5398: the link-destination and tag regexes rescanned an unterminated line quadratically, stripped tag-like text inside code spans, and the title was emitted unescaped, so '' became a tag and '__init__' emphasis. Read the visible text in one pass that keeps code spans, and backslash-escape Markdown and HTML specials in the heading. Issue: #2472 Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- src/fetch/src/mcp_server_fetch/title.py | 42 +++++++++++++++++++++---- src/fetch/tests/test_page_title.py | 33 +++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/fetch/src/mcp_server_fetch/title.py b/src/fetch/src/mcp_server_fetch/title.py index 5d58599137..bab279aed1 100644 --- a/src/fetch/src/mcp_server_fetch/title.py +++ b/src/fetch/src/mcp_server_fetch/title.py @@ -24,10 +24,10 @@ # escapes, Sphinx's "¶" permalink and other punctuation are ignored, but word # boundaries are kept, so "Rust" does not match "Trust". _WORD = re.compile(r"[^\W_]+") -# What a reader does not see on a Markdown line: a link's destination and any -# HTML tag (an autolink's URL included). A title found only there is not shown. -_LINK_DESTINATION = re.compile(r"\]\([^)]*\)") -_HTML_TAG = re.compile(r"<[^>]*>") +# Characters that Markdown or inline HTML would interpret in the emitted +# heading ("" read as a tag, "__init__" as emphasis, "<" as an entity), +# each escaped with a backslash so the heading shows the title as written. +_MARKDOWN_SPECIAL = re.compile(r"([\\`*_\[\]<>#&|~])") def _normalized(text: str) -> str: @@ -40,7 +40,36 @@ def _normalized(text: str) -> str: def _visible(line: str) -> str: - return _HTML_TAG.sub(" ", _LINK_DESTINATION.sub("]", line)) + """The text a reader sees on one Markdown line. + + A link's destination (`](...)`) and an HTML tag (an autolink's URL + included) are dropped; a code span's contents are kept, tag-like text + and all. One pass, so a long line with an unterminated `](`, `<` or + backtick costs linear time: each closer is searched for forward from + where it is needed, and once a search for one fails it is never repeated. + """ + out: list[str] = [] + missing: set[str] = set() + i, end = 0, len(line) + while i < end: + for opener, closer, keep in ( + ("`", "`", True), + ("<", ">", False), + ("](", ")", False), + ): + if closer in missing or not line.startswith(opener, i): + continue + j = line.find(closer, i + len(opener)) + if j == -1: + missing.add(closer) + continue + out.append(line[i + len(opener) : j] if keep else " ") + i = j + 1 + break + else: + out.append(line[i]) + i += 1 + return "".join(out) def _collapsed(text: str) -> str: @@ -105,4 +134,5 @@ def with_page_title(html: str, content: str) -> str: title = page_title(html) if title is None or _opens_with_title(content, title): return content - return f"# {title}\n\n{content.lstrip()}" + heading = _MARKDOWN_SPECIAL.sub(r"\\\1", title) + return f"# {heading}\n\n{content.lstrip()}" diff --git a/src/fetch/tests/test_page_title.py b/src/fetch/tests/test_page_title.py index f0c6b383c1..1aea9a2f59 100644 --- a/src/fetch/tests/test_page_title.py +++ b/src/fetch/tests/test_page_title.py @@ -175,5 +175,38 @@ def test_partial_words_do_not_match(title: str, content: str) -> None: assert with_page_title(html, content) == f"# {title}\n\n{content}" +def test_code_span_contents_count_as_visible() -> None: + # A heading's inline HTML example is text, not a tag, inside a code span. + content = "# The `` element\n\nBody." + assert ( + with_page_title("The <input> element", content) == content + ) + + +@pytest.mark.parametrize("opener", ["](", "<", "`"]) +def test_unterminated_markup_on_a_long_first_line(opener: str) -> None: + # Linear in the line's length: a quadratic scan of 200,000 openers would + # not finish in the test's time. The title is not on the line, so it is added. + content = opener * 200_000 + result = with_page_title("Title Here", content) + assert result == f"# Title Here\n\n{content}" + + +@pytest.mark.parametrize( + ("title_html", "heading"), + [ + ("Using <T>", "# Using \\"), + ("__init__ and *args", "# \\_\\_init\\_\\_ and \\*args"), + ( + "[Draft] R&D #3 `x` | a~b \\ c", + "# \\[Draft\\] R\\&D \\#3 \\`x\\` \\| a\\~b \\\\ c", + ), + ], +) +def test_heading_escapes_markdown_and_html(title_html: str, heading: str) -> None: + result = with_page_title(f"{title_html}", "Body.") + assert result == f"{heading}\n\nBody." + + def test_title_with_no_word_characters_is_not_matched() -> None: assert with_page_title("—", "Body.") == "# —\n\nBody." From 8b836f7c0291ac199ab4c7d13b7516a64f6462a5 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Wed, 28 Oct 2026 21:12:30 -0400 Subject: [PATCH 4/4] fix(fetch): keep literal and escaped angle brackets as visible text Copilot round 3 on #5398: the scanner dropped any <...> run, so a heading 'Comparing x < 5 and y > 3' did not match its own title and a duplicate heading was added. As in CommonMark, '<' opens a tag only before a letter, '/', '!' or '?', and a backslash-escaped character is text. Issue: #2472 Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- src/fetch/src/mcp_server_fetch/title.py | 14 +++++++++++++- src/fetch/tests/test_page_title.py | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/fetch/src/mcp_server_fetch/title.py b/src/fetch/src/mcp_server_fetch/title.py index bab279aed1..074387211f 100644 --- a/src/fetch/src/mcp_server_fetch/title.py +++ b/src/fetch/src/mcp_server_fetch/title.py @@ -39,12 +39,18 @@ def _normalized(text: str) -> str: return f" {' '.join(words)} " if words else "" +def _starts_tag(char: str) -> bool: + return char.isascii() and (char.isalpha() or char in "/!?") + + def _visible(line: str) -> str: """The text a reader sees on one Markdown line. A link's destination (`](...)`) and an HTML tag (an autolink's URL included) are dropped; a code span's contents are kept, tag-like text - and all. One pass, so a long line with an unterminated `](`, `<` or + and all. As in CommonMark, a `<` starts a tag only before a letter, `/`, + `!` or `?` (so "x < 5 and y > 3" is text), and a backslash-escaped + character is text. One pass, so a long line with an unterminated `](`, `<` or backtick costs linear time: each closer is searched for forward from where it is needed, and once a search for one fails it is never repeated. """ @@ -52,6 +58,10 @@ def _visible(line: str) -> str: missing: set[str] = set() i, end = 0, len(line) while i < end: + if line[i] == "\\" and i + 1 < end: + out.append(line[i + 1]) + i += 2 + continue for opener, closer, keep in ( ("`", "`", True), ("<", ">", False), @@ -59,6 +69,8 @@ def _visible(line: str) -> str: ): if closer in missing or not line.startswith(opener, i): continue + if opener == "<" and not _starts_tag(line[i + 1 : i + 2]): + continue j = line.find(closer, i + len(opener)) if j == -1: missing.add(closer) diff --git a/src/fetch/tests/test_page_title.py b/src/fetch/tests/test_page_title.py index 1aea9a2f59..be4978a31b 100644 --- a/src/fetch/tests/test_page_title.py +++ b/src/fetch/tests/test_page_title.py @@ -183,7 +183,20 @@ def test_code_span_contents_count_as_visible() -> None: ) -@pytest.mark.parametrize("opener", ["](", "<", "`"]) +@pytest.mark.parametrize( + ("title_html", "content"), + [ + # A "<" before a space is text, not a tag. + ("Comparing x < 5 and y > 3", "# Comparing x < 5 and y > 3\n\nBody."), + # Backslash-escaped brackets are text too. + ("Using <T>", "# Using \\\n\nBody."), + ], +) +def test_literal_angle_brackets_count_as_visible(title_html: str, content: str) -> None: + assert with_page_title(f"{title_html}", content) == content + + +@pytest.mark.parametrize("opener", ["](", " None: # Linear in the line's length: a quadratic scan of 200,000 openers would # not finish in the test's time. The title is not on the line, so it is added.