diff --git a/markdown_it/common/utils.py b/markdown_it/common/utils.py index 5f1bd24b..d9c1f2c6 100644 --- a/markdown_it/common/utils.py +++ b/markdown_it/common/utils.py @@ -216,6 +216,21 @@ def mdTrim(string: str) -> str: return string.strip(MD_TRIM_CHARS) +#: The characters trimmed from headings and paragraphs. +#: +#: Only ASCII whitespace may be removed from block content: CommonMark's +#: definition of whitespace does not include Unicode space characters such as +#: U+00A0 or U+3000, and upstream markdown-it preserves them (markdown-it +#: #1074, released in 14.2.0). :py:func:`mdTrim` is still used where upstream +#: trims with ``String.prototype.trim`` (table cells and fence info). +ASCII_TRIM_CHARS = " \t\n\r" + + +def asciiTrim(string: str) -> str: + """Strip leading and trailing ASCII whitespace from block content.""" + return string.strip(ASCII_TRIM_CHARS) + + # ////////////////////////////////////////////////////////////////////////////// diff --git a/markdown_it/rules_block/heading.py b/markdown_it/rules_block/heading.py index 201f32dd..4dd48189 100644 --- a/markdown_it/rules_block/heading.py +++ b/markdown_it/rules_block/heading.py @@ -4,7 +4,7 @@ import logging -from ..common.utils import isStrSpace, mdTrim +from ..common.utils import asciiTrim, isStrSpace from .state_block import StateBlock LOGGER = logging.getLogger(__name__) @@ -62,7 +62,7 @@ def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bo token.map = [startLine, state.line] token = state.push("inline", "", 0) - token.content = mdTrim(state.src[pos:maximum]) + token.content = asciiTrim(state.src[pos:maximum]) token.map = [startLine, state.line] token.children = [] diff --git a/markdown_it/rules_block/lheading.py b/markdown_it/rules_block/lheading.py index 57138039..add05ff4 100644 --- a/markdown_it/rules_block/lheading.py +++ b/markdown_it/rules_block/lheading.py @@ -1,7 +1,7 @@ # lheading (---, ==) import logging -from ..common.utils import mdTrim +from ..common.utils import asciiTrim from .state_block import StateBlock LOGGER = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> b # Didn't find valid underline return False - content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False)) + content = asciiTrim(state.getLines(startLine, nextLine, state.blkIndent, False)) state.line = nextLine + 1 diff --git a/markdown_it/rules_block/paragraph.py b/markdown_it/rules_block/paragraph.py index d27b07b1..ba582272 100644 --- a/markdown_it/rules_block/paragraph.py +++ b/markdown_it/rules_block/paragraph.py @@ -2,7 +2,7 @@ import logging -from ..common.utils import mdTrim +from ..common.utils import asciiTrim from .state_block import StateBlock LOGGER = logging.getLogger(__name__) @@ -48,7 +48,7 @@ def paragraph(state: StateBlock, startLine: int, endLine: int, silent: bool) -> nextLine += 1 - content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False)) + content = asciiTrim(state.getLines(startLine, nextLine, state.blkIndent, False)) state.line = nextLine diff --git a/tests/test_port/test_whitespace.py b/tests/test_port/test_whitespace.py index dee1b196..307379f2 100644 --- a/tests/test_port/test_whitespace.py +++ b/tests/test_port/test_whitespace.py @@ -6,8 +6,10 @@ drops those characters from the output and folds two distinct reference labels into one. -The expected values below are what ``markdown-it@14.1.0`` produces for the same -input. +The expected values below are what upstream markdown-it produces for the same +input: ``14.1.0`` for reference labels and fence info, and ``15.0.x`` for block +content, where only ASCII whitespace is trimmed from headings and paragraphs +(upstream #1074, released in 14.2.0). """ import pytest @@ -54,10 +56,37 @@ def test_character_survives_in_output(char): def test_real_whitespace_is_still_trimmed(): """Control: what CommonMark does call whitespace is still removed.""" md = MarkdownIt("js-default") - for char in ["\t", " ", "\xa0", " ", " "]: + for char in ["\t", " "]: assert md.render(f"# h{char}") == "
{char}x{char}
\n" + + +def test_non_ascii_trim_matches_upstream_fixture(): + """The fixture added upstream in markdown-it #1074 (issue #1067).""" + md = MarkdownIt("js-default") + src = ( + "# \u3000U+3000\u3000 \n" + "\n" + " \u3000U+3000\u3000 \n" + "=\n" + "\n" + " \u3000U+3000\u3000 \n" + ) + assert md.render(src) == ( + "\u3000U+3000\u3000
\n" + ) + + def test_fence_info_is_not_split_on_it(): """``str.split()`` would split the info string on U+0085.""" md = MarkdownIt("js-default")