FIX: preserve non-ASCII whitespace in headings and paragraphs

markdown_it/common/utils.py

@@ -216,6 +216,21 @@ def mdTrim(string: str) -> str:
     return string.strip(MD_TRIM_CHARS)
 
 
+#: The characters trimmed from headings and paragraphs.
+#:
+#: Only ASCII whitespace may be removed from block content: CommonMark's
+#: definition of whitespace does not include Unicode space characters such as
+#: U+00A0 or U+3000, and upstream markdown-it preserves them (markdown-it
+#: #1074, released in 14.2.0). :py:func:`mdTrim` is still used where upstream
+#: trims with ``String.prototype.trim`` (table cells and fence info).
+ASCII_TRIM_CHARS = " \t\n\r"
+
+
+def asciiTrim(string: str) -> str:
+    """Strip leading and trailing ASCII whitespace from block content."""
+    return string.strip(ASCII_TRIM_CHARS)
+
+
 # //////////////////////////////////////////////////////////////////////////////
 
 

markdown_it/rules_block/heading.py

@@ -4,7 +4,7 @@
 
 import logging
 
-from ..common.utils import isStrSpace, mdTrim
+from ..common.utils import asciiTrim, isStrSpace
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -62,7 +62,7 @@ def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bo
     token.map = [startLine, state.line]
 
     token = state.push("inline", "", 0)
-    token.content = mdTrim(state.src[pos:maximum])
+    token.content = asciiTrim(state.src[pos:maximum])
     token.map = [startLine, state.line]
     token.children = []
 

markdown_it/rules_block/lheading.py

@@ -1,7 +1,7 @@
 # lheading (---, ==)
 import logging
 
-from ..common.utils import mdTrim
+from ..common.utils import asciiTrim
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -66,7 +66,7 @@ def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> b
         # Didn't find valid underline
         return False
 
-    content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False))
+    content = asciiTrim(state.getLines(startLine, nextLine, state.blkIndent, False))
 
     state.line = nextLine + 1
 

markdown_it/rules_block/paragraph.py

@@ -2,7 +2,7 @@
 
 import logging
 
-from ..common.utils import mdTrim
+from ..common.utils import asciiTrim
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -48,7 +48,7 @@ def paragraph(state: StateBlock, startLine: int, endLine: int, silent: bool) ->
 
         nextLine += 1
 
-    content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False))
+    content = asciiTrim(state.getLines(startLine, nextLine, state.blkIndent, False))
 
     state.line = nextLine
 

tests/test_port/test_whitespace.py

@@ -6,8 +6,10 @@
 drops those characters from the output and folds two distinct reference labels
 into one.
 
-The expected values below are what ``markdown-it@14.1.0`` produces for the same
-input.
+The expected values below are what upstream markdown-it produces for the same
+input: ``14.1.0`` for reference labels and fence info, and ``15.0.x`` for block
+content, where only ASCII whitespace is trimmed from headings and paragraphs
+(upstream #1074, released in 14.2.0).
 """
 
 import pytest
@@ -54,10 +56,37 @@ def test_character_survives_in_output(char):
 def test_real_whitespace_is_still_trimmed():
     """Control: what CommonMark does call whitespace is still removed."""
     md = MarkdownIt("js-default")
-    for char in ["\t", " ", "\xa0", " ", "　"]:
+    for char in ["\t", " "]:
         assert md.render(f"# h{char}") == "<h1>h</h1>\n", repr(char)
 
 
+@pytest.mark.parametrize("char", ["\xa0", "\u3000"])
+def test_non_ascii_whitespace_is_preserved_in_blocks(char):
+    """Unicode whitespace is content, so it is not trimmed from blocks (#1074)."""
+    md = MarkdownIt("js-default")
+    assert md.render(f"# h{char}") == f"<h1>h{char}</h1>\n"
+    assert md.render(f"h{char}\n===") == f"<h1>h{char}</h1>\n"
+    assert md.render(f"{char}x{char}") == f"<p>{char}x{char}</p>\n"
+
+
+def test_non_ascii_trim_matches_upstream_fixture():
+    """The fixture added upstream in markdown-it #1074 (issue #1067)."""
+    md = MarkdownIt("js-default")
+    src = (
+        "# \u3000U+3000\u3000  \n"
+        "\n"
+        "  \u3000U+3000\u3000  \n"
+        "=\n"
+        "\n"
+        "  \u3000U+3000\u3000  \n"
+    )
+    assert md.render(src) == (
+        "<h1>\u3000U+3000\u3000</h1>\n"
+        "<h1>\u3000U+3000\u3000</h1>\n"
+        "<p>\u3000U+3000\u3000</p>\n"
+    )
+
+
 def test_fence_info_is_not_split_on_it():
     """``str.split()`` would split the info string on U+0085."""
     md = MarkdownIt("js-default")