From c12544648d5b101f934edc32cdd057c6418f9fdc Mon Sep 17 00:00:00 2001 From: KennyMcSimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:50:14 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Preserve=20edited=20s?= =?UTF-8?q?yntax=20tree=20token=20children?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + docs/using.md | 4 +++ markdown_it/tree.py | 11 +++++++- tests/test_tree.py | 63 ++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 040ff138..e68f8af2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* šŸ› Preserve inline and image child edits when converting a `SyntaxTreeNode` back to tokens. * āœØ Add `--enable-tables` to the CLI for file, standard input and interactive parsing in [#422](https://github.com/executablebooks/markdown-it-py/pull/422) * šŸ› Fix CLI interactive mode joining input lines with an extra newline, which split every line into its own paragraph and broke hard line breaks, in [#172](https://github.com/executablebooks/markdown-it-py/issues/172) * šŸ› Fix trimming and splitting with the Python whitespace set instead of the CommonMark one, which dropped U+001Cā€“U+001F and U+0085 from paragraphs, headings, table cells and fence info strings and let distinct reference labels resolve each other, in [#418](https://github.com/executablebooks/markdown-it-py/pull/418), thanks to [@Nexory](https://github.com/Nexory) diff --git a/docs/using.md b/docs/using.md index aae3e858..7cccab53 100644 --- a/docs/using.md +++ b/docs/using.md @@ -286,6 +286,10 @@ print(node[0]) node[0].next_sibling ``` +Use `node.to_tokens()` to recover the linear token stream after modifying the +tree's `children`, including those of inline and image nodes. This reuses the +original token objects and updates their child lists to match the tree. + ## Renderers +++ diff --git a/markdown_it/tree.py b/markdown_it/tree.py index ebe00d23..d436bbe6 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -91,13 +91,22 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]: return self.children[item] def to_tokens(self) -> list[Token]: - """Recover the linear token stream.""" + """Recover the linear token stream, including changes to the tree. + + The original tokens are reused, with their child lists updated to match + the tree. + """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: if node.type == "root": for child in node.children: recursive_collect_tokens(child, token_list) elif node.token: + if node.token.children is not None: + children: list[Token] = [] + for child in node.children: + recursive_collect_tokens(child, children) + node.token.children = children token_list.append(node.token) else: assert node.nester_tokens diff --git a/tests/test_tree.py b/tests/test_tree.py index 36bd0b67..7e5d5108 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -8,10 +8,71 @@ """ -def test_tree_to_tokens_conversion(): +def test_tree_to_tokens_conversion() -> None: tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) + serialized_tokens = [token.as_dict() for token in tokens] tokens_after_roundtrip = SyntaxTreeNode(tokens).to_tokens() assert tokens == tokens_after_roundtrip + assert [token.as_dict() for token in tokens_after_roundtrip] == serialized_tokens + assert all( + before is after + for before, after in zip(tokens, tokens_after_roundtrip, strict=True) + ) + + +def test_tree_to_tokens_reordered_children() -> None: + cases: list[tuple[str, tuple[int, ...], str]] = [ + ("one\n\ntwo", (), "

two

\n

one

\n"), + ( + "before *nested* after", + (0, 0), + "

afternestedbefore

\n", + ), + ( + "![before *nested* after](image.png)", + (0, 0, 0), + '

afternestedbefore

\n', + ), + ] + for source, path, expected in cases: + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + node = tree + for index in path: + node = node[index] + node.children.reverse() + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + + +def test_tree_to_tokens_cleared_children() -> None: + cases: list[tuple[str, tuple[int, ...], str]] = [ + ("before *nested* after", (0, 0), "

\n"), + ("before *nested* after", (0, 0, 1), "

before after

\n"), + ( + "![old alt](image.png)", + (0, 0, 0), + '

\n', + ), + ] + for source, path, expected in cases: + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + node = tree + for index in path: + node = node[index] + node.children.clear() + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + + +def test_tree_to_tokens_replaced_empty_inline_children() -> None: + md = MarkdownIt() + tree = SyntaxTreeNode(md.parseInline("")) + replacement = SyntaxTreeNode(md.parseInline("*new*")) + tree[0].children = replacement[0].children + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == "new" def test_property_passthrough(): From 6537b87a154610662aab729ae959133a0536f601 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:22:10 +0100 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20drop=20changelog?= =?UTF-8?q?=20entry=20(written=20at=20release)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e68f8af2..040ff138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,6 @@ ## Unreleased -* šŸ› Preserve inline and image child edits when converting a `SyntaxTreeNode` back to tokens. * āœØ Add `--enable-tables` to the CLI for file, standard input and interactive parsing in [#422](https://github.com/executablebooks/markdown-it-py/pull/422) * šŸ› Fix CLI interactive mode joining input lines with an extra newline, which split every line into its own paragraph and broke hard line breaks, in [#172](https://github.com/executablebooks/markdown-it-py/issues/172) * šŸ› Fix trimming and splitting with the Python whitespace set instead of the CommonMark one, which dropped U+001Cā€“U+001F and U+0085 from paragraphs, headings, table cells and fence info strings and let distinct reference labels resolve each other, in [#418](https://github.com/executablebooks/markdown-it-py/pull/418), thanks to [@Nexory](https://github.com/Nexory) From 0ea8702fe9b70d86ce580d1639ea199f7c631979 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:22:17 +0100 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20state=20that=20to=5F?= =?UTF-8?q?tokens=20updates=20inline=20children=20in=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/using.md | 2 ++ markdown_it/tree.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/docs/using.md b/docs/using.md index 4478a94c..b0415756 100644 --- a/docs/using.md +++ b/docs/using.md @@ -289,6 +289,8 @@ node[0].next_sibling Use `node.to_tokens()` to recover the linear token stream after modifying the tree's `children`, including those of inline and image nodes. This reuses the original token objects and updates their child lists to match the tree. +The child lists are replaced in place, so the token stream the tree was built +from (and any other tree built from it) sees the same edits. ## Renderers diff --git a/markdown_it/tree.py b/markdown_it/tree.py index d436bbe6..d43215e4 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -95,6 +95,11 @@ def to_tokens(self) -> list[Token]: The original tokens are reused, with their child lists updated to match the tree. + + The ``children`` of inline and image tokens are replaced in place. + These ``Token`` objects are shared with the token stream the tree was + built from, and with any other tree built from it, so those see the + edits too. Build the tree from ``copy.deepcopy(tokens)`` to avoid this. """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: From 793a76d482d6d46ef2619fb76690b64e6f72c87b Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 20:22:51 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20parametrize=20the=20?= =?UTF-8?q?to=5Ftokens=20tests=20and=20pin=20the=20in-place=20children=20u?= =?UTF-8?q?pdate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_tree.py | 75 +++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/tests/test_tree.py b/tests/test_tree.py index 7e5d5108..b06a80ba 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,3 +1,5 @@ +import pytest + from markdown_it import MarkdownIt from markdown_it.tree import SyntaxTreeNode @@ -8,7 +10,7 @@ """ -def test_tree_to_tokens_conversion() -> None: +def test_tree_to_tokens_conversion(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) serialized_tokens = [token.as_dict() for token in tokens] tokens_after_roundtrip = SyntaxTreeNode(tokens).to_tokens() @@ -20,8 +22,16 @@ def test_tree_to_tokens_conversion() -> None: ) -def test_tree_to_tokens_reordered_children() -> None: - cases: list[tuple[str, tuple[int, ...], str]] = [ +def _node_at(tree, path): + node = tree + for index in path: + node = node[index] + return node + + +@pytest.mark.parametrize( + "source,path,expected", + [ ("one\n\ntwo", (), "

two

\n

one

\n"), ( "before *nested* after", @@ -33,20 +43,20 @@ def test_tree_to_tokens_reordered_children() -> None: (0, 0, 0), '

afternestedbefore

\n', ), - ] - for source, path, expected in cases: - md = MarkdownIt() - tree = SyntaxTreeNode(md.parse(source)) - node = tree - for index in path: - node = node[index] - node.children.reverse() + ], + ids=["blocks", "inline", "image"], +) +def test_tree_to_tokens_reordered_children(source, path, expected): + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + _node_at(tree, path).children.reverse() - assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected -def test_tree_to_tokens_cleared_children() -> None: - cases: list[tuple[str, tuple[int, ...], str]] = [ +@pytest.mark.parametrize( + "source,path,expected", + [ ("before *nested* after", (0, 0), "

\n"), ("before *nested* after", (0, 0, 1), "

before after

\n"), ( @@ -54,19 +64,18 @@ def test_tree_to_tokens_cleared_children() -> None: (0, 0, 0), '

\n', ), - ] - for source, path, expected in cases: - md = MarkdownIt() - tree = SyntaxTreeNode(md.parse(source)) - node = tree - for index in path: - node = node[index] - node.children.clear() + ], + ids=["inline", "nested-inline", "image"], +) +def test_tree_to_tokens_cleared_children(source, path, expected): + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + _node_at(tree, path).children.clear() - assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected -def test_tree_to_tokens_replaced_empty_inline_children() -> None: +def test_tree_to_tokens_replaced_empty_inline_children(): md = MarkdownIt() tree = SyntaxTreeNode(md.parseInline("")) replacement = SyntaxTreeNode(md.parseInline("*new*")) @@ -75,6 +84,24 @@ def test_tree_to_tokens_replaced_empty_inline_children() -> None: assert md.renderer.render(tree.to_tokens(), md.options, {}) == "new" +def test_tree_to_tokens_updates_original_tokens(): + """Inline child edits are written in place on the shared tokens. + + This pins the documented behaviour of ``to_tokens``: the token stream the + tree was built from sees the edit as well. + """ + md = MarkdownIt() + tokens = md.parse("before *nested* after") + tree = SyntaxTreeNode(tokens) + tree[0][0].children.clear() + + tree.to_tokens() + + assert tokens[1].type == "inline" + assert tokens[1].children == [] + assert md.renderer.render(tokens, md.options, {}) == "

\n" + + def test_property_passthrough(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) heading_open = tokens[0] From 3349c41457185acab6449c365d305d5850b7b6a0 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 9 Nov 2026 21:32:28 +0100 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20say=20which=20trees?= =?UTF-8?q?=20see=20to=5Ftokens=20edits=20on=20shared=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The token stream a tree was built from sees the edits; another tree built from the same tokens keeps its own nodes and writes its own children back. --- docs/using.md | 4 +++- markdown_it/tree.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/using.md b/docs/using.md index b0415756..816bd75c 100644 --- a/docs/using.md +++ b/docs/using.md @@ -290,7 +290,9 @@ Use `node.to_tokens()` to recover the linear token stream after modifying the tree's `children`, including those of inline and image nodes. This reuses the original token objects and updates their child lists to match the tree. The child lists are replaced in place, so the token stream the tree was built -from (and any other tree built from it) sees the same edits. +from sees the same edits. Another tree built from the same tokens keeps its own +nodes, and its `to_tokens()` writes its own children back. +Build the tree from `copy.deepcopy(tokens)` to avoid sharing. ## Renderers diff --git a/markdown_it/tree.py b/markdown_it/tree.py index d43215e4..dab29ae1 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -98,8 +98,10 @@ def to_tokens(self) -> list[Token]: The ``children`` of inline and image tokens are replaced in place. These ``Token`` objects are shared with the token stream the tree was - built from, and with any other tree built from it, so those see the - edits too. Build the tree from ``copy.deepcopy(tokens)`` to avoid this. + built from, so that stream sees the edits too. + Another tree built from the same tokens keeps its own nodes, + and its ``to_tokens()`` writes its own children back. + Build the tree from ``copy.deepcopy(tokens)`` to avoid sharing. """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: