diff --git a/docs/using.md b/docs/using.md index ce3a4c08..816bd75c 100644 --- a/docs/using.md +++ b/docs/using.md @@ -286,6 +286,14 @@ print(node[0]) node[0].next_sibling ``` +Use `node.to_tokens()` to recover the linear token stream after modifying the +tree's `children`, including those of inline and image nodes. This reuses the +original token objects and updates their child lists to match the tree. +The child lists are replaced in place, so the token stream the tree was built +from sees the same edits. Another tree built from the same tokens keeps its own +nodes, and its `to_tokens()` writes its own children back. +Build the tree from `copy.deepcopy(tokens)` to avoid sharing. + ## Renderers +++ diff --git a/markdown_it/tree.py b/markdown_it/tree.py index ebe00d23..dab29ae1 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -91,13 +91,29 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]: return self.children[item] def to_tokens(self) -> list[Token]: - """Recover the linear token stream.""" + """Recover the linear token stream, including changes to the tree. + + The original tokens are reused, with their child lists updated to match + the tree. + + The ``children`` of inline and image tokens are replaced in place. + These ``Token`` objects are shared with the token stream the tree was + built from, so that stream sees the edits too. + Another tree built from the same tokens keeps its own nodes, + and its ``to_tokens()`` writes its own children back. + Build the tree from ``copy.deepcopy(tokens)`` to avoid sharing. + """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: if node.type == "root": for child in node.children: recursive_collect_tokens(child, token_list) elif node.token: + if node.token.children is not None: + children: list[Token] = [] + for child in node.children: + recursive_collect_tokens(child, children) + node.token.children = children token_list.append(node.token) else: assert node.nester_tokens diff --git a/tests/test_tree.py b/tests/test_tree.py index 36bd0b67..b06a80ba 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,3 +1,5 @@ +import pytest + from markdown_it import MarkdownIt from markdown_it.tree import SyntaxTreeNode @@ -10,8 +12,94 @@ def test_tree_to_tokens_conversion(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) + serialized_tokens = [token.as_dict() for token in tokens] tokens_after_roundtrip = SyntaxTreeNode(tokens).to_tokens() assert tokens == tokens_after_roundtrip + assert [token.as_dict() for token in tokens_after_roundtrip] == serialized_tokens + assert all( + before is after + for before, after in zip(tokens, tokens_after_roundtrip, strict=True) + ) + + +def _node_at(tree, path): + node = tree + for index in path: + node = node[index] + return node + + +@pytest.mark.parametrize( + "source,path,expected", + [ + ("one\n\ntwo", (), "

two

\n

one

\n"), + ( + "before *nested* after", + (0, 0), + "

afternestedbefore

\n", + ), + ( + "![before *nested* after](image.png)", + (0, 0, 0), + '

 afternestedbefore

\n', + ), + ], + ids=["blocks", "inline", "image"], +) +def test_tree_to_tokens_reordered_children(source, path, expected): + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + _node_at(tree, path).children.reverse() + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + + +@pytest.mark.parametrize( + "source,path,expected", + [ + ("before *nested* after", (0, 0), "

\n"), + ("before *nested* after", (0, 0, 1), "

before after

\n"), + ( + "![old alt](image.png)", + (0, 0, 0), + '

\n', + ), + ], + ids=["inline", "nested-inline", "image"], +) +def test_tree_to_tokens_cleared_children(source, path, expected): + md = MarkdownIt() + tree = SyntaxTreeNode(md.parse(source)) + _node_at(tree, path).children.clear() + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == expected + + +def test_tree_to_tokens_replaced_empty_inline_children(): + md = MarkdownIt() + tree = SyntaxTreeNode(md.parseInline("")) + replacement = SyntaxTreeNode(md.parseInline("*new*")) + tree[0].children = replacement[0].children + + assert md.renderer.render(tree.to_tokens(), md.options, {}) == "new" + + +def test_tree_to_tokens_updates_original_tokens(): + """Inline child edits are written in place on the shared tokens. + + This pins the documented behaviour of ``to_tokens``: the token stream the + tree was built from sees the edit as well. + """ + md = MarkdownIt() + tokens = md.parse("before *nested* after") + tree = SyntaxTreeNode(tokens) + tree[0][0].children.clear() + + tree.to_tokens() + + assert tokens[1].type == "inline" + assert tokens[1].children == [] + assert md.renderer.render(tokens, md.options, {}) == "

\n" def test_property_passthrough():