diff --git a/docs/using.md b/docs/using.md index ce3a4c08..816bd75c 100644 --- a/docs/using.md +++ b/docs/using.md @@ -286,6 +286,14 @@ print(node[0]) node[0].next_sibling ``` +Use `node.to_tokens()` to recover the linear token stream after modifying the +tree's `children`, including those of inline and image nodes. This reuses the +original token objects and updates their child lists to match the tree. +The child lists are replaced in place, so the token stream the tree was built +from sees the same edits. Another tree built from the same tokens keeps its own +nodes, and its `to_tokens()` writes its own children back. +Build the tree from `copy.deepcopy(tokens)` to avoid sharing. + ## Renderers +++ diff --git a/markdown_it/tree.py b/markdown_it/tree.py index ebe00d23..dab29ae1 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -91,13 +91,29 @@ def __getitem__(self, item: int | slice) -> Self | list[Self]: return self.children[item] def to_tokens(self) -> list[Token]: - """Recover the linear token stream.""" + """Recover the linear token stream, including changes to the tree. + + The original tokens are reused, with their child lists updated to match + the tree. + + The ``children`` of inline and image tokens are replaced in place. + These ``Token`` objects are shared with the token stream the tree was + built from, so that stream sees the edits too. + Another tree built from the same tokens keeps its own nodes, + and its ``to_tokens()`` writes its own children back. + Build the tree from ``copy.deepcopy(tokens)`` to avoid sharing. + """ def recursive_collect_tokens(node: Self, token_list: list[Token]) -> None: if node.type == "root": for child in node.children: recursive_collect_tokens(child, token_list) elif node.token: + if node.token.children is not None: + children: list[Token] = [] + for child in node.children: + recursive_collect_tokens(child, children) + node.token.children = children token_list.append(node.token) else: assert node.nester_tokens diff --git a/tests/test_tree.py b/tests/test_tree.py index 36bd0b67..b06a80ba 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,3 +1,5 @@ +import pytest + from markdown_it import MarkdownIt from markdown_it.tree import SyntaxTreeNode @@ -10,8 +12,94 @@ def test_tree_to_tokens_conversion(): tokens = MarkdownIt().parse(EXAMPLE_MARKDOWN) + serialized_tokens = [token.as_dict() for token in tokens] tokens_after_roundtrip = SyntaxTreeNode(tokens).to_tokens() assert tokens == tokens_after_roundtrip + assert [token.as_dict() for token in tokens_after_roundtrip] == serialized_tokens + assert all( + before is after + for before, after in zip(tokens, tokens_after_roundtrip, strict=True) + ) + + +def _node_at(tree, path): + node = tree + for index in path: + node = node[index] + return node + + +@pytest.mark.parametrize( + "source,path,expected", + [ + ("one\n\ntwo", (), "
two
\none
\n"), + ( + "before *nested* after", + (0, 0), + "afternestedbefore
\n", + ), + ( + "", + (0, 0, 0), + '
before after
\n"), + ( + "", + (0, 0, 0), + '