From 3dc2393bcdf4eecc61b49eb461c3ead6bc556a19 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 04:24:33 +0200 Subject: [PATCH 01/13] refactor(templates): parse a template into its markup, blocks and includes _inspect read a template into one flat markup in which every tag was a marker. Parsing now keeps apart its blocks, with what each holds, its includes and {{ block.super }}, and the flat markup is rebuilt from them; each file is parsed once. The verdicts are unchanged: equal to the former ones on 2,238 template files of Django projects and on 218,421 generated texts. --- src/coretrace_python/taint/templates.py | 129 ++++++++++++++++++++---- 1 file changed, 112 insertions(+), 17 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index 0607fe5..a85de9b 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -50,7 +50,7 @@ from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType -from typing import ClassVar, NamedTuple +from typing import ClassVar, NamedTuple, TypeAlias from coretrace_python.analysis import Analysis, AnalysisContext from coretrace_python.frontend import HIRBuildError, ParseError, build_hir @@ -271,27 +271,91 @@ def _text(path: Path) -> str | None: def _inspect(text: str) -> _Verdict: """What the template of ``text`` establishes (``_Verdict``).""" + parsed = _parse(text) + if not parsed.ok: + return _REFUSED + reading = _read_markup(_flattened(parsed.nodes)) + return _Verdict(reading.escapes, parsed.extends, parsed.includes, reading.open_end) + + +_Node: TypeAlias = "str | _Block | _Include | _Super" + + +class _Block(NamedTuple): + """A ``{% block %}``: its name, what it holds, and whether its ``{% endblock %}`` + closes it.""" + + name: str + nodes: tuple[_Node, ...] + closed: bool + + +class _Include(NamedTuple): + """An ``{% include %}`` of the template ``name``.""" + + name: str + + +class _Super(NamedTuple): + """``{{ block.super }}``, which renders the parent's definition of its block.""" + + +@dataclass(frozen=True) +class _Parsed: + """One template file as its syntax gives it: whether that syntax lets it escape + (``ok``: Django's own tags, libraries and escaping filters only); the templates it + extends and those it includes; and its ``nodes``, markup in which a variable is + ``_OUTPUT``, a ``url`` or ``static`` tag ``_URL`` and any other tag ``_TAG``, with its + blocks, includes and ``{{ block.super }}`` kept apart, and its blocks by name.""" + + ok: bool + extends: frozenset[str] = frozenset() + includes: frozenset[str] = frozenset() + nodes: tuple[_Node, ...] = () + blocks: Mapping[str, tuple[_Node, ...]] = field(default_factory=dict) + + +_NOT_ESCAPING = _Parsed(ok=False) + + +def _parse(text: str) -> _Parsed: + """The template of ``text`` (``_Parsed``).""" + extends: set[str] = set() includes: set[str] = set() - html: list[str] = [] + blocks: dict[str, tuple[_Node, ...]] = {} + # The blocks open around the position read, each with what it holds so far, under + # the nodes the template holds outside them. + nodes: list[_Node] = [] + opened: list[tuple[str, list[_Node]]] = [] skipping: str | None = None position = 0 text = text.replace(_OUTPUT, "").replace(_TAG, "") + + def held() -> list[_Node]: + return opened[-1][1] if opened else nodes + + def close() -> None: + name, inside = opened.pop() + block = _Block(name, tuple(inside), closed=True) + blocks.setdefault(name, block.nodes) + held().append(block) + for match in _SYNTAX.finditer(text): variable, tag = match.group(1), match.group(2) words = tag.split() if tag is not None else [] if skipping is not None: if words[:1] == [skipping]: if skipping == "endverbatim": - html.append(text[position : match.start()]) + held().append(text[position : match.start()]) skipping, position = None, match.end() continue - html.append(text[position : match.start()]) + held().append(text[position : match.start()]) position = match.end() if variable is not None: if not _escaping_filters(variable): - return _REFUSED - html.append(_OUTPUT) + return _NOT_ESCAPING + held().append(_Super() if variable.strip() == _BLOCK_SUPER else _OUTPUT) elif words and words[0] in ("comment", "verbatim"): skipping = f"end{words[0]}" if words[0] == "verbatim": @@ -299,24 +363,55 @@ def _inspect(text: str) -> _Verdict: elif words: name = words[0] if name not in _TAGS: - return _REFUSED + return _NOT_ESCAPING if name == "load" and not set(_loaded(words[1:])) <= _LIBRARIES: - return _REFUSED + return _NOT_ESCAPING if name == "autoescape" and words[1:] != ["on"]: - return _REFUSED + return _NOT_ESCAPING if name == "filter" and not _escaping_filters(tag.split(None, 1)[1] if len(words) > 1 else ""): - return _REFUSED + return _NOT_ESCAPING if name in ("extends", "include"): needed = _constant(words[1] if len(words) > 1 else "") if needed is None: - return _REFUSED - (extends if name == "extends" else includes).add(needed) - html.append(_OUTPUT if name in _PRINTING else _URL if name in ("url", "static") else _TAG) + return _NOT_ESCAPING + if name == "extends": + extends.add(needed) + held().append(_TAG) + else: + includes.add(needed) + held().append(_Include(needed)) + elif name == "block" and len(words) > 1: + opened.append((words[1], [])) + elif name == "endblock" and opened: + close() + else: + held().append(_OUTPUT if name in _PRINTING else _URL if name in ("url", "static") else _TAG) if skipping is not None: - return _REFUSED - html.append(text[position:]) - reading = _read_markup("".join(html)) - return _Verdict(reading.escapes, frozenset(extends), frozenset(includes), reading.open_end) + return _NOT_ESCAPING + held().append(text[position:]) + while opened: + name, inside = opened.pop() + block = _Block(name, tuple(inside), closed=False) + blocks.setdefault(name, block.nodes) + held().append(block) + return _Parsed(True, frozenset(extends), frozenset(includes), tuple(nodes), MappingProxyType(blocks)) + + +def _flattened(nodes: Iterable[_Node]) -> str: + """``nodes`` as one markup, each block, include and ``{{ block.super }}`` marked where + it stands as the tag or variable it is.""" + + parts: list[str] = [] + for node in nodes: + if isinstance(node, str): + parts.append(node) + elif isinstance(node, _Block): + parts.extend((_TAG, _flattened(node.nodes), _TAG if node.closed else "")) + elif isinstance(node, _Include): + parts.append(_TAG) + else: + parts.append(_OUTPUT) + return "".join(parts) def _loaded(words: list[str]) -> list[str]: From 373dda159cf40b93d3efbc84293ba313b5ee7409 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 04:24:33 +0200 Subject: [PATCH 02/13] fix(templates): read the page each template renders, its blocks and includes in place Each template was read on its own: a child's block placed inside its parent's {% block body %}{% endblock %}\n" + + +# The reproductions of #273, rendered with Django 5.1.15: the variable runs as script in +# F, as an event handler in H, and in H's variant when other/templates comes first. +@pytest.mark.parametrize( + "templates", + [ + { + "base.html": "\n", + "page.html": "{% extends 'base.html' %}{% block js %}var name = {{ name }};{% endblock %}\n", + }, + {"page.html": 'x\n', "q.html": '" onclick="\n'}, + { + "page.html": 'x\n', + "q.html": "quote: \n", + "other/templates/q.html": '" onclick="\n', + }, + ], + ids=["F-block-inside-parent-script", "H-include-inside-attribute-value", "H-include-of-a-name-two-files-have"], +) +def test_markup_a_block_or_an_include_puts_inside_a_construct_is_read_there( + tmp_path: Path, templates: dict[str, str] +) -> None: + assert xss(tmp_path, templates) == [("app/views.py", 5)] + + +def test_only_the_page_printing_inside_the_parent_script_is_refused(tmp_path: Path) -> None: + templates = { + "base.html": BASE_WITH_SCRIPT_BLOCK, + "page.html": "{% extends 'base.html' %}{% block body %}

{{ name }}

{% endblock %}\n", + "inert.html": "{% extends 'base.html' %}{% block js %}var count = 1;{% endblock %}\n", + "script.html": "{% extends 'base.html' %}{% block js %}var name = {{ name }};{% endblock %}\n", + } + + assert escaped(tmp_path, templates) == ["base.html", "inert.html", "page.html"] + + +@pytest.mark.parametrize( + "templates", + [ + { + "base.html": '{% block content %}{% endblock %}\n', + "page.html": "{% extends 'base.html' %}{% block bodyclass %}{{ name }} change-form{% endblock %}\n", + }, + { + "base.html": "\n", + "page.html": "{% extends 'base.html' %}{% block js %}{{ block.super }} var total = 2;{% endblock %}\n", + }, + {"page.html": 'x\n', "q.html": "prefix \n"}, + { + "base.html": "

{% block body %}{% endblock %}

\n", + "middle.html": "{% extends 'base.html' %}{% block body %}{% block inner %}{% endblock %}{% endblock %}\n", + "page.html": "{% extends 'middle.html' %}{% block inner %}{{ name }}{% endblock %}\n", + }, + ], + ids=["variable-in-a-quoted-class", "block-super-in-a-script", "include-without-a-quote-in-a-title", "nested-blocks"], +) +def test_a_page_whose_parts_stay_escaped_once_put_together_is_established( + tmp_path: Path, templates: dict[str, str] +) -> None: + assert escaped(tmp_path, templates) == sorted(templates) + + +@pytest.mark.parametrize( + "templates", + [ + { + "page.html": "{% include 'self.html' %}", + "self.html": 'x', + }, + { + "page.html": "{% include 'a.html' %}", + "a.html": "", + "b.html": "{{ name }}{% include 'a.html' %}", + }, + ], + ids=["itself-inside-an-attribute", "through-another-inside-a-script"], +) +def test_a_template_including_itself_inside_a_construct_is_not_established( + tmp_path: Path, templates: dict[str, str] +) -> None: + # Rendered twice, self.html puts an inside the title it is in, whose quote + # ends that title before the variable; b.html's variable lands inside a.html's script. + assert escaped(tmp_path, templates) == [] + + +def test_a_page_beyond_eight_mebibytes_is_not_established(tmp_path: Path) -> None: + # level{n}.html holds 2 ** n characters once put together: level23.html is 8 MiB, + # level24.html beyond, and the page including it is put together no further. + templates = {"level0.html": "x"} + for level in range(1, 25): + templates[f"level{level}.html"] = f"{{% include 'level{level - 1}.html' %}}" * 2 + templates["page.html"] = "{% include 'level24.html' %}" + + assert escaped(tmp_path, templates) == sorted(f"level{level}.html" for level in range(24)) From fb6b558af3db4b444562d541268c6377ee6db743 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 04:24:33 +0200 Subject: [PATCH 03/13] fix(templates): accept in HTML data an include of a name several files have Django renders the first file its loaders find, which the engine does not know. Such an include now inserts a marker standing for each file. The marker is refused in any attribute, tag, end tag, script or style block and comment, and right after "<", "") # ``<{{ name }}`` or `` frozenset[str]: extends: dict[str, set[str]] = {} includes: dict[str, set[str]] = {} open_ended: set[str] = set() + # The names a page inserts (``_INSERTED``), and those whose every file renders a page + # ending in HTML data, as an inserted markup must. + inserting: dict[str, set[str]] = {} + settled: set[str] = set() for name, paths in found.items(): verdicts = [_verdict(parsed[path]) for path in paths] - local[name] = all(verdict.escapes for verdict in verdicts) and all(map(composition.escapes, paths)) + pages = [composition.page(path) for path in paths] + readings = [None if page is None else _read_markup(page.markup) for page in pages] + local[name] = all(verdict.escapes for verdict in verdicts) and all(r is not None and r.escapes for r in readings) + inserting[name] = {inserted for page in pages if page is not None for inserted in page.inserted} + if all(r is not None and r.settled for r in readings): + settled.add(name) requires[name] = {needed for verdict in verdicts for needed in verdict.extends | verdict.includes} extends[name] = {extended for verdict in verdicts for extended in verdict.extends} includes[name] = {included for verdict in verdicts for included in verdict.includes} @@ -222,7 +239,9 @@ def escaped_templates(root: Path) -> frozenset[str]: open_ended.add(name) while grown := {name for name, extended in extends.items() if extended & open_ended} - open_ended: open_ended |= grown - escaped = {name for name, safe in local.items() if safe and not includes[name] & open_ended} + escaped = { + name for name, safe in local.items() if safe and not includes[name] & open_ended and inserting[name] <= settled + } changed = True while changed: kept = {name for name in escaped if requires[name] <= escaped} @@ -333,7 +352,7 @@ def _parse(text: str) -> _Parsed: opened: list[tuple[str, list[_Node]]] = [] skipping: str | None = None position = 0 - text = text.replace(_OUTPUT, "").replace(_TAG, "") + text = text.replace(_OUTPUT, "").replace(_TAG, "").replace(_INSERTED, "") def held() -> list[_Node]: return opened[-1][1] if opened else nodes @@ -426,12 +445,21 @@ class _Unrenderable(Exception): """A page Django could not render: a block holding a block of its own name.""" +class _Page(NamedTuple): + """The markup a template renders, and the names of the includes it inserts there + (``_INSERTED``), its own and those of the pages it includes.""" + + markup: str + inserted: frozenset[str] = frozenset() + + class _Composition: """The page each template file renders, put together as Django renders it (#273): its extends chain from the root, each block taking the definition of the most derived template defining it and ``{{ block.super }}`` the next definition up, each include - the page of the included template. A child's text outside its blocks is not part of - it. A page that cannot be put together in place is not established: a template it + the page of the included template, or, for a name several files have, an inserted + markup standing for each (``_INSERTED``). A child's text outside its blocks is not + part of it. A page that cannot be put together is not established: a template it needs is missing, refused by its own syntax or given by several files, it includes itself, a block of it holds itself, or it grows beyond ``_PAGE_LIMIT``, which ``too_large`` then lists.""" @@ -447,23 +475,19 @@ def __init__(self, found: Mapping[str, Sequence[Path]], parsed: Mapping[Path, _P for name in p.includes if len(paths := found.get(name, ())) == 1 } - self._pages: dict[Path, str | None] = {} + self._pages: dict[Path, _Page | None] = {} self.too_large: set[Path] = set() - def escapes(self, path: Path) -> bool: - """Whether the page ``path`` renders, put together, is read as escaping.""" - - page = self._page(path) - return page is not None and _read_markup(page).escapes + def page(self, path: Path) -> _Page | None: + """The page ``path`` renders, put together; None when it cannot be.""" - def _page(self, path: Path) -> str | None: if path in self._pages: return self._pages[path] # Depth first over the includes each page renders, without recursion: a page is # put together once the pages it includes are. stack = [path] waiting: set[Path] = set() - result: str | None = None + result: _Page | None = None while stack: current = stack[-1] if current in self._pages: @@ -505,35 +529,44 @@ def _chain(self, path: Path) -> list[_Parsed] | None: return chain if all(parsed.ok for parsed in chain) else None def _targets(self, chain: Sequence[_Parsed]) -> list[Path] | None: - """The files the page of ``chain`` includes, or None when one has not exactly one.""" + """The files the page of ``chain`` includes in place, those of a name one file + has; None when a name it includes has none.""" targets: list[Path] = [] try: for node in _rendered(chain): if isinstance(node, _Include): paths = self._found.get(node.name, ()) - if len(paths) != 1: + if not paths: return None - targets.append(paths[0]) + if len(paths) == 1: + targets.append(paths[0]) except _Unrenderable: return None return targets - def _render(self, path: Path, chain: Sequence[_Parsed]) -> str | None: + def _render(self, path: Path, chain: Sequence[_Parsed]) -> _Page | None: parts: list[str] = [] + inserted: set[str] = set() size = 0 for node in _rendered(chain): if isinstance(node, _Include): - included = self._pages[self._found[node.name][0]] - if included is None: - return None - node = included + paths = self._found[node.name] + if len(paths) > 1: + inserted.add(node.name) + node = _INSERTED + else: + included = self._pages[paths[0]] + if included is None: + return None + inserted |= included.inserted + node = included.markup parts.append(node) size += len(node) if size > _PAGE_LIMIT: self.too_large.add(path) return None - return "".join(parts) + return _Page("".join(parts), frozenset(inserted)) def _rendered(chain: Sequence[_Parsed]) -> Iterator[str | _Include]: @@ -580,11 +613,17 @@ def _constant(word: str) -> str | None: class _Reading(NamedTuple): """What one pass over a template's markup establishes: whether every output marker - stands where HTML escaping protects it, and whether the text ends inside a script or - style block, a tag or an end tag, which the markup after its include continues.""" + stands where HTML escaping protects it; whether the text ends inside a script or + style block, a tag or an end tag, which the markup after its include continues; and + whether it ends in HTML data, ``settled``: nothing of it left open, comment included, + and no ``<``, `` _Reading: @@ -598,7 +637,12 @@ def _read_markup(html: str) -> _Reading: include there may close the comment, a condition around its ``", + " None: + # Both files are harmless here: what matters is that either could end the construct. + templates = {"part.html": "x\n", "other/templates/part.html": "y\n", "page.html": page} + + assert escaped(tmp_path, templates) == ["part.html"] + + +@pytest.mark.parametrize( + "other", + ["\n", ["base.html"]), + ], + ids=["each-escapes", "one-runs-the-block-as-script"], +) +def test_a_page_extending_a_name_two_files_have_must_escape_with_each( + tmp_path: Path, other: str, expected: list[str] +) -> None: + templates = { + "base.html": "

{% block body %}{% endblock %}

\n", + "other/templates/base.html": other, + "page.html": "{% extends 'base.html' %}{% block body %}{{ name }}{% endblock %}\n", + } + + assert escaped(tmp_path, templates) == expected + + +@pytest.mark.parametrize( + ("page", "expected"), + [ + ("
{% include 'widget.html' %}
\n", ["base.html", "page.html", "widget.html"]), + ('
x\n', ["base.html", "widget.html"]), + ], + ids=["in-html-data", "in-an-attribute"], +) +def test_an_include_of_a_template_with_several_pages_stands_for_each( + tmp_path: Path, page: str, expected: list[str] +) -> None: + templates = { + "base.html": "

{% block body %}{% endblock %}

\n", + "other/templates/base.html": "
{% block body %}{% endblock %}
\n", + "widget.html": "{% extends 'base.html' %}{% block body %}x{% endblock %}", + "page.html": page, + } + + assert escaped(tmp_path, templates) == expected + + +def chain_of_names_two_files_have(levels: int, root: str = "x") -> dict[str, str]: + # level{n}.html, given by two files, extends level{n - 1}.html: each file of level n + # renders 2 ** n pages, one for each file each level below it may be. + templates: dict[str, str] = {} + for directory in ("app", "other"): + templates[f"{directory}/templates/level0.html"] = f"{root}{{% block body %}}{{% endblock %}}" + for level in range(1, levels + 1): + templates[f"{directory}/templates/level{level}.html"] = ( + f"{{% extends 'level{level - 1}.html' %}}{{% block body %}}{{{{ name }}}}{{% endblock %}}" + ) + return templates + + +def test_a_template_rendering_more_than_64_pages_is_not_established(tmp_path: Path) -> None: + assert escaped(tmp_path, chain_of_names_two_files_have(7)) == [f"level{level}.html" for level in range(7)] + + +def test_pages_of_a_template_beyond_sixteen_mebibytes_together_are_not_established(tmp_path: Path) -> None: + # Each of the 64 pages of level6.html holds 300,000 characters: 19.2 million together. + templates = chain_of_names_two_files_have(6, root="x" * 300_000) + + assert escaped(tmp_path, templates) == [f"level{level}.html" for level in range(6)] + + # Markup, a variable and a template tag, as the reader sees a page. PIECES = ["x", " ", "

", "

", "", '", "", "<", " Date: Fri, 23 Oct 2026 04:24:33 +0200 Subject: [PATCH 05/13] fix(templates): insert a marker for an include on the way to itself A template including itself, directly or through others, was not put together. Its recursive include now inserts a marker standing for its own page: accepted only in HTML data, and only when that page ends in data. Property tests prove it on generated templates: a text holding such a marker, once accepted, stays accepted unfolded level after level, and a template established stays established when its recursion is written out as templates including each other. --- src/coretrace_python/taint/templates.py | 26 +++++---- tests/test_template_page_composition.py | 74 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index 2c6d12e..c9b83f4 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -464,14 +464,16 @@ class _Composition: """The page each template file renders, put together as Django renders it (#273): its extends chain from the root, each block taking the definition of the most derived template defining it and ``{{ block.super }}`` the next definition up, each include - the page of the included template, or, for a name several files have or a template - rendering several pages, an inserted markup standing for each (``_INSERTED``). A + the page of the included template, or, for a name several files have, a template + rendering several pages or one on the way to itself, an inserted markup standing for + each (``_INSERTED``): a page inserted in HTML data reads as the markup it stands for + there, nested to any depth, provided that markup is accepted and ends in data. A name several files have in the chain gives a page for each, up to ``_VARIANT_LIMIT`` and ``_VARIANTS_SIZE_LIMIT`` together, beyond which ``too_many`` lists the template. A child's text outside its blocks is not part of a page. A page that cannot be put together is not established: a template it needs is missing or refused by its own - syntax, it includes or extends itself, a block of it holds itself, or it grows - beyond ``_PAGE_LIMIT``, which ``too_large`` then lists.""" + syntax, it extends itself, a block of it holds itself, or it grows beyond + ``_PAGE_LIMIT``, which ``too_large`` then lists.""" def __init__(self, found: Mapping[str, Sequence[Path]], parsed: Mapping[Path, _Parsed]) -> None: self._found = found @@ -505,15 +507,15 @@ def page(self, path: Path) -> _Page | None: continue chains = self._chains(current) targets = None if chains is None else self._targets(chains) - if targets is None or any(target in waiting for target in targets): + if targets is None: page = None - elif pending := [target for target in targets if target not in self._pages]: + elif pending := [target for target in targets if target not in self._pages and target not in waiting]: waiting.add(current) stack.extend(pending) continue else: assert chains is not None - page = self._render(current, chains) + page = self._render(current, chains, waiting) waiting.discard(current) stack.pop() if current == path: @@ -565,7 +567,10 @@ def _targets(self, chains: Iterable[Sequence[_Parsed]]) -> list[Path] | None: return None return targets - def _render(self, path: Path, chains: Iterable[Sequence[_Parsed]]) -> _Page | None: + def _render(self, path: Path, chains: Iterable[Sequence[_Parsed]], waiting: Container[Path]) -> _Page | None: + """The pages ``chains`` render; an include of a page being put together, on the + way to itself, inserted (``_INSERTED``).""" + markups: list[str] = [] inserted: set[str] = set() together = 0 @@ -575,11 +580,12 @@ def _render(self, path: Path, chains: Iterable[Sequence[_Parsed]]) -> _Page | No for node in _rendered(chain): if isinstance(node, _Include): paths = self._found[node.name] - included = self._pages[paths[0]] if len(paths) == 1 else None + recursive = len(paths) == 1 and paths[0] in waiting + included = self._pages[paths[0]] if len(paths) == 1 and not recursive else None if included is not None and len(included.markups) == 1: inserted |= included.inserted node = included.markups[0] - elif len(paths) == 1 and included is None: + elif len(paths) == 1 and included is None and not recursive: return None else: inserted.add(node.name) diff --git a/tests/test_template_page_composition.py b/tests/test_template_page_composition.py index d539ade..20da33f 100644 --- a/tests/test_template_page_composition.py +++ b/tests/test_template_page_composition.py @@ -281,6 +281,23 @@ def test_pages_of_a_template_beyond_sixteen_mebibytes_together_are_not_establish assert escaped(tmp_path, templates) == [f"level{level}.html" for level in range(6)] + +def test_a_template_including_itself_in_html_data_is_established(tmp_path: Path) -> None: + templates = { + "page.html": "{% include 'tree.html' %}", + "tree.html": "
  • {{ name }}
      {% if more %}{% include 'tree.html' %}{% endif %}
  • ", + } + + assert escaped(tmp_path, templates) == ["page.html", "tree.html"] + + +def test_a_template_including_itself_must_end_in_html_data(tmp_path: Path) -> None: + # Rendered twice, the comment it leaves open runs to the "-->" in the next title, and + # the variable after it names a tag. + templates = {"self.html": '{% if more %}{% include \'self.html\' %}{% endif %}
    ", "<", " bool: spliced = templates._read_markup(prefix + content + suffix) assert spliced.escapes, (prefix, content, suffix) assert spliced.settled or not marked.settled, (prefix, content, suffix) + + +def test_a_text_inserting_itself_is_accepted_only_if_its_unfoldings_are() -> None: + """A text holding the marker where it inserts itself, accepted and ending in HTML + data, stays accepted unfolded: the marker replaced by the text's own unfolding, level + after level, down to the text without it. Checked on every text of up to two pieces + around the marker.""" + + from coretrace_python.taint import templates + + pieces = [*PIECES, templates._OUTPUT, templates._TAG] + short = ["", *pieces, *("".join(pair) for pair in itertools.product(pieces, repeat=2))] + for before, after in itertools.product(short, repeat=2): + reading = templates._read_markup(before + templates._INSERTED + after) + if not (reading.escapes and reading.settled): + continue + unfolded = before + after + for _ in range(3): + assert templates._read_markup(unfolded).escapes, (before, after) + unfolded = before + unfolded + after + + +# Template pieces around a self-include: unfolded, the include gives way to the template's +# own text, level after level, until the condition around it, if any, no longer holds. +TEMPLATE_PIECES = ["x", " ", "{{ name }}", "<", "", '", "", "", "{% if c %}", "{% endif %}"] + + +def test_a_template_including_itself_is_established_only_if_its_unfoldings_are(tmp_path: Path) -> None: + """For generated templates including themselves, established means every unfolding + of the recursion, written out as templates including each other down to one that + includes nothing, is established too.""" + + from coretrace_python.taint.templates import escaped_templates + + rng = random.Random(273) + root = tmp_path / "project" + directory = root / "app" / "templates" + directory.mkdir(parents=True) + count, depth = 3_000, 4 + for index in range(count): + before = "".join(rng.choice(TEMPLATE_PIECES) for _ in range(rng.randint(0, 3))) + after = "".join(rng.choice(TEMPLATE_PIECES) for _ in range(rng.randint(0, 3))) + # Half of them include themselves under a condition, the others without one: a + # tag after the include would end a comment its markup leaves open. + opening, closing = ("{% if more %}", "{% endif %}") if index % 2 else ("", "") + written = { + f"t{index}.html": f"{{% include 't{index}.html' %}}", + f"t{index}-0.html": "", + **{f"t{index}-{level}.html": f"{{% include 't{index}-{level - 1}.html' %}}" for level in range(1, depth)}, + } + for name, included in written.items(): + (directory / name).write_text(f"{before}{opening}{included}{closing}{after}", encoding="utf-8") + established = escaped_templates(root) + recursive = [index for index in range(count) if f"t{index}.html" in established] + assert recursive + for index in recursive: + assert all(f"t{index}-{level}.html" in established for level in range(depth)), index From 1c692ed93d05a791c6af26daeaf8c6384c384172 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 04:24:33 +0200 Subject: [PATCH 06/13] feat(templates): list the pages that cannot be put together among templates not read A page beyond 8 MiB once put together, and a template with more than 64 pages or 16 MiB in all, are listed in Coverage.unread_templates with the reason. template_escaping returns them with the names of the templates established to escape; escaped_templates keeps its interface. --- src/coretrace_python/engine.py | 7 ++-- src/coretrace_python/taint/__init__.py | 2 + src/coretrace_python/taint/templates.py | 25 +++++++++++- tests/test_template_page_composition.py | 51 +++++++++++++++++++++---- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/src/coretrace_python/engine.py b/src/coretrace_python/engine.py index 63830e7..92bf25a 100644 --- a/src/coretrace_python/engine.py +++ b/src/coretrace_python/engine.py @@ -135,10 +135,10 @@ SecurityModelRegistry, TaintAnalysis, TaintKind, - escaped_templates, project_templates, registered_routes, request_processor, + template_escaping, unread_renders, ) from coretrace_python.taint.urls import flow_url @@ -532,7 +532,8 @@ def analyze_project( for name in sorted(analysable): for symbol, registered in _routes_of(analysable[name]).items(): routes.setdefault(symbol, registered) - escaped = escaped_templates(root) + escaping = template_escaping(root) + escaped = escaping.escaped templates = project_templates(root) clearing = models.clearing(escaped) template_calls = models.template_calls( @@ -678,7 +679,7 @@ def analyze_project( findings = [_sourced(finding, origins) for finding in findings] accepted = tuple(f for f in findings if policy.accepts(f)) kept, suppressed = partition(apply_policy(policy, findings), _text_of(sources)) - unread = list(templates.unread) + unread = [*templates.unread, *escaping.unread] for name in sorted(call_graphs): unread.extend(unread_renders(name, call_graphs[name], models, templates.names)) return ProjectAnalysis( diff --git a/src/coretrace_python/taint/__init__.py b/src/coretrace_python/taint/__init__.py index 81bb361..c00f379 100644 --- a/src/coretrace_python/taint/__init__.py +++ b/src/coretrace_python/taint/__init__.py @@ -42,6 +42,7 @@ escaped_templates, project_templates, request_processor, + template_escaping, unread_renders, ) @@ -85,5 +86,6 @@ "propagate_taint", "registered_routes", "request_processor", + "template_escaping", "unread_renders", ] diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index c9b83f4..d4c9dec 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -199,6 +199,21 @@ def escaped_templates(root: Path) -> frozenset[str]: + """The names of the templates under ``root`` that escape every variable they render + (``template_escaping``).""" + + return template_escaping(root).escaped + + +class TemplateEscaping(NamedTuple): + """The names of the templates established to escape, and the pages the engine could + not put together to establish it, as entries of the templates it could not read.""" + + escaped: frozenset[str] + unread: tuple[str, ...] = () + + +def template_escaping(root: Path) -> TemplateEscaping: """The names of the templates under ``root`` that escape every variable they render, as ``render_to_string`` names them (their path below a ``templates`` directory). A template including one that ends inside a script or style block, a tag or an end tag @@ -210,7 +225,7 @@ def escaped_templates(root: Path) -> frozenset[str]: found: dict[str, list[Path]] = {} for path in discover_files(root): if path.suffix == ".py" and _AUTOESCAPE_OFF.search(_read(path)): - return frozenset() + return TemplateEscaping(frozenset()) for name in _names(path.relative_to(root).parts): found.setdefault(name, []).append(path) parsed = {path: _parse_file(path) for paths in found.values() for path in paths} @@ -249,7 +264,11 @@ def escaped_templates(root: Path) -> frozenset[str]: kept = {name for name in escaped if requires[name] <= escaped} changed = kept != escaped escaped = kept - return frozenset(escaped) + unread = [ + *(f"{path.relative_to(root).as_posix()}, {_TOO_LARGE}" for path in composition.too_large), + *(f"{path.relative_to(root).as_posix()}, {_TOO_MANY}" for path in composition.too_many), + ] + return TemplateEscaping(frozenset(escaped), tuple(sorted(unread))) def _names(parts: tuple[str, ...]) -> Iterator[str]: @@ -445,6 +464,8 @@ def _flattened(nodes: Iterable[_Node]) -> str: # their size together: many small pages never reach the bound of one. _VARIANT_LIMIT = 64 _VARIANTS_SIZE_LIMIT = 16 * 1024 * 1024 +_TOO_LARGE = "beyond 8 MiB once its blocks and includes are put together" +_TOO_MANY = "more than 64 pages, or 16 MiB in all, with each file the templates it extends may be" class _Unrenderable(Exception): diff --git a/tests/test_template_page_composition.py b/tests/test_template_page_composition.py index 20da33f..7b8ed0a 100644 --- a/tests/test_template_page_composition.py +++ b/tests/test_template_page_composition.py @@ -64,6 +64,22 @@ def escaped(tmp_path: Path, templates: dict[str, str]) -> list[str]: return names +UNREAD_TEMPLATES = ( + "import json, sys\n" + "from pathlib import Path\n" + "from coretrace_python import engine\n" + "analysis = engine.analyze_project(Path(sys.argv[1]), [engine.BUNDLED_PLUGINS])\n" + "print(json.dumps(sorted(analysis.coverage.unread_templates)))\n" +) + + +def unread(tmp_path: Path, templates: dict[str, str]) -> list[str]: + completed = run([sys.executable, "-c", UNREAD_TEMPLATES, str(project(tmp_path, templates))]) + assert completed.returncode == 0, completed.stderr + entries: list[str] = json.loads(completed.stdout) + return entries + + def xss(tmp_path: Path, templates: dict[str, str]) -> list[tuple[str, int]]: root = project(tmp_path, templates) completed = run([sys.executable, "-m", "coretrace_python", "--check", str(root), "--format", "json"]) @@ -158,15 +174,24 @@ def test_a_template_including_itself_inside_a_construct_is_not_established( assert escaped(tmp_path, templates) == [] -def test_a_page_beyond_eight_mebibytes_is_not_established(tmp_path: Path) -> None: - # level{n}.html holds 2 ** n characters once put together: level23.html is 8 MiB, - # level24.html beyond, and the page including it is put together no further. - templates = {"level0.html": "x"} - for level in range(1, 25): +def doubling_includes() -> dict[str, str]: + # level{n}.html holds 2 ** n KiB once put together: level13.html is 8 MiB, + # level14.html beyond, and the page including it is put together no further. + templates = {"level0.html": "x" * 1024} + for level in range(1, 15): templates[f"level{level}.html"] = f"{{% include 'level{level - 1}.html' %}}" * 2 - templates["page.html"] = "{% include 'level24.html' %}" + templates["page.html"] = "{% include 'level14.html' %}" + return templates + + +def test_a_page_beyond_eight_mebibytes_is_not_established(tmp_path: Path) -> None: + assert escaped(tmp_path, doubling_includes()) == sorted(f"level{level}.html" for level in range(14)) + - assert escaped(tmp_path, templates) == sorted(f"level{level}.html" for level in range(24)) +def test_a_page_beyond_eight_mebibytes_is_listed_among_the_templates_not_read(tmp_path: Path) -> None: + assert unread(tmp_path, doubling_includes()) == [ + "app/templates/level14.html, beyond 8 MiB once its blocks and includes are put together" + ] # Two files give part.html. Django renders the first its loaders find, which the analyzer @@ -281,6 +306,18 @@ def test_pages_of_a_template_beyond_sixteen_mebibytes_together_are_not_establish assert escaped(tmp_path, templates) == [f"level{level}.html" for level in range(6)] +@pytest.mark.parametrize(("levels", "root"), [(7, "x"), (6, "x" * 300_000)], ids=["more-than-64", "beyond-16-mib"]) +def test_a_template_with_too_many_pages_is_listed_among_the_templates_not_read( + tmp_path: Path, levels: int, root: str +) -> None: + reason = "more than 64 pages, or 16 MiB in all, with each file the templates it extends may be" + + assert unread(tmp_path, {**chain_of_names_two_files_have(levels, root), "page.html": "x"}) == [ + f"app/templates/level{levels}.html, {reason}", + f"other/templates/level{levels}.html, {reason}", + ] + + def test_a_template_including_itself_in_html_data_is_established(tmp_path: Path) -> None: templates = { From 69208db4e3ebc452304badf85c053984c3fdff82 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 04:24:33 +0200 Subject: [PATCH 07/13] docs: describe how a page is put together to read Django templates The usage guide, the module's docstring and the changelog state how pages are put together, how a name several files have and a template including itself are read, and the bounds on a page. --- CHANGELOG.md | 8 ++++++++ docs/usage.md | 16 ++++++++++++++-- src/coretrace_python/taint/templates.py | 3 +++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a8c9b0..e45930a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,14 @@ instead of repeating its id (#270), and each result carries its confidence as the `confidence` property (#271). A flow reaching its sink inside a project function points at that sink as a related location, which the result's message links to (#271). +- Django templates are also read as each page is put together (#273): a variable a + child's `{% block %}` prints inside its parent's `