From 82793278e3dec01da5fba101f10ba2841e4bac4b Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 27 Oct 2026 23:27:52 +0100 Subject: [PATCH 1/6] refactor(templates): record the block each block of a template stands in The template read for the filter walk keeps, for each block, the block it stands in, None for one standing in none: what putting a render together over its extends chain needs to place the blocks a definition holds. Nothing reads it yet. --- src/coretrace_python/taint/templates.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index 1913ea4..f4cd9df 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -989,14 +989,16 @@ def overrides(self, name: str) -> bool | None: @dataclass(frozen=True) class _Template: """What one template file applies and references, the ``blocks`` it defines, and - among them the ``extended`` ones, which render ``{{ block.super }}``, and the loads - under which it applies a filter the engine could not resolve.""" + among them the ``extended`` ones, which render ``{{ block.super }}``, the loads under + which it applies a filter the engine could not resolve, and the block each block + stands in, None for one standing in none (#285).""" applications: tuple[_Application, ...] references: tuple[_Reference, ...] blocks: frozenset[str] extended: frozenset[str] = frozenset() unresolved: tuple[_Load, ...] = () + enclosing: Mapping[str, str | None] = field(default_factory=dict) def rendered(self, blocks: tuple[str, ...], overridden: frozenset[str]) -> bool: """Whether what stands in ``blocks`` renders when a template extending this one @@ -1275,6 +1277,7 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) opened: list[str] = [] defined: set[str] = set() extended: set[str] = set() + enclosing: dict[str, str | None] = {} scopes: list[dict[str, frozenset[str]]] = [{}] blocks: list[tuple[list[tuple[str, int, frozenset[str]]], set[str]]] = [] loaded: list[_Load] = [] @@ -1360,6 +1363,7 @@ def apply(head: frozenset[str], chain: list[tuple[str, int, frozenset[str]]]) -> else: values.setdefault(word, reached) if name == "block" and len(words) > 1: + enclosing.setdefault(words[1], opened[-1] if opened else None) opened.append(words[1]) defined.add(words[1]) elif name == "endblock" and opened: @@ -1388,7 +1392,7 @@ def apply(head: frozenset[str], chain: list[tuple[str, int, frozenset[str]]]) -> target = words[words.index("as", 1) + 1] scopes[-1][target] = values.get(words[1], frozenset()) if name == "regroup" else frozenset() return _Template( - tuple(applications), tuple(references), frozenset(defined), frozenset(extended), tuple(unresolved) + tuple(applications), tuple(references), frozenset(defined), frozenset(extended), tuple(unresolved), enclosing ) From 562aa3ef6f7e655d8bec04e5d9c664695694f4e8 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 27 Oct 2026 23:27:52 +0100 Subject: [PATCH 2/6] test(templates): pin what a chain keeps when its root is unknown, and its order A parent named by an expression or not found, and an extends cycle, keep every block placed, and the text outside the blocks of a template extending another never renders; the templates not found stay listed among the unread ones. A chain lists its filters in the order of a walk through its extends: what it applies, then what it includes from the root down. --- tests/test_template_block_placement.py | 84 ++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/test_template_block_placement.py diff --git a/tests/test_template_block_placement.py b/tests/test_template_block_placement.py new file mode 100644 index 0000000..94defb6 --- /dev/null +++ b/tests/test_template_block_placement.py @@ -0,0 +1,84 @@ +"""Acceptance tests for issue #285: a template's render is put together over its extends +chain as Django puts it. + +The root's blocks standing in no other block are placed; a placed block renders its most +derived definition, then, while a definition renders ``{{ block.super }}``, the next one +up; the blocks a rendered definition holds are placed in turn. A filter or an include +renders when the definition of the innermost block it stands in renders. A block no +template of the chain places never renders. When the chain reaches no known root, every +block counts as placed, so that no filter is dropped on a placement the engine cannot +establish. Each expectation below is what Django 5.1.15 applies to the same templates. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coretrace_python.taint.templates import project_templates + + +def write(root: Path, templates: dict[str, str]) -> Path: + for name, text in templates.items(): + path = root / "app" / "templates" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def described(root: Path, name: str) -> list[tuple[str, str, int, int, tuple[str, ...]]]: + """The filters rendering ``name`` reaches: the function, the template applying it, the + line and column it is applied at, and the names reaching its value.""" + + return [ + (str(f.symbol).rsplit(".", 1)[1], f.template, f.location.start_line, f.location.start_column, f.value) + for f in project_templates(root).filters[name] + ] + + +@pytest.mark.parametrize( + ("parent", "unread"), + [ + ("parent", "app/templates/page.html:1 names a template by an expression"), + ("'missing.html'", "app/templates/page.html:1 names 'missing.html', not found"), + ], + ids=["named-by-an-expression", "not-found"], +) +def test_a_parent_the_engine_cannot_follow_keeps_every_block_placed(tmp_path: Path, parent: str, unread: str) -> None: + # The text outside the blocks of a template extending another never renders. + root = write(tmp_path, {"page.html": "{% extends " + parent + " %}\n{% block body %}{{ v|upper }}{% endblock %}{{ v|striptags }}\n"}) + + assert described(root, "page.html") == [("upper", "page.html", 2, 22, ("v",))] + assert project_templates(root).unread == (unread,) + + +def test_an_extends_cycle_keeps_every_block_placed(tmp_path: Path) -> None: + # Django refuses to render either template; the engine keeps their filters. + root = write( + tmp_path, + { + "a.html": "{% extends 'b.html' %}\n{% block body %}{{ v|upper }}{% endblock %}\n", + "b.html": "{% extends 'a.html' %}\n{% block side %}{{ v|striptags }}{% endblock %}\n", + }, + ) + + assert described(root, "a.html") == [("upper", "a.html", 2, 22, ("v",)), ("striptags", "b.html", 2, 22, ("v",))] + assert described(root, "b.html") == [("striptags", "b.html", 2, 22, ("v",)), ("upper", "a.html", 2, 22, ("v",))] + + + +def test_a_chain_lists_its_filters_in_the_order_of_a_walk_through_its_extends(tmp_path: Path) -> None: + # What the chain applies from the template up, then what it includes from the root + # down: base.html's include comes before page.html's. + root = write( + tmp_path, + { + "base.html": "{% include 'b.html' %}{% block body %}{% endblock %}\n", + "page.html": "{% extends 'base.html' %}\n{% block body %}{% include 'a.html' %}{% endblock %}\n", + "a.html": "{{ v|upper }}\n", + "b.html": "{{ v|striptags }}\n", + }, + ) + + assert described(root, "page.html") == [("striptags", "b.html", 1, 6, ("v",)), ("upper", "a.html", 1, 6, ("v",))] From 4ecdeb94fe52b438477d5d589ba4e4dbd59fd8ff Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 27 Oct 2026 23:27:52 +0100 Subject: [PATCH 3/6] fix(templates): put each render together over its extends chain as Django places blocks Whether a filter or an include of an extended template rendered depended only on the blocks it stood in and those the templates extending it replaced, not on where the parents place their blocks (#285). A block no template of the chain places counted as rendered, and so did a block whose only placeholder a child replaces; a block a child redefines with {{ block.super }} inside a block it replaces did not render the parent's definition, nor what it includes. Each template's render is now put together over its extends chain: the root's blocks are placed, a placed block renders its most derived definition and, through {{ block.super }}, the next ones up, and the blocks a rendered definition holds are placed in turn. A filter or an include renders when the definition of the innermost block it stands in renders. When the chain reaches no known root, every block of the chain counts as placed, so that no filter is dropped on a placement that cannot be established. The fixpoint of #284 runs on the templates themselves, the pairs with the blocks overridden being no longer needed. The comparison with the walk path by path puts each render together over its chain, reading the blocks' nesting from the text; with this reference, 18 generated projects exceed the unchanged budget, so up to 20 may stay unverified. --- src/coretrace_python/taint/templates.py | 195 ++++++++++++++---------- tests/test_template_block_placement.py | 125 +++++++++++++++ tests/test_template_filter_walk.py | 133 ++++++++++++---- 3 files changed, 345 insertions(+), 108 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index f4cd9df..15e703b 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -53,7 +53,7 @@ import re from bisect import bisect_left from collections import deque -from collections.abc import Container, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType @@ -1000,21 +1000,6 @@ class _Template: unresolved: tuple[_Load, ...] = () enclosing: Mapping[str, str | None] = field(default_factory=dict) - def rendered(self, blocks: tuple[str, ...], overridden: frozenset[str]) -> bool: - """Whether what stands in ``blocks`` renders when a template extending this one - defines the ``overridden`` blocks: a template extending another renders only - its blocks, where they replace the blocks of that name in what it extends.""" - - extends = any(reference.extends for reference in self.references) - return not overridden.intersection(blocks) and (bool(blocks) or not extends) - - @property - def replaced(self) -> frozenset[str]: - """The blocks this template replaces in what it extends: those it defines - without rendering ``{{ block.super }}``, which keeps the parent's content.""" - - return self.blocks - self.extended - def project_templates(root: Path) -> ProjectTemplates: """What the templates under ``root`` call, and the templates they include or extend @@ -1058,7 +1043,7 @@ def project_templates(root: Path) -> ProjectTemplates: name: tuple( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in settled.get((name, frozenset()), ()) + for symbol, span, where, value, argument in settled.get(name, ()) ) ) for name in files @@ -1069,68 +1054,115 @@ def project_templates(root: Path) -> ProjectTemplates: # A filter a template's render reaches: the function, where it is applied, the template # applying it, and the names free in the rendered template reaching its value and argument. _Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]] -# A template, and the blocks the templates extending it override: what rendering the -# template applies depends on both. -_Pair = tuple[str, frozenset[str]] class _Render(NamedTuple): - """What rendering a pair applies itself, and the pairs its references render, each + """What rendering a template on its own applies, and the templates it includes, each with its reference.""" applied: tuple[_Reached, ...] - references: tuple[tuple[_Pair, _Reference], ...] + references: tuple[tuple[str, _Reference], ...] + + +def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Render: + """What rendering ``name`` on its own applies and includes, its extends chain put + together as Django puts it (#285). + + The chain runs from ``name`` up to its root, the template extending none. The root's + blocks standing in no other block are placed; a placed block renders its most derived + definition, then, while a definition renders ``{{ block.super }}``, the next one up; + the blocks a rendered definition holds are placed in turn. A filter or an include + renders when the definition of the innermost block it stands in renders; outside + every block, only in the root, Django ignoring a child's text outside its blocks. + + When the chain reaches no known root, as through a parent named by an expression, a + name several files have, a file the engine cannot read, or an extends cycle Django + refuses, every block of the chain is placed: no filter is dropped on a placement the + engine cannot establish. + + What the chain applies is listed from ``name`` up to the root, and what it includes + from the root down to ``name``: the order of a walk of each path through extends.""" + + chain: list[tuple[str, _Template]] = [] + seen: set[str] = set() + known = True + current: str | None = name + while True: + template = readable(current) + if current is None or template is None or current in seen: + known = False + break + chain.append((current, template)) + seen.add(current) + parent = next((reference for reference in template.references if reference.extends), None) + if parent is None: + break + current = parent.name + defining: dict[str, list[int]] = {} + for level, (_, template) in enumerate(chain): + for block in template.blocks: + defining.setdefault(block, []).append(level) + root = chain[-1][1] + pending = [block for block in root.blocks if root.enclosing.get(block) is None] if known else list(defining) + placed: set[str] = set() + shown: set[tuple[int, str]] = set() + while pending: + block = pending.pop() + if block in placed: + continue + placed.add(block) + for level in defining.get(block, []): + shown.add((level, block)) + template = chain[level][1] + pending.extend(inner for inner in template.blocks if template.enclosing.get(inner) == block) + if block not in template.extended: + break + + def renders(level: int, blocks: tuple[str, ...]) -> bool: + return (level, blocks[-1]) in shown if blocks else known and level == len(chain) - 1 + + applied = tuple( + (application.symbol, application.span, template_name, application.value, application.argument) + for level, (template_name, template) in enumerate(chain) + for application in template.applications + if renders(level, application.blocks) + ) + references = tuple( + (reference.name, reference) + for level in reversed(range(len(chain))) + for reference in chain[level][1].references + if not reference.extends + and reference.name is not None + and readable(reference.name) is not None + and renders(level, reference.blocks) + ) + return _Render(applied, references) -def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, _Render]: - """Every pair the render of a template reaches, each template rendered on its own. A - name several files have, or a file the engine could not read, renders nothing.""" +def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[str, _Render]: + """What rendering each template on its own applies and includes. A name several files + have, or a file the engine could not read, renders nothing.""" - def pair(name: str, overridden: frozenset[str]) -> _Pair | None: - paths = files.get(name, []) - return (name, overridden) if len(paths) == 1 and paths[0] in read else None + def readable(name: str | None) -> _Template | None: + paths = files.get(name or "", []) + return read[paths[0]] if len(paths) == 1 and paths[0] in read else None - renders: dict[_Pair, _Render] = {} - pending = [root for name in files if (root := pair(name, frozenset())) is not None] - while pending: - current = pending.pop() - if current in renders: - continue - name, overridden = current - template = read[files[name][0]] - applied = tuple( - (a.symbol, a.span, name, a.value, a.argument) for a in template.applications if template.rendered(a.blocks, overridden) - ) - references: list[tuple[_Pair, _Reference]] = [] - for reference in template.references: - if reference.name is None: - continue - if reference.extends: - below = pair(reference.name, overridden | template.replaced) - elif template.rendered(reference.blocks, overridden): - below = pair(reference.name, frozenset()) - else: - continue - if below is not None: - references.append((below, reference)) - pending.append(below) - renders[current] = _Render(applied, tuple(references)) - return renders + return {name: _page(name, readable) for name in files if readable(name) is not None} -def _components(renders: Mapping[_Pair, _Render]) -> list[list[_Pair]]: +def _components(renders: Mapping[str, _Render]) -> list[list[str]]: """The strongly connected components of the renders, each after every component its references reach: Tarjan's algorithm, without recursion.""" - index: dict[_Pair, int] = {} - low: dict[_Pair, int] = {} - stack: list[_Pair] = [] - on_stack: set[_Pair] = set() - components: list[list[_Pair]] = [] + index: dict[str, int] = {} + low: dict[str, int] = {} + stack: list[str] = [] + on_stack: set[str] = set() + components: list[list[str]] = [] for root in renders: if root in index: continue - # Each entry is a pair and the position of the next reference to follow. + # Each entry is a template and the position of the next reference to follow. work = [(root, 0)] while work: current, position = work.pop() @@ -1149,7 +1181,7 @@ def _components(renders: Mapping[_Pair, _Render]) -> list[list[_Pair]]: low[current] = min(low[current], index[below]) else: if low[current] == index[current]: - component: list[_Pair] = [] + component: list[str] = [] while not component or component[-1] != current: component.append(stack.pop()) on_stack.discard(component[-1]) @@ -1168,7 +1200,7 @@ def _mapped(found: Iterable[_Reached], reference: _Reference) -> Iterator[_Reach def _canonical(found: _Reached) -> tuple[str, int, int, str, list[str], list[str]]: - """The order of the filters a pair in a cycle reaches beyond what it applies.""" + """The order of the filters a template in a cycle reaches beyond what it applies.""" symbol, span, where, value, argument = found return where, span.start_line, span.start_column, str(symbol), sorted(value), sorted(argument) @@ -1193,11 +1225,12 @@ def _facts(names: Mapping[_Site, tuple[frozenset[str], frozenset[str]]], sites: return [(symbol, span, where, *names[symbol, span, where]) for symbol, span, where in sites] -def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, list[_Reached]]: - """The filters the render of each pair reaches, in order (#284). +def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[str, list[_Reached]]: + """The filters the render of each template reaches, in order (#284). - What a pair reaches is the least set holding what it applies, and what each of its - references reaches mapped through the reference's bindings. The sets are computed + What a template reaches is the least set holding what it applies, its extends chain + put together (#285), and what each template it includes reaches, mapped through the + reference's bindings. The sets are computed component by component, each after those its references reach, which are then final. No cycle is cut: a template including itself with other bindings reaches what Django renders at its next levels. @@ -1207,20 +1240,20 @@ def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> analysis reads those names position by position, joining what each brings, so the join loses nothing, and it keeps the facts few: bindings permuted around a cycle would otherwise make one fact of every combination of names. In a component, every - pair starts with no names and is recomputed from its references' facts; names only - join, and the applications and names of a project are finitely many, so the names of - an application grow at most once for each name. A pair is recomputed when a fact it - reads grew, until none grows. - - Once the facts are settled, a pair outside any cycle lists what it applies, then what - each reference reaches, in order, first occurrences only, as a walk of each path - would. A pair in a cycle has no such first occurrence, the cycle having no start: it - lists what it applies, then the rest by template, position, function and names. The - order thus depends only on the facts and the templates, never on the order of the - computation.""" + template starts with no names and is recomputed from its references' facts; names + only join, and the applications and names of a project are finitely many, so the + names of an application grow at most once for each name. A template is recomputed + when a fact it reads grew, until none grows. + + Once the facts are settled, a template outside any cycle lists what it applies, + then what each reference reaches, in order, first occurrences only, as a walk of + each path would. A template in a cycle has no such first occurrence, the cycle + having no start: it lists what it applies, then the rest by template, position, + function and names. The order thus depends only on the facts and the templates, + never on the order of the computation.""" renders = _renders(files, read) - reached: dict[_Pair, list[_Reached]] = {} + reached: dict[str, list[_Reached]] = {} for component in _components(renders): single = component[0] if len(component) == 1 and all(below != single for below, _ in renders[single].references): @@ -1231,12 +1264,12 @@ def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> reached[single] = list(found) continue members = set(component) - readers: dict[_Pair, list[_Pair]] = {member: [] for member in component} + readers: dict[str, list[str]] = {member: [] for member in component} for member in component: for below, _ in renders[member].references: if below in members: readers[below].append(member) - names: dict[_Pair, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component} + names: dict[str, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component} sizes = dict.fromkeys(component, 0) pending = deque(component) queued = set(component) diff --git a/tests/test_template_block_placement.py b/tests/test_template_block_placement.py index 94defb6..a8a397f 100644 --- a/tests/test_template_block_placement.py +++ b/tests/test_template_block_placement.py @@ -16,8 +16,43 @@ import pytest +from coretrace_python import engine +from coretrace_python.dependency import Advisory, AdvisoryEntryPoint, dump_advisories +from coretrace_python.findings import Severity +from coretrace_python.semantic.symbols import SymbolId from coretrace_python.taint.templates import project_templates +FILTERS = "python.django.template.defaultfilters" +STRIPTAGS = f"{FILTERS}.striptags" +ADVISORY = Advisory( + "CVE-2099-8501", + "django", + "<4.2.17", + "striptags is quadratic in its input", + Severity.MEDIUM, + entry_points=(AdvisoryEntryPoint(SymbolId(STRIPTAGS), "a template filter"),), +) +LOCK = """version = 1 + +[[package]] +name = "app" +version = "0.1.0" +source = { virtual = "." } +dependencies = [{ name = "django" }] + +[[package]] +name = "django" +version = "4.2.16" +source = { registry = "https://pypi.org/simple" } +""" +VIEW = ( + "from django.http import HttpRequest\n" + "from django.shortcuts import render\n\n" + "def page(request: HttpRequest):\n" + " return render(request, 'page.html', {'v': request.POST['bio']})\n" +) +PAGE = "{% extends 'base.html' %}\n{% block body %}hello{% endblock %}\n{% block extra %}{{ v|striptags }}{% endblock %}\n" + def write(root: Path, templates: dict[str, str]) -> Path: for name, text in templates.items(): @@ -37,6 +72,96 @@ def described(root: Path, name: str) -> list[tuple[str, str, int, int, tuple[str ] +@pytest.mark.parametrize( + ("base", "expected"), + [ + ("
{% block body %}{% endblock %}
\n", []), + ("
{% block body %}{% endblock %}{% block extra %}{% endblock %}
\n", [(5, "page.html", "3")]), + ], + ids=["a-block-no-parent-places", "control-the-parent-places-it"], +) +def test_input_reaches_a_filter_only_in_a_block_a_parent_places( + tmp_path: Path, base: str, expected: list[tuple[int, str, str]] +) -> None: + # Django renders "
hello
" without the parent's extra block, and never + # calls striptags; with it, striptags receives the request's bio. + root = write(tmp_path, {"base.html": base, "page.html": PAGE}) + for relative, text in {"uv.lock": LOCK, "app/__init__.py": "", "app/views.py": VIEW}.items(): + (root / relative).write_text(text, encoding="utf-8") + (root / "advisories.json").write_text(dump_advisories((ADVISORY,)), encoding="utf-8") + findings = engine.analyze_project(root, [engine.BUNDLED_PLUGINS]).findings + + assert sorted( + (f.span.start_line, f.metadata["through"], f.metadata["sink_line"]) + for f in findings + if f.rule_id == "exploitable-vulnerability" and f.metadata["advisory"] == ADVISORY.id + ) == expected + + +BASE = "{% block content %}{% block main %}{{ v|upper }}{% endblock %}{% endblock %}\n" +FORM = "{% extends 'base.html' %}\n{% block main %}{{ v|striptags }}{% endblock %}\n" + + +@pytest.mark.parametrize( + ("page", "expected"), + [ + ("{% extends 'form.html' %}\n{% block content %}hello{% endblock %}\n", []), + ("{% extends 'form.html' %}\n{% block title %}hello{% endblock %}\n", [("striptags", "form.html", 2, 22, ("v",))]), + ], + ids=["the-placeholder-replaced", "control-content-kept"], +) +def test_a_block_whose_only_placeholder_a_child_replaces_does_not_render( + tmp_path: Path, page: str, expected: list[tuple[str, str, int, int, tuple[str, ...]]] +) -> None: + # page.html replaces content, which held the only main placeholder: Django renders + # "hello", and form.html's main never; without that, it applies striptags. + root = write(tmp_path, {"base.html": BASE, "form.html": FORM, "page.html": page}) + + assert described(root, "page.html") == expected + + +HEADER_BASE = "{% block content %}{% block header %}{% block main_header %}{% include 'header.html' %}{% endblock %}{% endblock %}{% endblock %}\n" + + +@pytest.mark.parametrize( + ("header", "expected"), + [ + ("{{ block.super }}", [("upper", "header.html", 1, 6, ("v",))]), + ("hello", []), + ], + ids=["block-super-inside-a-replaced-block", "control-without-block-super"], +) +def test_a_parent_block_rendered_through_block_super_inside_a_replaced_block_renders( + tmp_path: Path, header: str, expected: list[tuple[str, str, int, int, tuple[str, ...]]] +) -> None: + # page.html replaces content but defines header inside it: through {{ block.super }}, + # Django renders the parent's header, its main_header and the include it holds. + page = "{% extends 'base.html' %}\n{% block content %}{% block header %}" + header + "{% endblock %}{% endblock %}\n" + root = write(tmp_path, {"base.html": HEADER_BASE, "page.html": page, "header.html": "{{ v|upper }}\n"}) + + assert described(root, "page.html") == expected + + +def test_a_chain_of_three_templates_renders_what_each_level_places(tmp_path: Path) -> None: + # base.html places content; mid.html's content places side; page.html extends side + # through {{ block.super }}. Its aside, which no template of the chain places, never + # renders: Django applies striptags, then mid.html's upper. + root = write( + tmp_path, + { + "base.html": "
{% block content %}{% endblock %}
\n", + "mid.html": "{% extends 'base.html' %}\n{% block content %}{% block side %}{{ v|upper }}{% endblock %}{% endblock %}\n", + "page.html": ( + "{% extends 'mid.html' %}\n" + "{% block side %}{{ block.super }}{{ v|striptags }}{% endblock %}\n" + "{% block aside %}{{ v|upper }}{% endblock %}\n" + ), + }, + ) + + assert described(root, "page.html") == [("striptags", "page.html", 2, 39, ("v",)), ("upper", "mid.html", 2, 41, ("v",))] + + @pytest.mark.parametrize( ("parent", "unread"), [ diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index eaf0f77..06f51dc 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -198,48 +198,126 @@ def test_a_child_its_parent_includes_back_gets_the_same_filters_whichever_is_wal # The walk path by path lets each cycle turn 2, then 3 times, within this many steps. REPEATS = (2, 3) BUDGET = 20_000 -# Projects the walk does not settle within its budget: 10 with this seed. -UNVERIFIED = 15 +# Projects the walk does not settle within its budget: 18 with this seed. +UNVERIFIED = 20 class Unverified(Exception): """The walk path by path exceeded its budget: it verifies nothing of the project.""" +# A block tag opening or closing, read from the template's text apart from the engine. +BLOCK_TAG = re.compile(r"\{%\s*(?:block\s+(\w+)|endblock)\b[^%]*%\}") + + +def enclosing(text: str) -> dict[str, str | None]: + """Each block of ``text``, with the block it stands in.""" + + found: dict[str, str | None] = {} + opened: list[str] = [] + for match in BLOCK_TAG.finditer(text): + if match.group(1): + found.setdefault(match.group(1), opened[-1] if opened else None) + opened.append(match.group(1)) + elif opened: + opened.pop() + return found + + +def rendered_page( + name: str, files: dict[str, list[Path]], read: dict[Path, Any] +) -> tuple[list[tuple[Any, ...]], list[Any]] | None: + """What rendering the template ``name`` on its own applies, and the includes it + renders, its extends chain put together as Django puts it (#285): the root's blocks + standing in no other block are placed, a placed block renders its most derived + definition and, through {{ block.super }}, the next ones up, and a rendered + definition places the blocks it holds. With no known root, every block is placed.""" + + def readable(template_name: str | None) -> Any: + paths = files.get(template_name or "", []) + return read[paths[0]] if len(paths) == 1 and paths[0] in read else None + + nested: dict[str, dict[str, str | None]] = {} + + def nesting(template_name: str) -> dict[str, str | None]: + if template_name not in nested: + nested[template_name] = enclosing(files[template_name][0].read_text(encoding="utf-8")) + return nested[template_name] + + if readable(name) is None: + return None + chain: list[tuple[str, Any]] = [] + known = True + current: str | None = name + while True: + template = readable(current) + if template is None or any(current == seen for seen, _ in chain): + known = False + break + chain.append((str(current), template)) + parent = next((reference for reference in template.references if reference.extends), None) + if parent is None: + break + current = parent.name + rendering: set[tuple[int, str]] = set() + + def place(block: str) -> None: + for level, (_, template) in enumerate(chain): + if block not in template.blocks: + continue + if (level, block) in rendering: + return + rendering.add((level, block)) + for inner, outer in nesting(chain[level][0]).items(): + if outer == block: + place(inner) + if block not in template.extended: + return + + for block in [b for b, outer in nesting(chain[-1][0]).items() if outer is None] if known else sorted({b for _, t in chain for b in t.blocks}): + place(block) + + def renders(level: int, blocks: tuple[str, ...]) -> bool: + return (level, blocks[-1]) in rendering if blocks else known and level == len(chain) - 1 + + applied = [ + (application.symbol, application.span, template_name, application.value, application.argument) + for level, (template_name, template) in enumerate(chain) + for application in template.applications + if renders(level, application.blocks) + ] + includes = [ + reference + for level in reversed(range(len(chain))) + for reference in chain[level][1].references + if not reference.extends and reference.name is not None and renders(level, reference.blocks) + ] + return applied, includes + + def unrolled( name: str, - files: dict[str, list[Path]], - read: dict[Path, Any], + pages: dict[str, tuple[list[tuple[Any, ...]], list[Any]] | None], repeats: int, budget: list[int], - visiting: tuple[tuple[str, frozenset[str]], ...] = (), - overridden: frozenset[str] = frozenset(), + visiting: tuple[str, ...] = (), ) -> Iterator[tuple[Any, ...]]: - """Every path through includes and extends, where each template, with the blocks - overridden, may come back ``repeats`` times on the way: what Django renders when the - project's conditions let each cycle turn that many times.""" + """Every path through includes, each template rendered over its extends chain as + ``pages`` gives it, where each template may come back ``repeats`` times on the way: + what Django renders when the project's conditions let each cycle turn that many times.""" - paths = files.get(name, []) - here = (name, overridden) - if len(paths) != 1 or paths[0] not in read or visiting.count(here) >= repeats: + if visiting.count(name) >= repeats: + return + page = pages.get(name) + if page is None: return budget[0] -= 1 if budget[0] < 0: raise Unverified - template = read[paths[0]] - for application in template.applications: - if template.rendered(application.blocks, overridden): - yield application.symbol, application.span, name, application.value, application.argument - for reference in template.references: - if reference.name is None: - continue - if reference.extends: - found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here), overridden | template.replaced) - elif template.rendered(reference.blocks, overridden): - found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here)) - else: - continue - for symbol, span, where, value, argument in found: + applied, includes = page + yield from applied + for reference in includes: + for symbol, span, where, value, argument in unrolled(reference.name, pages, repeats, budget, (*visiting, name)): yield symbol, span, where, reference.maps(value), reference.maps(argument) @@ -255,11 +333,12 @@ def unrolled_filters(root: Path, repeats: int) -> dict[str, list[TemplateFilter] for name in found[path]: files.setdefault(name, []).append(path) read = {path: templates._read_scopes(text, SourceId(str(path)), libraries) for path in paths if (text := templates._text(path))} + pages = {name: rendered_page(name, files, read) for name in files} return { name: list( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in unrolled(name, files, read, repeats, [BUDGET]) + for symbol, span, where, value, argument in unrolled(name, pages, repeats, [BUDGET]) ) ) for name in files From 68cbd83a25ae6cf463e529c93b82af943aca8365 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 27 Oct 2026 23:27:52 +0100 Subject: [PATCH 4/6] docs: state where a template's blocks render along its extends chain --- docs/usage.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index d6870a4..04a1dfe 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -605,9 +605,14 @@ certain of it: of includes is followed as far as Django may render it: a template reached again on the way applies its filters to what each include binds at every level, as a template including itself with other bindings, or a recursive tree, does (#284). -- The filter renders: a template extending another renders only its blocks, which - replace the blocks of that name in what it extends, unless the block renders - `{{ block.super }}`, which keeps the parent's content and the filters it applies. +- The filter renders: a template extending another renders only its blocks, and only + where its chain of parents places them (#285). The root's blocks are placed; a placed + block renders its most derived definition, and through `{{ block.super }}` the next + ones up, which keep the parent's content and the filters it applies; the blocks a + rendered definition holds are placed in turn. A block no template of the chain + places never renders, nor does a block whose only placeholder a child replaces. When + the chain cannot be followed to a root (a parent named by an expression, not found or + in several files, or an extends cycle), every block of the chain counts as placed. - One file has the template's name; of two files of the same name, which one renders depends on the loaders. From 33b25c7b8342e15f0d0262d38ad410f1b566e2a8 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 27 Oct 2026 23:27:52 +0100 Subject: [PATCH 5/6] docs(changelog): announce the blocks placed along the extends chain --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 892da79..be25874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ a parent including the child that extends it. A filter reached this way may be reported as `exploitable-vulnerability` where it was not. Where no cycle is reached, the filters and their order do not change. +- A template's render is put together over its extends chain as Django places its + blocks (#285). A filter in a block no template of the chain places, or in a block + whose only placeholder a child replaces, is no longer reached, and an advisory naming + it is no longer `exploitable-vulnerability` through it. A block a child redefines with + `{{ block.super }}` inside a block it replaces now renders the parent's definition and + what it includes. Where the chain cannot be followed to a root, every block counts as + placed, as before. ### Performance From cc98e801c1c6bb333d883ecbeedc6a5800aefb89 Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 29 Oct 2026 00:14:57 +0100 Subject: [PATCH 6/6] fix(templates): place the blocks of a chain in time linear in its blocks Each block placed looked up the blocks it holds among all the blocks of its template, so placing a template's blocks took time quadratic in their number: a 501 KB template of 16,000 empty blocks took more than 8 s to collect, where main takes 0.07 s, and the full analysis 11.6 s instead of 0.5 s. The blocks each block holds, and those standing in no block, are now indexed once per chain, while the blocks are listed. The same 16,000 blocks are placed in 0.08 s, and the full analysis takes 0.3 s. What is placed does not change. --- src/coretrace_python/taint/templates.py | 10 ++++----- tests/test_template_block_placement.py | 28 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index 15e703b..71b44c0 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -1099,11 +1099,12 @@ def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Ren break current = parent.name defining: dict[str, list[int]] = {} + held: dict[tuple[int, str | None], list[str]] = {} for level, (_, template) in enumerate(chain): for block in template.blocks: defining.setdefault(block, []).append(level) - root = chain[-1][1] - pending = [block for block in root.blocks if root.enclosing.get(block) is None] if known else list(defining) + held.setdefault((level, template.enclosing.get(block)), []).append(block) + pending = list(held.get((len(chain) - 1, None), ())) if known else list(defining) placed: set[str] = set() shown: set[tuple[int, str]] = set() while pending: @@ -1113,9 +1114,8 @@ def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Ren placed.add(block) for level in defining.get(block, []): shown.add((level, block)) - template = chain[level][1] - pending.extend(inner for inner in template.blocks if template.enclosing.get(inner) == block) - if block not in template.extended: + pending.extend(held.get((level, block), ())) + if block not in chain[level][1].extended: break def renders(level: int, blocks: tuple[str, ...]) -> bool: diff --git a/tests/test_template_block_placement.py b/tests/test_template_block_placement.py index a8a397f..c210aef 100644 --- a/tests/test_template_block_placement.py +++ b/tests/test_template_block_placement.py @@ -12,6 +12,9 @@ from __future__ import annotations +import json +import subprocess +import sys from pathlib import Path import pytest @@ -22,6 +25,7 @@ from coretrace_python.semantic.symbols import SymbolId from coretrace_python.taint.templates import project_templates +TIMEOUT = 60 FILTERS = "python.django.template.defaultfilters" STRIPTAGS = f"{FILTERS}.striptags" ADVISORY = Advisory( @@ -207,3 +211,27 @@ def test_a_chain_lists_its_filters_in_the_order_of_a_walk_through_its_extends(tm ) assert described(root, "page.html") == [("striptags", "b.html", 1, 6, ("v",)), ("upper", "a.html", 1, 6, ("v",))] + + +COUNT_FILTERS = ( + "import json, sys\n" + "from pathlib import Path\n" + "from coretrace_python.taint.templates import project_templates\n" + "print(json.dumps({name: len(found) for name, found in project_templates(Path(sys.argv[1])).filters.items()}))\n" +) + + +def test_the_blocks_of_a_template_are_placed_in_linear_time(tmp_path: Path) -> None: + # Looking up the blocks a placed block holds among all the template's blocks made + # placing 50,000 blocks take minutes. Only the last one applies a filter. + blocks = "".join(f"{{% block b{i} %}}{{% endblock %}}" for i in range(50_000)) + root = write(tmp_path, {"page.html": blocks + "{% block last %}{{ v|upper }}{% endblock %}"}) + try: + completed = subprocess.run( + [sys.executable, "-c", COUNT_FILTERS, str(root)], capture_output=True, text=True, check=False, timeout=TIMEOUT + ) + except subprocess.TimeoutExpired: + pytest.fail(f"placing the blocks did not end within {TIMEOUT} s") + + assert completed.returncode == 0, completed.stderr + assert json.loads(completed.stdout) == {"page.html": 1}