diff --git a/CHANGELOG.md b/CHANGELOG.md
index 892da79..be25874 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,13 @@
a parent including the child that extends it. A filter reached this way may be
reported as `exploitable-vulnerability` where it was not. Where no cycle is reached,
the filters and their order do not change.
+- A template's render is put together over its extends chain as Django places its
+ blocks (#285). A filter in a block no template of the chain places, or in a block
+ whose only placeholder a child replaces, is no longer reached, and an advisory naming
+ it is no longer `exploitable-vulnerability` through it. A block a child redefines with
+ `{{ block.super }}` inside a block it replaces now renders the parent's definition and
+ what it includes. Where the chain cannot be followed to a root, every block counts as
+ placed, as before.
### Performance
diff --git a/docs/usage.md b/docs/usage.md
index d6870a4..04a1dfe 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -605,9 +605,14 @@ certain of it:
of includes is followed as far as Django may render it: a template reached again on
the way applies its filters to what each include binds at every level, as a template
including itself with other bindings, or a recursive tree, does (#284).
-- The filter renders: a template extending another renders only its blocks, which
- replace the blocks of that name in what it extends, unless the block renders
- `{{ block.super }}`, which keeps the parent's content and the filters it applies.
+- The filter renders: a template extending another renders only its blocks, and only
+ where its chain of parents places them (#285). The root's blocks are placed; a placed
+ block renders its most derived definition, and through `{{ block.super }}` the next
+ ones up, which keep the parent's content and the filters it applies; the blocks a
+ rendered definition holds are placed in turn. A block no template of the chain
+ places never renders, nor does a block whose only placeholder a child replaces. When
+ the chain cannot be followed to a root (a parent named by an expression, not found or
+ in several files, or an extends cycle), every block of the chain counts as placed.
- One file has the template's name; of two files of the same name, which one renders
depends on the loaders.
diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py
index 1913ea4..71b44c0 100644
--- a/src/coretrace_python/taint/templates.py
+++ b/src/coretrace_python/taint/templates.py
@@ -53,7 +53,7 @@
import re
from bisect import bisect_left
from collections import deque
-from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from types import MappingProxyType
@@ -989,29 +989,16 @@ def overrides(self, name: str) -> bool | None:
@dataclass(frozen=True)
class _Template:
"""What one template file applies and references, the ``blocks`` it defines, and
- among them the ``extended`` ones, which render ``{{ block.super }}``, and the loads
- under which it applies a filter the engine could not resolve."""
+ among them the ``extended`` ones, which render ``{{ block.super }}``, the loads under
+ which it applies a filter the engine could not resolve, and the block each block
+ stands in, None for one standing in none (#285)."""
applications: tuple[_Application, ...]
references: tuple[_Reference, ...]
blocks: frozenset[str]
extended: frozenset[str] = frozenset()
unresolved: tuple[_Load, ...] = ()
-
- def rendered(self, blocks: tuple[str, ...], overridden: frozenset[str]) -> bool:
- """Whether what stands in ``blocks`` renders when a template extending this one
- defines the ``overridden`` blocks: a template extending another renders only
- its blocks, where they replace the blocks of that name in what it extends."""
-
- extends = any(reference.extends for reference in self.references)
- return not overridden.intersection(blocks) and (bool(blocks) or not extends)
-
- @property
- def replaced(self) -> frozenset[str]:
- """The blocks this template replaces in what it extends: those it defines
- without rendering ``{{ block.super }}``, which keeps the parent's content."""
-
- return self.blocks - self.extended
+ enclosing: Mapping[str, str | None] = field(default_factory=dict)
def project_templates(root: Path) -> ProjectTemplates:
@@ -1056,7 +1043,7 @@ def project_templates(root: Path) -> ProjectTemplates:
name: tuple(
dict.fromkeys(
TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument)))
- for symbol, span, where, value, argument in settled.get((name, frozenset()), ())
+ for symbol, span, where, value, argument in settled.get(name, ())
)
)
for name in files
@@ -1067,68 +1054,115 @@ def project_templates(root: Path) -> ProjectTemplates:
# A filter a template's render reaches: the function, where it is applied, the template
# applying it, and the names free in the rendered template reaching its value and argument.
_Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]]
-# A template, and the blocks the templates extending it override: what rendering the
-# template applies depends on both.
-_Pair = tuple[str, frozenset[str]]
class _Render(NamedTuple):
- """What rendering a pair applies itself, and the pairs its references render, each
+ """What rendering a template on its own applies, and the templates it includes, each
with its reference."""
applied: tuple[_Reached, ...]
- references: tuple[tuple[_Pair, _Reference], ...]
+ references: tuple[tuple[str, _Reference], ...]
+
+
+def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Render:
+ """What rendering ``name`` on its own applies and includes, its extends chain put
+ together as Django puts it (#285).
+
+ The chain runs from ``name`` up to its root, the template extending none. The root's
+ blocks standing in no other block are placed; a placed block renders its most derived
+ definition, then, while a definition renders ``{{ block.super }}``, the next one up;
+ the blocks a rendered definition holds are placed in turn. A filter or an include
+ renders when the definition of the innermost block it stands in renders; outside
+ every block, only in the root, Django ignoring a child's text outside its blocks.
+
+ When the chain reaches no known root, as through a parent named by an expression, a
+ name several files have, a file the engine cannot read, or an extends cycle Django
+ refuses, every block of the chain is placed: no filter is dropped on a placement the
+ engine cannot establish.
+
+ What the chain applies is listed from ``name`` up to the root, and what it includes
+ from the root down to ``name``: the order of a walk of each path through extends."""
+
+ chain: list[tuple[str, _Template]] = []
+ seen: set[str] = set()
+ known = True
+ current: str | None = name
+ while True:
+ template = readable(current)
+ if current is None or template is None or current in seen:
+ known = False
+ break
+ chain.append((current, template))
+ seen.add(current)
+ parent = next((reference for reference in template.references if reference.extends), None)
+ if parent is None:
+ break
+ current = parent.name
+ defining: dict[str, list[int]] = {}
+ held: dict[tuple[int, str | None], list[str]] = {}
+ for level, (_, template) in enumerate(chain):
+ for block in template.blocks:
+ defining.setdefault(block, []).append(level)
+ held.setdefault((level, template.enclosing.get(block)), []).append(block)
+ pending = list(held.get((len(chain) - 1, None), ())) if known else list(defining)
+ placed: set[str] = set()
+ shown: set[tuple[int, str]] = set()
+ while pending:
+ block = pending.pop()
+ if block in placed:
+ continue
+ placed.add(block)
+ for level in defining.get(block, []):
+ shown.add((level, block))
+ pending.extend(held.get((level, block), ()))
+ if block not in chain[level][1].extended:
+ break
+
+ def renders(level: int, blocks: tuple[str, ...]) -> bool:
+ return (level, blocks[-1]) in shown if blocks else known and level == len(chain) - 1
+
+ applied = tuple(
+ (application.symbol, application.span, template_name, application.value, application.argument)
+ for level, (template_name, template) in enumerate(chain)
+ for application in template.applications
+ if renders(level, application.blocks)
+ )
+ references = tuple(
+ (reference.name, reference)
+ for level in reversed(range(len(chain)))
+ for reference in chain[level][1].references
+ if not reference.extends
+ and reference.name is not None
+ and readable(reference.name) is not None
+ and renders(level, reference.blocks)
+ )
+ return _Render(applied, references)
-def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, _Render]:
- """Every pair the render of a template reaches, each template rendered on its own. A
- name several files have, or a file the engine could not read, renders nothing."""
+def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[str, _Render]:
+ """What rendering each template on its own applies and includes. A name several files
+ have, or a file the engine could not read, renders nothing."""
- def pair(name: str, overridden: frozenset[str]) -> _Pair | None:
- paths = files.get(name, [])
- return (name, overridden) if len(paths) == 1 and paths[0] in read else None
+ def readable(name: str | None) -> _Template | None:
+ paths = files.get(name or "", [])
+ return read[paths[0]] if len(paths) == 1 and paths[0] in read else None
- renders: dict[_Pair, _Render] = {}
- pending = [root for name in files if (root := pair(name, frozenset())) is not None]
- while pending:
- current = pending.pop()
- if current in renders:
- continue
- name, overridden = current
- template = read[files[name][0]]
- applied = tuple(
- (a.symbol, a.span, name, a.value, a.argument) for a in template.applications if template.rendered(a.blocks, overridden)
- )
- references: list[tuple[_Pair, _Reference]] = []
- for reference in template.references:
- if reference.name is None:
- continue
- if reference.extends:
- below = pair(reference.name, overridden | template.replaced)
- elif template.rendered(reference.blocks, overridden):
- below = pair(reference.name, frozenset())
- else:
- continue
- if below is not None:
- references.append((below, reference))
- pending.append(below)
- renders[current] = _Render(applied, tuple(references))
- return renders
+ return {name: _page(name, readable) for name in files if readable(name) is not None}
-def _components(renders: Mapping[_Pair, _Render]) -> list[list[_Pair]]:
+def _components(renders: Mapping[str, _Render]) -> list[list[str]]:
"""The strongly connected components of the renders, each after every component its
references reach: Tarjan's algorithm, without recursion."""
- index: dict[_Pair, int] = {}
- low: dict[_Pair, int] = {}
- stack: list[_Pair] = []
- on_stack: set[_Pair] = set()
- components: list[list[_Pair]] = []
+ index: dict[str, int] = {}
+ low: dict[str, int] = {}
+ stack: list[str] = []
+ on_stack: set[str] = set()
+ components: list[list[str]] = []
for root in renders:
if root in index:
continue
- # Each entry is a pair and the position of the next reference to follow.
+ # Each entry is a template and the position of the next reference to follow.
work = [(root, 0)]
while work:
current, position = work.pop()
@@ -1147,7 +1181,7 @@ def _components(renders: Mapping[_Pair, _Render]) -> list[list[_Pair]]:
low[current] = min(low[current], index[below])
else:
if low[current] == index[current]:
- component: list[_Pair] = []
+ component: list[str] = []
while not component or component[-1] != current:
component.append(stack.pop())
on_stack.discard(component[-1])
@@ -1166,7 +1200,7 @@ def _mapped(found: Iterable[_Reached], reference: _Reference) -> Iterator[_Reach
def _canonical(found: _Reached) -> tuple[str, int, int, str, list[str], list[str]]:
- """The order of the filters a pair in a cycle reaches beyond what it applies."""
+ """The order of the filters a template in a cycle reaches beyond what it applies."""
symbol, span, where, value, argument = found
return where, span.start_line, span.start_column, str(symbol), sorted(value), sorted(argument)
@@ -1191,11 +1225,12 @@ def _facts(names: Mapping[_Site, tuple[frozenset[str], frozenset[str]]], sites:
return [(symbol, span, where, *names[symbol, span, where]) for symbol, span, where in sites]
-def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, list[_Reached]]:
- """The filters the render of each pair reaches, in order (#284).
+def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[str, list[_Reached]]:
+ """The filters the render of each template reaches, in order (#284).
- What a pair reaches is the least set holding what it applies, and what each of its
- references reaches mapped through the reference's bindings. The sets are computed
+ What a template reaches is the least set holding what it applies, its extends chain
+ put together (#285), and what each template it includes reaches, mapped through the
+ reference's bindings. The sets are computed
component by component, each after those its references reach, which are then
final. No cycle is cut: a template including itself with other bindings reaches what
Django renders at its next levels.
@@ -1205,20 +1240,20 @@ def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) ->
analysis reads those names position by position, joining what each brings, so the
join loses nothing, and it keeps the facts few: bindings permuted around a cycle
would otherwise make one fact of every combination of names. In a component, every
- pair starts with no names and is recomputed from its references' facts; names only
- join, and the applications and names of a project are finitely many, so the names of
- an application grow at most once for each name. A pair is recomputed when a fact it
- reads grew, until none grows.
-
- Once the facts are settled, a pair outside any cycle lists what it applies, then what
- each reference reaches, in order, first occurrences only, as a walk of each path
- would. A pair in a cycle has no such first occurrence, the cycle having no start: it
- lists what it applies, then the rest by template, position, function and names. The
- order thus depends only on the facts and the templates, never on the order of the
- computation."""
+ template starts with no names and is recomputed from its references' facts; names
+ only join, and the applications and names of a project are finitely many, so the
+ names of an application grow at most once for each name. A template is recomputed
+ when a fact it reads grew, until none grows.
+
+ Once the facts are settled, a template outside any cycle lists what it applies,
+ then what each reference reaches, in order, first occurrences only, as a walk of
+ each path would. A template in a cycle has no such first occurrence, the cycle
+ having no start: it lists what it applies, then the rest by template, position,
+ function and names. The order thus depends only on the facts and the templates,
+ never on the order of the computation."""
renders = _renders(files, read)
- reached: dict[_Pair, list[_Reached]] = {}
+ reached: dict[str, list[_Reached]] = {}
for component in _components(renders):
single = component[0]
if len(component) == 1 and all(below != single for below, _ in renders[single].references):
@@ -1229,12 +1264,12 @@ def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) ->
reached[single] = list(found)
continue
members = set(component)
- readers: dict[_Pair, list[_Pair]] = {member: [] for member in component}
+ readers: dict[str, list[str]] = {member: [] for member in component}
for member in component:
for below, _ in renders[member].references:
if below in members:
readers[below].append(member)
- names: dict[_Pair, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component}
+ names: dict[str, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component}
sizes = dict.fromkeys(component, 0)
pending = deque(component)
queued = set(component)
@@ -1275,6 +1310,7 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library])
opened: list[str] = []
defined: set[str] = set()
extended: set[str] = set()
+ enclosing: dict[str, str | None] = {}
scopes: list[dict[str, frozenset[str]]] = [{}]
blocks: list[tuple[list[tuple[str, int, frozenset[str]]], set[str]]] = []
loaded: list[_Load] = []
@@ -1360,6 +1396,7 @@ def apply(head: frozenset[str], chain: list[tuple[str, int, frozenset[str]]]) ->
else:
values.setdefault(word, reached)
if name == "block" and len(words) > 1:
+ enclosing.setdefault(words[1], opened[-1] if opened else None)
opened.append(words[1])
defined.add(words[1])
elif name == "endblock" and opened:
@@ -1388,7 +1425,7 @@ def apply(head: frozenset[str], chain: list[tuple[str, int, frozenset[str]]]) ->
target = words[words.index("as", 1) + 1]
scopes[-1][target] = values.get(words[1], frozenset()) if name == "regroup" else frozenset()
return _Template(
- tuple(applications), tuple(references), frozenset(defined), frozenset(extended), tuple(unresolved)
+ tuple(applications), tuple(references), frozenset(defined), frozenset(extended), tuple(unresolved), enclosing
)
diff --git a/tests/test_template_block_placement.py b/tests/test_template_block_placement.py
new file mode 100644
index 0000000..c210aef
--- /dev/null
+++ b/tests/test_template_block_placement.py
@@ -0,0 +1,237 @@
+"""Acceptance tests for issue #285: a template's render is put together over its extends
+chain as Django puts it.
+
+The root's blocks standing in no other block are placed; a placed block renders its most
+derived definition, then, while a definition renders ``{{ block.super }}``, the next one
+up; the blocks a rendered definition holds are placed in turn. A filter or an include
+renders when the definition of the innermost block it stands in renders. A block no
+template of the chain places never renders. When the chain reaches no known root, every
+block counts as placed, so that no filter is dropped on a placement the engine cannot
+establish. Each expectation below is what Django 5.1.15 applies to the same templates.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from coretrace_python import engine
+from coretrace_python.dependency import Advisory, AdvisoryEntryPoint, dump_advisories
+from coretrace_python.findings import Severity
+from coretrace_python.semantic.symbols import SymbolId
+from coretrace_python.taint.templates import project_templates
+
+TIMEOUT = 60
+FILTERS = "python.django.template.defaultfilters"
+STRIPTAGS = f"{FILTERS}.striptags"
+ADVISORY = Advisory(
+ "CVE-2099-8501",
+ "django",
+ "<4.2.17",
+ "striptags is quadratic in its input",
+ Severity.MEDIUM,
+ entry_points=(AdvisoryEntryPoint(SymbolId(STRIPTAGS), "a template filter"),),
+)
+LOCK = """version = 1
+
+[[package]]
+name = "app"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [{ name = "django" }]
+
+[[package]]
+name = "django"
+version = "4.2.16"
+source = { registry = "https://pypi.org/simple" }
+"""
+VIEW = (
+ "from django.http import HttpRequest\n"
+ "from django.shortcuts import render\n\n"
+ "def page(request: HttpRequest):\n"
+ " return render(request, 'page.html', {'v': request.POST['bio']})\n"
+)
+PAGE = "{% extends 'base.html' %}\n{% block body %}hello{% endblock %}\n{% block extra %}{{ v|striptags }}{% endblock %}\n"
+
+
+def write(root: Path, templates: dict[str, str]) -> Path:
+ for name, text in templates.items():
+ path = root / "app" / "templates" / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="utf-8")
+ return root
+
+
+def described(root: Path, name: str) -> list[tuple[str, str, int, int, tuple[str, ...]]]:
+ """The filters rendering ``name`` reaches: the function, the template applying it, the
+ line and column it is applied at, and the names reaching its value."""
+
+ return [
+ (str(f.symbol).rsplit(".", 1)[1], f.template, f.location.start_line, f.location.start_column, f.value)
+ for f in project_templates(root).filters[name]
+ ]
+
+
+@pytest.mark.parametrize(
+ ("base", "expected"),
+ [
+ ("{% block body %}{% endblock %}\n", []),
+ ("{% block body %}{% endblock %}{% block extra %}{% endblock %}\n", [(5, "page.html", "3")]),
+ ],
+ ids=["a-block-no-parent-places", "control-the-parent-places-it"],
+)
+def test_input_reaches_a_filter_only_in_a_block_a_parent_places(
+ tmp_path: Path, base: str, expected: list[tuple[int, str, str]]
+) -> None:
+ # Django renders "hello" without the parent's extra block, and never
+ # calls striptags; with it, striptags receives the request's bio.
+ root = write(tmp_path, {"base.html": base, "page.html": PAGE})
+ for relative, text in {"uv.lock": LOCK, "app/__init__.py": "", "app/views.py": VIEW}.items():
+ (root / relative).write_text(text, encoding="utf-8")
+ (root / "advisories.json").write_text(dump_advisories((ADVISORY,)), encoding="utf-8")
+ findings = engine.analyze_project(root, [engine.BUNDLED_PLUGINS]).findings
+
+ assert sorted(
+ (f.span.start_line, f.metadata["through"], f.metadata["sink_line"])
+ for f in findings
+ if f.rule_id == "exploitable-vulnerability" and f.metadata["advisory"] == ADVISORY.id
+ ) == expected
+
+
+BASE = "{% block content %}{% block main %}{{ v|upper }}{% endblock %}{% endblock %}\n"
+FORM = "{% extends 'base.html' %}\n{% block main %}{{ v|striptags }}{% endblock %}\n"
+
+
+@pytest.mark.parametrize(
+ ("page", "expected"),
+ [
+ ("{% extends 'form.html' %}\n{% block content %}hello{% endblock %}\n", []),
+ ("{% extends 'form.html' %}\n{% block title %}hello{% endblock %}\n", [("striptags", "form.html", 2, 22, ("v",))]),
+ ],
+ ids=["the-placeholder-replaced", "control-content-kept"],
+)
+def test_a_block_whose_only_placeholder_a_child_replaces_does_not_render(
+ tmp_path: Path, page: str, expected: list[tuple[str, str, int, int, tuple[str, ...]]]
+) -> None:
+ # page.html replaces content, which held the only main placeholder: Django renders
+ # "hello", and form.html's main never; without that, it applies striptags.
+ root = write(tmp_path, {"base.html": BASE, "form.html": FORM, "page.html": page})
+
+ assert described(root, "page.html") == expected
+
+
+HEADER_BASE = "{% block content %}{% block header %}{% block main_header %}{% include 'header.html' %}{% endblock %}{% endblock %}{% endblock %}\n"
+
+
+@pytest.mark.parametrize(
+ ("header", "expected"),
+ [
+ ("{{ block.super }}", [("upper", "header.html", 1, 6, ("v",))]),
+ ("hello", []),
+ ],
+ ids=["block-super-inside-a-replaced-block", "control-without-block-super"],
+)
+def test_a_parent_block_rendered_through_block_super_inside_a_replaced_block_renders(
+ tmp_path: Path, header: str, expected: list[tuple[str, str, int, int, tuple[str, ...]]]
+) -> None:
+ # page.html replaces content but defines header inside it: through {{ block.super }},
+ # Django renders the parent's header, its main_header and the include it holds.
+ page = "{% extends 'base.html' %}\n{% block content %}{% block header %}" + header + "{% endblock %}{% endblock %}\n"
+ root = write(tmp_path, {"base.html": HEADER_BASE, "page.html": page, "header.html": "{{ v|upper }}\n"})
+
+ assert described(root, "page.html") == expected
+
+
+def test_a_chain_of_three_templates_renders_what_each_level_places(tmp_path: Path) -> None:
+ # base.html places content; mid.html's content places side; page.html extends side
+ # through {{ block.super }}. Its aside, which no template of the chain places, never
+ # renders: Django applies striptags, then mid.html's upper.
+ root = write(
+ tmp_path,
+ {
+ "base.html": "{% block content %}{% endblock %}\n",
+ "mid.html": "{% extends 'base.html' %}\n{% block content %}{% block side %}{{ v|upper }}{% endblock %}{% endblock %}\n",
+ "page.html": (
+ "{% extends 'mid.html' %}\n"
+ "{% block side %}{{ block.super }}{{ v|striptags }}{% endblock %}\n"
+ "{% block aside %}{{ v|upper }}{% endblock %}\n"
+ ),
+ },
+ )
+
+ assert described(root, "page.html") == [("striptags", "page.html", 2, 39, ("v",)), ("upper", "mid.html", 2, 41, ("v",))]
+
+
+@pytest.mark.parametrize(
+ ("parent", "unread"),
+ [
+ ("parent", "app/templates/page.html:1 names a template by an expression"),
+ ("'missing.html'", "app/templates/page.html:1 names 'missing.html', not found"),
+ ],
+ ids=["named-by-an-expression", "not-found"],
+)
+def test_a_parent_the_engine_cannot_follow_keeps_every_block_placed(tmp_path: Path, parent: str, unread: str) -> None:
+ # The text outside the blocks of a template extending another never renders.
+ root = write(tmp_path, {"page.html": "{% extends " + parent + " %}\n{% block body %}{{ v|upper }}{% endblock %}{{ v|striptags }}\n"})
+
+ assert described(root, "page.html") == [("upper", "page.html", 2, 22, ("v",))]
+ assert project_templates(root).unread == (unread,)
+
+
+def test_an_extends_cycle_keeps_every_block_placed(tmp_path: Path) -> None:
+ # Django refuses to render either template; the engine keeps their filters.
+ root = write(
+ tmp_path,
+ {
+ "a.html": "{% extends 'b.html' %}\n{% block body %}{{ v|upper }}{% endblock %}\n",
+ "b.html": "{% extends 'a.html' %}\n{% block side %}{{ v|striptags }}{% endblock %}\n",
+ },
+ )
+
+ assert described(root, "a.html") == [("upper", "a.html", 2, 22, ("v",)), ("striptags", "b.html", 2, 22, ("v",))]
+ assert described(root, "b.html") == [("striptags", "b.html", 2, 22, ("v",)), ("upper", "a.html", 2, 22, ("v",))]
+
+
+
+def test_a_chain_lists_its_filters_in_the_order_of_a_walk_through_its_extends(tmp_path: Path) -> None:
+ # What the chain applies from the template up, then what it includes from the root
+ # down: base.html's include comes before page.html's.
+ root = write(
+ tmp_path,
+ {
+ "base.html": "{% include 'b.html' %}{% block body %}{% endblock %}\n",
+ "page.html": "{% extends 'base.html' %}\n{% block body %}{% include 'a.html' %}{% endblock %}\n",
+ "a.html": "{{ v|upper }}\n",
+ "b.html": "{{ v|striptags }}\n",
+ },
+ )
+
+ assert described(root, "page.html") == [("striptags", "b.html", 1, 6, ("v",)), ("upper", "a.html", 1, 6, ("v",))]
+
+
+COUNT_FILTERS = (
+ "import json, sys\n"
+ "from pathlib import Path\n"
+ "from coretrace_python.taint.templates import project_templates\n"
+ "print(json.dumps({name: len(found) for name, found in project_templates(Path(sys.argv[1])).filters.items()}))\n"
+)
+
+
+def test_the_blocks_of_a_template_are_placed_in_linear_time(tmp_path: Path) -> None:
+ # Looking up the blocks a placed block holds among all the template's blocks made
+ # placing 50,000 blocks take minutes. Only the last one applies a filter.
+ blocks = "".join(f"{{% block b{i} %}}{{% endblock %}}" for i in range(50_000))
+ root = write(tmp_path, {"page.html": blocks + "{% block last %}{{ v|upper }}{% endblock %}"})
+ try:
+ completed = subprocess.run(
+ [sys.executable, "-c", COUNT_FILTERS, str(root)], capture_output=True, text=True, check=False, timeout=TIMEOUT
+ )
+ except subprocess.TimeoutExpired:
+ pytest.fail(f"placing the blocks did not end within {TIMEOUT} s")
+
+ assert completed.returncode == 0, completed.stderr
+ assert json.loads(completed.stdout) == {"page.html": 1}
diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py
index eaf0f77..06f51dc 100644
--- a/tests/test_template_filter_walk.py
+++ b/tests/test_template_filter_walk.py
@@ -198,48 +198,126 @@ def test_a_child_its_parent_includes_back_gets_the_same_filters_whichever_is_wal
# The walk path by path lets each cycle turn 2, then 3 times, within this many steps.
REPEATS = (2, 3)
BUDGET = 20_000
-# Projects the walk does not settle within its budget: 10 with this seed.
-UNVERIFIED = 15
+# Projects the walk does not settle within its budget: 18 with this seed.
+UNVERIFIED = 20
class Unverified(Exception):
"""The walk path by path exceeded its budget: it verifies nothing of the project."""
+# A block tag opening or closing, read from the template's text apart from the engine.
+BLOCK_TAG = re.compile(r"\{%\s*(?:block\s+(\w+)|endblock)\b[^%]*%\}")
+
+
+def enclosing(text: str) -> dict[str, str | None]:
+ """Each block of ``text``, with the block it stands in."""
+
+ found: dict[str, str | None] = {}
+ opened: list[str] = []
+ for match in BLOCK_TAG.finditer(text):
+ if match.group(1):
+ found.setdefault(match.group(1), opened[-1] if opened else None)
+ opened.append(match.group(1))
+ elif opened:
+ opened.pop()
+ return found
+
+
+def rendered_page(
+ name: str, files: dict[str, list[Path]], read: dict[Path, Any]
+) -> tuple[list[tuple[Any, ...]], list[Any]] | None:
+ """What rendering the template ``name`` on its own applies, and the includes it
+ renders, its extends chain put together as Django puts it (#285): the root's blocks
+ standing in no other block are placed, a placed block renders its most derived
+ definition and, through {{ block.super }}, the next ones up, and a rendered
+ definition places the blocks it holds. With no known root, every block is placed."""
+
+ def readable(template_name: str | None) -> Any:
+ paths = files.get(template_name or "", [])
+ return read[paths[0]] if len(paths) == 1 and paths[0] in read else None
+
+ nested: dict[str, dict[str, str | None]] = {}
+
+ def nesting(template_name: str) -> dict[str, str | None]:
+ if template_name not in nested:
+ nested[template_name] = enclosing(files[template_name][0].read_text(encoding="utf-8"))
+ return nested[template_name]
+
+ if readable(name) is None:
+ return None
+ chain: list[tuple[str, Any]] = []
+ known = True
+ current: str | None = name
+ while True:
+ template = readable(current)
+ if template is None or any(current == seen for seen, _ in chain):
+ known = False
+ break
+ chain.append((str(current), template))
+ parent = next((reference for reference in template.references if reference.extends), None)
+ if parent is None:
+ break
+ current = parent.name
+ rendering: set[tuple[int, str]] = set()
+
+ def place(block: str) -> None:
+ for level, (_, template) in enumerate(chain):
+ if block not in template.blocks:
+ continue
+ if (level, block) in rendering:
+ return
+ rendering.add((level, block))
+ for inner, outer in nesting(chain[level][0]).items():
+ if outer == block:
+ place(inner)
+ if block not in template.extended:
+ return
+
+ for block in [b for b, outer in nesting(chain[-1][0]).items() if outer is None] if known else sorted({b for _, t in chain for b in t.blocks}):
+ place(block)
+
+ def renders(level: int, blocks: tuple[str, ...]) -> bool:
+ return (level, blocks[-1]) in rendering if blocks else known and level == len(chain) - 1
+
+ applied = [
+ (application.symbol, application.span, template_name, application.value, application.argument)
+ for level, (template_name, template) in enumerate(chain)
+ for application in template.applications
+ if renders(level, application.blocks)
+ ]
+ includes = [
+ reference
+ for level in reversed(range(len(chain)))
+ for reference in chain[level][1].references
+ if not reference.extends and reference.name is not None and renders(level, reference.blocks)
+ ]
+ return applied, includes
+
+
def unrolled(
name: str,
- files: dict[str, list[Path]],
- read: dict[Path, Any],
+ pages: dict[str, tuple[list[tuple[Any, ...]], list[Any]] | None],
repeats: int,
budget: list[int],
- visiting: tuple[tuple[str, frozenset[str]], ...] = (),
- overridden: frozenset[str] = frozenset(),
+ visiting: tuple[str, ...] = (),
) -> Iterator[tuple[Any, ...]]:
- """Every path through includes and extends, where each template, with the blocks
- overridden, may come back ``repeats`` times on the way: what Django renders when the
- project's conditions let each cycle turn that many times."""
+ """Every path through includes, each template rendered over its extends chain as
+ ``pages`` gives it, where each template may come back ``repeats`` times on the way:
+ what Django renders when the project's conditions let each cycle turn that many times."""
- paths = files.get(name, [])
- here = (name, overridden)
- if len(paths) != 1 or paths[0] not in read or visiting.count(here) >= repeats:
+ if visiting.count(name) >= repeats:
+ return
+ page = pages.get(name)
+ if page is None:
return
budget[0] -= 1
if budget[0] < 0:
raise Unverified
- template = read[paths[0]]
- for application in template.applications:
- if template.rendered(application.blocks, overridden):
- yield application.symbol, application.span, name, application.value, application.argument
- for reference in template.references:
- if reference.name is None:
- continue
- if reference.extends:
- found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here), overridden | template.replaced)
- elif template.rendered(reference.blocks, overridden):
- found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here))
- else:
- continue
- for symbol, span, where, value, argument in found:
+ applied, includes = page
+ yield from applied
+ for reference in includes:
+ for symbol, span, where, value, argument in unrolled(reference.name, pages, repeats, budget, (*visiting, name)):
yield symbol, span, where, reference.maps(value), reference.maps(argument)
@@ -255,11 +333,12 @@ def unrolled_filters(root: Path, repeats: int) -> dict[str, list[TemplateFilter]
for name in found[path]:
files.setdefault(name, []).append(path)
read = {path: templates._read_scopes(text, SourceId(str(path)), libraries) for path in paths if (text := templates._text(path))}
+ pages = {name: rendered_page(name, files, read) for name in files}
return {
name: list(
dict.fromkeys(
TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument)))
- for symbol, span, where, value, argument in unrolled(name, files, read, repeats, [BUDGET])
+ for symbol, span, where, value, argument in unrolled(name, pages, repeats, [BUDGET])
)
)
for name in files