From bcacf395475a90e592c8f531bf80b13fa8a7a552 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 02:23:53 +0200 Subject: [PATCH 1/2] test: define persistent cache behavior --- tests/test_architecture.py | 2 +- tests/test_cache.py | 241 +++++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 tests/test_cache.py diff --git a/tests/test_architecture.py b/tests/test_architecture.py index ff94c27..7da12c0 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -42,7 +42,7 @@ } # Top-level modules that wire the pipeline together and may import any layer. -ROOT_MODULES = {"__init__", "__main__", "cli", "engine"} +ROOT_MODULES = {"__init__", "__main__", "cache", "cli", "engine"} def package_files() -> Iterator[tuple[Path, str, str]]: diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..d2e172c --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,241 @@ +"""Acceptance tests for the persistent cache (``docs/architecture.md`` §11, §38 Phase 10). + +``ProjectCache`` stores, per module, its function summaries, its call sites and its +findings under a key derived from the source text, the engine, schema and plugin +versions, the security models, the advisories, the dependency graph and the keys of the +modules it imports transitively. On a later run a module whose key is unchanged is +served from the cache and never lowered again; its summaries seed the project index, so +changing one file re-analyses that file and its importers only. + +Expected to remain red until ``coretrace_python.cache`` and the ``--cache`` option exist. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from coretrace_python import engine +from coretrace_python.cli import main +from coretrace_python.findings import Confidence, Finding, Severity +from coretrace_python.source import SourceId, SourceSpan + +try: + from coretrace_python.cache import CachedModule, ProjectCache, decode, encode +except ImportError as error: # pragma: no cover - red until the cache lands + MISSING = error +else: + MISSING = None + + +@pytest.fixture(autouse=True) +def require_cache() -> None: + if MISSING is not None: + pytest.fail(f"persistent cache is not implemented yet: {MISSING}") + + +REPO = Path(__file__).resolve().parent.parent +PLUGINS = REPO / "plugins" + +HELPERS = "import os\n\ndef execute(command):\n os.system(command)\n" +MAIN = "from app.helpers import execute\n\ndef run():\n execute(input())\n" +CLEAN = "def ok():\n return 1\n" + + +def project(root: Path, files: dict[str, str]) -> Path: + for relative, text in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def standard(root: Path) -> Path: + return project( + root / "src", + {"app/__init__.py": "", "app/helpers.py": HELPERS, "app/main.py": MAIN, "app/clean.py": CLEAN}, + ) + + +def rules(findings: tuple[Finding, ...]) -> list[tuple[str, str, int]]: + return sorted((Path(str(f.span.source_id)).name, f.rule_id, f.span.start_line) for f in findings) + + +# --------------------------------------------------------------------------- codec + + +def test_findings_and_summaries_round_trip_through_json() -> None: + from coretrace_python.interprocedural import ExternalCall, FunctionSummary + from coretrace_python.semantic.symbols import SymbolId + + span = SourceSpan(SourceId("/p/a.py"), 3, 5, 3, 9) + finding = Finding("r", "m", Severity.HIGH, Confidence.MEDIUM, span, "f", {"k": "v"}) + summary = FunctionSummary( + "f", + 2, + frozenset({0}), + (ExternalCall(SymbolId("python.os.system"), (frozenset({0}), frozenset()), frozenset({1}), span, None),), + False, + frozenset({SymbolId("python.builtins.input")}), + ) + + text = json.dumps(encode(CachedModule(("f",), {"f": summary}, (), (finding,)))) + restored = decode(json.loads(text)) + + assert restored.findings == (finding,) + assert restored.summaries["f"] == summary + assert restored.functions == ("f",) + + +# --------------------------------------------------------------------------- keys + + +def test_keys_depend_on_source_configuration_and_imports(tmp_path: Path) -> None: + root = standard(tmp_path) + first = engine.analyze_project(root, [PLUGINS]).keys + (root / "app" / "clean.py").write_text("def ok():\n return 2\n", encoding="utf-8") + changed_clean = engine.analyze_project(root, [PLUGINS]).keys + (root / "app" / "helpers.py").write_text(HELPERS + "\n", encoding="utf-8") + changed_helpers = engine.analyze_project(root, [PLUGINS]).keys + without_plugins = engine.analyze_project(root).keys + + assert set(first) == {"app", "app.helpers", "app.main", "app.clean"} + assert changed_clean["app.clean"] != first["app.clean"] + assert changed_clean["app.main"] == first["app.main"] + assert changed_helpers["app.helpers"] != changed_clean["app.helpers"] + assert changed_helpers["app.main"] != changed_clean["app.main"] + assert changed_helpers["app.clean"] == changed_clean["app.clean"] + assert without_plugins["app.clean"] != changed_helpers["app.clean"] + + +def test_keys_depend_on_dependency_files(tmp_path: Path) -> None: + root = standard(tmp_path) + before = engine.analyze_project(root, [PLUGINS]).keys + (root / "requirements.txt").write_text("pyyaml==5.3.1\n", encoding="utf-8") + after = engine.analyze_project(root, [PLUGINS]).keys + + assert all(after[name] != before[name] for name in before) + + +# --------------------------------------------------------------------------- reuse + + +def test_second_run_reuses_every_module_and_reports_the_same_findings(tmp_path: Path) -> None: + root = standard(tmp_path) + cache = ProjectCache(tmp_path / "cache") + + first = engine.analyze_project(root, [PLUGINS], cache=cache) + second = engine.analyze_project(root, [PLUGINS], cache=cache) + + assert first.reused == () + assert set(second.reused) == {"app", "app.helpers", "app.main", "app.clean"} + assert second.findings == first.findings + assert rules(second.findings) == [("main.py", "command-injection", 4)] + assert second.index == first.index + assert list((tmp_path / "cache").glob("*.json")) + + +def test_changing_a_file_reanalyses_it_and_its_importers_only(tmp_path: Path) -> None: + root = standard(tmp_path) + cache = ProjectCache(tmp_path / "cache") + engine.analyze_project(root, [PLUGINS], cache=cache) + + (root / "app" / "helpers.py").write_text("import os\n\ndef execute(command):\n os.system('ls')\n", encoding="utf-8") + result = engine.analyze_project(root, [PLUGINS], cache=cache) + + assert set(result.reused) == {"app", "app.clean"} + assert result.findings == () + + +def test_changing_an_importer_leaves_the_callee_cached(tmp_path: Path) -> None: + root = standard(tmp_path) + cache = ProjectCache(tmp_path / "cache") + engine.analyze_project(root, [PLUGINS], cache=cache) + + (root / "app" / "main.py").write_text("from app.helpers import execute\n\ndef run():\n execute('ls')\n", encoding="utf-8") + result = engine.analyze_project(root, [PLUGINS], cache=cache) + + assert set(result.reused) == {"app", "app.helpers", "app.clean"} + assert result.findings == () + + +def test_cached_modules_still_serve_project_plugins_and_correlation(tmp_path: Path) -> None: + root = project( + tmp_path / "src", + { + "requirements.txt": "pyyaml==5.3.1\n", + "app/__init__.py": "", + "app/config.py": "import yaml\n\ndef parse(text):\n return yaml.load(text)\n", + "app/main.py": "from app.config import parse\n\ndef run():\n parse(input())\n", + }, + ) + cache = ProjectCache(tmp_path / "cache") + + first = engine.analyze_project(root, [PLUGINS], cache=cache) + second = engine.analyze_project(root, [PLUGINS], cache=cache) + + assert set(second.reused) == {"app", "app.config", "app.main"} + assert rules(second.findings) == rules(first.findings) + assert ("config.py", "reachable-vulnerability", 4) in rules(second.findings) + assert ("main.py", "exploitable-vulnerability", 4) in rules(second.findings) + + +def test_unsupported_and_syntax_notes_are_cached_too(tmp_path: Path) -> None: + root = project(tmp_path / "src", {"nested.py": "def outer():\n def inner():\n pass\n return inner\n"}) + cache = ProjectCache(tmp_path / "cache") + + first = engine.analyze_project(root, [PLUGINS], cache=cache) + second = engine.analyze_project(root, [PLUGINS], cache=cache) + + assert rules(first.findings) == [("nested.py", "unsupported-syntax", 1)] + assert second.findings == first.findings + assert second.reused == ("nested",) + + +def test_corrupt_or_foreign_cache_entries_are_ignored(tmp_path: Path) -> None: + root = standard(tmp_path) + cache = ProjectCache(tmp_path / "cache") + engine.analyze_project(root, [PLUGINS], cache=cache) + for path in (tmp_path / "cache").glob("*.json"): + path.write_text("{not json", encoding="utf-8") + + result = engine.analyze_project(root, [PLUGINS], cache=cache) + + assert result.reused == () + assert rules(result.findings) == [("main.py", "command-injection", 4)] + + +def test_cache_directory_is_created_on_demand(tmp_path: Path) -> None: + root = standard(tmp_path) + cache = ProjectCache(tmp_path / "nested" / "cache") + + engine.analyze_project(root, [PLUGINS], cache=cache) + + assert (tmp_path / "nested" / "cache").is_dir() + + +# --------------------------------------------------------------------------- CLI + + +def test_check_accepts_a_cache_directory(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + root = standard(tmp_path) + cache_dir = tmp_path / "cache" + + first = main(["--check", str(root), "--plugins", str(PLUGINS), "--cache", str(cache_dir)]) + first_output = capsys.readouterr().out + second = main(["--check", str(root), "--plugins", str(PLUGINS), "--cache", str(cache_dir)]) + second_output = capsys.readouterr().out + + assert first == second == 1 + assert first_output == second_output + assert "command-injection" in second_output + + +def test_cache_option_requires_a_check(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + source = tmp_path / "x.py" + source.write_text("", encoding="utf-8") + + assert main(["--emit-ir", "--cache", str(tmp_path / "c"), str(source)]) == 2 + assert "--cache" in capsys.readouterr().err From 0f4e6e819cf1fece6da81fbdcd37f04f30e3428e Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 02:23:53 +0200 Subject: [PATCH 2/2] feat: add persistent per-module cache with incremental summaries --- README.md | 9 + src/coretrace_python/cache.py | 278 ++++++++++++++++++ src/coretrace_python/cli.py | 15 +- src/coretrace_python/engine.py | 151 ++++++++-- .../interprocedural/callgraph.py | 5 +- src/coretrace_python/plugins/api.py | 8 +- 6 files changed, 434 insertions(+), 32 deletions(-) create mode 100644 src/coretrace_python/cache.py diff --git a/README.md b/README.md index f04489b..31c37b2 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,15 @@ attacker-controlled data reaches such a call, the engine correlates the four fac one critical `exploitable-vulnerability` finding, judged like any other flow. The shipped advisory database is a small offline sample; a live OSV feed is future work. +`--cache DIR` keeps the results of a directory check on disk, one JSON entry per module. +The entry is keyed by the module's source, the engine and plugin versions, the plugin +code, the security models, the advisories, the dependency graph and the keys of the +modules it imports transitively. On the next run a module whose key is unchanged is +served from the cache: its function summaries seed the project index, its call sites +serve the project plugins and its findings are reported as they were, so editing one file +re-analyses that file and the modules that import it only. Entries are plain data, never +code; an unreadable entry is simply recomputed. + `--plugins` is a directory searched recursively for `plugin.toml` manifests and may be repeated. The repository ships a first set under `plugins/`: security models for the standard library, Flask, FastAPI and SQLAlchemy (`models/`), taint detectors for SQL diff --git a/src/coretrace_python/cache.py b/src/coretrace_python/cache.py new file mode 100644 index 0000000..f62bcf5 --- /dev/null +++ b/src/coretrace_python/cache.py @@ -0,0 +1,278 @@ +"""Persistent per-module cache (architecture §11, §38 Phase 10). + +A module's results are stored under a key derived from everything they depend on: its +source text and identity, the engine, schema and plugin API versions, the plugins and +their code, the security models, the advisories, the dependency graph, and the keys of +the project modules it imports transitively. A module whose key is unchanged on a later +run is served from the cache: its summaries seed the project index, its call sites serve +the project plugins and its findings are reported as they were. Entries are JSON, so a +tampered or foreign file can never execute anything; an unreadable entry is a miss. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from coretrace_python.findings import Confidence, Finding, Severity +from coretrace_python.interprocedural import ( + CallSite, + ExternalCall, + ExternalSymbol, + FunctionSummary, + KnownFunction, + ModuleGraph, + Target, + UnknownTarget, +) +from coretrace_python.semantic.symbols import SymbolId +from coretrace_python.source import SourceId, SourceSpan + +CACHE_FORMAT = 1 + + +@dataclass(frozen=True) +class CachedModule: + """Everything a later run needs from one module without lowering it again.""" + + functions: tuple[str, ...] + summaries: Mapping[str, FunctionSummary] + sites: tuple[CallSite, ...] + findings: tuple[Finding, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "summaries", MappingProxyType(dict(self.summaries))) + + +# --------------------------------------------------------------------------- keys + + +def fingerprint(*parts: str) -> str: + digest = hashlib.sha256() + for part in parts: + digest.update(part.encode("utf-8")) + digest.update(b"\0") + return digest.hexdigest() + + +def directory_fingerprint(directory: Path, suffixes: Iterable[str] = (".py", ".toml")) -> str: + """A digest of the source files under ``directory``, so edited plugin code misses.""" + + wanted = tuple(suffixes) + parts: list[str] = [] + for path in sorted(directory.rglob("*")): + if path.is_file() and path.suffix in wanted: + parts.append(str(path.relative_to(directory))) + parts.append(path.read_text(encoding="utf-8", errors="replace")) + return fingerprint(*parts) + + +def module_keys(graph: ModuleGraph, own: Mapping[str, str]) -> dict[str, str]: + """The key of each module: its own key plus those of every module it imports, + transitively, so a change anywhere below a module misses for that module too.""" + + keys: dict[str, str] = {} + for name in graph.modules: + closure: set[str] = set() + pending = [name] + while pending: + current = pending.pop() + for imported in graph.imports(current): + if imported in own and imported not in closure and imported != name: + closure.add(imported) + pending.append(imported) + keys[name] = fingerprint(own[name], *(own[m] for m in sorted(closure))) + return keys + + +# --------------------------------------------------------------------------- store + + +class ProjectCache: + """A directory of JSON entries, one per module key.""" + + def __init__(self, directory: Path) -> None: + self.directory = directory + + def load(self, key: str) -> CachedModule | None: + path = self.directory / f"{key}.json" + try: + with path.open(encoding="utf-8") as handle: + return decode(json.load(handle)) + except (OSError, ValueError, KeyError, TypeError, IndexError, AttributeError): + return None + + def store(self, key: str, module: CachedModule) -> None: + self.directory.mkdir(parents=True, exist_ok=True) + path = self.directory / f"{key}.json" + temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(encode(module)), encoding="utf-8") + temporary.replace(path) + + +# --------------------------------------------------------------------------- codec + + +def encode(module: CachedModule) -> dict[str, Any]: + return { + "format": CACHE_FORMAT, + "functions": list(module.functions), + "summaries": {name: _encode_summary(s) for name, s in module.summaries.items()}, + "sites": [_encode_site(site) for site in module.sites], + "findings": [_encode_finding(finding) for finding in module.findings], + } + + +def decode(data: Mapping[str, Any]) -> CachedModule: + if data["format"] != CACHE_FORMAT: + raise ValueError(f"unsupported cache format {data['format']!r}") + return CachedModule( + tuple(_string(name) for name in data["functions"]), + {_string(name): _decode_summary(s) for name, s in data["summaries"].items()}, + tuple(_decode_site(site) for site in data["sites"]), + tuple(_decode_finding(finding) for finding in data["findings"]), + ) + + +def _string(value: Any) -> str: + if not isinstance(value, str): + raise TypeError(f"expected a string, got {value!r}") + return value + + +def _integer(value: Any) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError(f"expected an integer, got {value!r}") + return value + + +def _indices(values: Any) -> frozenset[int]: + return frozenset(_integer(v) for v in values) + + +def _encode_span(span: SourceSpan) -> list[Any]: + return [str(span.source_id), span.start_line, span.start_column, span.end_line, span.end_column] + + +def _decode_span(data: Any) -> SourceSpan: + file, line, column, end_line, end_column = data + return SourceSpan( + SourceId(_string(file)), + _integer(line), + _integer(column), + None if end_line is None else _integer(end_line), + None if end_column is None else _integer(end_column), + ) + + +def _encode_finding(finding: Finding) -> dict[str, Any]: + return { + "rule": finding.rule_id, + "message": finding.message, + "severity": finding.severity.value, + "confidence": finding.confidence.value, + "span": _encode_span(finding.span), + "function": finding.function, + "metadata": dict(finding.metadata), + } + + +def _decode_finding(data: Mapping[str, Any]) -> Finding: + function = data["function"] + return Finding( + _string(data["rule"]), + _string(data["message"]), + Severity(data["severity"]), + Confidence(data["confidence"]), + _decode_span(data["span"]), + None if function is None else _string(function), + {_string(k): _string(v) for k, v in data["metadata"].items()}, + ) + + +def _encode_call(call: ExternalCall) -> dict[str, Any]: + return { + "symbol": str(call.symbol), + "arguments": [sorted(deps) for deps in call.argument_dependencies], + "keywords": sorted(call.keyword_dependencies), + "location": _encode_span(call.location), + "call_site": None if call.call_site is None else _encode_span(call.call_site), + } + + +def _decode_call(data: Mapping[str, Any]) -> ExternalCall: + site = data["call_site"] + return ExternalCall( + SymbolId(_string(data["symbol"])), + tuple(_indices(deps) for deps in data["arguments"]), + _indices(data["keywords"]), + _decode_span(data["location"]), + None if site is None else _decode_span(site), + ) + + +def _encode_summary(summary: FunctionSummary) -> dict[str, Any]: + return { + "name": summary.name, + "parameters": summary.parameters, + "returns": sorted(summary.return_dependencies), + "external_calls": [_encode_call(call) for call in summary.external_calls], + "unsupported": summary.unsupported, + "return_externals": sorted(str(s) for s in summary.return_externals), + } + + +def _decode_summary(data: Mapping[str, Any]) -> FunctionSummary: + return FunctionSummary( + _string(data["name"]), + _integer(data["parameters"]), + _indices(data["returns"]), + tuple(_decode_call(call) for call in data["external_calls"]), + bool(data["unsupported"]), + frozenset(SymbolId(_string(s)) for s in data["return_externals"]), + ) + + +def _encode_target(target: Target) -> dict[str, Any]: + if isinstance(target, KnownFunction): + return {"kind": "known", "name": target.name} + if isinstance(target, ExternalSymbol): + return {"kind": "external", "symbol": str(target.symbol)} + return {"kind": "unknown"} + + +def _decode_target(data: Mapping[str, Any]) -> Target: + kind = data["kind"] + if kind == "known": + return KnownFunction(_string(data["name"])) + if kind == "external": + return ExternalSymbol(SymbolId(_string(data["symbol"]))) + if kind == "unknown": + return UnknownTarget() + raise ValueError(f"unknown call target kind {kind!r}") + + +def _encode_site(site: CallSite) -> dict[str, Any]: + return { + "caller": site.caller, + "location": _encode_span(site.location), + "target": _encode_target(site.target), + "arguments": site.arguments, + "keywords": site.keywords, + } + + +def _decode_site(data: Mapping[str, Any]) -> CallSite: + return CallSite( + _string(data["caller"]), + _decode_span(data["location"]), + _decode_target(data["target"]), + _integer(data["arguments"]), + _integer(data["keywords"]), + ) diff --git a/src/coretrace_python/cli.py b/src/coretrace_python/cli.py index 5ff2487..7f33d7e 100644 --- a/src/coretrace_python/cli.py +++ b/src/coretrace_python/cli.py @@ -6,6 +6,7 @@ from coretrace_python import engine from coretrace_python.analysis import AnalysisError +from coretrace_python.cache import ProjectCache from coretrace_python.cfg import CFGError from coretrace_python.frontend import HIRBuildError, ParseError, build_hir from coretrace_python.ir.lowering import LoweringError, lower_module @@ -70,6 +71,14 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="with --check, the report format (default: text)", ) + parser.add_argument( + "--cache", + type=Path, + default=None, + metavar="DIR", + help="with --check on a directory, keep per-module results under DIR and reuse " + "them for modules unchanged since the previous run", + ) return parser @@ -81,6 +90,9 @@ def main(argv: list[str] | None = None) -> int: if args.format is not None and not args.check: print("error: --format only applies to --check", file=sys.stderr) return EXIT_ERROR + if args.cache is not None and not (args.check and args.path.is_dir()): + print("error: --cache only applies to --check on a directory", file=sys.stderr) + return EXIT_ERROR if args.path.is_dir() and args.emit_ir: print(f"error: {args.path} is a directory; --emit-ir needs a file", file=sys.stderr) @@ -92,7 +104,8 @@ def main(argv: list[str] | None = None) -> int: print(format_module(lower_module(build_hir(source), ssa=args.ssa))) if args.check: if args.path.is_dir(): - findings = engine.analyze_project(args.path, args.plugins).findings + cache = None if args.cache is None else ProjectCache(args.cache) + findings = engine.analyze_project(args.path, args.plugins, cache=cache).findings else: findings = engine.check(SourceManager().load_file(args.path), args.plugins) print(render(args.format or "text", engine.report(findings)), end="") diff --git a/src/coretrace_python/engine.py b/src/coretrace_python/engine.py index 146f83c..26be07c 100644 --- a/src/coretrace_python/engine.py +++ b/src/coretrace_python/engine.py @@ -6,9 +6,10 @@ from __future__ import annotations -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path +from types import MappingProxyType from typing import ClassVar from coretrace_python import __version__ @@ -19,20 +20,30 @@ AnyAnalysis, TransformationPass, ) +from coretrace_python.cache import ( + CachedModule, + ProjectCache, + directory_fingerprint, + fingerprint, + module_keys, +) from coretrace_python.cfg import CFGAnalysis, CFGError, DominanceAnalysis, PostDominanceAnalysis from coretrace_python.dependency import ( DEPENDENCY_FILES, + Advisory, DependencyAnalysis, DependencyGraph, parse_dependencies, ) from coretrace_python.dependency.correlation import advisory_sinks, affected_symbols, correlate -from coretrace_python.findings import Confidence, Finding, Severity +from coretrace_python.findings import FINDING_SCHEMA_VERSION, Confidence, Finding, Severity from coretrace_python.findings.refutation import RefutationAnalysis from coretrace_python.frontend import HIRBuildError, ParseError, build_hir -from coretrace_python.hir import nodes +from coretrace_python.hir import HIR_SCHEMA_VERSION, nodes from coretrace_python.interprocedural import ( + CallGraph, CallGraphAnalysis, + CallSite, FunctionSummary, ModuleGraph, ProjectSummaries, @@ -51,6 +62,7 @@ ) from coretrace_python.ir.ssa import SSAAnalysis from coretrace_python.plugins import ( + PLUGIN_API_VERSION, Plugin, PluginRegistry, ProjectContext, @@ -62,6 +74,7 @@ from coretrace_python.semantic import SEMANTIC_ANALYSES from coretrace_python.semantic.imports import ImportAnalysis, ImportResolutionError, ImportTable from coretrace_python.semantic.scopes import ScopeError +from coretrace_python.semantic.symbols import SymbolId from coretrace_python.source import SourceFile, SourceManager, SourceSpan from coretrace_python.taint import ( ModelTable, @@ -115,12 +128,18 @@ def run(cls, ctx: AnalysisContext) -> None: pass +def _no_keys() -> Mapping[str, str]: + return MappingProxyType({}) + + @dataclass(frozen=True) class ProjectAnalysis: graph: ModuleGraph index: SummaryIndex findings: tuple[Finding, ...] dependencies: DependencyGraph = field(default_factory=DependencyGraph) + keys: Mapping[str, str] = field(default_factory=_no_keys) + reused: tuple[str, ...] = () def build_manager( @@ -171,11 +190,15 @@ def resolve_dependencies(root: Path, sources: SourceManager) -> DependencyGraph: def analyze_project( - root: Path, plugin_roots: Sequence[Path] = (), plugins: Sequence[Plugin] = () + root: Path, + plugin_roots: Sequence[Path] = (), + plugins: Sequence[Plugin] = (), + cache: ProjectCache | None = None, ) -> ProjectAnalysis: """Analyse every Python file under ``root`` with a shared summary index (§21) and the dependency graph of its manifests (§26). ``plugins`` adds plugin instances to the - ones discovered under ``plugin_roots``.""" + ones discovered under ``plugin_roots``. With a ``cache``, modules whose key is + unchanged since a previous run are served from it (§11).""" sources = SourceManager() findings: list[Finding] = [] @@ -199,10 +222,8 @@ def analyze_project( advisories = tuple(a for plugin in all_plugins for a in plugin.advisories) affected = affected_symbols(dependencies, advisories) models = plugin_models(all_plugins).extended(*advisory_sinks(affected)) - index = SummaryIndex() for manager in managers.values(): manager.provide(SecurityModelAnalysis, models) - manager.provide(ProjectSummaries, index) manager.provide(DependencyAnalysis, dependencies) imports: dict[str, ImportTable] = {} @@ -218,42 +239,118 @@ def analyze_project( {name: files[name] for name in analysable}, {name: modules[name] for name in analysable}, imports ) + configuration = _configuration_key(registry, plugins, models, advisories, dependencies) + keys = module_keys( + graph, + {name: fingerprint(configuration, str(files[name].source_id), name, files[name].text) for name in analysable}, + ) + cached: dict[str, CachedModule] = {} + if cache is not None: + for name in analysable: + entry = cache.load(keys[name]) + if entry is not None: + cached[name] = entry + fresh = {name: manager for name, manager in analysable.items() if name not in cached} + + index = SummaryIndex() + cached_summaries = { + project_symbol(name, function): summary + for name, entry in cached.items() + for function, summary in entry.summaries.items() + } + for manager in fresh.values(): + manager.provide(ProjectSummaries, index) for _ in range(MAX_PROJECT_ITERATIONS): updated = SummaryIndex( { - project_symbol(name, function): summary - for name, manager in analysable.items() - for function, summary in _summaries_of(manager).items() + **cached_summaries, + **{ + project_symbol(name, function): summary + for name, manager in fresh.items() + for function, summary in _summaries_of(manager).items() + }, } ) if updated == index: break index = updated - for manager in analysable.values(): + for manager in fresh.values(): manager.run(ProjectSummariesUpdated) manager.provide(ProjectSummaries, index) module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin)) + call_graphs: dict[str, CallGraph] = {} for name in sorted(analysable): - manager = analysable[name] - module_findings, supported = _check_module(manager, module_plugins) - findings.extend(module_findings) - if affected: - graph_of_module = manager.get(CallGraphAnalysis) - for function in supported: - findings.extend( - correlate( - graph_of_module.name_of(function), - manager.get(TaintAnalysis, function).flows, - manager.get(RefutationAnalysis, function), - affected, - ) - ) - context = ProjectContext(graph, dependencies, advisories, analysable) + entry = cached.get(name) + if entry is None: + entry = _analyse_module(analysable[name], module_plugins, affected) + if cache is not None: + cache.store(keys[name], entry) + else: + sites: dict[str, list[CallSite]] = {function: [] for function in entry.functions} + for site in entry.sites: + sites.setdefault(site.caller, []).append(site) + call_graphs[name] = CallGraph({}, {f: tuple(s) for f, s in sites.items()}, frozenset()) + findings.extend(entry.findings) + context = ProjectContext(graph, dependencies, advisories, analysable, call_graphs) for plugin in all_plugins: if isinstance(plugin, ProjectPlugin): findings.extend(plugin.analyze_project(context)) - return ProjectAnalysis(graph, index, tuple(findings), dependencies) + return ProjectAnalysis( + graph, index, tuple(findings), dependencies, MappingProxyType(keys), tuple(sorted(cached)) + ) + + +def _configuration_key( + registry: PluginRegistry, + plugins: Sequence[Plugin], + models: ModelTable, + advisories: tuple[Advisory, ...], + dependencies: DependencyGraph, +) -> str: + """Everything a module's results depend on besides the project sources (§11).""" + + return fingerprint( + __version__, + str(HIR_SCHEMA_VERSION), + str(FINDING_SCHEMA_VERSION), + str(PLUGIN_API_VERSION), + *( + f"{loaded.manifest.name}={loaded.manifest.version}:{directory_fingerprint(loaded.directory)}" + for loaded in registry + ), + *(f"{type(p).__module__}.{type(p).__qualname__}" for p in plugins), + repr(models), + repr(advisories), + repr(dependencies.requirements), + repr(dependencies.errors), + ) + + +def _analyse_module( + manager: AnalysisManager, plugins: tuple[Plugin, ...], affected: Mapping[SymbolId, Advisory] +) -> CachedModule: + """One module's findings, summaries and call sites: what the cache keeps (§11).""" + + findings, supported = _check_module(manager, plugins) + graph = manager.get(CallGraphAnalysis) + correlated: list[Finding] = [] + if affected: + for function in supported: + correlated.extend( + correlate( + graph.name_of(function), + manager.get(TaintAnalysis, function).flows, + manager.get(RefutationAnalysis, function), + affected, + ) + ) + return CachedModule( + graph.functions, + _summaries_of(manager), + tuple(site for function in graph.functions for site in graph.sites(function)), + (*findings, *correlated), + ) def _summaries_of(manager: AnalysisManager) -> dict[str, FunctionSummary]: diff --git a/src/coretrace_python/interprocedural/callgraph.py b/src/coretrace_python/interprocedural/callgraph.py index db8b386..1ec5d23 100644 --- a/src/coretrace_python/interprocedural/callgraph.py +++ b/src/coretrace_python/interprocedural/callgraph.py @@ -61,7 +61,8 @@ def __init__( ) -> None: self._symbols = {name: MappingProxyType(dict(found)) for name, found in (symbols or {}).items()} self.definitions: Mapping[str, nodes.Function] = MappingProxyType(dict(definitions)) - self.functions = tuple(definitions) + # A graph rebuilt from cached call sites has sites but no definitions. + self.functions = tuple(dict.fromkeys((*definitions, *sites))) self._names = {function.span: name for name, function in definitions.items()} self.unsupported = unsupported self._sites = MappingProxyType(dict(sites)) @@ -72,7 +73,7 @@ def __init__( for found in sites.values(): for site in found: if isinstance(site.target, KnownFunction): - callers[site.target.name].add(site.caller) + callers.setdefault(site.target.name, set()).add(site.caller) self._callers = {name: frozenset(found) for name, found in callers.items()} def name_of(self, function: nodes.Function) -> str: diff --git a/src/coretrace_python/plugins/api.py b/src/coretrace_python/plugins/api.py index c2efcd8..4989524 100644 --- a/src/coretrace_python/plugins/api.py +++ b/src/coretrace_python/plugins/api.py @@ -54,7 +54,8 @@ def analyze(self, ctx: PluginContext) -> Sequence[Finding]: class ProjectContext: """What a project-scoped plugin sees: the module graph, the dependency graph, the - advisories every plugin contributed, and each module's imports and call graph.""" + advisories every plugin contributed, and each module's imports and call graph. The + engine passes the call graphs of modules it served from its cache.""" def __init__( self, @@ -62,11 +63,13 @@ def __init__( dependencies: DependencyGraph, advisories: tuple[Advisory, ...], managers: Mapping[str, AnalysisManager], + call_graphs: Mapping[str, CallGraph] | None = None, ) -> None: self.graph = graph self.dependencies = dependencies self.advisories = advisories self._managers = managers + self._call_graphs = dict(call_graphs or {}) @property def modules(self) -> tuple[str, ...]: @@ -76,7 +79,8 @@ def imports(self, module: str) -> ImportTable: return self._managers[module].get(ImportAnalysis) def call_graph(self, module: str) -> CallGraph: - return self._managers[module].get(CallGraphAnalysis) + cached = self._call_graphs.get(module) + return cached if cached is not None else self._managers[module].get(CallGraphAnalysis) class ProjectPlugin(Plugin):