diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7351bda..eeae8ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,5 +16,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - run: python -m pip install -e ".[dev]" - - run: ruff check . - - run: pytest + - run: python -m mypy + - run: python -m ruff check . + - run: python -m pytest diff --git a/README.md b/README.md index 4846ea6..c236a38 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,26 @@ A standalone, Python-specific static analysis frontend for CoreTrace. -The first milestone parses Python and lowers a deliberately small language subset to a -deterministic, Python-aware intermediate representation (PyIR). CFG construction, SSA, -data-flow, and security rules are intentionally deferred. +The analyzer loads source through a source manager, adapts parsed Python into a +parser-independent high-level representation (PyHIR), and lowers a deliberately small +language subset to deterministic PyIR. CFG construction, SSA, data-flow, and security rules +are intentionally deferred. + +```text +Python source -> SourceManager -> parser -> PyHIR -> semantic imports -> PyIR +``` + +The long-term engine architecture and incremental migration plan are recorded in +[`docs/architecture.md`](docs/architecture.md). ## Development ```bash python -m venv .venv python -m pip install -e ".[dev]" -pytest +python -m mypy +python -m pytest +python -m ruff check . ``` ## Emit PyIR diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..ddecefa --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,93 @@ +# CoreTrace Python Analysis Engine Architecture + +## Product direction + +CoreTrace Python Analyzer is evolving from a security-specific source scanner into an +extensible static-analysis engine. Security detectors will be plugins built on shared, +language-level analysis infrastructure. + +The target data path is: + +```text +Python project + -> source discovery and SourceManager + -> parser (Tree-sitter target; stdlib ast adapter initially) + -> parser-independent PyHIR + -> scopes, imports, and canonical symbols + -> control-flow graph + -> SSA PyIR + -> shared analysis DAG + -> models, detectors, and proof/refutation plugins + -> normalized findings + -> CLI, JSON, and SARIF reporters +``` + +## Architectural boundaries + +1. Source text and locations belong to `source`; all later representations reference stable + source IDs and spans. +2. Parser-specific objects stop at `frontend`. Semantic analysis and lowering consume PyHIR, + never `ast.*` or Tree-sitter nodes. +3. `semantic` owns scopes, imports, and symbols. PyIR lowering consumes semantic results rather + than resolving names itself. +4. CFG is built before SSA. SSA construction is a transformation over non-SSA control flow. +5. Analyses are immutable providers managed through a dependency DAG. Results are lazy, cached, + and invalidated explicitly after transformations. +6. Plugins consume versioned, typed result APIs. They do not access engine internals or rebuild + shared analyses. +7. Framework plugins provide models and facts; detectors consume generic facts. The taint engine + is global and multi-kind. +8. Findings store lightweight IDs. Reporters consume normalized findings and never initiate + analysis. + +## Migration strategy + +The migration is incremental so every merged branch leaves the CLI usable. + +### Phase 1: frontend foundation + +- Introduce `SourceManager`, `SourceId`, and `SourceSpan`. +- Introduce parser-independent PyHIR. +- Adapt Python's standard-library AST into PyHIR. +- Route existing PyIR lowering through PyHIR without changing emitted output. + +Tree-sitter is intentionally behind the parser boundary. It can replace the initial AST adapter +once PyHIR is stable, without affecting semantic analysis or PyIR. + +### Phase 2: semantic layer + +- Move import binding and canonical symbol resolution into `semantic`. +- Add lexical scopes, local/global/nonlocal classification, closures, and class scopes. +- Publish immutable semantic results keyed by stable IDs. + +### Phase 3: control flow and SSA + +- Build per-function CFGs with explicit terminators and graph integrity checks. +- Add dominators and dominance frontiers. +- Transform local reads and writes into SSA with phi nodes and def-use chains. + +### Phase 4: shared analysis infrastructure + +- Add typed analysis providers and dependency declarations. +- Add lazy evaluation, caching, and transformation invalidation. +- Validate the framework with constant propagation and abstract values. + +### Later phases + +- Plugin manifests, registry, and versioned API. +- Global multi-kind taint and security model registry. +- Call graph, function summaries, module graph, and interprocedural fixpoints. +- Framework models, detectors, proof/refutation, correlation, and reporters. +- Persistent caching, incremental parsing, parallelism, and memory eviction. + +## Current milestone acceptance criteria + +The frontend-foundation milestone is complete when: + +- the CLI loads files through `SourceManager`; +- source locations use `SourceSpan` with stable `SourceId` values; +- supported Python syntax is represented in PyHIR; +- PyIR lowering no longer imports or accepts `ast` nodes; +- existing `--emit-ir` output remains deterministic; +- existing tests pass alongside focused source and PyHIR tests. + diff --git a/pyproject.toml b/pyproject.toml index e4a1ceb..fb27a90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [] [project.optional-dependencies] dev = [ + "mypy>=1.15", "pytest>=8.0", "ruff>=0.9", ] @@ -32,3 +33,7 @@ testpaths = ["tests"] line-length = 100 target-version = "py311" +[tool.mypy] +python_version = "3.11" +strict = true +files = ["src"] diff --git a/src/coretrace_python/cli.py b/src/coretrace_python/cli.py index 75efa82..fc03e2d 100644 --- a/src/coretrace_python/cli.py +++ b/src/coretrace_python/cli.py @@ -4,10 +4,12 @@ import sys from pathlib import Path +from coretrace_python.frontend.ast_adapter import HIRBuildError, build_module from coretrace_python.frontend.imports import ImportResolutionError from coretrace_python.frontend.lowering import LoweringError, lower_module -from coretrace_python.frontend.parser import ParseError, parse_file +from coretrace_python.frontend.parser import ParseError, parse_source_file from coretrace_python.ir.printer import format_module +from coretrace_python.source import SourceManager def build_parser() -> argparse.ArgumentParser: @@ -31,9 +33,18 @@ def main(argv: list[str] | None = None) -> int: return 2 try: - tree = parse_file(args.path) - module = lower_module(tree, filename=str(args.path)) - except (OSError, ParseError, ImportResolutionError, LoweringError) as error: + source = SourceManager().load_file(args.path) + syntax_tree = parse_source_file(source) + hir_module = build_module(source, syntax_tree) + module = lower_module(hir_module) + except ( + OSError, + UnicodeError, + ParseError, + HIRBuildError, + ImportResolutionError, + LoweringError, + ) as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/src/coretrace_python/frontend/ast_adapter.py b/src/coretrace_python/frontend/ast_adapter.py new file mode 100644 index 0000000..ebd860f --- /dev/null +++ b/src/coretrace_python/frontend/ast_adapter.py @@ -0,0 +1,176 @@ +"""Adapter from Python's standard-library AST to parser-independent PyHIR.""" + +from __future__ import annotations + +import ast +from typing import NoReturn + +from coretrace_python.hir import nodes +from coretrace_python.source import SourceFile, SourceSpan + + +class HIRBuildError(Exception): + """A source-located failure to represent parsed syntax as PyHIR.""" + + +_BINARY_OPERATORS = { + ast.Add: "add", + ast.Sub: "sub", + ast.Mult: "mul", + ast.Div: "div", + ast.FloorDiv: "floor_div", + ast.Mod: "mod", + ast.Pow: "pow", + ast.LShift: "lshift", + ast.RShift: "rshift", + ast.BitOr: "bit_or", + ast.BitXor: "bit_xor", + ast.BitAnd: "bit_and", + ast.MatMult: "matmul", +} +_UNARY_OPERATORS = {ast.Invert: "invert", ast.Not: "not", ast.UAdd: "pos", ast.USub: "neg"} +_COMPARE_OPERATORS = { + ast.Eq: "eq", + ast.NotEq: "not_eq", + ast.Lt: "lt", + ast.LtE: "lt_eq", + ast.Gt: "gt", + ast.GtE: "gt_eq", + ast.Is: "is", + ast.IsNot: "is_not", + ast.In: "in", + ast.NotIn: "not_in", +} + + +class AstHIRBuilder: + """Build PyHIR while containing every dependency on ``ast`` in this adapter.""" + + def __init__(self, source: SourceFile) -> None: + self._source = source + + def span(self, node: ast.AST) -> SourceSpan: + end_line = getattr(node, "end_lineno", None) + end_offset = getattr(node, "end_col_offset", None) + return SourceSpan( + source_id=self._source.source_id, + start_line=getattr(node, "lineno", 1), + start_column=getattr(node, "col_offset", 0) + 1, + end_line=end_line, + end_column=end_offset + 1 if end_offset is not None else None, + ) + + def fail(self, node: ast.AST, message: str | None = None) -> NoReturn: + span = self.span(node) + detail = message or f"unsupported syntax: {type(node).__name__}" + raise HIRBuildError(f"{span.display()}: {detail}") + + def expression(self, node: ast.expr) -> nodes.Expression: + span = self.span(node) + if isinstance(node, ast.Name): + return nodes.Name(node.id, span) + if isinstance(node, ast.Constant): + return nodes.Constant(node.value, span) + if isinstance(node, ast.BinOp): + operator = _BINARY_OPERATORS.get(type(node.op)) + if operator is None: + self.fail(node.op) + return nodes.BinaryOp(operator, self.expression(node.left), self.expression(node.right), span) + if isinstance(node, ast.UnaryOp): + operator = _UNARY_OPERATORS.get(type(node.op)) + if operator is None: + self.fail(node.op) + return nodes.UnaryOp(operator, self.expression(node.operand), span) + if isinstance(node, ast.Compare): + if len(node.ops) != 1 or len(node.comparators) != 1: + self.fail(node, "chained comparisons are not supported yet") + operator = _COMPARE_OPERATORS.get(type(node.ops[0])) + if operator is None: + self.fail(node.ops[0]) + return nodes.Compare( + operator, + self.expression(node.left), + self.expression(node.comparators[0]), + span, + ) + if isinstance(node, ast.Attribute): + return nodes.Attribute(self.expression(node.value), node.attr, span) + if isinstance(node, ast.Subscript): + return nodes.Subscript(self.expression(node.value), self.expression(node.slice), span) + if isinstance(node, ast.Call): + arguments = tuple(self.expression(argument) for argument in node.args) + keywords = tuple( + nodes.Keyword(keyword.arg, self.expression(keyword.value), self.span(keyword)) + for keyword in node.keywords + ) + return nodes.Call(self.expression(node.func), arguments, keywords, span) + self.fail(node) + + def statement(self, node: ast.stmt) -> nodes.Statement: + span = self.span(node) + if isinstance(node, ast.Assign): + if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name): + self.fail(node, "only assignment to one local name is supported") + target = nodes.Name(node.targets[0].id, self.span(node.targets[0])) + return nodes.Assign(target, self.expression(node.value), span) + if isinstance(node, ast.Return): + value = self.expression(node.value) if node.value is not None else None + return nodes.Return(value, span) + if isinstance(node, ast.Expr): + return nodes.ExpressionStatement(self.expression(node.value), span) + if isinstance(node, ast.Pass): + return nodes.Pass(span) + if isinstance(node, ast.Import): + aliases = tuple( + nodes.ImportAlias(alias.name, alias.asname, self.span(alias)) for alias in node.names + ) + return nodes.Import(aliases, span) + if isinstance(node, ast.ImportFrom): + aliases = tuple( + nodes.ImportAlias(alias.name, alias.asname, self.span(alias)) for alias in node.names + ) + return nodes.ImportFrom(node.module, aliases, node.level, span) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return self.function(node) + self.fail(node) + + def function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> nodes.Function: + if node.decorator_list: + self.fail(node, "decorated functions are not supported yet") + arguments = node.args + has_advanced_arguments = ( + arguments.posonlyargs or arguments.kwonlyargs or arguments.vararg or arguments.kwarg + ) + if has_advanced_arguments or arguments.defaults or arguments.kw_defaults: + self.fail(arguments, "only required positional arguments are supported") + parameters = tuple( + nodes.Parameter(argument.arg, self.span(argument)) for argument in arguments.args + ) + body = tuple(self.statement(statement) for statement in node.body) + return nodes.Function( + name=node.name, + parameters=parameters, + body=body, + is_async=isinstance(node, ast.AsyncFunctionDef), + span=self.span(node), + ) + + def module(self, tree: ast.Module) -> nodes.Module: + body = tuple(self.statement(statement) for statement in tree.body) + if tree.body: + first_span = self.span(tree.body[0]) + last_span = self.span(tree.body[-1]) + span = SourceSpan( + self._source.source_id, + first_span.start_line, + first_span.start_column, + last_span.end_line, + last_span.end_column, + ) + else: + span = SourceSpan(self._source.source_id, 1, 1, 1, 1) + return nodes.Module(body, span) + + +def build_module(source: SourceFile, tree: ast.Module) -> nodes.Module: + return AstHIRBuilder(source).module(tree) diff --git a/src/coretrace_python/frontend/imports.py b/src/coretrace_python/frontend/imports.py index 46adf16..889319a 100644 --- a/src/coretrace_python/frontend/imports.py +++ b/src/coretrace_python/frontend/imports.py @@ -2,10 +2,10 @@ from __future__ import annotations -import ast from collections.abc import Iterator, Mapping from dataclasses import dataclass +from coretrace_python.hir import nodes from coretrace_python.ir.symbol import SymbolId @@ -32,35 +32,33 @@ def resolve(self, name: str) -> SymbolId | None: return self._bindings.get(name) -def _error(filename: str, node: ast.AST, message: str) -> ImportResolutionError: - line = getattr(node, "lineno", 1) - column = getattr(node, "col_offset", 0) + 1 - return ImportResolutionError(f"{filename}:{line}:{column}: {message}") +def _error(node: nodes.ImportFrom, message: str) -> ImportResolutionError: + return ImportResolutionError(f"{node.span.display()}: {message}") -def collect_imports(tree: ast.Module, filename: str = "") -> ImportBindings: +def collect_imports(module: nodes.Module) -> ImportBindings: """Collect supported top-level imports without importing any modules.""" bindings: dict[str, SymbolId] = {} - for statement in tree.body: - if isinstance(statement, ast.Import): + for statement in module.body: + if isinstance(statement, nodes.Import): for alias in statement.names: - if alias.asname is None: + if alias.as_name is None: local_name = alias.name.partition(".")[0] canonical_path = local_name else: - local_name = alias.asname + local_name = alias.as_name canonical_path = alias.name bindings[local_name] = SymbolId.from_python_path(canonical_path) - elif isinstance(statement, ast.ImportFrom): + elif isinstance(statement, nodes.ImportFrom): if statement.level: - raise _error(filename, statement, "relative imports are not supported yet") + raise _error(statement, "relative imports are not supported yet") if statement.module is None: - raise _error(filename, statement, "import module is missing") + raise _error(statement, "import module is missing") for alias in statement.names: if alias.name == "*": - raise _error(filename, statement, "wildcard imports are not supported") - local_name = alias.asname or alias.name + raise _error(statement, "wildcard imports are not supported") + local_name = alias.as_name or alias.name canonical_path = f"{statement.module}.{alias.name}" bindings[local_name] = SymbolId.from_python_path(canonical_path) return ImportBindings(bindings) diff --git a/src/coretrace_python/frontend/lowering.py b/src/coretrace_python/frontend/lowering.py index a85e9c6..ca5e6cc 100644 --- a/src/coretrace_python/frontend/lowering.py +++ b/src/coretrace_python/frontend/lowering.py @@ -1,10 +1,12 @@ +"""Lower parser-independent PyHIR into analysis-oriented PyIR.""" + from __future__ import annotations -import ast from dataclasses import dataclass, field +from typing import NoReturn from coretrace_python.frontend.imports import ImportBindings, collect_imports -from coretrace_python.frontend.symbols import SymbolId +from coretrace_python.hir import nodes from coretrace_python.ir.model import ( BasicBlock, BinaryOp, @@ -18,73 +20,34 @@ LoadLocal, ModuleIR, Return, - SourceLocation, StoreLocal, Symbol, UnaryOp, Value, ValueInstruction, ) +from coretrace_python.ir.symbol import SymbolId class LoweringError(Exception): - """Raised when source uses syntax outside the current PyIR subset.""" - - -_BINARY_OPERATORS = { - ast.Add: "add", - ast.Sub: "sub", - ast.Mult: "mul", - ast.Div: "div", - ast.FloorDiv: "floor_div", - ast.Mod: "mod", - ast.Pow: "pow", - ast.LShift: "lshift", - ast.RShift: "rshift", - ast.BitOr: "bit_or", - ast.BitXor: "bit_xor", - ast.BitAnd: "bit_and", - ast.MatMult: "matmul", -} -_UNARY_OPERATORS = {ast.Invert: "invert", ast.Not: "not", ast.UAdd: "pos", ast.USub: "neg"} -_COMPARE_OPERATORS = { - ast.Eq: "eq", - ast.NotEq: "not_eq", - ast.Lt: "lt", - ast.LtE: "lt_eq", - ast.Gt: "gt", - ast.GtE: "gt_eq", - ast.Is: "is", - ast.IsNot: "is_not", - ast.In: "in", - ast.NotIn: "not_in", -} + """Raised when PyHIR uses constructs outside the current PyIR subset.""" @dataclass class _FunctionLowerer: - filename: str imports: ImportBindings next_value_id: int = 0 - locals: dict[str, Value] = field(default_factory=dict) - block: BasicBlock = field(default_factory=lambda: BasicBlock("entry")) + locals: set[str] = field(default_factory=set) parameters: dict[str, Value] = field(default_factory=dict) + block: BasicBlock = field(default_factory=lambda: BasicBlock("entry")) - def location(self, node: ast.AST) -> SourceLocation: - return SourceLocation( - filename=self.filename, - line=getattr(node, "lineno", 1), - column=getattr(node, "col_offset", 0) + 1, - end_line=getattr(node, "end_lineno", None), - end_column=(getattr(node, "end_col_offset", 0) + 1) - if getattr(node, "end_col_offset", None) is not None - else None, - ) - - def fail(self, node: ast.AST, message: str | None = None) -> None: - location = self.location(node) + def fail( + self, + node: nodes.Statement | nodes.Expression, + message: str | None = None, + ) -> NoReturn: detail = message or f"unsupported syntax: {type(node).__name__}" - raise LoweringError(f"{location.filename}:{location.line}:{location.column}: {detail}") + raise LoweringError(f"{node.span.display()}: {detail}") def new_value(self) -> Value: value = Value(self.next_value_id) @@ -98,133 +61,98 @@ def emit(self, instruction: ValueInstruction) -> Value: def emit_effect(self, instruction: StoreLocal | Return) -> None: self.block.instructions.append(instruction) - def imported_symbol(self, node: ast.expr) -> SymbolId | None: - if isinstance(node, ast.Name): - if node.id in self.locals or node.id in self.parameters: + def imported_symbol(self, node: nodes.Expression) -> SymbolId | None: + if isinstance(node, nodes.Name): + if node.identifier in self.locals or node.identifier in self.parameters: return None - return self.imports.resolve(node.id) - if isinstance(node, ast.Attribute): + return self.imports.resolve(node.identifier) + if isinstance(node, nodes.Attribute): parent = self.imported_symbol(node.value) - return parent.attribute(node.attr) if parent is not None else None + return parent.attribute(node.name) if parent is not None else None return None - def expression(self, node: ast.expr) -> Value: - location = self.location(node) + def expression(self, node: nodes.Expression) -> Value: imported_symbol = self.imported_symbol(node) if imported_symbol is not None: - return self.emit(Symbol(self.new_value(), location, imported_symbol)) - if isinstance(node, ast.Name): - if node.id in self.locals: - return self.emit(LoadLocal(self.new_value(), location, node.id)) - if node.id in self.parameters: - return self.parameters[node.id] - return self.emit(Global(self.new_value(), location, node.id)) - if isinstance(node, ast.Constant): - return self.emit(Constant(self.new_value(), location, node.value)) - if isinstance(node, ast.BinOp): - operator = _BINARY_OPERATORS.get(type(node.op)) - if operator is None: - self.fail(node.op) + return self.emit(Symbol(self.new_value(), node.span, imported_symbol)) + if isinstance(node, nodes.Name): + if node.identifier in self.locals: + return self.emit(LoadLocal(self.new_value(), node.span, node.identifier)) + if node.identifier in self.parameters: + return self.parameters[node.identifier] + return self.emit(Global(self.new_value(), node.span, node.identifier)) + if isinstance(node, nodes.Constant): + return self.emit(Constant(self.new_value(), node.span, node.value)) + if isinstance(node, nodes.BinaryOp): left = self.expression(node.left) right = self.expression(node.right) - return self.emit( - BinaryOp( - self.new_value(), - location, - operator, - left, - right, - ) - ) - if isinstance(node, ast.UnaryOp): - operator = _UNARY_OPERATORS.get(type(node.op)) - if operator is None: - self.fail(node.op) + return self.emit(BinaryOp(self.new_value(), node.span, node.operator, left, right)) + if isinstance(node, nodes.UnaryOp): operand = self.expression(node.operand) - return self.emit(UnaryOp(self.new_value(), location, operator, operand)) - if isinstance(node, ast.Compare): - if len(node.ops) != 1 or len(node.comparators) != 1: - self.fail(node, "chained comparisons are not supported yet") - operator = _COMPARE_OPERATORS.get(type(node.ops[0])) - if operator is None: - self.fail(node.ops[0]) + return self.emit(UnaryOp(self.new_value(), node.span, node.operator, operand)) + if isinstance(node, nodes.Compare): left = self.expression(node.left) - right = self.expression(node.comparators[0]) - return self.emit( - Compare( - self.new_value(), - location, - operator, - left, - right, - ) - ) - if isinstance(node, ast.Call): + right = self.expression(node.right) + return self.emit(Compare(self.new_value(), node.span, node.operator, left, right)) + if isinstance(node, nodes.Call): if node.keywords: self.fail(node, "keyword arguments are not supported yet") - callee = self.expression(node.func) - arguments = tuple(self.expression(argument) for argument in node.args) - return self.emit(Call(self.new_value(), location, callee, arguments)) - if isinstance(node, ast.Attribute): + callee = self.expression(node.callee) + arguments = tuple(self.expression(argument) for argument in node.arguments) + return self.emit(Call(self.new_value(), node.span, callee, arguments)) + if isinstance(node, nodes.Attribute): object_value = self.expression(node.value) - return self.emit(GetAttr(self.new_value(), location, object_value, node.attr)) - if isinstance(node, ast.Subscript): + return self.emit(GetAttr(self.new_value(), node.span, object_value, node.name)) + if isinstance(node, nodes.Subscript): object_value = self.expression(node.value) - key = self.expression(node.slice) - return self.emit(GetItem(self.new_value(), location, object_value, key)) + key = self.expression(node.key) + return self.emit(GetItem(self.new_value(), node.span, object_value, key)) self.fail(node) - def statement(self, node: ast.stmt) -> None: - if isinstance(node, ast.Assign): - if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name): - self.fail(node, "only assignment to one local name is supported") - name = node.targets[0].id + def statement(self, node: nodes.Statement) -> None: + if isinstance(node, nodes.Assign): value = self.expression(node.value) - self.emit_effect(StoreLocal(None, self.location(node), name, value)) - self.locals[name] = value + self.emit_effect(StoreLocal(None, node.span, node.target.identifier, value)) + self.locals.add(node.target.identifier) return - if isinstance(node, ast.Return): - value = self.expression(node.value) if node.value is not None else None - self.emit_effect(Return(None, self.location(node), value)) + if isinstance(node, nodes.Return): + return_value = self.expression(node.value) if node.value is not None else None + self.emit_effect(Return(None, node.span, return_value)) return - if isinstance(node, ast.Expr): - self.expression(node.value) + if isinstance(node, nodes.ExpressionStatement): + self.expression(node.expression) return - if isinstance(node, ast.Pass): + if isinstance(node, nodes.Pass): return self.fail(node) - def function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> FunctionIR: - if node.decorator_list: - self.fail(node, "decorated functions are not supported yet") - has_advanced_arguments = ( - node.args.posonlyargs or node.args.kwonlyargs or node.args.vararg or node.args.kwarg - ) - if has_advanced_arguments: - self.fail(node.args, "only positional arguments are supported") - parameters = tuple(self.new_value() for _ in node.args.args) - argument_names = (argument.arg for argument in node.args.args) - self.parameters.update(zip(argument_names, parameters, strict=True)) + def function(self, node: nodes.Function) -> FunctionIR: + parameters = tuple(self.new_value() for _ in node.parameters) + parameter_names = (parameter.name for parameter in node.parameters) + self.parameters.update(zip(parameter_names, parameters, strict=True)) for statement in node.body: self.statement(statement) - return FunctionIR(node.name, parameters, (self.block,), self.location(node)) + return FunctionIR(node.name, parameters, (self.block,), node.span) -def lower_module(tree: ast.Module, filename: str = "") -> ModuleIR: - imports = collect_imports(tree, filename) +def lower_module(module: nodes.Module) -> ModuleIR: + imports = collect_imports(module) functions: list[FunctionIR] = [] - for statement in tree.body: - if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): - functions.append(_FunctionLowerer(filename, imports).function(statement)) - elif isinstance(statement, (ast.Import, ast.ImportFrom)): + for statement in module.body: + if isinstance(statement, nodes.Function): + functions.append(_FunctionLowerer(imports).function(statement)) + elif isinstance(statement, (nodes.Import, nodes.ImportFrom)): continue - elif isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant): + elif ( + isinstance(statement, nodes.ExpressionStatement) + and isinstance(statement.expression, nodes.Constant) + and isinstance(statement.expression.value, str) + ): # Permit module docstrings. continue else: - location = SourceLocation.from_ast(filename, statement) raise LoweringError( - f"{filename}:{location.line}:{location.column}: " + f"{statement.span.display()}: " f"unsupported module syntax: {type(statement).__name__}" ) return ModuleIR(tuple(functions)) diff --git a/src/coretrace_python/frontend/parser.py b/src/coretrace_python/frontend/parser.py index 5adb370..13c6c4e 100644 --- a/src/coretrace_python/frontend/parser.py +++ b/src/coretrace_python/frontend/parser.py @@ -3,6 +3,8 @@ import ast from pathlib import Path +from coretrace_python.source import SourceFile, SourceManager + class ParseError(Exception): """A source-located Python parsing failure.""" @@ -19,7 +21,9 @@ def parse_source(source: str, filename: str = "") -> ast.Module: def parse_file(path: str | Path) -> ast.Module: - source_path = Path(path) - source = source_path.read_text(encoding="utf-8") - return parse_source(source, filename=str(source_path)) + source = SourceManager().load_file(path) + return parse_source_file(source) + +def parse_source_file(source: SourceFile) -> ast.Module: + return parse_source(source.text, filename=str(source.source_id)) diff --git a/src/coretrace_python/hir/__init__.py b/src/coretrace_python/hir/__init__.py new file mode 100644 index 0000000..4083bf0 --- /dev/null +++ b/src/coretrace_python/hir/__init__.py @@ -0,0 +1,5 @@ +"""Parser-independent high-level representation of Python source.""" + +from coretrace_python.hir.nodes import Module + +__all__ = ["Module"] diff --git a/src/coretrace_python/hir/nodes.py b/src/coretrace_python/hir/nodes.py new file mode 100644 index 0000000..b90c89a --- /dev/null +++ b/src/coretrace_python/hir/nodes.py @@ -0,0 +1,145 @@ +"""Immutable nodes for the Python high-level intermediate representation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TypeAlias + +from coretrace_python.source import SourceSpan + + +@dataclass(frozen=True, slots=True) +class Parameter: + name: str + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Name: + identifier: str + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Constant: + value: object + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class BinaryOp: + operator: str + left: Expression + right: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class UnaryOp: + operator: str + operand: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Compare: + operator: str + left: Expression + right: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Attribute: + value: Expression + name: str + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Subscript: + value: Expression + key: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Keyword: + name: str | None + value: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Call: + callee: Expression + arguments: tuple[Expression, ...] + keywords: tuple[Keyword, ...] + span: SourceSpan + + +Expression: TypeAlias = Name | Constant | BinaryOp | UnaryOp | Compare | Attribute | Subscript | Call + + +@dataclass(frozen=True, slots=True) +class Assign: + target: Name + value: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Return: + value: Expression | None + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class ExpressionStatement: + expression: Expression + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Pass: + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class ImportAlias: + name: str + as_name: str | None + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Import: + names: tuple[ImportAlias, ...] + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class ImportFrom: + module: str | None + names: tuple[ImportAlias, ...] + level: int + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class Function: + name: str + parameters: tuple[Parameter, ...] + body: tuple[Statement, ...] + is_async: bool + span: SourceSpan + + +Statement: TypeAlias = Assign | Return | ExpressionStatement | Pass | Import | ImportFrom | Function + + +@dataclass(frozen=True, slots=True) +class Module: + body: tuple[Statement, ...] + span: SourceSpan + diff --git a/src/coretrace_python/ir/__init__.py b/src/coretrace_python/ir/__init__.py index 4e2c35e..9b5f406 100644 --- a/src/coretrace_python/ir/__init__.py +++ b/src/coretrace_python/ir/__init__.py @@ -1,6 +1,6 @@ """Python-aware intermediate representation.""" -from coretrace_python.ir.model import BasicBlock, FunctionIR, ModuleIR, SourceLocation, Value - -__all__ = ["BasicBlock", "FunctionIR", "ModuleIR", "SourceLocation", "Value"] +from coretrace_python.ir.model import BasicBlock, FunctionIR, ModuleIR, Value +from coretrace_python.source import SourceSpan +__all__ = ["BasicBlock", "FunctionIR", "ModuleIR", "SourceSpan", "Value"] diff --git a/src/coretrace_python/ir/model.py b/src/coretrace_python/ir/model.py index 0119845..dca96ed 100644 --- a/src/coretrace_python/ir/model.py +++ b/src/coretrace_python/ir/model.py @@ -1,23 +1,10 @@ from __future__ import annotations -import ast from dataclasses import dataclass from typing import TypeAlias from coretrace_python.ir.symbol import SymbolId - - -@dataclass(frozen=True) -class SourceLocation: - filename: str - line: int - column: int - end_line: int | None = None - end_column: int | None = None - - @classmethod - def from_ast(cls, filename: str, node: ast.AST) -> SourceLocation: - return cls(filename, getattr(node, "lineno", 1), getattr(node, "col_offset", 0) + 1) +from coretrace_python.source import SourceSpan @dataclass(frozen=True) @@ -28,28 +15,28 @@ class Value: @dataclass(frozen=True) class Constant: result: Value - location: SourceLocation + location: SourceSpan value: object @dataclass(frozen=True) class Global: result: Value - location: SourceLocation + location: SourceSpan name: str @dataclass(frozen=True) class Symbol: result: Value - location: SourceLocation + location: SourceSpan symbol_id: SymbolId @dataclass(frozen=True) class BinaryOp: result: Value - location: SourceLocation + location: SourceSpan operator: str left: Value right: Value @@ -58,7 +45,7 @@ class BinaryOp: @dataclass(frozen=True) class UnaryOp: result: Value - location: SourceLocation + location: SourceSpan operator: str operand: Value @@ -66,7 +53,7 @@ class UnaryOp: @dataclass(frozen=True) class Compare: result: Value - location: SourceLocation + location: SourceSpan operator: str left: Value right: Value @@ -75,7 +62,7 @@ class Compare: @dataclass(frozen=True) class Call: result: Value - location: SourceLocation + location: SourceSpan callee: Value arguments: tuple[Value, ...] @@ -83,7 +70,7 @@ class Call: @dataclass(frozen=True) class GetAttr: result: Value - location: SourceLocation + location: SourceSpan object: Value attribute: str @@ -91,7 +78,7 @@ class GetAttr: @dataclass(frozen=True) class GetItem: result: Value - location: SourceLocation + location: SourceSpan object: Value key: Value @@ -99,21 +86,21 @@ class GetItem: @dataclass(frozen=True) class Return: result: None - location: SourceLocation + location: SourceSpan value: Value | None @dataclass(frozen=True) class LoadLocal: result: Value - location: SourceLocation + location: SourceSpan name: str @dataclass(frozen=True) class StoreLocal: result: None - location: SourceLocation + location: SourceSpan name: str value: Value @@ -149,7 +136,7 @@ class FunctionIR: name: str parameters: tuple[Value, ...] blocks: tuple[BasicBlock, ...] - location: SourceLocation + location: SourceSpan @dataclass(frozen=True) diff --git a/src/coretrace_python/source/__init__.py b/src/coretrace_python/source/__init__.py new file mode 100644 index 0000000..bdba88f --- /dev/null +++ b/src/coretrace_python/source/__init__.py @@ -0,0 +1,7 @@ +"""Source storage and location primitives.""" + +from coretrace_python.source.manager import SourceManager +from coretrace_python.source.model import SourceFile, SourceId, SourceSpan + +__all__ = ["SourceFile", "SourceId", "SourceManager", "SourceSpan"] + diff --git a/src/coretrace_python/source/manager.py b/src/coretrace_python/source/manager.py new file mode 100644 index 0000000..cfdf59f --- /dev/null +++ b/src/coretrace_python/source/manager.py @@ -0,0 +1,49 @@ +"""Ownership and loading of source files.""" + +from __future__ import annotations + +from pathlib import Path + +from coretrace_python.source.model import SourceFile, SourceId + + +class SourceManager: + """Own source text and return one canonical object for each source ID.""" + + def __init__(self) -> None: + self._sources: dict[SourceId, SourceFile] = {} + + def add_source(self, name: str, text: str) -> SourceFile: + source_id = SourceId(name) + existing = self._sources.get(source_id) + if existing is not None: + if existing.text != text: + raise ValueError(f"source {name!r} already exists with different text") + return existing + + source = SourceFile(source_id=source_id, text=text) + self._sources[source_id] = source + return source + + def load_file(self, path: str | Path) -> SourceFile: + resolved_path = Path(path).resolve() + source_id = SourceId(str(resolved_path)) + existing = self._sources.get(source_id) + if existing is not None: + return existing + + # utf-8-sig accepts ordinary UTF-8 and strips a leading UTF-8 BOM. + text = resolved_path.read_text(encoding="utf-8-sig") + source = SourceFile(source_id=source_id, text=text, path=resolved_path) + self._sources[source_id] = source + return source + + def get(self, source_id: SourceId) -> SourceFile: + try: + return self._sources[source_id] + except KeyError as error: + raise KeyError(f"unknown source ID: {source_id}") from error + + def __len__(self) -> int: + return len(self._sources) + diff --git a/src/coretrace_python/source/model.py b/src/coretrace_python/source/model.py new file mode 100644 index 0000000..e809141 --- /dev/null +++ b/src/coretrace_python/source/model.py @@ -0,0 +1,55 @@ +"""Immutable source files and locations shared by engine layers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True, order=True) +class SourceId: + """Stable identity for a source unit within an analysis invocation.""" + + value: str + + def __post_init__(self) -> None: + if not self.value: + raise ValueError("source ID cannot be empty") + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True) +class SourceSpan: + """One-based, half-open source range.""" + + source_id: SourceId + start_line: int + start_column: int + end_line: int | None = None + end_column: int | None = None + + def __post_init__(self) -> None: + if self.start_line < 1 or self.start_column < 1: + raise ValueError("source span start must be one-based") + if (self.end_line is None) != (self.end_column is None): + raise ValueError("source span end line and column must be provided together") + if self.end_line is not None and self.end_column is not None: + if self.end_line < self.start_line: + raise ValueError("source span cannot end before it starts") + if self.end_line == self.start_line and self.end_column < self.start_column: + raise ValueError("source span cannot end before it starts") + + def display(self) -> str: + return f"{self.source_id}:{self.start_line}:{self.start_column}" + + +@dataclass(frozen=True) +class SourceFile: + """Decoded source text and its identity.""" + + source_id: SourceId + text: str + path: Path | None = None + diff --git a/tests/test_hir_builder.py b/tests/test_hir_builder.py new file mode 100644 index 0000000..9b58d56 --- /dev/null +++ b/tests/test_hir_builder.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import ast + +from coretrace_python.frontend.ast_adapter import build_module +from coretrace_python.hir import nodes +from coretrace_python.source import SourceManager + + +def build(source_text: str) -> nodes.Module: + source = SourceManager().add_source("example.py", source_text) + return build_module(source, ast.parse(source.text, filename=str(source.source_id))) + + +def test_builds_parser_independent_function_and_expression_nodes() -> None: + module = build("def add(a, b):\n return a + b\n") + + function = module.body[0] + assert isinstance(function, nodes.Function) + assert tuple(parameter.name for parameter in function.parameters) == ("a", "b") + returned = function.body[0] + assert isinstance(returned, nodes.Return) + assert isinstance(returned.value, nodes.BinaryOp) + assert returned.value.operator == "add" + + +def test_preserves_source_spans() -> None: + module = build("def answer():\n return 42\n") + function = module.body[0] + assert isinstance(function, nodes.Function) + returned = function.body[0] + + assert returned.span.source_id.value == "example.py" + assert returned.span.start_line == 2 + assert returned.span.start_column == 5 + + +def test_represents_imports_without_ast_nodes() -> None: + module = build("from os import system as run\n") + imported = module.body[0] + + assert isinstance(imported, nodes.ImportFrom) + assert imported.module == "os" + assert imported.names == ( + nodes.ImportAlias("system", "run", imported.names[0].span), + ) + + +def test_represents_calls_attributes_and_subscripts() -> None: + module = build('def read(request):\n return request.args["name"]\n') + function = module.body[0] + assert isinstance(function, nodes.Function) + returned = function.body[0] + assert isinstance(returned, nodes.Return) + assert isinstance(returned.value, nodes.Subscript) + assert isinstance(returned.value.value, nodes.Attribute) diff --git a/tests/test_imports.py b/tests/test_imports.py index b8ba75a..ff91741 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -2,6 +2,7 @@ import pytest +from coretrace_python.frontend.ast_adapter import build_module from coretrace_python.frontend.imports import ( ImportBindings, ImportResolutionError, @@ -9,10 +10,13 @@ ) from coretrace_python.frontend.parser import parse_source from coretrace_python.ir.symbol import SymbolId +from coretrace_python.source import SourceManager def bindings_for(source: str) -> ImportBindings: - return collect_imports(parse_source(source, "imports.py"), "imports.py") + source_file = SourceManager().add_source("imports.py", source) + tree = parse_source(source_file.text, str(source_file.source_id)) + return collect_imports(build_module(source_file, tree)) def test_collects_plain_and_aliased_imports() -> None: diff --git a/tests/test_ir.py b/tests/test_ir.py index 4a95989..177fd30 100644 --- a/tests/test_ir.py +++ b/tests/test_ir.py @@ -1,14 +1,16 @@ +from coretrace_python.frontend.ast_adapter import build_module from coretrace_python.frontend.lowering import lower_module from coretrace_python.frontend.parser import parse_source from coretrace_python.ir.model import BinaryOp, Return +from coretrace_python.source import SourceManager def test_lowering_preserves_locations() -> None: - tree = parse_source("def add(a, b):\n return a + b\n", "add.py") - module = lower_module(tree, "add.py") + source = SourceManager().add_source("add.py", "def add(a, b):\n return a + b\n") + tree = parse_source(source.text, str(source.source_id)) + module = lower_module(build_module(source, tree)) instructions = module.functions[0].blocks[0].instructions assert isinstance(instructions[0], BinaryOp) - assert instructions[0].location.line == 2 + assert instructions[0].location.start_line == 2 assert isinstance(instructions[1], Return) - assert instructions[1].location.filename == "add.py" - + assert instructions[1].location.source_id.value == "add.py" diff --git a/tests/test_source_manager.py b/tests/test_source_manager.py new file mode 100644 index 0000000..5d5f874 --- /dev/null +++ b/tests/test_source_manager.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coretrace_python.source import SourceId, SourceManager, SourceSpan + + +def test_add_source_returns_the_same_object_for_identical_input() -> None: + manager = SourceManager() + first = manager.add_source("memory.py", "value = 1\n") + second = manager.add_source("memory.py", "value = 1\n") + + assert first is second + assert len(manager) == 1 + + +def test_add_source_rejects_conflicting_text() -> None: + manager = SourceManager() + manager.add_source("memory.py", "value = 1\n") + + with pytest.raises(ValueError, match="already exists with different text"): + manager.add_source("memory.py", "value = 2\n") + + +def test_load_file_uses_a_resolved_path_as_its_stable_id(tmp_path: Path) -> None: + path = tmp_path / "module.py" + path.write_text("value = 1\n", encoding="utf-8") + + source = SourceManager().load_file(path) + + assert source.source_id == SourceId(str(path.resolve())) + assert source.path == path.resolve() + + +def test_load_file_accepts_a_utf8_bom(tmp_path: Path) -> None: + path = tmp_path / "module.py" + path.write_text("\ufeffvalue = 1\n", encoding="utf-8") + + source = SourceManager().load_file(path) + + assert source.text == "value = 1\n" + + +def test_source_span_uses_one_based_positions() -> None: + with pytest.raises(ValueError, match="one-based"): + SourceSpan(SourceId("module.py"), start_line=0, start_column=1) + + +def test_source_span_formats_its_start_location() -> None: + span = SourceSpan(SourceId("module.py"), start_line=3, start_column=7) + assert span.display() == "module.py:3:7" +