From c78df7e1bcbf20a67dabe68f6d938934d18d9d2d Mon Sep 17 00:00:00 2001 From: Duc Le Date: Mon, 10 Aug 2026 23:12:01 +0700 Subject: [PATCH 1/3] feat(parser): add Rust support (roadmap #16b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tree-sitter parser for Rust modules, types, traits, functions, methods, imports, qualified references, trait inheritance/implementations, and Cargo workspaces. All AST traversal is iterative and each file is extracted transactionally, so one adversarial file cannot erase healthy siblings. Two defects found while relanding the original contribution are fixed here. Linear entity indexing. `add_entity` guarded the per-package entity list with `if fqn not in package_entities` — a linear scan of a list that grows to tens of thousands of entries — and the cross-file merge repeated the same test in a generator expression. A companion `set` per package makes both O(1). On a 5.2 MB generated single-package file this is 70.2s -> 3.0s (23x) with identical entity (79,500) and edge (63,600) counts. The membership set is reset alongside the per-file `packages` dict so no state leaks between files. Complete `#[cfg(test)]` exclusion. Rust unit tests live inline, so path-based exclusion never sees them, and matching only the literal `cfg(test)` text on an outer attribute of an item with a body left four shapes leaking. A probe crate with 2 production structs and 10 test-only entities yielded 10 entities (8 test-only) before and yields exactly the 2 production structs now: * compound predicates — the cfg predicate tree is evaluated rather than string-compared, so `cfg(all(test, ...))` and `cfg(any(test, ...))` match while `cfg(not(test))` correctly stays production; * inner `#![cfg(test)]` on a file or module body, previously invisible because only `attribute_item` was inspected; * out-of-line `#[cfg(test)] mod helpers;`, whose backing `helpers.rs` was later parsed as an independent production file — files are now visited in declaring-module-first order so the module path can be excluded; * `#[cfg(test)] use ...`, which gave every production entity in the file a phantom import of mockall/proptest/rstest and inflated its fan-out. No input size cap is added. `go.py` and `typescript.py` both keep a 1 MB `_MAX_FILE_BYTES` for minified and vendored bundles; Rust does not need one now that the quadratic index is gone. Co-Authored-By: Tony Nguyen Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 1 + src/arcade_agent/parsers/__init__.py | 5 + src/arcade_agent/parsers/base.py | 12 + src/arcade_agent/parsers/rust.py | 934 ++++++++++++++++++ tests/fixtures/rust_workspace/Cargo.toml | 7 + .../rust_workspace/crates/alpha/Cargo.toml | 4 + .../rust_workspace/crates/alpha/src/lib.rs | 13 + .../rust_workspace/crates/beta/Cargo.toml | 7 + .../rust_workspace/crates/beta/src/lib.rs | 7 + tests/fixtures/rust_workspace/src/lib.rs | 11 + tests/test_parsers/test_rust.py | 652 ++++++++++++ 11 files changed, 1653 insertions(+) create mode 100644 src/arcade_agent/parsers/rust.py create mode 100644 tests/fixtures/rust_workspace/Cargo.toml create mode 100644 tests/fixtures/rust_workspace/crates/alpha/Cargo.toml create mode 100644 tests/fixtures/rust_workspace/crates/alpha/src/lib.rs create mode 100644 tests/fixtures/rust_workspace/crates/beta/Cargo.toml create mode 100644 tests/fixtures/rust_workspace/crates/beta/src/lib.rs create mode 100644 tests/fixtures/rust_workspace/src/lib.rs create mode 100644 tests/test_parsers/test_rust.py diff --git a/pyproject.toml b/pyproject.toml index afdb022..a64ad65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ languages = [ "tree-sitter-c>=0.23.0", "tree-sitter-cpp>=0.23.0", "tree-sitter-kotlin>=1.1.0", + "tree-sitter-rust>=0.23.0", ] mcp = [ "mcp[cli]>=1.0,<2", diff --git a/src/arcade_agent/parsers/__init__.py b/src/arcade_agent/parsers/__init__.py index ce1c445..1e971f1 100644 --- a/src/arcade_agent/parsers/__init__.py +++ b/src/arcade_agent/parsers/__init__.py @@ -26,6 +26,11 @@ except ImportError: pass +try: + import arcade_agent.parsers.rust # noqa: F401 +except ImportError: + pass + from arcade_agent.parsers.base import LanguageParser, get_parser __all__ = ["LanguageParser", "get_parser"] diff --git a/src/arcade_agent/parsers/base.py b/src/arcade_agent/parsers/base.py index 056e55e..a64445a 100644 --- a/src/arcade_agent/parsers/base.py +++ b/src/arcade_agent/parsers/base.py @@ -9,6 +9,18 @@ class LanguageParser(ABC): """Abstract base class for language-specific parsers.""" + def __init__(self, exclude_tests: bool = True) -> None: + """Create a parser. + + Args: + exclude_tests: Whether test code should be kept out of the graph. + File-level exclusion happens during discovery; this flag lets a + parser additionally drop *inline* test constructs that live in + production files (e.g. Rust's ``#[cfg(test)] mod tests``). + Parsers for languages without inline tests ignore it. + """ + self.exclude_tests = exclude_tests + @property @abstractmethod def language(self) -> str: diff --git a/src/arcade_agent/parsers/rust.py b/src/arcade_agent/parsers/rust.py new file mode 100644 index 0000000..d7ac009 --- /dev/null +++ b/src/arcade_agent/parsers/rust.py @@ -0,0 +1,934 @@ +"""Rust parser using tree-sitter. + +The parser follows Rust's file-module convention (``lib.rs``/``main.rs`` at +the crate root, ``mod.rs`` for a directory module) and extracts structs, +enums, unions, traits, type aliases, functions, and methods. A second pass +resolves ``use`` declarations, same-module references, qualified paths, and +trait inheritance/implementations into dependency edges. Cargo workspaces are +kept intact and each member crate gets a stable graph prefix. + +Rust unit tests are conventionally written inline as ``#[cfg(test)] mod tests`` +inside the production file, so path-based test exclusion never sees them. When +``exclude_tests`` is set (the default), test-only code is left out of the graph; +with ``exclude_tests=False`` it is extracted like any other code. Five shapes +are recognized: + +* ``#[cfg(test)]`` on any item (module, type, function, impl, ``use``); +* compound predicates such as ``#[cfg(all(test, feature = "x"))]`` and + ``#[cfg(any(test, fuzzing))]`` (``#[cfg(not(test))]`` stays production); +* an inner ``#![cfg(test)]`` at the top of a file or module body, which makes + the whole container test-only; +* out-of-line ``#[cfg(test)] mod helpers;``, whose backing ``helpers.rs`` / + ``helpers/mod.rs`` file is skipped instead of being parsed as production; +* ``#[cfg(test)] use ...``, so dev-dependency imports (mockall, proptest, + rstest, ...) never appear on production entities. +""" + +from __future__ import annotations + +import logging +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import tree_sitter_rust as tsrust +from tree_sitter import Language, Node, Parser + +from arcade_agent.parsers.base import LanguageParser, register_parser +from arcade_agent.parsers.graph import DependencyGraph, Edge, Entity + +RUST_LANGUAGE = Language(tsrust.language()) +logger = logging.getLogger(__name__) + +_TYPE_ITEMS = { + "struct_item": "struct", + "enum_item": "enum", + "union_item": "union", + "trait_item": "trait", + "type_item": "type", +} + + +@dataclass(frozen=True) +class _Import: + path: tuple[str, ...] + alias: str + wildcard: bool = False + + @property + def display(self) -> str: + suffix = "::*" if self.wildcard else "" + return "::".join(self.path) + suffix + + +@dataclass +class _References: + simple: set[str] + qualified: set[tuple[str, ...]] + + +@dataclass +class _PendingMethod: + name: str + references: _References + + +@dataclass +class _PendingImpl: + owner_path: tuple[str, ...] + trait_path: tuple[str, ...] + generic_parameters: frozenset[str] + module: tuple[str, ...] + crate: tuple[str, ...] + imports: list[_Import] + rel_path: str + methods: list[_PendingMethod] + + +def _get_text(node: Node | None) -> str: + raw_text = None if node is None else node.text + return "" if raw_text is None else raw_text.decode(errors="replace") + + +def _identifier(text: str) -> str: + """Normalize raw Rust identifiers (``r#type`` -> ``type``).""" + return text[2:] if text.startswith("r#") else text + + +def _cargo_data(directory: Path) -> dict[str, Any]: + manifest = directory / "Cargo.toml" + try: + if not manifest.is_file(): + return {} + with manifest.open("rb") as manifest_file: + return tomllib.load(manifest_file) + except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError): + return {} + + +def _crate_context(file_path: Path, root: Path, is_workspace: bool) -> tuple[Path, tuple[str, ...]]: + """Return the source root and graph prefix for the file's Cargo crate.""" + if not is_workspace: + conventional_source = root / "src" + if conventional_source in file_path.parents: + return conventional_source, () + return root, () + + current = file_path.parent + crate_dir: Path | None = None + while current == root or root in current.parents: + if (current / "Cargo.toml").is_file(): + crate_dir = current + break + if current == root: + break + current = current.parent + if crate_dir is None: + return root, () + + package = _cargo_data(crate_dir).get("package", {}) + crate_name = str(package.get("name", crate_dir.name)).replace("-", "_") + source_root = crate_dir / "src" + if source_root not in file_path.parents: + source_root = crate_dir + return source_root, (_identifier(crate_name),) + + +def _module_name(file_path: Path, source_root: Path, crate: tuple[str, ...]) -> tuple[str, ...]: + """Return the Rust module path represented by a source file.""" + rel = file_path.relative_to(source_root) + parts = list(rel.parts) + stem = Path(parts[-1]).stem + directory = parts[:-1] + if stem in {"lib", "main"}: + return (*crate, *directory) + if stem == "mod": + return (*crate, *directory) + return tuple([*crate, *directory, stem]) + + +def _path_segments(node: Node | None) -> tuple[str, ...]: + """Extract ``a::b::Name`` segments from a path-like AST node.""" + if node is None: + return () + + segments: list[str] = [] + stack = [node] + leaf_types = { + "identifier", + "type_identifier", + "crate", + "self", + "super", + "metavariable", + } + while stack: + current = stack.pop() + if current.type in leaf_types: + segments.append(_identifier(_get_text(current))) + continue + + path = current.child_by_field_name("path") + name = current.child_by_field_name("name") + if path is not None or name is not None: + # LIFO order: visit the path before its final name. + if name is not None: + stack.append(name) + if path is not None: + stack.append(path) + continue + + # use_wildcard has no named fields in tree-sitter-rust 0.24. + if current.type == "use_wildcard" and current.named_children: + stack.append(current.named_children[0]) + continue + + # References through generic paths only need the base type name. + if current.type == "generic_type": + generic_type = current.child_by_field_name("type") + if generic_type is not None: + stack.append(generic_type) + + return tuple(segments) + + +def _flatten_use(node: Node, prefix: tuple[str, ...] = ()) -> list[_Import]: + """Flatten a Rust use tree into concrete imports.""" + imports: list[_Import] = [] + pending: list[tuple[Node, tuple[str, ...]]] = [(node, prefix)] + while pending: + current, current_prefix = pending.pop() + + if current.type == "use_declaration": + argument = current.child_by_field_name("argument") + if argument is not None: + pending.append((argument, current_prefix)) + continue + + if current.type == "scoped_use_list": + path = ( + *current_prefix, + *_path_segments(current.child_by_field_name("path")), + ) + use_list = current.child_by_field_name("list") + if use_list is not None: + pending.extend((child, path) for child in reversed(use_list.named_children)) + continue + + if current.type == "use_list": + pending.extend((child, current_prefix) for child in reversed(current.named_children)) + continue + + if current.type == "self" and current_prefix: + imports.append(_Import(path=current_prefix, alias=current_prefix[-1])) + continue + + if current.type == "use_as_clause": + path = ( + *current_prefix, + *_path_segments(current.child_by_field_name("path")), + ) + if not path: + continue + alias_node = current.child_by_field_name("alias") + alias = _identifier(_get_text(alias_node)) if alias_node is not None else path[-1] + imports.append(_Import(path=path, alias=alias)) + continue + + if current.type == "use_wildcard": + path = (*current_prefix, *_path_segments(current)) + if path: + imports.append(_Import(path=path, alias="*", wildcard=True)) + continue + + path = (*current_prefix, *_path_segments(current)) + if path: + imports.append(_Import(path=path, alias=path[-1])) + + return imports + + +_ATTRIBUTE_ITEMS = {"attribute_item", "inner_attribute_item"} +_COMMENT_ITEMS = {"line_comment", "block_comment"} + + +def _cfg_predicate_enables_test(token_tree: Node) -> bool: + """Whether a ``cfg(...)`` predicate is satisfied by the ``test`` cfg flag. + + ``cfg(test)``, ``cfg(all(test, unix))`` and ``cfg(any(test, fuzzing))`` all + gate test-only code, so the whole predicate tree is searched rather than + string-matching ``cfg(test)``. ``not(...)`` sub-predicates are skipped: + ``#[cfg(not(test))]`` marks production-only code and must not be dropped. + """ + children = token_tree.named_children + index = 0 + while index < len(children): + child = children[index] + following = children[index + 1] if index + 1 < len(children) else None + if child.type == "identifier" and following is not None and following.type == "token_tree": + # A predicate function: all(...), any(...), not(...). + if _get_text(child) != "not" and _cfg_predicate_enables_test(following): + return True + index += 2 + continue + if child.type == "identifier" and _get_text(child) == "test": + return True + if child.type == "token_tree" and _cfg_predicate_enables_test(child): + return True + index += 1 + return False + + +def _is_cfg_test_attribute(node: Node) -> bool: + """Return whether an attribute gates its item on the ``test`` cfg flag.""" + if node.type not in _ATTRIBUTE_ITEMS: + return False + for child in node.named_children: + if child.type != "attribute": + continue + parts = child.named_children + if not parts or _get_text(parts[0]) != "cfg": + continue + if any( + part.type == "token_tree" and _cfg_predicate_enables_test(part) for part in parts[1:] + ): + return True + return False + + +def _has_inner_cfg_test(container: Node) -> bool: + """Whether a file or module body is wholly test-only via ``#![cfg(test)]``.""" + return any( + child.type == "inner_attribute_item" and _is_cfg_test_attribute(child) + for child in container.named_children + ) + + +def _extract_imports(container: Node, exclude_tests: bool) -> list[_Import]: + """Collect ``use`` declarations, optionally dropping test-only ones. + + ``#[cfg(test)] use mockall::predicate::Eq;`` is a dev-dependency import. It + is attached to no production entity, so including it would give every + entity in the file phantom coupling to the test-only crate. + """ + imports: list[_Import] = [] + pending_attributes: list[Node] = [] + for child in container.named_children: + if child.type == "attribute_item": + pending_attributes.append(child) + continue + if child.type in _COMMENT_ITEMS or child.type == "inner_attribute_item": + continue + is_cfg_test = any(_is_cfg_test_attribute(item) for item in pending_attributes) + pending_attributes.clear() + if child.type != "use_declaration": + continue + if is_cfg_test and exclude_tests: + continue + imports.extend(_flatten_use(child)) + return imports + + +def _module_file_order(file_path: Path) -> tuple[int, int, str]: + """Order files so a module's declaring file is always parsed before it. + + ``#[cfg(test)] mod helpers;`` can only exclude ``helpers.rs`` if the + declaring file (``lib.rs``/``mod.rs``/the parent module file) has already + been visited. Parents are either shallower or, at equal depth, a crate/ + directory root. + """ + return ( + len(file_path.parts), + 0 if file_path.stem in {"lib", "main", "mod"} else 1, + str(file_path), + ) + + +def _child_module_files(module_dir: Path, name: str) -> tuple[Path, Path]: + """Candidate files backing an out-of-line ``mod ;`` declaration.""" + return module_dir / f"{name}.rs", module_dir / name / "mod.rs" + + +def _references(node: Node) -> _References: + simple: set[str] = set() + qualified: set[tuple[str, ...]] = set() + stack = [node] + while stack: + current = stack.pop() + if current.type in {"scoped_identifier", "scoped_type_identifier"}: + path = _path_segments(current) + if len(path) > 1: + qualified.add(path) + # The leading segment drives import-alias resolution. The + # immediate owner prefix preserves references such as + # ``Type::associated_item`` without recomputing every nested + # scoped path (which is quadratic for generated long paths). + simple.add(path[0]) + if len(path) > 2: + qualified.add(path[:-1]) + continue + elif current.type in {"identifier", "type_identifier"}: + simple.add(_identifier(_get_text(current))) + stack.extend(current.named_children) + return _References(simple=simple, qualified=qualified) + + +def _base_type_path(node: Node | None) -> tuple[str, ...]: + """Get the implemented type path without generic arguments.""" + if node is None: + return () + + stack = [node] + wrapper_types = {"reference_type", "pointer_type", "array_type", "slice_type"} + while stack: + current = stack.pop() + # Wrapper nodes can contain lifetimes before the actual type. Follow + # the explicit type field so ``&'a mut T`` resolves to T, not ``a``. + if current.type in wrapper_types: + wrapped_type = current.child_by_field_name("type") + if wrapped_type is not None: + stack.append(wrapped_type) + continue + + direct = _path_segments(current) + if direct: + return direct + stack.extend(reversed(current.named_children)) + return () + + +def _generic_type_parameters(node: Node) -> frozenset[str]: + """Return type parameter names declared by an impl (excluding lifetimes).""" + parameters = node.child_by_field_name("type_parameters") + if parameters is None: + return frozenset() + return frozenset( + _identifier(_get_text(name)) + for parameter in parameters.named_children + if parameter.type == "type_parameter" + for name in [parameter.child_by_field_name("name")] + if name is not None + ) + + +def _normalize_path( + path: tuple[str, ...], + current: tuple[str, ...], + crate: tuple[str, ...] = (), +) -> tuple[str, ...]: + if not path: + return () + parts = list(path) + if parts[0] == "crate": + return (*crate, *parts[1:]) + if parts[0] == "self": + return (*current, *parts[1:]) + if parts[0] == "super": + base = list(current) + while parts and parts[0] == "super": + if base: + base.pop() + parts.pop(0) + return (*base, *parts) + return (*current, *parts) + + +def _deduplicate(edges: list[Edge]) -> list[Edge]: + seen: set[tuple[str, str, str]] = set() + unique: list[Edge] = [] + for edge in edges: + key = (edge.source, edge.target, edge.relation) + if edge.source != edge.target and key not in seen: + seen.add(key) + unique.append(edge) + return unique + + +@register_parser +class RustParser(LanguageParser): + """Rust source code parser using tree-sitter. + + Honors ``exclude_tests`` (default ``True``) by skipping inline test-only + code, which path-based test exclusion cannot reach. See the module + docstring for the recognized ``#[cfg(test)]`` shapes. + """ + + @property + def language(self) -> str: + return "rust" + + @property + def file_extensions(self) -> list[str]: + return [".rs"] + + def parse(self, files: list[Path], root: Path) -> DependencyGraph: + parser = Parser(RUST_LANGUAGE) + root = root.resolve() + + entities: dict[str, Entity] = {} + packages: dict[str, list[str]] = {} + # Companion membership index for ``packages``. ``fqn not in list`` is + # linear, which made ``add_entity`` quadratic in the number of entities + # per package (minutes on multi-MB generated sources). + package_members: dict[str, set[str]] = {} + entity_refs: dict[str, _References] = {} + entity_imports: dict[str, list[_Import]] = {} + entity_crates: dict[str, tuple[str, ...]] = {} + pending_impls: list[_PendingImpl] = [] + pending_trait_bounds: list[ + tuple[str, tuple[str, ...], tuple[str, ...], tuple[str, ...], list[_Import]] + ] = [] + is_workspace = "workspace" in _cargo_data(root) + # Files backing an out-of-line ``#[cfg(test)] mod x;`` declaration. + # Populated while visiting the declaring file, which ``_module_file_order`` + # guarantees is visited first. + excluded_module_files: set[Path] = set() + + def add_entity( + *, + name: str, + kind: str, + module: tuple[str, ...], + rel_path: str, + imports: list[_Import], + node: Node | None, + crate: tuple[str, ...], + owner: str | None = None, + fqn_override: str | None = None, + references: _References | None = None, + ) -> str: + package = ".".join(module) + fqn = fqn_override or (f"{package}.{name}" if package else name) + entities[fqn] = Entity( + fqn=fqn, + name=name, + package=package, + file_path=rel_path, + kind=kind, + language="rust", + imports=[item.display for item in imports], + properties={"owner": owner} if owner else {}, + ) + package_entities = packages.setdefault(package, []) + members = package_members.setdefault(package, set()) + if fqn not in members: + members.add(fqn) + package_entities.append(fqn) + if references is not None: + entity_refs[fqn] = references + elif node is not None: + entity_refs[fqn] = _references(node) + else: + entity_refs[fqn] = _References(set(), set()) + entity_imports[fqn] = imports + entity_crates[fqn] = crate + return fqn + + def visit_container( + container: Node, + module: tuple[str, ...], + rel_path: str, + file_stem: str, + crate: tuple[str, ...], + module_dir: Path, + is_file_root: bool = False, + ) -> None: + container_stack = [(container, module, module_dir, is_file_root)] + while container_stack: + ( + current_container, + current_module, + current_dir, + current_is_file_root, + ) = container_stack.pop() + + # ``#![cfg(test)]`` at the top of a file or module body makes the + # whole container test-only; there is no outer attribute to see. + if self.exclude_tests and _has_inner_cfg_test(current_container): + continue + + imports = _extract_imports(current_container, self.exclude_tests) + direct_entities = 0 + nested_containers: list[tuple[Node, tuple[str, ...], Path, bool]] = [] + pending_attributes: list[Node] = [] + + for node in current_container.named_children: + if node.type == "attribute_item": + pending_attributes.append(node) + continue + + # Comments between an attribute and its item must not + # clear the pending attribute list (tree-sitter emits + # line_comment/block_comment as named children). + if node.type in _COMMENT_ITEMS: + continue + + is_cfg_test = any( + _is_cfg_test_attribute(attribute) for attribute in pending_attributes + ) + pending_attributes.clear() + + # Rust unit tests live inline, so path-based test + # exclusion in ``ingest`` cannot see them. Drop + # ``#[cfg(test)]`` items only when the caller asked + # for test code to be excluded. + if is_cfg_test and self.exclude_tests: + # An out-of-line ``#[cfg(test)] mod helpers;`` carries no + # body here — its code lives in a sibling file that would + # otherwise be parsed as an independent production file. + if node.type == "mod_item" and node.child_by_field_name("body") is None: + name_node = node.child_by_field_name("name") + if name_node is not None: + excluded_module_files.update( + _child_module_files( + current_dir, _identifier(_get_text(name_node)) + ) + ) + continue + + if node.type in _TYPE_ITEMS: + name_node = node.child_by_field_name("name") + if name_node is None: + continue + name = _identifier(_get_text(name_node)) + owner = add_entity( + name=name, + kind=_TYPE_ITEMS[node.type], + module=current_module, + rel_path=rel_path, + imports=imports, + node=node, + crate=crate, + ) + direct_entities += 1 + if node.type == "trait_item": + bounds = node.child_by_field_name("bounds") + if bounds is not None: + for bound in bounds.named_children: + bound_path = _base_type_path(bound) + if bound_path: + pending_trait_bounds.append( + ( + owner, + bound_path, + current_module, + crate, + imports, + ) + ) + body = node.child_by_field_name("body") + if body is not None: + for member in body.named_children: + if member.type not in { + "function_item", + "function_signature_item", + }: + continue + method_name = member.child_by_field_name("name") + if method_name is None: + continue + normalized_name = _identifier(_get_text(method_name)) + add_entity( + name=normalized_name, + kind="method", + module=current_module, + rel_path=rel_path, + imports=imports, + node=member, + crate=crate, + owner=owner, + fqn_override=f"{owner}.{normalized_name}", + ) + elif node.type == "function_item": + name_node = node.child_by_field_name("name") + if name_node is not None: + add_entity( + name=_identifier(_get_text(name_node)), + kind="function", + module=current_module, + rel_path=rel_path, + imports=imports, + node=node, + crate=crate, + ) + direct_entities += 1 + elif node.type == "impl_item": + type_node = node.child_by_field_name("type") + trait_node = node.child_by_field_name("trait") + body = node.child_by_field_name("body") + owner_path = _base_type_path(type_node) + if not owner_path or body is None: + continue + methods: list[_PendingMethod] = [] + for member in body.named_children: + if member.type != "function_item": + continue + method_name = member.child_by_field_name("name") + if method_name is None: + continue + name = _identifier(_get_text(method_name)) + methods.append( + _PendingMethod( + name=name, + references=_references(member), + ) + ) + pending_impls.append( + _PendingImpl( + owner_path=owner_path, + trait_path=_base_type_path(trait_node), + generic_parameters=_generic_type_parameters(node), + module=current_module, + crate=crate, + imports=imports, + rel_path=rel_path, + methods=methods, + ) + ) + elif node.type == "mod_item": + name_node = node.child_by_field_name("name") + body = node.child_by_field_name("body") + if name_node is not None and body is not None: + child_name = _identifier(_get_text(name_node)) + child_module = (*current_module, child_name) + nested_containers.append( + (body, child_module, current_dir / child_name, False) + ) + + container_stack.extend(reversed(nested_containers)) + if current_is_file_root and direct_entities == 0: + module_name = ".".join(current_module) + name = current_module[-1] if current_module else file_stem + fqn = module_name or file_stem + add_entity( + name=name, + kind="module", + module=current_module[:-1] if current_module else (), + rel_path=rel_path, + imports=imports, + node=current_container, + crate=crate, + fqn_override=fqn, + ) + + all_entities: dict[str, Entity] = {} + all_packages: dict[str, list[str]] = {} + all_package_members: dict[str, set[str]] = {} + all_entity_refs: dict[str, _References] = {} + all_entity_imports: dict[str, list[_Import]] = {} + all_entity_crates: dict[str, tuple[str, ...]] = {} + all_pending_impls: list[_PendingImpl] = [] + all_pending_trait_bounds: list[ + tuple[str, tuple[str, ...], tuple[str, ...], tuple[str, ...], list[_Import]] + ] = [] + + resolved_files: list[Path] = [] + for candidate in files: + try: + resolved_files.append(candidate.resolve()) + except OSError: + resolved_files.append(candidate) + resolved_files.sort(key=_module_file_order) + + for source_file in resolved_files: + if source_file in excluded_module_files: + continue + + # Extract each file transactionally. If a malformed or adversarial + # file trips an unexpected parser edge case, discard its partial + # state and preserve entities from healthy sibling files. + entities = {} + packages = {} + package_members = {} + entity_refs = {} + entity_imports = {} + entity_crates = {} + pending_impls = [] + pending_trait_bounds = [] + try: + rel_path = str(source_file.relative_to(root)) + tree = parser.parse(source_file.read_bytes()) + source_root, crate = _crate_context(source_file, root, is_workspace) + module_dir = ( + source_file.parent + if source_file.stem in {"lib", "main", "mod"} + else source_file.parent / source_file.stem + ) + visit_container( + tree.root_node, + _module_name(source_file, source_root, crate), + rel_path, + source_file.stem, + crate, + module_dir, + is_file_root=True, + ) + except Exception as error: + logger.warning( + "Skipping Rust source after extraction failure (%s): %s", + type(error).__name__, + source_file, + ) + continue + + all_entities.update(entities) + all_entity_refs.update(entity_refs) + all_entity_imports.update(entity_imports) + all_entity_crates.update(entity_crates) + all_pending_impls.extend(pending_impls) + all_pending_trait_bounds.extend(pending_trait_bounds) + for package, fqns in packages.items(): + package_entities = all_packages.setdefault(package, []) + members = all_package_members.setdefault(package, set()) + for fqn in fqns: + if fqn not in members: + members.add(fqn) + package_entities.append(fqn) + + entities = all_entities + packages = all_packages + package_members = all_package_members + entity_refs = all_entity_refs + entity_imports = all_entity_imports + entity_crates = all_entity_crates + pending_impls = all_pending_impls + pending_trait_bounds = all_pending_trait_bounds + + non_member_entities = [e for e in entities.values() if e.kind != "method"] + by_module_name = {(e.package, e.name): e.fqn for e in non_member_entities} + by_simple_name: dict[str, list[str]] = {} + for entity in non_member_entities: + by_simple_name.setdefault(entity.name, []).append(entity.fqn) + + def resolve( + path: tuple[str, ...], + current_package: str, + crate: tuple[str, ...] = (), + ) -> str | None: + current = tuple(part for part in current_package.split(".") if part) + candidates: list[tuple[str, ...]] = [] + if path and path[0] in {"crate", "self", "super"}: + candidates.append(_normalize_path(path, current, crate)) + else: + candidates.extend([(*current, *path), path]) + for candidate in candidates: + fqn = ".".join(candidate) + if fqn in entities: + return fqn + # A qualified path that did not match is external or unresolved; + # falling back by its final name could link ``std::io::Error`` to + # an unrelated local ``Error``. Keep the unique-name fallback only + # for simple paths (notably owners imported through a glob). + if len(path) == 1: + matches = by_simple_name.get(path[-1], []) + if len(matches) == 1: + return matches[0] + return None + + def expand_alias(path: tuple[str, ...], imports: list[_Import]) -> tuple[str, ...]: + if not path: + return path + for item in imports: + if not item.wildcard and path[0] == item.alias: + return (*item.path, *path[1:]) + return path + + # Resolve impl owners only after all type declarations are known. This + # correctly attaches impls written in sibling modules and avoids graph + # entities for external/blanket owners such as std::io::Error, Box, + # or &'a mut T. + edges: list[Edge] = [] + owner_kinds = {"struct", "enum", "union", "type"} + + for ( + bound_owner, + bound_path, + bound_module, + bound_crate, + bound_imports, + ) in pending_trait_bounds: + package = ".".join(bound_module) + bound_target = resolve(expand_alias(bound_path, bound_imports), package, bound_crate) + if bound_target and entities[bound_target].kind == "trait": + bound_name = "::".join(bound_path) + if bound_name not in entities[bound_owner].interfaces: + entities[bound_owner].interfaces.append(bound_name) + edges.append(Edge(bound_owner, bound_target, "extends")) + + for pending in pending_impls: + package = ".".join(pending.module) + owner_path = expand_alias(pending.owner_path, pending.imports) + if len(owner_path) == 1 and owner_path[0] in pending.generic_parameters: + continue + resolved_owner = resolve(owner_path, package, pending.crate) + if resolved_owner is None or entities[resolved_owner].kind not in owner_kinds: + continue + + owner_module = tuple( + part for part in entities[resolved_owner].package.split(".") if part + ) + for method in pending.methods: + add_entity( + name=method.name, + kind="method", + module=owner_module, + rel_path=pending.rel_path, + imports=pending.imports, + node=None, + crate=pending.crate, + owner=resolved_owner, + fqn_override=f"{resolved_owner}.{method.name}", + references=method.references, + ) + + if pending.trait_path: + trait_path = expand_alias(pending.trait_path, pending.imports) + trait_target = resolve(trait_path, package, pending.crate) + if trait_target and entities[trait_target].kind == "trait": + trait_name = "::".join(pending.trait_path) + if trait_name not in entities[resolved_owner].interfaces: + entities[resolved_owner].interfaces.append(trait_name) + edges.append(Edge(resolved_owner, trait_target, "implements")) + + for fqn, entity in entities.items(): + refs = entity_refs.get(fqn, _References(set(), set())) + imports = entity_imports.get(fqn, []) + crate = entity_crates.get(fqn, ()) + + for item in imports: + if item.wildcard: + module_path = _normalize_path( + item.path, + tuple(part for part in entity.package.split(".") if part), + crate, + ) + wildcard_module = ".".join(module_path) + for ref in refs.simple: + wildcard_target = by_module_name.get((wildcard_module, ref)) + if wildcard_target: + edges.append(Edge(fqn, wildcard_target, "import")) + continue + if item.alias not in refs.simple and entity.kind != "module": + continue + import_target = resolve(item.path, entity.package, crate) + if import_target: + edges.append(Edge(fqn, import_target, "import")) + + for ref in refs.simple: + same_module_target = by_module_name.get((entity.package, ref)) + if same_module_target: + edges.append(Edge(fqn, same_module_target, "uses")) + + for qualified_path in refs.qualified: + qualified_target = resolve( + expand_alias(qualified_path, imports), entity.package, crate + ) + if qualified_target: + edges.append(Edge(fqn, qualified_target, "uses")) + + return DependencyGraph( + entities=entities, + edges=_deduplicate(edges), + packages=packages, + ) diff --git a/tests/fixtures/rust_workspace/Cargo.toml b/tests/fixtures/rust_workspace/Cargo.toml new file mode 100644 index 0000000..d8748ea --- /dev/null +++ b/tests/fixtures/rust_workspace/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] +members = ["crates/alpha", "crates/beta"] + +[package] +name = "workspace-root" +version = "0.1.0" +edition = "2021" diff --git a/tests/fixtures/rust_workspace/crates/alpha/Cargo.toml b/tests/fixtures/rust_workspace/crates/alpha/Cargo.toml new file mode 100644 index 0000000..f0c18cc --- /dev/null +++ b/tests/fixtures/rust_workspace/crates/alpha/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "alpha" +version = "0.1.0" +edition = "2021" diff --git a/tests/fixtures/rust_workspace/crates/alpha/src/lib.rs b/tests/fixtures/rust_workspace/crates/alpha/src/lib.rs new file mode 100644 index 0000000..ed28ecb --- /dev/null +++ b/tests/fixtures/rust_workspace/crates/alpha/src/lib.rs @@ -0,0 +1,13 @@ +//! First workspace member. + +pub trait Greeter { + fn greet(&self) -> String; +} + +pub struct AlphaGreeter; + +impl Greeter for AlphaGreeter { + fn greet(&self) -> String { + "alpha".to_string() + } +} diff --git a/tests/fixtures/rust_workspace/crates/beta/Cargo.toml b/tests/fixtures/rust_workspace/crates/beta/Cargo.toml new file mode 100644 index 0000000..4ccd1a5 --- /dev/null +++ b/tests/fixtures/rust_workspace/crates/beta/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "beta" +version = "0.1.0" +edition = "2021" + +[dependencies] +alpha = { path = "../alpha" } diff --git a/tests/fixtures/rust_workspace/crates/beta/src/lib.rs b/tests/fixtures/rust_workspace/crates/beta/src/lib.rs new file mode 100644 index 0000000..275befb --- /dev/null +++ b/tests/fixtures/rust_workspace/crates/beta/src/lib.rs @@ -0,0 +1,7 @@ +//! Second workspace member, depending on the first. + +use alpha::AlphaGreeter; + +pub struct BetaClient { + greeter: AlphaGreeter, +} diff --git a/tests/fixtures/rust_workspace/src/lib.rs b/tests/fixtures/rust_workspace/src/lib.rs new file mode 100644 index 0000000..e802124 --- /dev/null +++ b/tests/fixtures/rust_workspace/src/lib.rs @@ -0,0 +1,11 @@ +//! Root crate of the workspace. + +pub struct RootService { + pub name: String, +} + +impl RootService { + pub fn new(name: String) -> Self { + Self { name } + } +} diff --git a/tests/test_parsers/test_rust.py b/tests/test_parsers/test_rust.py new file mode 100644 index 0000000..4f1a247 --- /dev/null +++ b/tests/test_parsers/test_rust.py @@ -0,0 +1,652 @@ +"""Tests for the Rust parser.""" + +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_rust") +import arcade_agent.parsers.rust as rust_parser # noqa: E402 +from arcade_agent.parsers.rust import RustParser # noqa: E402 +from arcade_agent.tools.ingest import ingest # noqa: E402 +from arcade_agent.tools.parse import parse # noqa: E402 + + +def _project(tmp_path): + (tmp_path / "models.rs").write_text( + "pub struct User { pub id: u64 }\n" + "pub enum Role { Admin, Member }\n" + "pub struct Error;\n" + "pub struct T;\n" + "pub type UserId = u64;\n" + ) + (tmp_path / "repository.rs").write_text( + "use crate::models::User;\n" + "pub trait Resource {}\n" + "pub trait Repository: Resource {\n" + " fn find(&self, id: u64) -> Option;\n" + "}\n" + "pub struct Store;\n" + "impl Repository for Store {\n" + " fn find(&self, id: u64) -> Option { None }\n" + "}\n" + ) + (tmp_path / "service").mkdir() + (tmp_path / "service" / "mod.rs").write_text( + "use crate::models::{Role, User};\n" + "use crate::repository::Repository as Repo;\n" + "pub struct UserService { repo: R }\n" + "pub struct MemoryRepository;\n" + "impl Repo for MemoryRepository {\n" + " fn find(&self, id: u64) -> Option { None }\n" + "}\n" + "impl UserService {\n" + " pub fn get(&self, id: u64) -> Option { self.repo.find(id) }\n" + " pub fn role(&self) -> Role { Role::Member }\n" + "}\n" + ) + (tmp_path / "lib.rs").write_text( + "pub mod models;\n" + "pub mod repository;\n" + "pub mod service;\n" + "pub mod impls;\n" + "pub use service::UserService;\n" + ) + (tmp_path / "impls.rs").write_text( + "use crate::models::User as Account;\n" + "pub trait DisplayAccount { fn display(&self); }\n" + "impl DisplayAccount for Account { fn display(&self) {} }\n" + "pub trait ExternalOwnerHook { fn hook(&self); }\n" + "impl ExternalOwnerHook for std::io::Error { fn hook(&self) {} }\n" + "pub trait Forward { fn forward(&self); }\n" + "impl<'a, T> Forward for &'a mut T { fn forward(&self) {} }\n" + "#[cfg(unix)] impl Account { fn platform(&self) {} }\n" + "#[cfg(windows)] impl Account { fn platform(&self) {} }\n" + ) + return sorted(tmp_path.rglob("*.rs")) + + +def test_rust_parser_properties(): + parser = RustParser() + assert parser.language == "rust" + assert parser.file_extensions == [".rs"] + + +def test_rust_parser_extracts_types_functions_and_methods(tmp_path): + graph = RustParser().parse(_project(tmp_path), tmp_path) + + assert graph.entities["models.User"].kind == "struct" + assert graph.entities["models.Role"].kind == "enum" + assert graph.entities["models.UserId"].kind == "type" + assert graph.entities["repository.Repository"].kind == "trait" + assert graph.entities["repository.Repository.find"].kind == "method" + assert graph.entities["repository.Store.find"].properties["owner"] == "repository.Store" + assert graph.entities["service.UserService.get"].kind == "method" + assert graph.entities["service.UserService.get"].language == "rust" + + +def test_rust_parser_uses_rust_file_module_conventions(tmp_path): + graph = RustParser().parse(_project(tmp_path), tmp_path) + + assert "models" in graph.packages + assert "repository" in graph.packages + assert "service" in graph.packages + assert graph.entities["service.UserService"].file_path == "service/mod.rs" + assert "lib" in graph.entities # module-only crate root remains visible + + +def test_rust_parser_resolves_imports_references_and_trait_impls(tmp_path): + graph = RustParser().parse(_project(tmp_path), tmp_path) + edges = {(edge.source, edge.target, edge.relation) for edge in graph.edges} + + assert ("service.UserService.get", "models.User", "import") in edges + assert ("service.UserService.role", "models.Role", "import") in edges + assert ("repository.Store", "repository.Repository", "implements") in edges + assert ("service.MemoryRepository", "repository.Repository", "implements") in edges + assert ("repository.Repository", "repository.Resource", "extends") in edges + assert ("repository.Repository.find", "models.User", "import") in edges + + +def test_rust_parser_handles_inline_modules_and_empty_input(tmp_path): + source = tmp_path / "lib.rs" + source.write_text( + "mod internal {\n" + " pub struct Config;\n" + " impl Config { pub fn load() -> Self { Self } }\n" + "}\n" + ) + + graph = RustParser().parse([source], tmp_path) + assert "internal.Config" in graph.entities + assert "internal.Config.load" in graph.entities + + empty = RustParser().parse([], tmp_path) + assert empty.num_entities == 0 + assert empty.num_edges == 0 + + +@pytest.mark.parametrize( + "poisoned_source", + [ + "pub type Poison = " + "::".join(["a"] * 1_200 + ["T"]) + ";", + "use " + "a::{" * 1_200 + "T" + "}" * 1_200 + ";", + "mod nested {" * 1_200 + "pub struct Deep;" + "}" * 1_200, + "struct R; trait Marker {} impl Marker for " + "&" * 1_200 + "R {}", + "struct R; trait Marker {} impl Marker for " + + "(" * 1_200 + + "R" + + ")" * 1_200 + + " {}", + ], + ids=[ + "qualified-path", + "nested-use", + "inline-modules", + "wrapped-type", + "parenthesized-type", + ], +) +def test_rust_parser_handles_deep_ast_without_losing_sibling_files(tmp_path, poisoned_source): + """Machine-generated nesting in one file must not abort full analysis.""" + poisoned = tmp_path / "poisoned.rs" + poisoned.write_text(poisoned_source) + valid = tmp_path / "valid.rs" + valid.write_text("pub struct Survives;\n") + + graph = RustParser().parse([poisoned, valid], tmp_path) + assert "valid.Survives" in graph.entities + + +def test_rust_parser_discards_partial_state_when_one_file_fails(tmp_path, monkeypatch): + poisoned = tmp_path / "poisoned.rs" + poisoned.write_text("pub struct Poisoned;\n") + valid = tmp_path / "valid.rs" + valid.write_text("pub struct Survives;\n") + original_extract_imports = rust_parser._extract_imports + + def fail_for_poisoned_file(container, exclude_tests): + if b"Poisoned" in container.text: + raise RuntimeError("synthetic extraction failure") + return original_extract_imports(container, exclude_tests) + + monkeypatch.setattr(rust_parser, "_extract_imports", fail_for_poisoned_file) + graph = RustParser().parse([poisoned, valid], tmp_path) + + assert "poisoned.Poisoned" not in graph.entities + assert "valid.Survives" in graph.entities + + +_CFG_TEST_SOURCE = ( + "pub struct Production;\n" + "#[cfg(test)]\n" + "mod tests {\n" + " struct Fixture;\n" + " fn helper() {}\n" + "}\n" + "#[cfg(not(test))]\n" + "mod runtime { pub struct Included; }\n" +) + + +def test_rust_parser_skips_cfg_test_inline_modules(tmp_path): + source = tmp_path / "lib.rs" + source.write_text(_CFG_TEST_SOURCE) + + graph = RustParser().parse([source], tmp_path) + assert "Production" in graph.entities + assert "runtime.Included" in graph.entities + assert all(not fqn.startswith("tests") for fqn in graph.entities) + assert "tests" not in graph.packages + + +def test_rust_parser_keeps_cfg_test_modules_when_tests_are_not_excluded(tmp_path): + source = tmp_path / "lib.rs" + source.write_text(_CFG_TEST_SOURCE) + + graph = RustParser(exclude_tests=False).parse([source], tmp_path) + assert "Production" in graph.entities + assert "runtime.Included" in graph.entities + assert "tests.Fixture" in graph.entities + assert "tests.helper" in graph.entities + + +def test_rust_parser_skips_nested_items_under_cfg_test_module(tmp_path): + """Everything below a #[cfg(test)] module is dropped, not just its head.""" + source = tmp_path / "lib.rs" + source.write_text( + "pub struct Production;\n" + "#[cfg(test)]\n" + "mod tests {\n" + " mod inner {\n" + " pub struct DeepFixture;\n" + " }\n" + " impl super::Production {\n" + " fn only_for_tests(&self) {}\n" + " }\n" + "}\n" + ) + + graph = RustParser().parse([source], tmp_path) + assert "Production" in graph.entities + assert all("Fixture" not in fqn for fqn in graph.entities) + assert "Production.only_for_tests" not in graph.entities + + +def test_parse_tool_threads_exclude_tests_to_the_rust_parser(tmp_path): + source = tmp_path / "lib.rs" + source.write_text(_CFG_TEST_SOURCE) + + excluded = parse(str(tmp_path), language="rust", use_cache=False) + included = parse(str(tmp_path), language="rust", use_cache=False, exclude_tests=False) + + assert "tests.Fixture" not in excluded.entities + assert "tests.Fixture" in included.entities + + +def test_rust_parser_does_not_silently_drop_large_files(tmp_path): + """No parser caps input size; a >1 MB crate file must still be extracted.""" + # Bulk is comments so the file crosses 1 MB without a huge entity count. + filler = "// {}\n".format("padding " * 12) * 12_000 + source = tmp_path / "lib.rs" + source.write_text(f"pub struct Head;\n{filler}pub struct Tail;\n") + assert source.stat().st_size > 1_000_000 + + graph = RustParser().parse([source], tmp_path) + assert "Head" in graph.entities + assert "Tail" in graph.entities + + +def test_rust_parser_skips_cfg_test_with_comment_between_attribute_and_item(tmp_path): + """A comment between #[cfg(test)] and the item must not break exclusion.""" + source = tmp_path / "lib.rs" + source.write_text( + "pub struct Production;\n" + "#[cfg(test)]\n" + "// unit tests for this module\n" + "mod tests {\n" + " struct Fixture;\n" + " fn helper() {}\n" + "}\n" + "#[cfg(test)]\n" + "/// Doc comment should also not break exclusion\n" + "mod doc_tests {\n" + " struct DocFixture;\n" + "}\n" + ) + + graph = RustParser().parse([source], tmp_path) + assert "Production" in graph.entities + assert all(not fqn.startswith("tests") for fqn in graph.entities) + assert all(not fqn.startswith("doc_tests") for fqn in graph.entities) + + +def test_rust_parser_skips_cfg_test_on_non_mod_items(tmp_path): + """#[cfg(test)] on functions, structs, and impls must also be excluded.""" + source = tmp_path / "lib.rs" + source.write_text( + "pub struct Production;\n" + "#[cfg(test)]\n" + "fn test_only_helper() {}\n" + "#[cfg(test)]\n" + "struct TestFixture { x: u64 }\n" + "#[cfg(test)]\n" + "impl TestFixture { fn setup(&self) {} }\n" + "pub fn real_function() {}\n" + ) + + graph = RustParser().parse([source], tmp_path) + assert "Production" in graph.entities + assert "real_function" in graph.entities + assert "test_only_helper" not in graph.entities + assert all("TestFixture" not in fqn for fqn in graph.entities) + + +def test_rust_parser_handles_large_files_without_cap(tmp_path): + """Files larger than 1MB must still be parsed (no silent cap).""" + source = tmp_path / "large.rs" + # Generate a file > 1MB with valid Rust content + padding = "// padding\n" * 100_000 # ~1.1MB of comments + source.write_text(padding + "pub struct LargeFile;\n") + + graph = RustParser().parse([source], tmp_path) + assert "large.LargeFile" in graph.entities + + +def test_rust_parser_tolerates_invalid_cargo_manifest_encoding(tmp_path): + (tmp_path / "Cargo.toml").write_bytes(b"\xff\xfe") + source = tmp_path / "lib.rs" + source.write_text("pub struct StillParsed;\n") + + repo = ingest(str(tmp_path), language="rust") + graph = RustParser().parse([source], tmp_path) + + assert repo.source_files == [source] + assert "StillParsed" in graph.entities + + +def test_rust_parser_handles_ripgrep_impl_owner_regressions(tmp_path): + # Reduced from ripgrep 227381d: sibling impls from globset/serde_impl.rs, + # external owners from matcher/src/lib.rs, and reference blanket impls from + # ignore/src/walk.rs and searcher/src/sink.rs. + graph = RustParser().parse(_project(tmp_path), tmp_path) + edges = {(edge.source, edge.target, edge.relation) for edge in graph.edges} + + display = graph.entities["models.User.display"] + assert display.properties["owner"] == "models.User" + assert display.file_path == "impls.rs" + assert ("models.User", "impls.DisplayAccount", "implements") in edges + + # External and generic blanket impls must not manufacture local owners. + assert "std.io.Error.hook" not in graph.entities + assert "models.Error.hook" not in graph.entities + assert "impls.a.forward" not in graph.entities + assert "impls.T.forward" not in graph.entities + assert "models.T.forward" not in graph.entities + assert all( + entity.properties["owner"] in graph.entities + for entity in graph.entities.values() + if entity.kind == "method" + ) + + # Mutually exclusive cfg impls collapse to one graph method without + # duplicating package membership. + assert "models.User.platform" in graph.entities + assert all(len(fqns) == len(set(fqns)) for fqns in graph.packages.values()) + + +def test_rust_parser_handles_union_raw_identifiers_and_function_modifiers(tmp_path): + source = tmp_path / "advanced.rs" + source.write_text( + "pub union Payload { integer: u64, float: f64 }\n" + "pub struct r#type;\n" + "pub async fn fetch() {}\n" + "pub unsafe fn unchecked() {}\n" + 'pub extern "C" fn exported() {}\n' + ) + + graph = RustParser().parse([source], tmp_path) + assert graph.entities["advanced.Payload"].kind == "union" + assert graph.entities["advanced.type"].kind == "struct" + assert graph.entities["advanced.fetch"].kind == "function" + assert graph.entities["advanced.unchecked"].kind == "function" + assert graph.entities["advanced.exported"].kind == "function" + + +def test_rust_parser_resolves_super_glob_imports_in_inline_modules(tmp_path): + source = tmp_path / "lib.rs" + source.write_text( + "mod models { pub struct Config; }\n" + "mod service {\n" + " use super::models::*;\n" + " pub fn load(_: Config) {}\n" + "}\n" + ) + + graph = RustParser().parse([source], tmp_path) + edges = {(edge.source, edge.target, edge.relation) for edge in graph.edges} + assert ("service.load", "models.Config", "import") in edges + + +def test_rust_is_auto_detected_by_ingest_and_parse(tmp_path): + files = _project(tmp_path) + + repo = ingest(str(tmp_path)) + assert repo.language == "rust" + assert repo.source_files == files + + graph = parse(str(tmp_path), use_cache=False) + assert "models.User" in graph.entities + + +def test_rust_cargo_workspace_keeps_member_crates_and_crate_paths(tmp_path): + (tmp_path / "Cargo.toml").write_text('[workspace]\nmembers = ["app", "worker"]\n') + for crate in ("app", "worker"): + source = tmp_path / crate / "src" + source.mkdir(parents=True) + (tmp_path / crate / "Cargo.toml").write_text( + f'[package]\nname = "{crate}"\nversion = "0.1.0"\n' + ) + (tmp_path / "worker" / "src" / "lib.rs").write_text("pub struct Worker;\n") + (tmp_path / "app" / "src" / "lib.rs").write_text( + "use worker::Worker;\npub struct App { worker: Worker }\n" + ) + + repo = ingest(str(tmp_path), language="rust") + assert repo.path == tmp_path + assert len(repo.source_files) == 2 + + graph = parse( + str(repo.path), + language="rust", + files=[str(path) for path in repo.source_files], + use_cache=False, + ) + assert "app.App" in graph.entities + assert "worker.Worker" in graph.entities + assert ("app.App", "worker.Worker", "import") in { + (edge.source, edge.target, edge.relation) for edge in graph.edges + } + + +def test_rust_direct_parse_uses_single_crate_src_as_module_root(tmp_path): + (tmp_path / "Cargo.toml").write_text('[package]\nname = "single-crate"\nversion = "0.1.0"\n') + source = tmp_path / "src" + source.mkdir() + (source / "lib.rs").write_text("pub struct RootType;\n") + (source / "service.rs").write_text("pub struct Service;\n") + + graph = parse(str(tmp_path), language="rust", use_cache=False) + assert "RootType" in graph.entities + assert "service.Service" in graph.entities + assert "src.RootType" not in graph.entities + + +def test_rust_parser_skips_compound_cfg_test_predicates(tmp_path): + """all(test, ...) / any(test, ...) gate test-only code just like cfg(test).""" + source = tmp_path / "lib.rs" + source.write_text( + "pub struct Production;\n" + '#[cfg(all(test, feature = "slow"))]\n' + "mod all_tests {\n" + " struct AllFixture;\n" + " fn all_helper() {}\n" + "}\n" + "#[cfg(any(test, fuzzing))]\n" + "struct AnyFixture;\n" + "#[cfg(all(unix, any(test, miri)))]\n" + "fn nested_test_helper() {}\n" + ) + + graph = RustParser().parse([source], tmp_path) + + assert "Production" in graph.entities + assert all("Fixture" not in fqn for fqn in graph.entities) + assert "all_tests.all_helper" not in graph.entities + assert "nested_test_helper" not in graph.entities + + +def test_rust_parser_keeps_cfg_not_test_items(tmp_path): + """#[cfg(not(test))] marks production-only code and must survive.""" + source = tmp_path / "lib.rs" + source.write_text( + "#[cfg(not(test))]\n" + "pub struct ProductionOnly;\n" + "#[cfg(all(not(test), unix))]\n" + "pub fn production_only_helper() {}\n" + ) + + graph = RustParser().parse([source], tmp_path) + + assert "ProductionOnly" in graph.entities + assert "production_only_helper" in graph.entities + + +def test_rust_parser_skips_files_gated_by_inner_cfg_test(tmp_path): + """A file whose top is #![cfg(test)] contributes nothing to the graph.""" + source = tmp_path / "lib.rs" + source.write_text( + "#![cfg(test)]\n" + "pub struct Fixture;\n" + "pub fn helper() {}\n" + "mod nested { pub struct NestedFixture; }\n" + ) + + assert RustParser().parse([source], tmp_path).entities == {} + + included = RustParser(exclude_tests=False).parse([source], tmp_path) + assert "Fixture" in included.entities + + +def test_rust_parser_skips_module_bodies_gated_by_inner_cfg_test(tmp_path): + source = tmp_path / "lib.rs" + source.write_text( + "pub struct Production;\n" + "mod harness {\n" + " #![cfg(test)]\n" + " pub struct Fixture;\n" + " pub fn helper() {}\n" + "}\n" + ) + + graph = RustParser().parse([source], tmp_path) + + assert "Production" in graph.entities + assert "harness.Fixture" not in graph.entities + assert "harness.helper" not in graph.entities + + +def test_rust_parser_skips_files_behind_out_of_line_cfg_test_modules(tmp_path): + """#[cfg(test)] mod helpers; must also exclude the helpers.rs file itself.""" + (tmp_path / "Cargo.toml").write_text('[package]\nname = "probe"\nversion = "0.1.0"\n') + source = tmp_path / "src" + source.mkdir() + (source / "lib.rs").write_text( + "pub struct Production;\n" + "#[cfg(test)]\n" + "mod helpers;\n" + "#[cfg(test)]\n" + "mod deep;\n" + "pub mod real;\n" + ) + (source / "helpers.rs").write_text("pub struct HelperFixture;\n") + (source / "deep").mkdir() + (source / "deep" / "mod.rs").write_text("pub struct DeepFixture;\n") + (source / "real.rs").write_text("pub struct RealThing;\n") + + graph = parse(str(tmp_path), language="rust", use_cache=False) + + assert "Production" in graph.entities + assert "real.RealThing" in graph.entities + assert all("Fixture" not in fqn for fqn in graph.entities) + + included = parse(str(tmp_path), language="rust", use_cache=False, exclude_tests=False) + assert "helpers.HelperFixture" in included.entities + assert "deep.DeepFixture" in included.entities + + +def test_rust_parser_skips_nested_out_of_line_cfg_test_modules(tmp_path): + """The excluded path is relative to the declaring module, not the crate root.""" + (tmp_path / "Cargo.toml").write_text('[package]\nname = "probe"\nversion = "0.1.0"\n') + source = tmp_path / "src" + (source / "engine").mkdir(parents=True) + (source / "lib.rs").write_text("pub mod engine;\n") + (source / "engine" / "mod.rs").write_text( + "pub struct Engine;\n#[cfg(test)]\nmod fixtures;\n" + ) + (source / "engine" / "fixtures.rs").write_text("pub struct EngineFixture;\n") + + graph = parse(str(tmp_path), language="rust", use_cache=False) + + assert "engine.Engine" in graph.entities + assert all("Fixture" not in fqn for fqn in graph.entities) + + +def test_rust_parser_drops_cfg_test_use_declarations(tmp_path): + """Dev-dependency imports must not attach to production entities.""" + source = tmp_path / "lib.rs" + source.write_text( + "use std::collections::HashMap;\n" + "#[cfg(test)]\n" + "use mockall::predicate::Eq;\n" + "#[cfg(test)]\n" + "use proptest::prelude::*;\n" + "pub struct Prod { map: HashMap }\n" + ) + + graph = RustParser().parse([source], tmp_path) + + assert graph.entities["Prod"].imports == ["std::collections::HashMap"] + + included = RustParser(exclude_tests=False).parse([source], tmp_path) + assert "mockall::predicate::Eq" in included.entities["Prod"].imports + + +def test_rust_parser_leaks_no_test_entities_across_cfg_test_shapes(tmp_path): + """End-to-end probe: every recognized #[cfg(test)] shape in one crate.""" + (tmp_path / "Cargo.toml").write_text('[package]\nname = "probe"\nversion = "0.1.0"\n') + source = tmp_path / "src" + source.mkdir() + (source / "lib.rs").write_text( + "use std::collections::HashMap;\n" + "#[cfg(test)]\n" + "use mockall::predicate::Eq;\n" + "pub struct Prod { map: HashMap }\n" + "#[cfg(test)]\n" + "mod tests { struct Fixture; fn helper() {} }\n" + '#[cfg(all(test, feature = "x"))]\n' + "mod all_tests { struct AllFixture; fn all_helper() {} }\n" + "#[cfg(any(test, fuzzing))]\n" + "struct AnyFixture;\n" + "#[cfg(test)]\n" + "mod outofline;\n" + "mod inner_test_file;\n" + ) + (source / "outofline.rs").write_text( + "pub struct OutOfLineFixture;\npub fn out_of_line_helper() {}\n" + ) + (source / "inner_test_file.rs").write_text( + "#![cfg(test)]\npub struct InnerFixture;\npub fn inner_helper() {}\n" + ) + + graph = parse(str(tmp_path), language="rust", use_cache=False) + + assert set(graph.entities) == {"Prod"} + assert graph.entities["Prod"].imports == ["std::collections::HashMap"] + + +def test_rust_parser_lists_each_package_entity_once(tmp_path): + """The package index is de-duplicated (via a set, not a linear scan).""" + source = tmp_path / "lib.rs" + source.write_text( + "pub struct Repo;\n" + "impl Repo { pub fn load(&self) {} pub fn save(&self) {} }\n" + "impl Repo { pub fn load(&self) {} }\n" + ) + + graph = RustParser().parse([source], tmp_path) + + for package, fqns in graph.packages.items(): + assert len(fqns) == len(set(fqns)), package + + +def test_rust_workspace_fixture_ingests_every_member_crate(): + """A Cargo [workspace] root must not be narrowed to its own src/ directory.""" + workspace = Path(__file__).resolve().parents[1] / "fixtures" / "rust_workspace" + + repo = ingest(str(workspace), language="rust") + + assert repo.path == workspace + assert sorted(str(path.relative_to(workspace)) for path in repo.source_files) == [ + "crates/alpha/src/lib.rs", + "crates/beta/src/lib.rs", + "src/lib.rs", + ] + + graph = parse( + str(repo.path), + language="rust", + files=[str(path) for path in repo.source_files], + use_cache=False, + ) + assert "workspace_root.RootService" in graph.entities + assert "alpha.AlphaGreeter" in graph.entities + assert "beta.BetaClient" in graph.entities From d933608f435893387ce9be3ce24cc7ae900b0fb1 Mon Sep 17 00:00:00 2001 From: Duc Le Date: Mon, 10 Aug 2026 23:12:20 +0700 Subject: [PATCH 2/3] fix(source): honor exclude_tests in parsers, cache keys, and Cargo workspaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three wiring gaps kept the Rust parser's inline test exclusion from ever taking effect through the public tools. `exclude_tests` never reached a parser. It only drove *path* filtering during discovery, so `_parse_one` left every parser on its default and inline `#[cfg(test)]` exclusion was dead on arrival. `get_parser` returns a fresh instance per call, so setting the flag there cannot leak between calls. `analyze` now forwards its own `exclude_tests` to `parse` as well. `cache_key` ignored `exclude_tests`. Inline exclusion changes the graph for an *identical* file list, so the explicit `files=` path could return a cached graph of the wrong shape. The flag now takes part in the key, `.rs` joins the tracked suffixes, and Cargo manifests are hashed for Rust parses — crate names and module layout come from `Cargo.toml`, which no `.rs` mtime reflects. The manifest probe matches composite language keys such as `rust|`, which is what `source.parse` actually passes. `_detect_source_root` narrowed Cargo workspaces to the root crate's `src`. On the new three-crate fixture that ingested 1 of 3 files; the `[workspace]` probe now runs before the generic source-root candidates and returns the workspace root, ingesting all 3. Co-Authored-By: Tony Nguyen Co-Authored-By: Claude Opus 5 (1M context) --- src/arcade_agent/cache.py | 40 ++++++++++++++++++++------ src/arcade_agent/source/ingest.py | 22 +++++++++++++- src/arcade_agent/source/parse.py | 20 +++++++++---- src/arcade_agent/tools/analyze.py | 1 + tests/test_cache.py | 48 +++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 14 deletions(-) diff --git a/src/arcade_agent/cache.py b/src/arcade_agent/cache.py index 91d222f..f5d492d 100644 --- a/src/arcade_agent/cache.py +++ b/src/arcade_agent/cache.py @@ -20,7 +20,18 @@ def _cache_dir(project_root: Path) -> Path: return project_root / _CACHE_DIR -def cache_key(source_path: str, language: str | None, files: list[str] | None) -> str: +_SOURCE_SUFFIXES = { + ".java", ".py", ".c", ".cpp", ".h", ".hpp", ".ts", ".tsx", ".js", ".jsx", + ".go", ".kt", ".kts", ".rs", +} + + +def cache_key( + source_path: str, + language: str | None, + files: list[str] | None, + exclude_tests: bool = True, +) -> str: """Compute a cache key from source path, language, and file mtimes. The key is a SHA-256 hash of the sorted file paths and their modification @@ -31,6 +42,9 @@ def cache_key(source_path: str, language: str | None, files: list[str] | None) - source_path: Root directory of the project. language: Language being parsed (or None for auto-detect). files: Specific files to parse, or None to discover all. + exclude_tests: Whether inline test code is excluded. Parsers that honor + it (Rust) produce a different graph for the same file list, so it + must take part in the key. Returns: A hex digest string usable as a cache filename. @@ -39,17 +53,27 @@ def cache_key(source_path: str, language: str | None, files: list[str] | None) - hasher = hashlib.sha256() hasher.update(str(root).encode()) hasher.update((language or "auto").encode()) + hasher.update(b"tests:excluded" if exclude_tests else b"tests:included") if files: - file_paths = sorted(files) + file_paths = set(files) else: # Hash all source-like files under root - file_paths = sorted(str(f) for f in root.rglob("*") if f.is_file() and f.suffix in { - ".java", ".py", ".c", ".cpp", ".h", ".hpp", ".ts", ".tsx", ".js", ".jsx", - ".go", ".kt", ".kts", - }) - - for fp in file_paths: + file_paths = { + str(f) for f in root.rglob("*") if f.is_file() and f.suffix in _SOURCE_SUFFIXES + } + + # Rust module layout and crate names come from Cargo manifests, so editing + # one changes the graph without touching any .rs file. + tracks_rust = language is None or "rust" in language or any( + fp.endswith(".rs") for fp in file_paths + ) + if tracks_rust: + file_paths.update( + str(manifest) for manifest in root.rglob("Cargo.toml") if manifest.is_file() + ) + + for fp in sorted(file_paths): p = Path(fp) hasher.update(fp.encode()) if p.exists(): diff --git a/src/arcade_agent/source/ingest.py b/src/arcade_agent/source/ingest.py index 248fd30..7b6c71a 100644 --- a/src/arcade_agent/source/ingest.py +++ b/src/arcade_agent/source/ingest.py @@ -5,6 +5,7 @@ import shutil import subprocess import tempfile +import tomllib from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING @@ -60,6 +61,7 @@ def cleanup(self) -> None: "c": [".c", ".h", ".cpp", ".hpp", ".cc", ".cxx"], "go": [".go"], "kotlin": [".kt", ".kts"], + "rust": [".rs"], } # Reverse mapping @@ -135,12 +137,30 @@ def _detect_languages( return sorted(found) +def _is_cargo_workspace(path: Path) -> bool: + """Whether *path* holds a Cargo.toml declaring a ``[workspace]``.""" + manifest = path / "Cargo.toml" + try: + if not manifest.is_file(): + return False + with manifest.open("rb") as manifest_file: + return "workspace" in tomllib.load(manifest_file) + except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError): + return False + + def _detect_source_root(path: Path, language: str | None = None) -> Path: """Detect the main source root directory. Prefers a language-specific Maven/Gradle root when *language* is set. Falls back to well-known roots, then the project root. """ + # A Cargo workspace holds a root crate plus member crates under arbitrary + # directories. Narrowing it to the root's ``src`` would silently drop every + # member, so this probe must run before the generic source-root candidates. + if language == "rust" and _is_cargo_workspace(path): + return path + if language: preferred = _LANG_PREFERRED_ROOTS.get(language) if preferred and (path / preferred).is_dir(): @@ -327,7 +347,7 @@ def ingest( Args: source: Git repo URL or local directory path. language: Override language detection (java, python, typescript, c, go, - kotlin, or "multi" to ingest every detected language). + kotlin, rust, or "multi" to ingest every detected language). languages: Explicit language list for polyglot ingest (e.g. ["java", "kotlin"]). Mutually exclusive with *language*. work_dir: Directory to clone into. Uses temp dir if None. Ignored when diff --git a/src/arcade_agent/source/parse.py b/src/arcade_agent/source/parse.py index e9b2cc8..ef97ab5 100644 --- a/src/arcade_agent/source/parse.py +++ b/src/arcade_agent/source/parse.py @@ -105,8 +105,13 @@ def _parse_one( file_paths: list[Path], root: Path, use_cache: bool, + exclude_tests: bool = True, ) -> DependencyGraph: + # ``get_parser`` returns a fresh instance per call, so setting the flag here + # cannot leak across languages or calls. Path-based filtering alone cannot + # reach inline test code such as Rust's ``#[cfg(test)] mod tests``. parser = get_parser(language) + parser.exclude_tests = exclude_tests if not file_paths: return DependencyGraph() if use_cache and hasattr(parser, "parse_incremental"): @@ -172,7 +177,9 @@ def parse( explicit list is authoritative and is not filtered. use_cache: If True, return cached results when source files haven't changed. exclude_tests: Exclude test/vendor/build directories during automatic - file discovery (default: True). + file discovery (default: True). Also passed to the parser, so + parsers that recognize *inline* test constructs (Rust's + ``#[cfg(test)]``) leave them out of the graph as well. exclude_dirs: Additional exact project-relative directories to exclude during automatic discovery. Explicit files are always honored. @@ -203,7 +210,7 @@ def parse( cache_lang = f"{cache_lang or 'auto'}|{exclusion_namespace}" if use_cache: - key = cache_key(source_path, cache_lang, files) + key = cache_key(source_path, cache_lang, files, exclude_tests) cached = get_cached_graph(source_path, key) if cached is not None: return cached @@ -233,16 +240,19 @@ def parse( ) if len(resolved) == 1: - graph = _parse_one(resolved[0], per_language[resolved[0]], root, use_cache) + graph = _parse_one( + resolved[0], per_language[resolved[0]], root, use_cache, exclude_tests + ) else: graphs = [ - _parse_one(lang, per_language[lang], root, use_cache) for lang in resolved + _parse_one(lang, per_language[lang], root, use_cache, exclude_tests) + for lang in resolved ] graphs = [g for g in graphs if g.num_entities or g.num_edges] graph = merge_and_relink(*graphs) if graphs else DependencyGraph() if use_cache: - key = cache_key(source_path, cache_lang, files) + key = cache_key(source_path, cache_lang, files, exclude_tests) put_cached_graph(source_path, key, graph) return graph diff --git a/src/arcade_agent/tools/analyze.py b/src/arcade_agent/tools/analyze.py index e0161f3..793441a 100644 --- a/src/arcade_agent/tools/analyze.py +++ b/src/arcade_agent/tools/analyze.py @@ -90,6 +90,7 @@ def _run_sync_pipeline( language=repository.language or language, files=[str(path) for path in repository.source_files], use_cache=use_cache, + exclude_tests=exclude_tests, ) if on_stage is not None: on_stage("graph", graph) diff --git a/tests/test_cache.py b/tests/test_cache.py index 4f6906f..12c92dc 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -60,6 +60,54 @@ def test_cache_key_changes_with_file_modification(tmp_project): assert k1 != k2 +def test_cache_key_tracks_rust_source_files(tmp_project): + rust_file = tmp_project / "src" / "lib.rs" + rust_file.write_text("pub struct Before;") + k1 = cache_key(str(tmp_project), "rust", None) + rust_file.write_text("pub struct After;") + k2 = cache_key(str(tmp_project), "rust", None) + assert k1 != k2 + + +def test_cache_key_tracks_cargo_manifests_with_explicit_rust_files(tmp_project): + rust_file = tmp_project / "src" / "lib.rs" + rust_file.write_text("pub struct App;") + manifest = tmp_project / "Cargo.toml" + manifest.write_text('[package]\nname = "before"\n') + files = [str(rust_file)] + k1 = cache_key(str(tmp_project), "rust", files) + + manifest.write_text('[package]\nname = "after"\n') + newer = manifest.stat().st_mtime + 2 + os.utime(manifest, (newer, newer)) + k2 = cache_key(str(tmp_project), "rust", files) + + assert k1 != k2 + + +def test_cache_key_tracks_cargo_manifests_for_composite_language_keys(tmp_project): + """source.parse passes keys like "rust|", not a bare language.""" + rust_file = tmp_project / "src" / "lib.rs" + rust_file.write_text("pub struct App;") + manifest = tmp_project / "Cargo.toml" + manifest.write_text('[package]\nname = "before"\n') + k1 = cache_key(str(tmp_project), "rust|excl:default", None) + + manifest.write_text('[package]\nname = "after"\n') + newer = manifest.stat().st_mtime + 2 + os.utime(manifest, (newer, newer)) + k2 = cache_key(str(tmp_project), "rust|excl:default", None) + + assert k1 != k2 + + +def test_cache_key_changes_with_exclude_tests(tmp_project): + """Rust graphs differ by exclude_tests, so cached graphs must not collide.""" + k1 = cache_key(str(tmp_project), "rust", None) + k2 = cache_key(str(tmp_project), "rust", None, exclude_tests=False) + assert k1 != k2 + + def test_cache_miss_returns_none(tmp_project): result = get_cached_graph(str(tmp_project), "nonexistent_key") assert result is None From 059e68a70a98a8a45bafe5aa0d19fcbb8cbecc26 Mon Sep 17 00:00:00 2001 From: Duc Le Date: Mon, 10 Aug 2026 23:12:20 +0700 Subject: [PATCH 3/3] docs: document Rust support and the parser hardening workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Rust to the supported-language lists across the README, ROADMAP, the reusable analysis workflow, the analyze action, the MCP tool docstrings, and the self-analysis CLI help. Ports the contributor process files from the original Rust contribution, with one claim corrected: `docs/BUG_CATALOG.md` asserted that per-parser input caps "diverge from the other parsers", but `parsers/go.py` and `parsers/typescript.py` both define `_MAX_FILE_BYTES = 1_000_000`. The rule now says what it means — a cap is a legitimate performance tool for input that is not human-authored, but never a substitute for fixing the underlying algorithm. Records two new reusable failure classes: quadratic membership tests on de-duplicated ordered collections, and annotation-gated test exclusion leaking through its less common syntactic shapes. Co-Authored-By: Tony Nguyen Co-Authored-By: Claude Opus 5 (1M context) --- .../harden-tree-sitter-parsers/SKILL.md | 51 +++++++++ .../architecture-analysis-reusable.yml | 2 +- README.md | 12 ++- ROADMAP.md | 8 +- actions/analyze/action.yml | 2 +- docs/BUG_CATALOG.md | 100 ++++++++++++++++++ src/arcade_agent/ci/run_self_analysis.py | 2 +- src/arcade_agent/tools/adapters/mcp.py | 4 +- 8 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 .github/skills/harden-tree-sitter-parsers/SKILL.md create mode 100644 docs/BUG_CATALOG.md diff --git a/.github/skills/harden-tree-sitter-parsers/SKILL.md b/.github/skills/harden-tree-sitter-parsers/SKILL.md new file mode 100644 index 0000000..2e0bc05 --- /dev/null +++ b/.github/skills/harden-tree-sitter-parsers/SKILL.md @@ -0,0 +1,51 @@ +--- +name: harden-tree-sitter-parsers +description: Harden new or changed tree-sitter parsers against adversarial nesting, malformed files, partial-state leaks, and stale non-source cache inputs. +reliability: validated-2x +--- + +# Harden Tree-sitter Parsers + +Use this skill for new parser implementations, parser reviews, recursion failures, +or changes to AST traversal and linking. + +## Required workflow + +1. Inventory every AST traversal helper and classify it as iterative or recursive. +2. Replace source-depth recursion with an explicit stack or queue. Preserve traversal + order deliberately and avoid repeated tuple/list copying where practical. +3. Extract each file into isolated temporary state. Merge entities, edges, imports, + packages, and pending links only after the file succeeds. +4. Log skipped files with the failure class; do not silently discard valid siblings. +5. Add adversarial fixtures deeper than `sys.getrecursionlimit()` for every distinct + traversal shape. Each fixture must be parsed beside a valid sibling file. +6. Test cache invalidation for manifests or configuration that changes graph identity. +7. Time the parser on one large generated single-package file before declaring it done. + A de-duplicated ordered collection guarded by `if x not in list` is quadratic and only + shows up at scale (see `docs/BUG_CATALOG.md` #3). +8. Run, in order: + - focused parser and cache tests; + - Ruff and the full test suite; + - a large real repository for the target language, reporting wall-clock time; + - arcade-agent self-analysis before/after, reporting metric and smell deltas. +9. Record any newly discovered reusable failure class in `docs/BUG_CATALOG.md`. + +## Acceptance invariants + +- No `RecursionError` for valid tree-sitter AST depth within the configured file limit. +- One malformed or adversarial file cannot erase healthy sibling entities. +- No partial entities from a failed file enter the final graph. +- No dangling edges, missing method owners, or duplicate package membership. +- Parse time grows linearly, not quadratically, with entity count. +- Relevant non-source inputs invalidate cached graphs, and so do flags such as + `exclude_tests` that change the graph for an identical file list. +- Test exclusion covers every syntactic shape the language offers, imports included. +- Correctness and explicit failure behavior take precedence over cosmetic metric gains. + +## Evidence + +- Kotlin follow-up `b7effc5`: iterative deep-expression traversal and sibling survival. +- Rust PR #18: iterative path/use/module/type traversal, transactional file extraction, + Cargo-aware cache invalidation, and adversarial regression matrix. +- Rust reland: linear package membership (70.2 s -> 3.0 s on a 5.2 MB generated file, with + identical entity and edge counts) and full `#[cfg(test)]` shape coverage. diff --git a/.github/workflows/architecture-analysis-reusable.yml b/.github/workflows/architecture-analysis-reusable.yml index 47540df..996fdc0 100644 --- a/.github/workflows/architecture-analysis-reusable.yml +++ b/.github/workflows/architecture-analysis-reusable.yml @@ -24,7 +24,7 @@ on: type: string default: "." language: - description: Optional language override (java, python, typescript, c, go, kotlin, multi). + description: Optional language override (java, python, typescript, c, go, kotlin, rust, multi). required: false type: string default: "" diff --git a/README.md b/README.md index 54f4aba..c3ffb54 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,16 @@ smell burden, or another architectural pressure. - TypeScript/JavaScript (full support) - Go (full support) - Kotlin (structural support via optional `[languages]` extra; import + inheritance graph) +- Rust (structural support via optional `[languages]` extra; structs, enums, unions, traits, + type aliases, functions, methods, imports, qualified references, trait + inheritance/implementations, and Cargo workspaces) + +Rust unit tests live *inline* in production files, so path-based test exclusion +cannot see them. With `exclude_tests=True` (the default) the Rust parser also +drops `#[cfg(test)]` items — including `cfg(all(test, ...))` / `cfg(any(test, ...))`, +inner `#![cfg(test)]` files and module bodies, the file behind an out-of-line +`#[cfg(test)] mod helpers;`, and `#[cfg(test)] use ...` dev-dependency imports. +Pass `exclude_tests=False` to `ingest`/`parse`/`analyze` to keep them. ## Example: ARCADE Core @@ -381,7 +391,7 @@ arcade-agent ports and extends the capabilities of the original [ARCADE](https:/ | 6 quality metrics | Done | RCI, TurboMQ, BasicMQ, IntraConnectivity, InterConnectivity, TwoWayPairRatio | | Balanced architecture score | Done | Derived reporting score combining core metrics, principle signals, and smell burden | | A2A architecture comparison | Done | Hungarian algorithm on Jaccard similarity | -| Multi-language parsing | Done | Java, Python, C/C++, TypeScript/JavaScript, Go (full); Kotlin (structural); polyglot merge+relink via `languages=[...]` / `language="multi"` (cross-language edges within the JVM family only) | +| Multi-language parsing | Done | Java, Python, C/C++, TypeScript/JavaScript, Go (full); Kotlin, Rust (structural); polyglot merge+relink via `languages=[...]` / `language="multi"` (cross-language edges within the JVM family only) | | 5 export formats | Done | HTML, DOT, JSON, RSF, Mermaid | | LLM concern extraction | Done | Claude CLI for semantic BCO/SPF detection | | MCP server | Done | Expose tools to AI agents via Model Context Protocol with session store | diff --git a/ROADMAP.md b/ROADMAP.md index 893b254..0f6b4dd 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -43,7 +43,9 @@ Handle real-world polyglot monorepos. - [x] **15. TypeScript/JS parser** — Shipped in #8 (`parsers/typescript.py`). - [x] **16a. Go parser** — Shipped alongside TS/JS in #8 (`parsers/go.py`). - [x] **16a2. Kotlin parser** — Shipped (`parsers/kotlin.py`) for JVM/Kotlin-first repos (e.g. embabel-agent). -- [ ] **16b. Rust parser** — Still open. High-demand language for agent-assisted development. +- [x] **16b. Rust parser** — Shipped (`parsers/rust.py`): modules, types, traits, functions, + methods, imports, qualified references, trait relationships, Cargo workspaces, and inline + `#[cfg(test)]` exclusion. - [x] **17. Incremental parsing** — Content-hash extract cache shipped in #9 (`incremental.py`), wired for the Python parser only; extending to the other two-pass parsers is follow-up. - [x] **18. Cross-language dependency tracking** — MVP: multi-language ingest/parse (`languages=[...]` / `language="multi"`) merges per-language graphs and relinks import/extends/implements across FQN space. Relinking is **family-scoped**: the `jvm` family (Java↔Kotlin) is the supported and validated pair; every other language is its own family and is merged without cross-language edges. Extending relinking to further families (and broader RPC/IDL bridges — gRPC stubs, OpenAPI) remains follow-up. @@ -60,6 +62,6 @@ Work everywhere agents work. | Priority | Items | Rationale | |----------|-------|-----------| -| **Done** | 1–10, 12, 13, 14, 15, 16a, 16a2, 17, 18 (MVP) | Phases 1–2 + TS/JS & Go & Kotlin parsers, incremental parsing (Python), `diff_impact`, `context_for_task`, `api_surface`, polyglot merge+relink, `changelog_architecture` | -| **Now** | 11, 16b | Component ownership, Rust parser | +| **Done** | 1–10, 12, 13, 14, 15, 16a, 16a2, 16b, 17, 18 (MVP) | Phases 1–2 + TS/JS & Go & Kotlin & Rust parsers, incremental parsing (Python), `diff_impact`, `context_for_task`, `api_surface`, polyglot merge+relink, `changelog_architecture` | +| **Now** | 11 | Component ownership | | **Then** | 19–22 | Ecosystem breadth (OpenAI / LangChain / Claude SDK / IDE) | diff --git a/actions/analyze/action.yml b/actions/analyze/action.yml index 5c47400..a9eef9c 100644 --- a/actions/analyze/action.yml +++ b/actions/analyze/action.yml @@ -24,7 +24,7 @@ inputs: default: "." language: description: > - Optional language override (java, python, typescript, c, go, kotlin, or multi + Optional language override (java, python, typescript, c, go, kotlin, rust, or multi for every detected language with cross-language edge relinking). required: false default: "" diff --git a/docs/BUG_CATALOG.md b/docs/BUG_CATALOG.md new file mode 100644 index 0000000..0d02e39 --- /dev/null +++ b/docs/BUG_CATALOG.md @@ -0,0 +1,100 @@ +# Parser Robustness Bug Catalog + +Reliability: `validated-2x` + +This is the living catalog for parser failure classes that can abort or distort +whole-repository analysis. Entries use reproducible fixtures and design-time +prevention rules so the same defect is not rediscovered language by language. + +## Design-time checklist + +- Traverse untrusted AST depth with explicit stacks or queues, never Python recursion. +- Put every file extraction behind a transactional boundary: publish its entities only + after extraction succeeds. +- Test nesting deeper than `sys.getrecursionlimit()` for every traversal shape. +- Pair each poisoned input with a healthy sibling file and assert the sibling survives. +- Track non-source inputs such as manifests when they affect graph identity or cache keys. +- Do not add per-parser input caps (file size, node counts) as a stand-in for robustness: + they drop real code silently and never fix the traversal or complexity defect they appear + to mitigate. The per-file exception boundary is the backstop. Caps are a legitimate + *performance* tool for input that is genuinely not human-authored — `parsers/go.py` and + `parsers/typescript.py` both keep a 1 MB `_MAX_FILE_BYTES` for vendored and minified + bundles — but adopt one only after the underlying algorithm is linear, and say so + explicitly rather than claiming other parsers have no cap. +- Run focused tests, the full suite, a large real repository, and arcade-agent's own + self-analysis before publishing parser changes. + +## 1. Kotlin deep-expression traversal aborted repository analysis + +- **Symptom:** A machine-generated expression with thousands of nested parentheses + raised `RecursionError`; valid sibling files disappeared because parsing aborted. +- **Root cause:** Recursive AST descent treated source nesting as trusted call-stack depth. +- **Detection:** Parse a deeply nested Kotlin file beside a valid file and assert the + valid entity remains in the graph. +- **Fix:** Replace recursive descent with explicit stacks and isolate failures per file. +- **Prevention:** Apply the parser hardening skill to every new or materially changed + tree-sitter traversal. +- First encountered: Kotlin parser follow-up `b7effc5`. +- **Pattern note:** First confirmed instance of cross-language AST depth fragility. + +## 2. Rust path/use/module/type traversals repeated the recursion defect + +- **Symptom:** Roughly 1,000 nested path segments, use groups, inline modules, or type + wrappers raised `RecursionError` and killed analysis for healthy sibling files. +- **Root cause:** Four helpers used recursive descent even though `_references` already + demonstrated the safe iterative pattern; extraction also ran outside the file-level + exception boundary. +- **Detection:** Parameterize all four AST shapes above the Python recursion limit and + parse each beside a valid Rust file. +- **Fix:** Use explicit LIFO worklists, publish per-file extraction state transactionally, + and log-and-skip unexpected file-level failures. +- **Prevention:** Require the shared adversarial matrix and self-dogfood before parser PRs. +- First encountered: Rust parser PR #18 review, 2026-07-21. +- **Pattern note:** Second confirmed cross-language instance. Keep the class on the + design checklist; wait for a third instance before naming a broader meta-pattern. + +## 3. Rust package membership index was quadratic in entities per package + +- **Symptom:** A 5.2 MB generated `.rs` file (79,500 entities) took 70 s to parse. Parse + time grew with the square of the entity count, so large real crates looked like hangs. +- **Root cause:** `add_entity` guarded the per-package entity list with `if fqn not in + package_entities`, a linear scan of a list that grows to tens of thousands of entries; + the cross-file merge repeated the same `not in` test inside a generator expression. +- **Detection:** Generate one large single-package source file and time the parse; the + entity count is a fine proxy for the input size the cap was hiding. +- **Fix:** Keep a companion `set` next to each ordered list purely for membership, and + reset it wherever the list it shadows is reset (the Rust parser clears per-file state + each iteration — a stale set would leak entities across files). +- **Prevention:** For any de-duplicated *ordered* collection, pair the list with a set at + the moment it is introduced. Treat "an input cap makes this fast enough" as a signal + that a container is being scanned linearly. +- First encountered: Rust parser reland, 2026-08-10 (70.2 s → 3.0 s, 23x, identical + entity and edge counts). +- **Pattern note:** The mirror image of class 1/2 — not a crash, a silent complexity cliff + that an input cap conceals instead of fixing. + +## 4. Conditional-compilation test gating leaks through its less common shapes + +- **Symptom:** With `exclude_tests=True`, a Rust probe crate with 2 production structs and + 10 test-only entities still yielded 10 entities, 8 of them test-only. Every production + entity also carried a phantom `mockall` import, inflating fan-out and inventing coupling + to dev-only crates. +- **Root cause:** The exclusion matched one syntactic shape (`#[cfg(test)]` normalizing to + exactly that text, on an outer `attribute_item`, attached to an item with a body). Four + other shapes bypassed it: compound predicates (`cfg(all(test, ...))`, `cfg(any(test, ...))`), + inner `#![cfg(test)]` on a file or module body, out-of-line `#[cfg(test)] mod x;` whose + backing file was later parsed as independent production source, and `#[cfg(test)] use ...` + which was collected by an import pass that ran before attributes were inspected. +- **Detection:** Build one probe crate containing *every* shape and assert the production + entity set exactly, not just the absence of one fixture name. Assert on `entity.imports` + too — import leaks are invisible in entity counts. +- **Fix:** Evaluate the cfg predicate tree rather than string-matching (while leaving + `not(test)` alone, which marks production-only code), inspect inner attributes, make the + import pass attribute-aware, and record out-of-line test module paths so their files are + skipped — which requires visiting a module's declaring file before the module's own file. +- **Prevention:** When a language gates test code by annotation rather than by path, + enumerate the annotation's full grammar before implementing the filter; a single + normalized string comparison is a smell. +- First encountered: Rust parser reland, 2026-08-10. +- **Pattern note:** Applies to any annotation-gated exclusion (Go build tags, + C/C++ `#ifdef`), not only Rust. diff --git a/src/arcade_agent/ci/run_self_analysis.py b/src/arcade_agent/ci/run_self_analysis.py index f0bb6d5..cc8ec67 100644 --- a/src/arcade_agent/ci/run_self_analysis.py +++ b/src/arcade_agent/ci/run_self_analysis.py @@ -48,7 +48,7 @@ def main() -> None: "--language", default="", help=( - "Optional language override (java, python, typescript, c, go, kotlin, " + "Optional language override (java, python, typescript, c, go, kotlin, rust, " "or multi for every detected language)" ), ) diff --git a/src/arcade_agent/tools/adapters/mcp.py b/src/arcade_agent/tools/adapters/mcp.py index 91a2eda..2dda534 100644 --- a/src/arcade_agent/tools/adapters/mcp.py +++ b/src/arcade_agent/tools/adapters/mcp.py @@ -209,7 +209,7 @@ def ingest( Args: source: Git repo URL or local directory path. language: Override language detection (java, python, c, typescript, - go, kotlin, or "multi" for every detected language). + go, kotlin, rust, or "multi" for every detected language). languages: Explicit polyglot language list (e.g. ["java", "kotlin"]). Mutually exclusive with language. work_dir: Directory to clone into. Uses temp dir if None. @@ -262,7 +262,7 @@ def parse( ingest. An ingest session carries its selected files and languages into this parse call unless explicitly overridden. Its files are not re-filtered by parse-level exclusion options. - language: Language to parse (java, python, c, typescript, go, kotlin), + language: Language to parse (java, python, c, typescript, go, kotlin, rust), or "multi" to parse every detected language and relink cross-language edges. languages: Explicit polyglot language list (e.g. ["java", "kotlin"]).