diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f224..9e232f31fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,12 @@ jobs: - name: Run tests run: uv run --frozen pytest tests/ -q --tb=short + - name: Check semantic provider extension + run: | + uv run --frozen ruff check graphify_semantic_providers tests/test_semantic_provider_*.py + uv run --frozen pyright graphify_semantic_providers + uv run --frozen graphify-semantic list + - name: Verify install works end-to-end run: | uv run --frozen graphify --help @@ -99,7 +105,7 @@ jobs: - name: bandit (static security analysis) continue-on-error: true - run: uv run --frozen bandit -r graphify -ll + run: uv run --frozen bandit -r graphify graphify_semantic_providers -ll - name: pip-audit (dependency vulnerabilities) continue-on-error: true diff --git a/README.md b/README.md index 0c14d207c9..b1bb025338 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,11 @@ graphify prs --triage # AI ranks your review queue (uses whatever b graphify prs --conflicts # PRs sharing graph communities — merge-order risk ``` +For opt-in compiler-resolved symbols, references, implementations, and call +hierarchies, see the [semantic provider guide](docs/SEMANTIC-PROVIDERS.md). +Language-server evidence is bounded, provenance-tagged, and merged into a +separate output graph; native tree-sitter extraction remains the default. + See the [full command reference](#full-command-reference) below. --- diff --git a/docs/SEMANTIC-PROVIDERS.md b/docs/SEMANTIC-PROVIDERS.md new file mode 100644 index 0000000000..e02f3f3dd9 --- /dev/null +++ b/docs/SEMANTIC-PROVIDERS.md @@ -0,0 +1,125 @@ +# Optional semantic provider guide + +## What this adds + +Graphify already parses popular web languages locally with tree-sitter. This +extension adds optional compiler/language-server evidence. The AST graph remains +the always-available baseline; provider facts are additive, provenance-tagged +evidence and never replace native extraction. + +| Provider | Languages | Evidence level in this repository | +|---|---|---| +| rust-analyzer | Rust | real-tool smoke + protocol tests | +| typescript-language-server | TypeScript, JavaScript, JSX/TSX | real-tool smoke + protocol tests | +| Pyright | Python | real-tool smoke + protocol tests | +| Eclipse JDT LS | Java | protocol tests; external binary not exercised in CI | +| JetBrains Kotlin LSP | Kotlin | protocol tests; external binary not exercised in CI | +| csharp-ls | C# | protocol tests; external binary not exercised in CI | +| gopls | Go | protocol tests; external binary not exercised in CI | +| Phpactor | PHP | protocol tests; external binary not exercised in CI | +| Ruby LSP | Ruby | protocol tests; external binary not exercised in CI | + +Provider projects: [rust-analyzer](https://github.com/rust-lang/rust-analyzer), +[TypeScript language server](https://github.com/typescript-language-server/typescript-language-server), +[Eclipse JDT LS](https://github.com/eclipse-jdtls/eclipse.jdt.ls), +[Kotlin LSP](https://github.com/Kotlin/kotlin-lsp), +[csharp-ls](https://github.com/razzmatazz/csharp-language-server), +[Pyright](https://github.com/microsoft/pyright), +[gopls](https://github.com/golang/tools/tree/master/gopls), +[Phpactor](https://github.com/phpactor/phpactor), and +[Ruby LSP](https://github.com/Shopify/ruby-lsp). + +Production images should pin and verify each selected provider version. Do not +install every language server in every image: construct domain-specific images +from the same registry contract. + +Validated with real local tools during development: + +- `rust-analyzer 1.95.0` on a real Cargo workspace; +- `typescript-language-server 6.0.0` with `TypeScript 6.0.3` on a real strict + TypeScript workspace. +- `Pyright 1.1.409` on a real two-file Python workspace. + +TypeScript 7.0.2 was rejected by the tested language-server release during +initialization, which is why container builds must pin the compiler/server pair +rather than installing unbounded `latest` versions. Java, Kotlin, C#, Go, PHP +and Ruby use the same protocol-tested bounded LSP contract, but remain honestly +marked as not real-tool-tested until their optional binaries join the integration +matrix. + +`auto` selection requires both a matching source file and a project marker. This +prevents a polyglot repository from launching language servers for incidental +examples or vendored snippets. Explicit `--provider` selection intentionally +overrides that convenience check. + +## Safety and failure behavior + +- Native AST extraction is never disabled by a provider result. +- Provider subprocesses use argv execution with `shell=False`. +- Provider subprocesses receive an allowlisted toolchain environment rather + than an unfiltered copy of unrelated credentials. +- The LSP client advertises no workspace-edit support and rejects edit requests. +- The workspace, file count, source size, symbol count, RPC message size, + request count and timeout are bounded. +- Source symlinks escaping the workspace are ignored. +- Output contains symbols, locations and relationships, not source text, + process environment, server stderr or model chain-of-thought. +- Missing providers return `unavailable`; exhausted limits return + `budget_exhausted`. Neither condition silently expands a budget. +- Enrichment is additive and writes a separate output graph by default. +- A semantic symbol is merged into an AST node only on one unambiguous + `(source_file, label)` match. Ambiguous matches stay separate. +- Every fact includes provider kind, run ID, timestamp, confidence and source + range metadata. A registered profile is never described as real-tool proof. + +Language servers are external executables and may invoke project tooling (for +example, compiler checks or build scripts). Run them only on trusted workspaces, +or inside an appropriately isolated environment. Installing or running a +language server is never part of Graphify's default extraction path. + +## Commands + +```bash +uv sync +uv run graphify-semantic list +uv run graphify extract /path/to/repo --code-only +uv run graphify-semantic run /path/to/repo \ + --provider auto \ + --max-files 200 \ + --max-symbols 5000 \ + --max-relationship-requests 500 \ + --request-timeout 20 \ + --out /path/to/repo/graphify-out/semantic-runs.json +uv run graphify-semantic merge \ + /path/to/repo/graphify-out/graph.json \ + /path/to/repo/graphify-out/semantic-runs.json \ + --out /path/to/repo/graphify-out/graph.semantic.json +``` + +Select explicit providers when `auto` is too broad: + +```bash +uv run graphify-semantic run . \ + --provider rust-analyzer \ + --provider typescript-language-server \ + --out graphify-out/semantic-runs.json +``` + +## Adding another language + +Custom manifests are trusted configuration because they choose an executable. +They are strictly shape-checked and commands must be argv arrays. + +```json +{ + "name": "dart-analysis-server", + "languages": ["dart"], + "extensions": [".dart"], + "command": ["dart", "language-server", "--protocol=lsp"], + "binary_env": "GRAPHIFY_SEMANTIC_DART_BINARY", + "project_markers": ["pubspec.yaml"], + "initialization_options": {} +} +``` + +No runner or merger change is needed. diff --git a/graphify_semantic_providers/__init__.py b/graphify_semantic_providers/__init__.py new file mode 100644 index 0000000000..72be1af63d --- /dev/null +++ b/graphify_semantic_providers/__init__.py @@ -0,0 +1,18 @@ +"""Optional language-server semantic evidence for Graphify graphs. + +Graphify's native AST extraction remains the always-available baseline. This +package adds bounded, local evidence from supported language servers. Providers +never replace or weaken native extraction. +""" + +from .contracts import ProviderKind, ProviderRun, ProviderSpec, ProviderStatus +from .registry import ProviderRegistry, builtin_registry + +__all__ = [ + "ProviderRegistry", + "ProviderKind", + "ProviderRun", + "ProviderSpec", + "ProviderStatus", + "builtin_registry", +] diff --git a/graphify_semantic_providers/cli.py b/graphify_semantic_providers/cli.py new file mode 100644 index 0000000000..b9f71d0631 --- /dev/null +++ b/graphify_semantic_providers/cli.py @@ -0,0 +1,182 @@ +"""Command line interface for optional semantic provider runs.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from graphify.paths import write_json_atomic + +from .contracts import ProviderKind, ProviderRun, ProviderStatus +from .lsp import discover_files, resolve_command, run_provider +from .merge import merge_runs +from .registry import ProviderRegistry, builtin_registry + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="graphify-semantic") + parser.add_argument( + "--manifest", + action="append", + type=Path, + default=[], + help="operator-trusted custom provider JSON manifest", + ) + sub = parser.add_subparsers(dest="command", required=True) + + list_parser = sub.add_parser("list", help="list built-in and custom providers") + list_parser.add_argument("--json", action="store_true") + + run_parser = sub.add_parser("run", help="collect bounded local semantic evidence") + run_parser.add_argument("path", type=Path) + run_parser.add_argument("--provider", action="append", default=[]) + run_parser.add_argument("--out", type=Path, required=True) + run_parser.add_argument("--max-files", type=int, default=200) + run_parser.add_argument("--max-symbols", type=int, default=5_000) + run_parser.add_argument("--max-relationship-requests", type=int, default=500) + run_parser.add_argument("--request-timeout", type=float, default=20.0) + + merge_parser = sub.add_parser("merge", help="add provider results to a separate graph file") + merge_parser.add_argument("graph", type=Path) + merge_parser.add_argument("runs", type=Path) + merge_parser.add_argument("--out", type=Path, required=True) + + args = parser.parse_args(argv) + try: + registry = _registry(args.manifest) + if args.command == "list": + return _list(registry, args.json) + if args.command == "run": + return _run(args, registry) + if args.command == "merge": + return _merge(args) + except (KeyError, OSError, ValueError, json.JSONDecodeError) as exc: + print(f"graphify-semantic: {exc}", file=sys.stderr) + return 2 + return 2 + + +def _registry(manifests: list[Path]) -> ProviderRegistry: + registry = builtin_registry() + for path in manifests: + registry.load_manifest(path.resolve()) + return registry + + +def _list(registry: ProviderRegistry, as_json: bool) -> int: + rows = [ + { + "name": spec.name, + "languages": list(spec.languages), + "extensions": list(spec.extensions), + "available": resolve_command(spec) is not None, + "description": spec.description, + } + for spec in registry.all() + ] + if as_json: + print(json.dumps(rows, indent=2, sort_keys=True)) + else: + for row in rows: + availability = "available" if row["available"] else "not installed" + print(f"{row['name']}: {', '.join(row['languages'])} ({availability})") + return 0 + + +def _run(args: argparse.Namespace, registry: ProviderRegistry) -> int: + root = args.path.resolve() + if not root.is_dir(): + raise ValueError(f"workspace does not exist: {root}") + requested = args.provider or ["auto"] + if "auto" in requested: + specs = [spec for spec in registry.for_workspace(root) if discover_files(root, spec, 1)] + else: + specs = [registry.get(name) for name in requested] + runs = [ + run_provider( + spec, + root, + max_files=_positive_int(args.max_files, "max-files"), + max_symbols=_positive_int(args.max_symbols, "max-symbols"), + max_relationship_requests=_positive_int( + args.max_relationship_requests, "max-relationship-requests" + ), + request_timeout=_positive_float(args.request_timeout, "request-timeout"), + ) + for spec in specs + ] + payload = { + "contract": "graphify-semantic-providers/v1", + "workspace": root.name, + "runs": [run.to_dict() for run in runs], + } + args.out.parent.mkdir(parents=True, exist_ok=True) + write_json_atomic(args.out.resolve(), payload, indent=2) + completed = sum( + run.status in {ProviderStatus.COMPLETED, ProviderStatus.BUDGET_EXHAUSTED} for run in runs + ) + print(f"wrote {args.out}: {completed}/{len(runs)} provider runs produced bounded evidence") + return 0 if completed or not runs else 1 + + +def _merge(args: argparse.Namespace) -> int: + graph = _read_json(args.graph, 512 * 1024 * 1024) + payload = _read_json(args.runs, 256 * 1024 * 1024) + raw_runs = payload.get("runs", []) if isinstance(payload, dict) else [] + runs: list[ProviderRun] = [] + for raw in raw_runs: + if not isinstance(raw, dict): + continue + try: + status = ProviderStatus(raw.get("status")) + except ValueError: + continue + try: + provider_kind = ProviderKind(raw.get("provider_kind", "semantic")) + except ValueError: + continue + runs.append( + ProviderRun( + provider=str(raw.get("provider", "unknown")), + status=status, + provider_kind=provider_kind, + version=str(raw.get("version", "unknown")), + run_id=str(raw.get("run_id", "")) or "legacy-unknown", + timestamp=str(raw.get("timestamp", "")) or "unknown", + nodes=raw.get("nodes", []) if isinstance(raw.get("nodes"), list) else [], + edges=raw.get("edges", []) if isinstance(raw.get("edges"), list) else [], + ) + ) + merged = merge_runs(graph, runs) + args.out.parent.mkdir(parents=True, exist_ok=True) + write_json_atomic(args.out.resolve(), merged, indent=2) + print(f"wrote additive semantic graph: {args.out}") + return 0 + + +def _read_json(path: Path, max_bytes: int) -> Any: + resolved = path.resolve() + if not resolved.is_file(): + raise ValueError(f"file does not exist: {resolved}") + if resolved.stat().st_size > max_bytes: + raise ValueError(f"file exceeds size limit: {resolved}") + return json.loads(resolved.read_bytes()) + + +def _positive_int(value: int, name: str) -> int: + if value <= 0: + raise ValueError(f"{name} must be positive") + return value + + +def _positive_float(value: float, name: str) -> float: + if value <= 0: + raise ValueError(f"{name} must be positive") + return value + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/graphify_semantic_providers/contracts.py b/graphify_semantic_providers/contracts.py new file mode 100644 index 0000000000..602a11dc5e --- /dev/null +++ b/graphify_semantic_providers/contracts.py @@ -0,0 +1,76 @@ +"""Small, stable contracts shared by semantic provider plugins.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from enum import Enum +from typing import Any +from uuid import uuid4 + + +class ProviderKind(str, Enum): + """Evidence class emitted by a provider run.""" + + SEMANTIC = "semantic" + + +class ProviderStatus(str, Enum): + """A provider outcome that never silently erases the AST baseline.""" + + COMPLETED = "completed" + UNAVAILABLE = "unavailable" + FAILED = "failed" + BUDGET_EXHAUSTED = "budget_exhausted" + + +@dataclass(frozen=True) +class ProviderSpec: + """Trusted execution and language metadata for one provider plugin. + + ``command`` is an argv tuple, not a shell command. Custom manifests are an + operator-trusted boundary because they choose an executable. + """ + + name: str + languages: tuple[str, ...] + extensions: tuple[str, ...] + command: tuple[str, ...] + binary_env: str + project_markers: tuple[str, ...] = () + initialization_options: dict[str, Any] = field(default_factory=dict) + description: str = "" + + def __post_init__(self) -> None: + if not self.name or not self.command or not self.command[0]: + raise ValueError("provider name and command are required") + if not self.languages or not self.extensions: + raise ValueError("provider languages and extensions are required") + if any(not suffix.startswith(".") for suffix in self.extensions): + raise ValueError("provider extensions must begin with '.'") + + +@dataclass +class ProviderRun: + """Graphify-compatible evidence fragment plus explicit bounded status.""" + + provider: str + status: ProviderStatus + provider_kind: ProviderKind = ProviderKind.SEMANTIC + version: str = "unknown" + run_id: str = field(default_factory=lambda: f"provider-{uuid4().hex}") + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + nodes: list[dict[str, Any]] = field(default_factory=list) + edges: list[dict[str, Any]] = field(default_factory=list) + files_considered: int = 0 + files_processed: int = 0 + requests: int = 0 + relationship_requests: int = 0 + reason_code: str | None = None + warnings: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + result = asdict(self) + result["status"] = self.status.value + result["provider_kind"] = self.provider_kind.value + return result diff --git a/graphify_semantic_providers/lsp.py b/graphify_semantic_providers/lsp.py new file mode 100644 index 0000000000..3afadd5775 --- /dev/null +++ b/graphify_semantic_providers/lsp.py @@ -0,0 +1,745 @@ +"""Bounded, local Language Server Protocol runner. + +The runner collects observable symbol/call/reference facts. It does not store +source text, model reasoning, environment variables, or server stderr. It uses +argv execution without a shell and constrains every source path to the selected +workspace root. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import queue +import shutil + +# The bounded LSP adapter intentionally owns a local provider process. +import subprocess # nosec B404 +import threading +import time +from pathlib import Path +from typing import Any, Protocol + +from graphify.security import sanitize_metadata + +from .contracts import ProviderKind, ProviderRun, ProviderSpec, ProviderStatus + + +MAX_MESSAGE_BYTES = 8 * 1024 * 1024 +MAX_SOURCE_BYTES = 2 * 1024 * 1024 +_PROVIDER_ENV_ALLOWLIST = { + "APPDATA", + "BUNDLE_GEMFILE", + "CARGO_HOME", + "COMSPEC", + "DOTNET_ROOT", + "GEM_HOME", + "GEM_PATH", + "GOMODCACHE", + "GOPATH", + "GOROOT", + "HOME", + "JAVA_HOME", + "KOTLIN_HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "LOCALAPPDATA", + "NODE_PATH", + "NUGET_PACKAGES", + "PATH", + "PATHEXT", + "PHP_INI_SCAN_DIR", + "PYTHONPATH", + "RBENV_ROOT", + "RUSTUP_HOME", + "RUSTUP_TOOLCHAIN", + "SHELL", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "USER", + "USERNAME", + "VIRTUAL_ENV", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", +} + + +class RpcTransport(Protocol): + notifications: list[dict[str, Any]] + + def notify(self, method: str, params: dict[str, Any]) -> None: ... + + def request(self, method: str, params: dict[str, Any], timeout: float) -> Any: ... + + def close(self, timeout: float = 2.0) -> None: ... + + +class StdioJsonRpc: + """Minimal LSP JSON-RPC transport with message and time bounds.""" + + def __init__(self, command: tuple[str, ...], root: Path) -> None: + # Provider manifests are operator-trusted configuration and commands are argv-only. + self._process = subprocess.Popen( # nosec B603 + list(command), + cwd=root, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + shell=False, + env=_provider_environment(), + ) + if self._process.stdin is None or self._process.stdout is None: + raise RuntimeError("language server stdio was not created") + self._stdin = self._process.stdin + self._stdout = self._process.stdout + self._messages: queue.Queue[dict[str, Any] | BaseException] = queue.Queue() + self._write_lock = threading.Lock() + self._next_id = 1 + self.notifications: list[dict[str, Any]] = [] + self._reader_thread = threading.Thread(target=self._reader, daemon=True) + self._reader_thread.start() + + def _reader(self) -> None: + try: + while True: + headers: dict[str, str] = {} + while True: + raw = self._stdout.readline() + if not raw: + return + if raw in {b"\r\n", b"\n"}: + break + text = raw.decode("ascii", errors="strict").strip() + if ":" not in text: + raise RuntimeError("malformed LSP header") + key, value = text.split(":", 1) + headers[key.lower().strip()] = value.strip() + length = int(headers.get("content-length", "0")) + if length <= 0 or length > MAX_MESSAGE_BYTES: + raise RuntimeError("invalid or oversized LSP message") + body = self._stdout.read(length) + if len(body) != length: + raise RuntimeError("truncated LSP message") + message = json.loads(body) + if isinstance(message, dict): + self._messages.put(message) + except BaseException as exc: # noqa: BLE001 - delivered to waiting request. + self._messages.put(exc) + + def _send(self, message: dict[str, Any]) -> None: + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + if len(body) > MAX_MESSAGE_BYTES: + raise RuntimeError("outbound LSP message exceeds limit") + framed = f"Content-Length: {len(body)}\r\n\r\n".encode("ascii") + body + with self._write_lock: + self._stdin.write(framed) + self._stdin.flush() + + def notify(self, method: str, params: dict[str, Any]) -> None: + self._send({"jsonrpc": "2.0", "method": method, "params": params}) + + def request(self, method: str, params: dict[str, Any], timeout: float) -> Any: + request_id = self._next_id + self._next_id += 1 + self._send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"LSP request timed out: {method}") + try: + message = self._messages.get(timeout=remaining) + except queue.Empty as exc: + raise TimeoutError(f"LSP request timed out: {method}") from exc + if isinstance(message, BaseException): + raise RuntimeError("language server transport failed") from message + if message.get("id") == request_id: + if "error" in message: + error = message.get("error") or {} + code = error.get("code", "unknown") + detail = str(error.get("message", "server error")).replace("\n", " ")[:160] + raise RuntimeError(f"LSP request failed ({method}, code={code}): {detail}") + return message.get("result") + if "method" in message and "id" in message: + self._answer_server_request(message) + elif "method" in message: + if len(self.notifications) < 1_000: + self.notifications.append(message) + + def _answer_server_request(self, message: dict[str, Any]) -> None: + method = str(message.get("method", "")) + if method == "workspace/configuration": + items = (message.get("params") or {}).get("items") or [] + result: Any = [{} for _ in items] if isinstance(items, list) else [] + elif method == "workspace/workspaceFolders": + result = [] + elif method == "workspace/applyEdit": + result = { + "applied": False, + "failureReason": "semantic provider client is read-only", + } + elif method == "window/showDocument": + result = {"success": False} + else: + result = None + self._send({"jsonrpc": "2.0", "id": message["id"], "result": result}) + + def close(self, timeout: float = 2.0) -> None: + if self._process.poll() is None: + try: + self.request("shutdown", {}, timeout) + self.notify("exit", {}) + self._process.wait(timeout=timeout) + except (RuntimeError, TimeoutError, subprocess.TimeoutExpired): + self._process.terminate() + try: + self._process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + self._process.kill() + + +def resolve_command(spec: ProviderSpec) -> tuple[str, ...] | None: + """Resolve only the executable; arguments remain immutable provider data.""" + + override = os.environ.get(spec.binary_env, "").strip() + binary = override or spec.command[0] + resolved = shutil.which(binary) if not Path(binary).is_absolute() else binary + if not resolved or not Path(resolved).is_file(): + return None + # Do not resolve executable symlinks. Toolchain multiplexers such as rustup + # select the real binary from argv[0]; resolving ``rust-analyzer`` to the + # ``rustup`` target would start rustup with no subcommand and silently break + # LSP startup. + return (str(Path(resolved).absolute()), *spec.command[1:]) + + +def discover_files(root: Path, spec: ProviderSpec, max_files: int) -> list[Path]: + root = root.resolve() + result: list[Path] = [] + ignored = {".git", "node_modules", "target", "dist", "build", ".venv", "vendor"} + for candidate in sorted(root.rglob("*")): + if len(result) >= max_files: + break + if not candidate.is_file() or candidate.suffix.lower() not in spec.extensions: + continue + relative = candidate.relative_to(root) + if any(part in ignored for part in relative.parts): + continue + resolved = candidate.resolve() + if not resolved.is_relative_to(root) or candidate.stat().st_size > MAX_SOURCE_BYTES: + continue + result.append(resolved) + return result + + +def run_provider( + spec: ProviderSpec, + root: Path, + *, + max_files: int = 200, + max_symbols: int = 5_000, + max_relationship_requests: int = 500, + request_timeout: float = 20.0, + transport_factory: Any = StdioJsonRpc, +) -> ProviderRun: + """Run one provider without making it a prerequisite for native Graphify.""" + + root = root.resolve() + command = resolve_command(spec) + if command is None and transport_factory is StdioJsonRpc: + return ProviderRun( + provider=spec.name, + status=ProviderStatus.UNAVAILABLE, + reason_code="binary_not_found", + ) + files = discover_files(root, spec, max_files) + if not files: + return ProviderRun( + provider=spec.name, + status=ProviderStatus.COMPLETED, + reason_code="no_matching_files", + ) + + run = ProviderRun( + provider=spec.name, + provider_kind=ProviderKind.SEMANTIC, + status=ProviderStatus.COMPLETED, + ) + run.files_considered = len(files) + transport: RpcTransport | None = None + try: + transport = transport_factory(command or spec.command, root) + if transport is None: + raise RuntimeError("provider transport was not created") + initialize = transport.request( + "initialize", + { + "processId": os.getpid(), + "rootUri": root.as_uri(), + "workspaceFolders": [{"uri": root.as_uri(), "name": root.name}], + "capabilities": { + "workspace": { + "applyEdit": False, + "workspaceEdit": {"documentChanges": False}, + }, + "textDocument": { + "documentSymbol": {"hierarchicalDocumentSymbolSupport": True}, + "references": {}, + "implementation": {}, + "callHierarchy": {}, + }, + }, + "initializationOptions": spec.initialization_options, + }, + request_timeout, + ) + run.requests += 1 + if isinstance(initialize, dict): + run.version = str(((initialize.get("serverInfo") or {}).get("version") or "unknown")) + capabilities = initialize.get("capabilities") or {} + else: + capabilities = {} + transport.notify("initialized", {}) + + budget_exhausted = False + for path in files: + if len(run.nodes) >= max_symbols: + budget_exhausted = True + break + source = path.read_text(encoding="utf-8", errors="replace") + language_id = _language_id(path, spec) + uri = path.as_uri() + transport.notify( + "textDocument/didOpen", + { + "textDocument": { + "uri": uri, + "languageId": language_id, + "version": 1, + "text": source, + } + }, + ) + symbols = transport.request( + "textDocument/documentSymbol", + {"textDocument": {"uri": uri}}, + request_timeout, + ) + run.requests += 1 + flattened = _flatten_symbols(symbols, uri) + remaining = max_symbols - len(run.nodes) + if len(flattened) > remaining: + budget_exhausted = True + flattened = flattened[:remaining] + _append_symbols(run, flattened, root, spec) + run.files_processed += 1 + + for symbol in flattened: + if run.relationship_requests >= max_relationship_requests: + budget_exhausted = True + break + position = _position(symbol) + if capabilities.get("referencesProvider"): + refs = transport.request( + "textDocument/references", + { + "textDocument": {"uri": symbol["uri"]}, + "position": position, + "context": {"includeDeclaration": False}, + }, + request_timeout, + ) + run.requests += 1 + run.relationship_requests += 1 + budget_exhausted |= _append_locations( + run, symbol, refs, root, spec, "references", max_symbols + ) + if ( + capabilities.get("implementationProvider") + and run.relationship_requests < max_relationship_requests + ): + impls = transport.request( + "textDocument/implementation", + {"textDocument": {"uri": symbol["uri"]}, "position": position}, + request_timeout, + ) + run.requests += 1 + run.relationship_requests += 1 + budget_exhausted |= _append_locations( + run, symbol, impls, root, spec, "implemented_by", max_symbols + ) + if ( + capabilities.get("callHierarchyProvider") + and run.relationship_requests < max_relationship_requests + ): + prepared = transport.request( + "textDocument/prepareCallHierarchy", + {"textDocument": {"uri": symbol["uri"]}, "position": position}, + request_timeout, + ) + run.requests += 1 + run.relationship_requests += 1 + if ( + isinstance(prepared, list) + and prepared + and run.relationship_requests < max_relationship_requests + ): + calls = transport.request( + "callHierarchy/outgoingCalls", + {"item": prepared[0]}, + request_timeout, + ) + run.requests += 1 + run.relationship_requests += 1 + budget_exhausted |= _append_calls( + run, symbol, calls, root, spec, max_symbols + ) + if run.relationship_requests >= max_relationship_requests: + budget_exhausted = True + break + transport.notify("textDocument/didClose", {"textDocument": {"uri": uri}}) + + if budget_exhausted: + run.status = ProviderStatus.BUDGET_EXHAUSTED + run.reason_code = "semantic_budget_exhausted" + _deduplicate(run) + return run + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + run.status = ProviderStatus.FAILED + run.reason_code = type(exc).__name__.lower() + run.warnings.append(str(exc)[:240]) + return run + finally: + if transport is not None: + transport.close() + + +def _flatten_symbols(payload: Any, default_uri: str) -> list[dict[str, Any]]: + if not isinstance(payload, list): + return [] + result: list[dict[str, Any]] = [] + + def walk(item: Any, parent: str | None = None) -> None: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + return + raw_location = item.get("location") + location: dict[str, Any] = raw_location if isinstance(raw_location, dict) else {} + uri = str(location.get("uri") or default_uri) + item_range = item.get("selectionRange") or item.get("range") or location.get("range") or {} + record = { + "name": item["name"], + "kind": item.get("kind", 0), + "uri": uri, + "range": item_range, + "detail": str(item.get("detail", ""))[:240], + "parent_key": parent, + } + record["key"] = _symbol_key(record) + result.append(record) + children = item.get("children") or [] + if isinstance(children, list): + for child in children: + walk(child, record["key"]) + + for value in payload: + walk(value) + return result + + +def _append_symbols( + run: ProviderRun, symbols: list[dict[str, Any]], root: Path, spec: ProviderSpec +) -> None: + by_key: dict[str, str] = {} + for symbol in symbols: + source_file = _relative_uri(symbol["uri"], root) + if source_file is None: + continue + node_id = _node_id(spec.name, symbol["key"]) + by_key[symbol["key"]] = node_id + line = _line(symbol.get("range")) + run.nodes.append( + { + "id": node_id, + "label": symbol["name"], + "file_type": "code", + "source_file": source_file, + "source_location": f"L{line}" if line else "", + "metadata": _evidence_metadata( + run, + { + "semantic_kind": str(symbol.get("kind", 0)), + "semantic_detail": symbol.get("detail", ""), + "semantic_language": _language_id(Path(source_file), spec), + "semantic_range": _source_range(symbol.get("range")), + }, + ), + } + ) + for symbol in symbols: + parent_id = by_key.get(str(symbol.get("parent_key", ""))) + child_id = by_key.get(symbol["key"]) + if parent_id and child_id: + source_file = _relative_uri(symbol["uri"], root) or "" + run.edges.append( + _edge( + parent_id, + child_id, + "contains", + run, + source_file, + f"L{_line(symbol.get('range'))}" if _line(symbol.get("range")) else "", + ) + ) + + +def _append_locations( + run: ProviderRun, + source_symbol: dict[str, Any], + locations: Any, + root: Path, + spec: ProviderSpec, + relation: str, + max_symbols: int, +) -> bool: + if isinstance(locations, dict): + locations = [locations] + if not isinstance(locations, list): + return False + exhausted = False + existing_ids = {str(node.get("id", "")) for node in run.nodes} + source_id = _node_id(spec.name, source_symbol["key"]) + for location in locations[:100]: + if not isinstance(location, dict): + continue + uri = location.get("uri") or location.get("targetUri") + source_file = _relative_uri(str(uri or ""), root) + if source_file is None: + continue + location_range = location.get("range") or location.get("targetSelectionRange") or {} + line = _line(location_range) + target_id = _node_id(spec.name, f"file:{source_file}") + if target_id not in existing_ids: + if len(existing_ids) >= max_symbols: + exhausted = True + continue + run.nodes.append( + { + "id": target_id, + "label": Path(source_file).name, + "file_type": "code", + "source_file": source_file, + "source_location": f"L{line}" if line else "", + "metadata": _evidence_metadata( + run, + { + "semantic_kind": "file", + "semantic_range": _source_range(location_range), + }, + ), + } + ) + existing_ids.add(target_id) + run.edges.append( + _edge( + source_id, + target_id, + relation, + run, + source_file, + f"L{line}" if line else "", + ) + ) + return exhausted + + +def _append_calls( + run: ProviderRun, + source_symbol: dict[str, Any], + calls: Any, + root: Path, + spec: ProviderSpec, + max_symbols: int, +) -> bool: + if not isinstance(calls, list): + return False + exhausted = False + existing_ids = {str(node.get("id", "")) for node in run.nodes} + source_id = _node_id(spec.name, source_symbol["key"]) + for call in calls[:100]: + target = call.get("to") if isinstance(call, dict) else None + if not isinstance(target, dict) or not isinstance(target.get("name"), str): + continue + source_file = _relative_uri(str(target.get("uri", "")), root) + if source_file is None: + continue + target_symbol = { + "name": target["name"], + "uri": target.get("uri", ""), + "range": target.get("selectionRange") or target.get("range") or {}, + } + key = _symbol_key(target_symbol) + target_id = _node_id(spec.name, key) + line = _line(target_symbol["range"]) + if target_id not in existing_ids: + if len(existing_ids) >= max_symbols: + exhausted = True + continue + run.nodes.append( + { + "id": target_id, + "label": target["name"], + "file_type": "code", + "source_file": source_file, + "source_location": f"L{line}" if line else "", + "metadata": _evidence_metadata( + run, + { + "semantic_kind": str(target.get("kind", 0)), + "semantic_range": _source_range(target_symbol["range"]), + }, + ), + } + ) + existing_ids.add(target_id) + run.edges.append( + _edge( + source_id, + target_id, + "calls", + run, + source_file, + f"L{line}" if line else "", + ) + ) + return exhausted + + +def _edge( + source: str, + target: str, + relation: str, + run: ProviderRun, + source_file: str, + source_location: str = "", +) -> dict[str, Any]: + return { + "source": source, + "target": target, + "relation": relation, + "confidence": "EXTRACTED", + "weight": 1.0, + "source_file": source_file, + "source_location": source_location, + "context": "language_server", + "metadata": _evidence_metadata(run), + } + + +def _evidence_metadata(run: ProviderRun, extra: dict[str, Any] | None = None) -> dict[str, Any]: + metadata: dict[str, Any] = { + "semantic_provider": run.provider, + "evidence_provider": run.provider, + "evidence_provider_kind": run.provider_kind.value, + "evidence_run_id": run.run_id, + "evidence_timestamp": run.timestamp, + "evidence_confidence": "EXTRACTED_SEMANTIC", + } + metadata.update(extra or {}) + return sanitize_metadata(metadata) + + +def _source_range(value: Any) -> str: + if not isinstance(value, dict): + return "" + raw_start = value.get("start") + raw_end = value.get("end") + start: dict[str, Any] = raw_start if isinstance(raw_start, dict) else {} + end: dict[str, Any] = raw_end if isinstance(raw_end, dict) else {} + values = ( + start.get("line"), + start.get("character"), + end.get("line"), + end.get("character"), + ) + if any(isinstance(item, bool) or not isinstance(item, int) or item < 0 for item in values): + return "" + coordinates = [item for item in values if isinstance(item, int) and not isinstance(item, bool)] + line_start, column_start, line_end, column_end = coordinates + return f"L{line_start + 1}:C{column_start + 1}-L{line_end + 1}:C{column_end + 1}" + + +def _symbol_key(symbol: dict[str, Any]) -> str: + start = (symbol.get("range") or {}).get("start") or {} + return f"{symbol.get('uri', '')}:{start.get('line', -1)}:{start.get('character', -1)}:{symbol.get('name', '')}" + + +def _node_id(provider: str, key: str) -> str: + digest = hashlib.sha256(f"{provider}:{key}".encode()).hexdigest()[:20] + return f"semantic_{digest}" + + +def _position(symbol: dict[str, Any]) -> dict[str, int]: + start = (symbol.get("range") or {}).get("start") or {} + line = start.get("line", 0) + character = start.get("character", 0) + return { + "line": line if isinstance(line, int) and not isinstance(line, bool) and line >= 0 else 0, + "character": character + if isinstance(character, int) and not isinstance(character, bool) and character >= 0 + else 0, + } + + +def _line(value: Any) -> int: + line = ((value or {}).get("start") or {}).get("line", -1) if isinstance(value, dict) else -1 + return line + 1 if isinstance(line, int) and not isinstance(line, bool) and line >= 0 else 0 + + +def _relative_uri(uri: str, root: Path) -> str | None: + if not uri.startswith("file://"): + return None + from urllib.parse import unquote, urlparse + + path = Path(unquote(urlparse(uri).path)).resolve() + if not path.is_relative_to(root): + return None + return path.relative_to(root).as_posix() + + +def _language_id(path: Path, spec: ProviderSpec) -> str: + suffix = path.suffix.lower() + if suffix in {".js", ".jsx", ".mjs", ".cjs"}: + return "javascriptreact" if suffix == ".jsx" else "javascript" + if suffix in {".ts", ".tsx"}: + return "typescriptreact" if suffix == ".tsx" else "typescript" + return spec.languages[0] + + +def _deduplicate(run: ProviderRun) -> None: + nodes: dict[str, dict[str, Any]] = {} + for node in run.nodes: + nodes.setdefault(str(node.get("id", "")), node) + edges: dict[tuple[str, str, str, str], dict[str, Any]] = {} + for edge in run.edges: + key = ( + str(edge.get("source", "")), + str(edge.get("target", "")), + str(edge.get("relation", "")), + str(edge.get("source_file", "")), + ) + edges.setdefault(key, edge) + run.nodes = list(nodes.values()) + run.edges = list(edges.values()) + + +def _provider_environment() -> dict[str, str]: + """Forward runtime configuration without copying unrelated credentials.""" + + return { + key: value for key, value in os.environ.items() if key.upper() in _PROVIDER_ENV_ALLOWLIST + } diff --git a/graphify_semantic_providers/merge.py b/graphify_semantic_providers/merge.py new file mode 100644 index 0000000000..0123e0ef43 --- /dev/null +++ b/graphify_semantic_providers/merge.py @@ -0,0 +1,118 @@ +"""Merge provider fragments into a Graphify graph without replacing AST truth.""" + +from __future__ import annotations + +import copy +from collections import defaultdict +from typing import Any, Iterable + +from .contracts import ProviderRun + + +def merge_runs(base: dict[str, Any], runs: Iterable[ProviderRun]) -> dict[str, Any]: + """Return a new graph document with semantic evidence additively merged. + + A provider symbol is reconciled to a native AST symbol only when + ``(source_file, label)`` identifies exactly one native node. Ambiguity + leaves the semantic node separate rather than guessing. + """ + + runs = tuple(runs) + result = copy.deepcopy(base) + edge_key = "links" if "links" in result and "edges" not in result else "edges" + result.setdefault("nodes", []) + result.setdefault(edge_key, []) + nodes = result["nodes"] + edges = result[edge_key] + by_id = { + str(node.get("id")): node for node in nodes if isinstance(node, dict) and node.get("id") + } + native_index: dict[tuple[str, str], list[str]] = defaultdict(list) + for node_id, node in by_id.items(): + key = _reconciliation_key(node) + if key: + native_index[key].append(node_id) + + remap: dict[str, str] = {} + added_ids: set[str] = set() + for run in runs: + for node in run.nodes: + incoming_id = str(node.get("id", "")) + if not incoming_id: + continue + incoming_key = _reconciliation_key(node) + candidates = native_index.get(incoming_key, []) if incoming_key is not None else [] + if len(candidates) == 1: + canonical = candidates[0] + remap[incoming_id] = canonical + _record_evidence(by_id[canonical], run, node) + elif incoming_id not in by_id and incoming_id not in added_ids: + copied = copy.deepcopy(node) + _record_evidence(copied, run, node) + nodes.append(copied) + by_id[incoming_id] = copied + added_ids.add(incoming_id) + + seen_edges = { + ( + str(edge.get("source", "")), + str(edge.get("target", "")), + str(edge.get("relation", "")), + str(edge.get("source_location", "")), + ) + for edge in edges + if isinstance(edge, dict) + } + for run in runs: + for edge in run.edges: + copied = copy.deepcopy(edge) + copied["source"] = remap.get(str(copied.get("source", "")), copied.get("source")) + copied["target"] = remap.get(str(copied.get("target", "")), copied.get("target")) + if copied.get("source") not in by_id or copied.get("target") not in by_id: + continue + key = ( + str(copied.get("source", "")), + str(copied.get("target", "")), + str(copied.get("relation", "")), + str(copied.get("source_location", "")), + ) + if key not in seen_edges: + seen_edges.add(key) + edges.append(copied) + + graph_meta = result.setdefault("graph", {}) + if isinstance(graph_meta, dict): + materialized = [run for run in runs if run.nodes or run.edges] + semantic = sorted({run.provider for run in materialized}) + graph_meta["semantic_providers"] = semantic + graph_meta["evidence_provider_contract"] = "graphify-semantic-providers/v1" + if semantic: + graph_meta["semantic_provider_contract"] = "graphify-semantic-providers/v1" + return result + + +def _reconciliation_key(node: dict[str, Any]) -> tuple[str, str] | None: + source_file = str(node.get("source_file", "")).replace("\\", "/").lstrip("./") + label = str(node.get("label", "")).strip().casefold() + if not source_file or not label: + return None + return source_file, label + + +def _record_evidence(target: dict[str, Any], run: ProviderRun, incoming: dict[str, Any]) -> None: + evidence = target.setdefault("semantic_evidence", []) + if not isinstance(evidence, list): + evidence = [] + target["semantic_evidence"] = evidence + raw_metadata = incoming.get("metadata") + metadata = raw_metadata if isinstance(raw_metadata, dict) else {} + record = { + "provider": run.provider, + "provider_kind": run.provider_kind.value, + "run_id": run.run_id, + "timestamp": run.timestamp, + "source_location": str(incoming.get("source_location", "")), + "kind": str(metadata.get("semantic_kind", "")), + } + if record not in evidence and len(evidence) < 16: + evidence.append(record) diff --git a/graphify_semantic_providers/registry.py b/graphify_semantic_providers/registry.py new file mode 100644 index 0000000000..dcf04beeef --- /dev/null +++ b/graphify_semantic_providers/registry.py @@ -0,0 +1,214 @@ +"""Pluggable language-provider registry. + +Adding a language is data: register a :class:`ProviderSpec` or load a bounded +JSON manifest. The runner and Graphify merger do not need language-specific +changes. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .contracts import ProviderSpec + + +_BUILTINS = ( + ProviderSpec( + name="rust-analyzer", + languages=("rust",), + extensions=(".rs",), + command=("rust-analyzer",), + binary_env="GRAPHIFY_SEMANTIC_RUST_BINARY", + project_markers=("Cargo.toml",), + description="Rust definitions, references, implementations and call hierarchy.", + ), + ProviderSpec( + name="typescript-language-server", + languages=("typescript", "javascript"), + extensions=(".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"), + command=("typescript-language-server", "--stdio"), + binary_env="GRAPHIFY_SEMANTIC_TYPESCRIPT_BINARY", + project_markers=("tsconfig.json", "jsconfig.json", "package.json"), + description="TypeScript and JavaScript language-service semantics.", + ), + ProviderSpec( + name="eclipse-jdtls", + languages=("java",), + extensions=(".java",), + command=("jdtls",), + binary_env="GRAPHIFY_SEMANTIC_JAVA_BINARY", + project_markers=("pom.xml", "build.gradle", "build.gradle.kts"), + description="Java semantics through Eclipse JDT Language Server.", + ), + ProviderSpec( + name="kotlin-lsp", + languages=("kotlin",), + extensions=(".kt", ".kts"), + command=("kotlin-lsp",), + binary_env="GRAPHIFY_SEMANTIC_KOTLIN_BINARY", + project_markers=("build.gradle.kts", "settings.gradle.kts", "pom.xml"), + description="Kotlin/JVM semantics through JetBrains' official Kotlin LSP.", + ), + ProviderSpec( + name="csharp-ls", + languages=("csharp",), + extensions=(".cs",), + command=("csharp-ls",), + binary_env="GRAPHIFY_SEMANTIC_CSHARP_BINARY", + project_markers=("*.sln", "*.slnx", "*.csproj"), + description="C# semantics through Roslyn-backed csharp-ls.", + ), + ProviderSpec( + name="pyright", + languages=("python",), + extensions=(".py", ".pyi"), + command=("pyright-langserver", "--stdio"), + binary_env="GRAPHIFY_SEMANTIC_PYTHON_BINARY", + project_markers=("pyproject.toml", "pyrightconfig.json", "setup.cfg"), + description="Python type and reference evidence through Pyright.", + ), + ProviderSpec( + name="gopls", + languages=("go",), + extensions=(".go",), + command=("gopls", "serve"), + binary_env="GRAPHIFY_SEMANTIC_GO_BINARY", + project_markers=("go.mod", "go.work"), + description="Go definitions, references and call hierarchy.", + ), + ProviderSpec( + name="phpactor", + languages=("php",), + extensions=(".php",), + command=("phpactor", "language-server"), + binary_env="GRAPHIFY_SEMANTIC_PHP_BINARY", + project_markers=("composer.json",), + description="PHP language-server semantics through open-source Phpactor.", + ), + ProviderSpec( + name="ruby-lsp", + languages=("ruby",), + extensions=(".rb", ".rake"), + command=("ruby-lsp",), + binary_env="GRAPHIFY_SEMANTIC_RUBY_BINARY", + project_markers=("Gemfile",), + description="Ruby language-server semantics.", + ), +) + + +class ProviderRegistry: + """Deterministic provider lookup with explicit duplicate rejection.""" + + def __init__(self) -> None: + self._providers: dict[str, ProviderSpec] = {} + + def register(self, spec: ProviderSpec) -> None: + if spec.name in self._providers: + raise ValueError(f"provider already registered: {spec.name}") + self._providers[spec.name] = spec + + def get(self, name: str) -> ProviderSpec: + try: + return self._providers[name] + except KeyError as exc: + raise KeyError(f"unknown provider: {name}") from exc + + def all(self) -> tuple[ProviderSpec, ...]: + return tuple(self._providers[name] for name in sorted(self._providers)) + + def for_path(self, path: Path) -> tuple[ProviderSpec, ...]: + suffix = path.suffix.lower() + return tuple(spec for spec in self.all() if suffix in spec.extensions) + + def for_workspace(self, root: Path) -> tuple[ProviderSpec, ...]: + """Return providers whose project markers exist in ``root``. + + Markers are deliberately shallow and may be exact names or glob + patterns. Auto-selection uses them to avoid launching every language + server for incidental vendored/example files; explicit selection can + still run a provider without a marker. + """ + + root = root.resolve() + selected: list[ProviderSpec] = [] + for spec in self.all(): + if not spec.project_markers: + selected.append(spec) + continue + if any(any(root.glob(marker)) for marker in spec.project_markers): + selected.append(spec) + return tuple(selected) + + def load_manifest(self, path: Path) -> ProviderSpec: + """Load one operator-trusted plugin manifest with strict shape limits.""" + + raw = path.read_bytes() + if len(raw) > 64 * 1024: + raise ValueError("provider manifest exceeds 64 KiB") + doc = json.loads(raw) + if not isinstance(doc, dict): + raise ValueError("provider manifest must be an object") + allowed = { + "name", + "languages", + "extensions", + "command", + "binary_env", + "project_markers", + "initialization_options", + "description", + } + unknown = set(doc) - allowed + if unknown: + raise ValueError(f"unknown provider manifest fields: {sorted(unknown)}") + spec = ProviderSpec( + name=_string(doc, "name"), + languages=_strings(doc, "languages"), + extensions=_strings(doc, "extensions"), + command=_strings(doc, "command"), + binary_env=_string(doc, "binary_env"), + project_markers=_strings(doc, "project_markers", required=False), + initialization_options=_object(doc, "initialization_options"), + description=str(doc.get("description", "")), + ) + self.register(spec) + return spec + + +def builtin_registry() -> ProviderRegistry: + registry = ProviderRegistry() + for spec in _BUILTINS: + registry.register(spec) + return registry + + +def _string(doc: dict[str, Any], key: str) -> str: + value = doc.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"provider manifest field {key!r} must be a non-empty string") + return value + + +def _strings(doc: dict[str, Any], key: str, *, required: bool = True) -> tuple[str, ...]: + value = doc.get(key) + if value is None and not required: + return () + if ( + not isinstance(value, list) + or not value + or not all(isinstance(item, str) and item for item in value) + ): + raise ValueError(f"provider manifest field {key!r} must be a non-empty string array") + if len(value) > 64: + raise ValueError(f"provider manifest field {key!r} exceeds 64 entries") + return tuple(value) + + +def _object(doc: dict[str, Any], key: str) -> dict[str, Any]: + value = doc.get(key, {}) + if not isinstance(value, dict): + raise ValueError(f"provider manifest field {key!r} must be an object") + return value diff --git a/pyproject.toml b/pyproject.toml index 15ea9dd57c..089ab82618 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", [project.scripts] graphify = "graphify.__main__:main" graphify-mcp = "graphify.serve:_main" +graphify-semantic = "graphify_semantic_providers.cli:main" [dependency-groups] dev = [ @@ -124,7 +125,7 @@ dev = [ package = true [tool.setuptools] -packages = ["graphify", "graphify.extractors", "graphify.exporters"] +packages = ["graphify", "graphify.extractors", "graphify.exporters", "graphify_semantic_providers"] include-package-data = false [tool.setuptools.package-data] @@ -155,6 +156,6 @@ target-version = "py310" select = ["E9", "F63", "F7", "F82"] [tool.pyright] -include = ["graphify", "tests"] +include = ["graphify", "graphify_semantic_providers", "tests"] pythonVersion = "3.10" typeCheckingMode = "basic" diff --git a/tests/test_semantic_provider_cli.py b/tests/test_semantic_provider_cli.py new file mode 100644 index 0000000000..179309b459 --- /dev/null +++ b/tests/test_semantic_provider_cli.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from graphify_semantic_providers import cli +from graphify_semantic_providers.contracts import ProviderRun, ProviderSpec, ProviderStatus +from graphify_semantic_providers.registry import ProviderRegistry + + +def _provider(name: str, extension: str) -> ProviderSpec: + return ProviderSpec( + name=name, + languages=(name,), + extensions=(extension,), + command=(name,), + binary_env=f"GRAPHIFY_SEMANTIC_{name.upper()}_BINARY", + ) + + +def test_explicit_provider_does_not_also_select_auto(tmp_path, monkeypatch) -> None: + registry = ProviderRegistry() + registry.register(_provider("chosen", ".chosen")) + registry.register(_provider("other", ".other")) + calls: list[str] = [] + + monkeypatch.setattr(cli, "_registry", lambda manifests: registry) + monkeypatch.setattr(cli, "discover_files", lambda root, spec, limit: [root / "fixture"]) + + def fake_run(spec, root, **kwargs): + calls.append(spec.name) + return ProviderRun(provider=spec.name, status=ProviderStatus.COMPLETED) + + monkeypatch.setattr(cli, "run_provider", fake_run) + out = tmp_path / "runs.json" + result = cli.main(["run", str(tmp_path), "--provider", "chosen", "--out", str(out)]) + + assert result == 0 + assert calls == ["chosen"] + + +def test_omitted_provider_keeps_auto_discovery(tmp_path, monkeypatch) -> None: + registry = ProviderRegistry() + registry.register(_provider("chosen", ".chosen")) + registry.register(_provider("other", ".other")) + calls: list[str] = [] + + monkeypatch.setattr(cli, "_registry", lambda manifests: registry) + monkeypatch.setattr( + cli, + "discover_files", + lambda root, spec, limit: [root / "fixture"] if spec.name == "other" else [], + ) + + def fake_run(spec, root, **kwargs): + calls.append(spec.name) + return ProviderRun(provider=spec.name, status=ProviderStatus.COMPLETED) + + monkeypatch.setattr(cli, "run_provider", fake_run) + out = tmp_path / "runs.json" + result = cli.main(["run", str(tmp_path), "--out", str(out)]) + + assert result == 0 + assert calls == ["other"] diff --git a/tests/test_semantic_provider_lsp.py b/tests/test_semantic_provider_lsp.py new file mode 100644 index 0000000000..ee0415e884 --- /dev/null +++ b/tests/test_semantic_provider_lsp.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from graphify_semantic_providers.contracts import ProviderStatus +from graphify_semantic_providers.contracts import ProviderSpec +from graphify_semantic_providers.lsp import ( + _provider_environment, + discover_files, + resolve_command, + run_provider, +) +from graphify_semantic_providers.registry import builtin_registry + + +class FakeTransport: + instances: list["FakeTransport"] = [] + + def __init__(self, command: tuple[str, ...], root: Path) -> None: + self.command = command + self.root = root + self.notifications: list[dict[str, Any]] = [] + self.closed = False + self.requests: list[str] = [] + FakeTransport.instances.append(self) + + def notify(self, method: str, params: dict[str, Any]) -> None: + self.notifications.append({"method": method, "params": params}) + + def request(self, method: str, params: dict[str, Any], timeout: float) -> Any: + self.requests.append(method) + source_uri = (self.root / "src" / "lib.rs").as_uri() + if method == "initialize": + return { + "serverInfo": {"version": "test-1"}, + "capabilities": { + "referencesProvider": True, + "implementationProvider": True, + "callHierarchyProvider": True, + }, + } + if method == "textDocument/documentSymbol": + return [ + { + "name": "App", + "kind": 5, + "range": _range(0), + "selectionRange": _range(0), + "children": [ + { + "name": "run", + "kind": 6, + "range": _range(1), + "selectionRange": _range(1), + } + ], + } + ] + if method == "textDocument/references": + return [{"uri": source_uri, "range": _range(2)}] + if method == "textDocument/implementation": + return [] + if method == "textDocument/prepareCallHierarchy": + return [ + {"name": "run", "uri": source_uri, "range": _range(1), "selectionRange": _range(1)} + ] + if method == "callHierarchy/outgoingCalls": + return [ + { + "to": { + "name": "helper", + "kind": 12, + "uri": source_uri, + "range": _range(4), + "selectionRange": _range(4), + }, + "fromRanges": [_range(2)], + } + ] + if method == "shutdown": + return None + raise AssertionError(f"unexpected request: {method}") + + def close(self, timeout: float = 2.0) -> None: + self.closed = True + + +def _range(line: int) -> dict[str, dict[str, int]]: + return { + "start": {"line": line, "character": 0}, + "end": {"line": line, "character": 4}, + } + + +def test_generic_lsp_runner_emits_symbols_calls_references_and_containment(tmp_path: Path) -> None: + source = tmp_path / "src" / "lib.rs" + source.parent.mkdir() + source.write_text("struct App;\nfn run() {}\n", encoding="utf-8") + spec = builtin_registry().get("rust-analyzer") + + run = run_provider( + spec, + tmp_path, + max_relationship_requests=20, + transport_factory=FakeTransport, + ) + + assert run.status is ProviderStatus.COMPLETED + assert run.version == "test-1" + assert run.files_processed == 1 + assert {node["label"] for node in run.nodes} >= {"App", "run", "helper", "lib.rs"} + assert {edge["relation"] for edge in run.edges} >= {"contains", "references", "calls"} + assert all(node.get("source_file") == "src/lib.rs" for node in run.nodes) + assert all(node["metadata"]["evidence_provider"] == "rust-analyzer" for node in run.nodes) + assert all(node["metadata"]["evidence_run_id"] == run.run_id for node in run.nodes) + assert all(node["metadata"]["evidence_timestamp"] == run.timestamp for node in run.nodes) + assert FakeTransport.instances[-1].closed is True + + +class PortableTransport: + """Small capability-neutral server used to prove every profile's LSP shape.""" + + instances: list["PortableTransport"] = [] + + def __init__(self, command: tuple[str, ...], root: Path) -> None: + self.command = command + self.root = root + self.notifications: list[dict[str, Any]] = [] + self.current_uri = "" + PortableTransport.instances.append(self) + + def notify(self, method: str, params: dict[str, Any]) -> None: + self.notifications.append({"method": method, "params": params}) + if method == "textDocument/didOpen": + self.current_uri = params["textDocument"]["uri"] + + def request(self, method: str, params: dict[str, Any], timeout: float) -> Any: + if method == "initialize": + return {"serverInfo": {"version": "portable-test"}, "capabilities": {}} + if method == "textDocument/documentSymbol": + return [ + { + "name": "EntryPoint", + "kind": 12, + "range": _range(0), + "selectionRange": _range(0), + } + ] + if method == "shutdown": + return None + raise AssertionError(f"unexpected request: {method}") + + def close(self, timeout: float = 2.0) -> None: + return None + + +@pytest.mark.parametrize( + ("provider_name", "relative_path", "language_id"), + [ + ("rust-analyzer", "src/lib.rs", "rust"), + ("typescript-language-server", "src/app.ts", "typescript"), + ("typescript-language-server", "src/app.jsx", "javascriptreact"), + ("eclipse-jdtls", "src/App.java", "java"), + ("kotlin-lsp", "src/App.kt", "kotlin"), + ("csharp-ls", "src/App.cs", "csharp"), + ("pyright", "src/app.py", "python"), + ("gopls", "src/app.go", "go"), + ("phpactor", "src/App.php", "php"), + ("ruby-lsp", "src/app.rb", "ruby"), + ], +) +def test_builtin_profiles_share_the_bounded_lsp_contract( + tmp_path: Path, provider_name: str, relative_path: str, language_id: str +) -> None: + source = tmp_path / relative_path + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text("fixture\n", encoding="utf-8") + + run = run_provider( + builtin_registry().get(provider_name), + tmp_path, + max_relationship_requests=10, + transport_factory=PortableTransport, + ) + + assert run.status is ProviderStatus.COMPLETED + assert [node["label"] for node in run.nodes] == ["EntryPoint"] + opened = next( + notification + for notification in PortableTransport.instances[-1].notifications + if notification["method"] == "textDocument/didOpen" + ) + assert opened["params"]["textDocument"]["languageId"] == language_id + assert run.nodes[0]["metadata"]["semantic_language"] == language_id + assert run.nodes[0]["metadata"]["evidence_provider_kind"] == "semantic" + + +def test_provider_budget_is_explicit_not_silently_expanded(tmp_path: Path) -> None: + source = tmp_path / "src" / "lib.rs" + source.parent.mkdir() + source.write_text("struct App;\n", encoding="utf-8") + run = run_provider( + builtin_registry().get("rust-analyzer"), + tmp_path, + max_relationship_requests=1, + transport_factory=FakeTransport, + ) + assert run.status is ProviderStatus.BUDGET_EXHAUSTED + assert run.reason_code == "semantic_budget_exhausted" + assert run.relationship_requests == 1 + assert run.requests == 3 # initialize + document symbols + one relationship + + +def test_symbol_budget_also_bounds_relationship_targets(tmp_path: Path) -> None: + source = tmp_path / "src" / "lib.rs" + source.parent.mkdir() + source.write_text("struct App;\nfn run() {}\n", encoding="utf-8") + run = run_provider( + builtin_registry().get("rust-analyzer"), + tmp_path, + max_symbols=2, + max_relationship_requests=20, + transport_factory=FakeTransport, + ) + assert run.status is ProviderStatus.BUDGET_EXHAUSTED + assert len(run.nodes) <= 2 + + +def test_discovery_excludes_build_outputs_and_out_of_root_symlinks(tmp_path: Path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src" / "main.go").write_text("package main", encoding="utf-8") + (tmp_path / "vendor").mkdir() + (tmp_path / "vendor" / "ignored.go").write_text("package ignored", encoding="utf-8") + outside = tmp_path.parent / "outside.go" + outside.write_text("package outside", encoding="utf-8") + try: + (tmp_path / "src" / "escape.go").symlink_to(outside) + except OSError: + pass + files = discover_files(tmp_path, builtin_registry().get("gopls"), 10) + assert [path.relative_to(tmp_path).as_posix() for path in files] == ["src/main.go"] + + +def test_command_resolution_preserves_toolchain_proxy_symlink( + tmp_path: Path, monkeypatch, requires_symlinks +) -> None: + target = Path("/usr/bin/true") + alias = tmp_path / "language-server-proxy" + alias.symlink_to(target) + monkeypatch.setenv("TEST_LSP_BINARY", str(alias)) + spec = ProviderSpec( + name="test", + languages=("test",), + extensions=(".test",), + command=("ignored", "--stdio"), + binary_env="TEST_LSP_BINARY", + ) + assert resolve_command(spec) == (str(alias.absolute()), "--stdio") + + +def test_provider_environment_does_not_forward_unrelated_credentials(monkeypatch) -> None: + monkeypatch.setenv("PATH", "/usr/bin") + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "secret-canary") + monkeypatch.setenv("CLOUDFLARE_API_TOKEN", "secret-canary") + + environment = _provider_environment() + + assert environment["PATH"] == "/usr/bin" + assert "AWS_ACCESS_KEY_ID" not in environment + assert "CLOUDFLARE_API_TOKEN" not in environment diff --git a/tests/test_semantic_provider_merge.py b/tests/test_semantic_provider_merge.py new file mode 100644 index 0000000000..7137b3c715 --- /dev/null +++ b/tests/test_semantic_provider_merge.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from graphify_semantic_providers.contracts import ProviderRun, ProviderStatus +from graphify_semantic_providers.merge import merge_runs + + +def _run(nodes, edges=()): + return ProviderRun( + provider="test-lsp", + status=ProviderStatus.COMPLETED, + nodes=list(nodes), + edges=list(edges), + ) + + +def test_unique_native_symbol_is_enriched_not_duplicated() -> None: + base = { + "nodes": [ + { + "id": "native_run", + "label": "run", + "source_file": "src/app.ts", + "file_type": "code", + } + ], + "links": [], + "graph": {}, + } + run = _run( + [ + { + "id": "semantic_run", + "label": "run", + "source_file": "src/app.ts", + "source_location": "L7", + "file_type": "code", + "metadata": {"semantic_kind": "method"}, + } + ] + ) + merged = merge_runs(base, [run]) + assert [node["id"] for node in merged["nodes"]] == ["native_run"] + evidence = merged["nodes"][0]["semantic_evidence"] + assert len(evidence) == 1 + assert evidence[0]["provider"] == "test-lsp" + assert evidence[0]["provider_kind"] == "semantic" + assert evidence[0]["run_id"] + assert evidence[0]["timestamp"] + assert evidence[0]["source_location"] == "L7" + assert evidence[0]["kind"] == "method" + + +def test_ambiguous_native_match_keeps_semantic_node_separate() -> None: + base = { + "nodes": [ + {"id": "a", "label": "run", "source_file": "src/app.ts", "file_type": "code"}, + {"id": "b", "label": "run", "source_file": "src/app.ts", "file_type": "code"}, + ], + "edges": [], + } + run = _run([{"id": "s", "label": "run", "source_file": "src/app.ts", "file_type": "code"}]) + merged = merge_runs(base, [run]) + assert {node["id"] for node in merged["nodes"]} == {"a", "b", "s"} + + +def test_edges_are_remapped_to_reconciled_native_nodes() -> None: + base = { + "nodes": [ + {"id": "native", "label": "run", "source_file": "src/app.ts", "file_type": "code"} + ], + "edges": [], + } + run = _run( + [ + {"id": "semantic", "label": "run", "source_file": "src/app.ts", "file_type": "code"}, + {"id": "helper", "label": "helper", "source_file": "src/help.ts", "file_type": "code"}, + ], + [{"source": "semantic", "target": "helper", "relation": "calls"}], + ) + merged = merge_runs(base, [run]) + assert merged["edges"] == [{"source": "native", "target": "helper", "relation": "calls"}] + + +def test_generator_of_runs_is_not_consumed_before_edges_and_metadata() -> None: + run = _run( + [ + {"id": "source", "label": "source", "source_file": "src/app.py"}, + {"id": "target", "label": "target", "source_file": "src/app.py"}, + ], + [{"source": "source", "target": "target", "relation": "calls"}], + ) + + merged = merge_runs( + {"nodes": [], "edges": [], "graph": {}}, + (value for value in [run]), + ) + + assert merged["edges"][0]["relation"] == "calls" + assert merged["graph"]["semantic_providers"] == ["test-lsp"] diff --git a/tests/test_semantic_provider_registry.py b/tests/test_semantic_provider_registry.py new file mode 100644 index 0000000000..bc27003c14 --- /dev/null +++ b/tests/test_semantic_provider_registry.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from graphify_semantic_providers.registry import ProviderRegistry, builtin_registry + + +def test_builtin_registry_covers_popular_web_languages() -> None: + languages = {language for spec in builtin_registry().all() for language in spec.languages} + assert { + "rust", + "typescript", + "javascript", + "java", + "kotlin", + "csharp", + "python", + "go", + "php", + "ruby", + } <= languages + + +def test_registry_dispatches_javascript_without_typescript_rework() -> None: + providers = builtin_registry().for_path(Path("src/app.jsx")) + assert [provider.name for provider in providers] == ["typescript-language-server"] + + +def test_workspace_auto_selection_requires_project_marker(tmp_path: Path) -> None: + registry = builtin_registry() + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.py").write_text("value = 1\n", encoding="utf-8") + assert "pyright" not in {spec.name for spec in registry.for_workspace(tmp_path)} + + (tmp_path / "pyproject.toml").write_text("[project]\nname='fixture'\n", encoding="utf-8") + assert "pyright" in {spec.name for spec in registry.for_workspace(tmp_path)} + + +def test_workspace_marker_globs_cover_csharp_projects(tmp_path: Path) -> None: + (tmp_path / "Product.csproj").write_text("\n", encoding="utf-8") + assert "csharp-ls" in {spec.name for spec in builtin_registry().for_workspace(tmp_path)} + + +def test_operator_manifest_adds_language_as_data(tmp_path: Path) -> None: + manifest = tmp_path / "dart.json" + manifest.write_text( + json.dumps( + { + "name": "dart-analysis-server", + "languages": ["dart"], + "extensions": [".dart"], + "command": ["dart", "language-server", "--protocol=lsp"], + "binary_env": "GRAPHIFY_SEMANTIC_DART_BINARY", + "project_markers": ["pubspec.yaml"], + "initialization_options": {}, + } + ), + encoding="utf-8", + ) + registry = ProviderRegistry() + spec = registry.load_manifest(manifest) + assert spec.languages == ("dart",) + assert registry.for_path(Path("lib/main.dart")) == (spec,) + + +def test_manifest_rejects_shell_string_and_unknown_fields(tmp_path: Path) -> None: + manifest = tmp_path / "unsafe.json" + manifest.write_text( + json.dumps( + { + "name": "unsafe", + "languages": ["x"], + "extensions": [".x"], + "command": "server --stdio; rm -rf /", + "binary_env": "X", + "unexpected": True, + } + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="unknown provider manifest fields"): + ProviderRegistry().load_manifest(manifest)