From 2c369c3779b6419806e5a1b42a1eb55407282a29 Mon Sep 17 00:00:00 2001 From: Vikrant Date: Thu, 20 Aug 2026 18:24:15 +0530 Subject: [PATCH 1/3] Make TLDRGraph run via agent only --- .agents/rules/tldrgraph.md | 31 + .agents/workflows/tldrgraph-init.md | 128 ++++ .claude/commands/tldrgraph-init.md | 128 ++++ .cursor/commands/tldrgraph-init.md | 128 ++++ .gitignore | 10 +- AGENTS.md | 28 + AGENT_CONTRACT.md | 37 +- README.md | 105 ++- tests/conftest.py | 62 +- tests/test_agent_loop.py | 66 +- tests/test_auto_agent.py | 863 ++++++++++++++++++++++ tests/test_dynamic_layers.py | 142 ++-- tests/test_extractors.py | 9 +- tests/test_flow_engine.py | 2 +- tests/test_hash_gate.py | 6 +- tests/test_hierarchy.py | 11 +- tests/test_index_write_through.py | 6 +- tests/test_layer_config.py | 21 + tests/test_layers.py | 19 +- tests/test_scan_smoke.py | 7 +- tests/test_visualizer.py | 7 +- tldrgraph/__init__.py | 5 + tldrgraph/agent_commands.py | 391 ++++++++++ tldrgraph/agent_runner.py | 328 +++++++++ tldrgraph/cli.py | 1031 +++++++++++++++++++++------ tldrgraph/graph_loader.py | 33 +- tldrgraph/hierarchy.py | 4 +- tldrgraph/installer.py | 424 +++++------ tldrgraph/layer_config.py | 19 +- tldrgraph/layers.py | 39 +- tldrgraph/paths.py | 89 +++ tldrgraph/propose_layers.py | 574 +++++++-------- 32 files changed, 3831 insertions(+), 922 deletions(-) create mode 100644 .agents/rules/tldrgraph.md create mode 100644 .agents/workflows/tldrgraph-init.md create mode 100644 .claude/commands/tldrgraph-init.md create mode 100644 .cursor/commands/tldrgraph-init.md create mode 100644 AGENTS.md create mode 100644 tests/test_auto_agent.py create mode 100644 tldrgraph/agent_commands.py create mode 100644 tldrgraph/agent_runner.py create mode 100644 tldrgraph/paths.py diff --git a/.agents/rules/tldrgraph.md b/.agents/rules/tldrgraph.md new file mode 100644 index 0000000..131dc80 --- /dev/null +++ b/.agents/rules/tldrgraph.md @@ -0,0 +1,31 @@ +--- +name: tldrgraph +description: TLDRGraph architecture graph: trace before you edit +--- + +## TLDRGraph + +This repository is mapped into architectural layers designed from its own source, +with per-symbol intents you can search and trace. + +**Before planning or implementing a feature**, trace it instead of grepping: + +```bash +tldrgraph query "" # semantic search + end-to-end flow +tldrgraph trace "" "" # exact path between two symbols +tldrgraph layers # node counts per layer +tldrgraph dead-code # review candidates, never a delete list +``` + +Those are read-only and never trigger enrichment. + +**To build or continue the graph**, run `tldrgraph init`, do what the `NEXT ACTION` +block prints, and run it again -- repeat until `status: done`. It has no template +fallback: if this repository has no architecture yet, it will stop and ask you to +design one from the code. Do not skip reading the files. + +Full workflow: `.claude/commands/tldrgraph-init.md` (identical copies live in every +other agent directory). Schema: `.tldrgraph/AGENT_CONTRACT.md`. + +`tldrgraph dead-code` lists **review candidates, not confirmed dead code**. +`unreviewed` means "not enough evidence to conclude" and is never removable. diff --git a/.agents/workflows/tldrgraph-init.md b/.agents/workflows/tldrgraph-init.md new file mode 100644 index 0000000..7ddfa53 --- /dev/null +++ b/.agents/workflows/tldrgraph-init.md @@ -0,0 +1,128 @@ +--- +name: tldrgraph-init +description: Build or continue this repository's TLDRGraph architecture graph (layers, extraction, enrichment) +--- + +# TLDRGraph: build this repository's architecture graph + +One command, run repeatedly until it says DONE. `tldrgraph init` never guesses: +it stops and tells you exactly what it needs. + +```bash +tldrgraph init +``` + +Read the `NEXT ACTION` block it prints, do what it says, then run `tldrgraph init` +again. Repeat until the output says `status: done`. There are only three things +it can ask for. + +## 1. `status: needs_layers` + +TLDRGraph ships **no layer templates** and will not invent an architecture. +Design one from this repository. + +1. Read `.tldrgraph/propose_layers_request.json`. It carries the symbols and + files extraction already found -- a starting point, not a substitute for + opening the code. +2. **Open real source files**: entry points first, then a representative file + from each cluster in the evidence. Work out what this codebase actually does + and where responsibility changes hands. +3. Write `.tldrgraph/propose_layers_response.json`: + +```json +{ + "utility_id": "", + "layers": [ + { + "id": "short_machine_id", + "name": "Layer 1: Human Friendly Name", + "order": 1, + "description": "One sentence on what lives here", + "rules": [ + {"file_contains": ["substring"], "exclude_file": ["optional"]}, + {"label_contains": ["SymbolNamePart"]} + ] + } + ] +} +``` + +4. Run `tldrgraph init` again. + +### What a layer set looks like + +Sketches from other codebases, to show the *shape* of an answer. They are not a +menu and none of them will fit this repository -- read the code and name what you +actually find. + +- A web app might split presentation from request handling from domain logic + from persistence, with background jobs and deployment config as their own tiers. +- A CLI tool might split the command surface from the processing engine from + local state, with adapters to outside systems separate again. +- A library might split its public API from the core implementation from its + data types, with backend adapters separate. +- A data pipeline might split ingestion from transformation from model training + from serving. + +The useful question is not "which of these is it?" but "where does responsibility +change hands in *this* code, and what would a new engineer need named?" + +### Rules that hold for any answer + +- 3 to 6 layers, plus exactly one catch-all whose `id` equals `utility_id` and + whose `rules` are `[]`. +- Unique `id` and `name` per layer; sequential integer `order` from 1. +- Rule keys: `file_contains`, `exclude_file`, `path_regex`, `label_contains`, + `exclude_label`, `label_ends_with`, `type_in`, `id_prefix`. Values are lists of + strings. Rules are evaluated in `order` and the first match wins. +- Derive rules from paths and symbol names you actually saw. A rule matching + nothing is worse than no rule; a rule matching everything collapses the map. + +## 2. `status: needs_confirmation` + +The output shows how many nodes need enrichment and how many agent round-trips +that implies. **Ask the user whether to proceed, and show them that estimate.** +Do not decide for them. + +- They agree: `tldrgraph init --yes` +- Smaller first pass: `tldrgraph init --yes --limit 100` +- They decline: stop. The graph is already built and queryable. + +## 3. `status: needs_enrichment` + +1. Read `.tldrgraph/enrichment_request.yaml`. +2. **Open the source file of every node in it.** This is the entire point: an + intent paraphrased from a symbol name poisons semantic search with + confident-sounding noise. +3. Write `.tldrgraph/enrichment_response.yaml` -- a *different* file from the + request, which is regenerated on every run: + +```yaml +- id: "" + intent: | + What this symbol does, why it exists, and its execution logic. + input_fields: [caseId, remarks] + output_fields: [status, disposition] + calls: [ApplicationsService, pension_cases] +``` + +4. Run `tldrgraph init --yes` again. It applies the response and hands you the + next batch, until there is nothing left. + +**Copy every `id` verbatim.** A constructed id matches nothing, is dropped, and +gets reported back to you -- but the work is wasted. + +**Never invent `fields` or `calls`.** Omit what you cannot verify in the code: an +empty list is a correct answer, a wrong `calls` entry becomes a real wrong edge. + +## Once it says DONE + +```bash +tldrgraph query "" +tldrgraph trace "" "" +tldrgraph layers +tldrgraph ui --serve +``` + +Read-only, and they never trigger enrichment. Full schema: +`.tldrgraph/AGENT_CONTRACT.md`. diff --git a/.claude/commands/tldrgraph-init.md b/.claude/commands/tldrgraph-init.md new file mode 100644 index 0000000..7ddfa53 --- /dev/null +++ b/.claude/commands/tldrgraph-init.md @@ -0,0 +1,128 @@ +--- +name: tldrgraph-init +description: Build or continue this repository's TLDRGraph architecture graph (layers, extraction, enrichment) +--- + +# TLDRGraph: build this repository's architecture graph + +One command, run repeatedly until it says DONE. `tldrgraph init` never guesses: +it stops and tells you exactly what it needs. + +```bash +tldrgraph init +``` + +Read the `NEXT ACTION` block it prints, do what it says, then run `tldrgraph init` +again. Repeat until the output says `status: done`. There are only three things +it can ask for. + +## 1. `status: needs_layers` + +TLDRGraph ships **no layer templates** and will not invent an architecture. +Design one from this repository. + +1. Read `.tldrgraph/propose_layers_request.json`. It carries the symbols and + files extraction already found -- a starting point, not a substitute for + opening the code. +2. **Open real source files**: entry points first, then a representative file + from each cluster in the evidence. Work out what this codebase actually does + and where responsibility changes hands. +3. Write `.tldrgraph/propose_layers_response.json`: + +```json +{ + "utility_id": "", + "layers": [ + { + "id": "short_machine_id", + "name": "Layer 1: Human Friendly Name", + "order": 1, + "description": "One sentence on what lives here", + "rules": [ + {"file_contains": ["substring"], "exclude_file": ["optional"]}, + {"label_contains": ["SymbolNamePart"]} + ] + } + ] +} +``` + +4. Run `tldrgraph init` again. + +### What a layer set looks like + +Sketches from other codebases, to show the *shape* of an answer. They are not a +menu and none of them will fit this repository -- read the code and name what you +actually find. + +- A web app might split presentation from request handling from domain logic + from persistence, with background jobs and deployment config as their own tiers. +- A CLI tool might split the command surface from the processing engine from + local state, with adapters to outside systems separate again. +- A library might split its public API from the core implementation from its + data types, with backend adapters separate. +- A data pipeline might split ingestion from transformation from model training + from serving. + +The useful question is not "which of these is it?" but "where does responsibility +change hands in *this* code, and what would a new engineer need named?" + +### Rules that hold for any answer + +- 3 to 6 layers, plus exactly one catch-all whose `id` equals `utility_id` and + whose `rules` are `[]`. +- Unique `id` and `name` per layer; sequential integer `order` from 1. +- Rule keys: `file_contains`, `exclude_file`, `path_regex`, `label_contains`, + `exclude_label`, `label_ends_with`, `type_in`, `id_prefix`. Values are lists of + strings. Rules are evaluated in `order` and the first match wins. +- Derive rules from paths and symbol names you actually saw. A rule matching + nothing is worse than no rule; a rule matching everything collapses the map. + +## 2. `status: needs_confirmation` + +The output shows how many nodes need enrichment and how many agent round-trips +that implies. **Ask the user whether to proceed, and show them that estimate.** +Do not decide for them. + +- They agree: `tldrgraph init --yes` +- Smaller first pass: `tldrgraph init --yes --limit 100` +- They decline: stop. The graph is already built and queryable. + +## 3. `status: needs_enrichment` + +1. Read `.tldrgraph/enrichment_request.yaml`. +2. **Open the source file of every node in it.** This is the entire point: an + intent paraphrased from a symbol name poisons semantic search with + confident-sounding noise. +3. Write `.tldrgraph/enrichment_response.yaml` -- a *different* file from the + request, which is regenerated on every run: + +```yaml +- id: "" + intent: | + What this symbol does, why it exists, and its execution logic. + input_fields: [caseId, remarks] + output_fields: [status, disposition] + calls: [ApplicationsService, pension_cases] +``` + +4. Run `tldrgraph init --yes` again. It applies the response and hands you the + next batch, until there is nothing left. + +**Copy every `id` verbatim.** A constructed id matches nothing, is dropped, and +gets reported back to you -- but the work is wasted. + +**Never invent `fields` or `calls`.** Omit what you cannot verify in the code: an +empty list is a correct answer, a wrong `calls` entry becomes a real wrong edge. + +## Once it says DONE + +```bash +tldrgraph query "" +tldrgraph trace "" "" +tldrgraph layers +tldrgraph ui --serve +``` + +Read-only, and they never trigger enrichment. Full schema: +`.tldrgraph/AGENT_CONTRACT.md`. diff --git a/.cursor/commands/tldrgraph-init.md b/.cursor/commands/tldrgraph-init.md new file mode 100644 index 0000000..7ddfa53 --- /dev/null +++ b/.cursor/commands/tldrgraph-init.md @@ -0,0 +1,128 @@ +--- +name: tldrgraph-init +description: Build or continue this repository's TLDRGraph architecture graph (layers, extraction, enrichment) +--- + +# TLDRGraph: build this repository's architecture graph + +One command, run repeatedly until it says DONE. `tldrgraph init` never guesses: +it stops and tells you exactly what it needs. + +```bash +tldrgraph init +``` + +Read the `NEXT ACTION` block it prints, do what it says, then run `tldrgraph init` +again. Repeat until the output says `status: done`. There are only three things +it can ask for. + +## 1. `status: needs_layers` + +TLDRGraph ships **no layer templates** and will not invent an architecture. +Design one from this repository. + +1. Read `.tldrgraph/propose_layers_request.json`. It carries the symbols and + files extraction already found -- a starting point, not a substitute for + opening the code. +2. **Open real source files**: entry points first, then a representative file + from each cluster in the evidence. Work out what this codebase actually does + and where responsibility changes hands. +3. Write `.tldrgraph/propose_layers_response.json`: + +```json +{ + "utility_id": "", + "layers": [ + { + "id": "short_machine_id", + "name": "Layer 1: Human Friendly Name", + "order": 1, + "description": "One sentence on what lives here", + "rules": [ + {"file_contains": ["substring"], "exclude_file": ["optional"]}, + {"label_contains": ["SymbolNamePart"]} + ] + } + ] +} +``` + +4. Run `tldrgraph init` again. + +### What a layer set looks like + +Sketches from other codebases, to show the *shape* of an answer. They are not a +menu and none of them will fit this repository -- read the code and name what you +actually find. + +- A web app might split presentation from request handling from domain logic + from persistence, with background jobs and deployment config as their own tiers. +- A CLI tool might split the command surface from the processing engine from + local state, with adapters to outside systems separate again. +- A library might split its public API from the core implementation from its + data types, with backend adapters separate. +- A data pipeline might split ingestion from transformation from model training + from serving. + +The useful question is not "which of these is it?" but "where does responsibility +change hands in *this* code, and what would a new engineer need named?" + +### Rules that hold for any answer + +- 3 to 6 layers, plus exactly one catch-all whose `id` equals `utility_id` and + whose `rules` are `[]`. +- Unique `id` and `name` per layer; sequential integer `order` from 1. +- Rule keys: `file_contains`, `exclude_file`, `path_regex`, `label_contains`, + `exclude_label`, `label_ends_with`, `type_in`, `id_prefix`. Values are lists of + strings. Rules are evaluated in `order` and the first match wins. +- Derive rules from paths and symbol names you actually saw. A rule matching + nothing is worse than no rule; a rule matching everything collapses the map. + +## 2. `status: needs_confirmation` + +The output shows how many nodes need enrichment and how many agent round-trips +that implies. **Ask the user whether to proceed, and show them that estimate.** +Do not decide for them. + +- They agree: `tldrgraph init --yes` +- Smaller first pass: `tldrgraph init --yes --limit 100` +- They decline: stop. The graph is already built and queryable. + +## 3. `status: needs_enrichment` + +1. Read `.tldrgraph/enrichment_request.yaml`. +2. **Open the source file of every node in it.** This is the entire point: an + intent paraphrased from a symbol name poisons semantic search with + confident-sounding noise. +3. Write `.tldrgraph/enrichment_response.yaml` -- a *different* file from the + request, which is regenerated on every run: + +```yaml +- id: "" + intent: | + What this symbol does, why it exists, and its execution logic. + input_fields: [caseId, remarks] + output_fields: [status, disposition] + calls: [ApplicationsService, pension_cases] +``` + +4. Run `tldrgraph init --yes` again. It applies the response and hands you the + next batch, until there is nothing left. + +**Copy every `id` verbatim.** A constructed id matches nothing, is dropped, and +gets reported back to you -- but the work is wasted. + +**Never invent `fields` or `calls`.** Omit what you cannot verify in the code: an +empty list is a correct answer, a wrong `calls` entry becomes a real wrong edge. + +## Once it says DONE + +```bash +tldrgraph query "" +tldrgraph trace "" "" +tldrgraph layers +tldrgraph ui --serve +``` + +Read-only, and they never trigger enrichment. Full schema: +`.tldrgraph/AGENT_CONTRACT.md`. diff --git a/.gitignore b/.gitignore index 5823d21..3a041ee 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ htmlcov/ # Graphify / TLDRGraph analysis outputs & state graphify-out/ -.tldrgraph/ +# .tldrgraph/ # superseded by the TLDRGraph block below .codechakra/ scratch/ @@ -82,3 +82,11 @@ Thumbs.db !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json + +# BEGIN TLDRGRAPH +# TLDRGraph analysis state. Generated artifacts are ignored; the agent +# contract and layer map are committed so the whole team shares them. +.tldrgraph/* +!.tldrgraph/AGENT_CONTRACT.md +!.tldrgraph/layers.config.yaml +# END TLDRGRAPH diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2a6d519 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ + +## TLDRGraph + +This repository is mapped into architectural layers designed from its own source, +with per-symbol intents you can search and trace. + +**Before planning or implementing a feature**, trace it instead of grepping: + +```bash +tldrgraph query "" # semantic search + end-to-end flow +tldrgraph trace "" "" # exact path between two symbols +tldrgraph layers # node counts per layer +tldrgraph dead-code # review candidates, never a delete list +``` + +Those are read-only and never trigger enrichment. + +**To build or continue the graph**, run `tldrgraph init`, do what the `NEXT ACTION` +block prints, and run it again -- repeat until `status: done`. It has no template +fallback: if this repository has no architecture yet, it will stop and ask you to +design one from the code. Do not skip reading the files. + +Full workflow: `.claude/commands/tldrgraph-init.md` (identical copies live in every +other agent directory). Schema: `.tldrgraph/AGENT_CONTRACT.md`. + +`tldrgraph dead-code` lists **review candidates, not confirmed dead code**. +`unreviewed` means "not enough evidence to conclude" and is never removable. + diff --git a/AGENT_CONTRACT.md b/AGENT_CONTRACT.md index bc3ae20..d9c48ba 100644 --- a/AGENT_CONTRACT.md +++ b/AGENT_CONTRACT.md @@ -2,9 +2,10 @@ **Audience: the coding agent with this repository open** (Claude Code, Cursor, Antigravity). -TLDRGraph builds a 6-layer architectural graph from the graphify AST export. Structure -*within* a layer, and the high-volume deterministic seams between layers, are extracted -automatically. What cannot be extracted automatically is: +TLDRGraph builds an architectural graph from the graphify AST export. The layer set +itself is designed by you, reading this repository. TLDRGraph ships no layer templates. +Structure *within* a layer, and the high-volume deterministic seams between layers, are +extracted automatically. What cannot be extracted automatically is: - indirect dispatch, queue / event hops, dynamically-built routes; - the natural-language **intent** that makes semantic search work at all. @@ -15,6 +16,34 @@ and a path (`snippet` is never populated), so it guesses. You do not have to gue --- +## Start here: `tldrgraph init` + +One command does everything, and it is resumable: + +```bash +tldrgraph init +``` + +It runs every deterministic step — extraction, classification, indexing, applying whatever +you last wrote — then stops with a `NEXT ACTION` block the moment it needs judgement only +you can supply. Do what the block says and run it again. Repeat until `status: done`. + +| status | what it wants | +| --- | --- | +| `needs_layers` | Read the code and design this repository's architecture. **TLDRGraph ships no layer templates**; nothing will be applied for you. The request carries sketches of how other kinds of codebase divide — for shape only, never to copy. | +| `needs_confirmation` | Enrichment costs the user tokens. Show them the estimate and ask. Then `tldrgraph init --yes`. | +| `needs_enrichment` | A batch to open, read and describe, per the schema below. | +| `done` | Nothing left. Use `query` / `trace` / `layers`. | + +`--json` gives you the same thing machine-readably. The sections below document the file +formats `init` reads and writes; the underlying `queue-enrichment` / `apply-enrichment` +commands remain available for scripting. + +**Copy every `id` verbatim from the request.** A constructed id matches nothing, is +dropped, and will be reported back to you — but the work is wasted. + +--- + ## The loop ```bash @@ -213,7 +242,7 @@ It then records the ids in the cursor so the next `queue-enrichment` moves on. ## Related commands ```bash -tldrgraph scan . # rebuild layers + index from graphify-out/ +tldrgraph init # everything, resumable (scan/enrich are aliases) tldrgraph query "pension approval" # semantic search + end-to-end flow trace tldrgraph trace AaoDeskView pension_cases tldrgraph layers # node counts per layer diff --git a/README.md b/README.md index ce5fd54..3f78bc9 100644 --- a/README.md +++ b/README.md @@ -26,16 +26,22 @@ Modern codebases are messy. Microservices, multi-layer abstractions, dynamic API --- -## 🏛️ Dynamic Architectural Layers +## 🏛️ Agent-Designed Architectural Layers -Rather than forcing every project into a rigid model, TLDRGraph automatically detects your codebase archetype or synthesizes dynamic architectural layers directly into `.tldrgraph/layers.config.yaml`: +TLDRGraph does not pick your architecture from a menu, and **it ships no layer +templates at all**. On the first run it hands the repository to your coding agent +— with the symbols it just extracted, not merely a directory listing — and the +layer set the agent designs is written to `.tldrgraph/layers.config.yaml`, named +after your codebase's own concepts. -- **Full-Stack Web:** Presentation & UI $\rightarrow$ API Gateway $\rightarrow$ Domain Services $\rightarrow$ Data & Persistence $\rightarrow$ Async Tasks $\rightarrow$ DevOps -- **Backend APIs:** API & Handlers $\rightarrow$ Domain Services $\rightarrow$ Persistence & Repositories $\rightarrow$ Background Jobs $\rightarrow$ Utilities -- **CLI Applications:** CLI & Commands $\rightarrow$ Core Flow Engine $\rightarrow$ Storage & Index $\rightarrow$ Agent Loop & UI $\rightarrow$ Utilities -- **Libraries & SDKs:** Public API $\rightarrow$ Core Engine $\rightarrow$ Types & Models $\rightarrow$ Adapters $\rightarrow$ Utilities +The agent is given *ideas*, not a template: a handful of one-line sketches of how +different kinds of codebase can divide, explicitly labelled as belonging to other +repositories, followed by the real question — *where does responsibility change +hands in this code?* ---- +If no agent answers, TLDRGraph stops and asks. An unconfigured repository has a +single `Unclassified` bucket, not six confident guesses: a generic layer set is +wrong everywhere it looks right. ## 🚀 Quickstart @@ -45,11 +51,32 @@ pip install tldrgraph ``` *(For optional local ONNX neural embeddings: `pip install "tldrgraph[embeddings]"`)* -### 2. Scan & Classify Repository +### 2. Build the graph — one command +```bash +tldrgraph init +``` + +`init` is a resumable state machine. It runs every step it can — extraction, +classification, indexing, enrichment — and stops with a `NEXT ACTION` block the +moment it needs judgement only an agent can supply. Do what the block says, run +it again, repeat until it prints `status: done`. + +It can ask for exactly three things: + +| status | what it needs | +| --- | --- | +| `needs_layers` | Read the code and design the architecture. No template will be applied for you. | +| `needs_confirmation` | Shows how many nodes need enrichment and how many agent rounds that is. **Your agent asks you before spending tokens.** | +| `needs_enrichment` | A batch of nodes to open, read, and describe. | + ```bash -tldrgraph scan . +tldrgraph init --yes # proceed past the estimate +tldrgraph init --yes --limit 100 # smaller first pass +tldrgraph init --json # machine-readable status for agents ``` -This runs Graphify AST extraction, classifies architectural layers, builds offline local search indices, and creates `.tldrgraph/layers.yaml`. + +Your agent can drive the whole thing with the installed `/tldrgraph-init` +command. `scan` and `enrich` are aliases for `init`, kept for existing scripts. ### 3. Explore the Architecture Visually ```bash @@ -81,30 +108,68 @@ Surfaces orphaned components, unreferenced models, and unused files for human re --- -## 🤖 AI Assistant Rules (`tldrgraph install`) +## 🤖 Works with any coding agent + +TLDRGraph is driven **by** your agent, not the other way around. It never needs +to launch one, so there are no per-tool flags, auth or headless quirks to get +wrong — any agent that can read a file, read source, and run a shell command can +drive it. + +Every tool gets the **same two artifacts and no more**: one body of instructions +and one `tldrgraph-init` command, byte-identical everywhere. -TLDRGraph supports seamless pairing with AI coding assistants (Claude Code, Cursor, and Antigravity): +| Artifact | Where | +| --- | --- | +| **Instructions** | `AGENTS.md` — the cross-tool standard, read by Claude Code, Cursor, opencode, Codex, Gemini CLI, Zed and Copilot | +| | `.agents/rules/`, `.clinerules/`, `.windsurf/rules/` — only for tools not known to read AGENTS.md | +| **Command** | `.claude/commands/`, `.cursor/commands/`, `.agents/workflows/`, `.clinerules/workflows/`, `.windsurf/workflows/`, `.opencode/command/`, `.roo/commands/`, `.kilocode/workflows/`, `.goosehints/` | + +Tools with a marker directory are installed only when the repo shows them in +use; `tldrgraph install --all-agents` writes them all. Adding a tool is one row +in `TARGETS` in [agent_commands.py](tldrgraph/agent_commands.py) — **paths only, +never execution code.** + +No tool gets special treatment. Earlier versions shipped a Claude-only skill file +*plus* a `CLAUDE.md` section *plus* a Cursor rule *plus* an Antigravity rule, each +worded differently and each a different length; they contradicted each other +within a release. `tldrgraph install` deletes those on sight. + +### Letting TLDRGraph launch an agent itself + +Off by default, and opt-in per run: ```bash -tldrgraph install +tldrgraph init --yes --agent-cli ``` -Automatically writes agent instructions pointing to `.tldrgraph/AGENT_CONTRACT.md`: -- `.claude/skills/tldrgraph/SKILL.md` + delimited section in `CLAUDE.md` (Claude Code) -- `.cursor/rules/tldrgraph.mdc` (Cursor) -- `.agents/rules/tldrgraph.md` (Antigravity) - ---- +This shells out to `claude`, `cursor-agent` or `gemini` if one is on `PATH`. It +is genuinely useful in a plain terminal with no agent attached, but it is not the +default: agent CLIs differ per tool, block with no output while they think, and +some IDEs ship no working CLI at all. ## 📁 Artifacts & Output Formats -All project state is kept in `.tldrgraph/` (with automatic fallback to `.codechakra/`): +Scanning a repository adds **one** directory, `.tldrgraph/` — graphify's raw export is +kept inside it rather than in a second top-level `graphify-out/`: + - `.tldrgraph/graph.json` : Persisted multi-layer graph snapshot with cross-layer edges. +- `.tldrgraph/layers.config.yaml`: The agent-designed layer definition. **Commit this.** +- `.tldrgraph/AGENT_CONTRACT.md`: The request/response contract. **Commit this.** - `.tldrgraph/layers.yaml`: Layer distribution and node definitions. - `.tldrgraph/flows.yaml` : Exported trace paths. +- `.tldrgraph/graphify_graph.json`, `.tldrgraph/graphify_manifest.json`: graphify's raw + AST export and file manifest (renamed so they cannot collide with the enriched snapshot). +- `.tldrgraph/graphify/`: graphify's own AST cache. - `.tldrgraph/tldrgraph.db`: Local SQLite content-hash cache for zero-token incremental updates. - `.tldrgraph/TLDRGRAPH_VISUALIZER.html`: Standalone zero-dependency visualizer. +`tldrgraph install` (and every `scan`) adds a managed block to your `.gitignore` that +ignores the generated artifacts while keeping `layers.config.yaml` and `AGENT_CONTRACT.md` +committable, so your whole team shares one architecture map. + +Upgrading from an older version? A leftover `graphify-out/` is no longer read or written; +`scan` will point it out so you can delete it. + --- ## 🙏 Acknowledgements & Upstream Credits diff --git a/tests/conftest.py b/tests/conftest.py index 4d2c48d..a7550f9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ Hermetic fixtures for the TLDRGraph regression suite. Everything the tests touch lives under pytest's ``tmp_path``. Nothing here reads -the real repository, the real ``graphify-out/`` directory, or the real +the real repository, the real ``.tldrgraph/`` state directory, or the real ``.tldrgraph/`` state, and nothing makes a network call. """ @@ -30,8 +30,24 @@ sys.path.remove(_PKG_PARENT) sys.path.insert(0, _PKG_PARENT) +from tldrgraph import paths # noqa: E402 from tldrgraph.classifier import LayerType # noqa: E402 from tldrgraph.graph_loader import GraphLoader # noqa: E402 +from tldrgraph.layer_config import save_layer_config # noqa: E402 +from tldrgraph.layers import DEFAULT_LAYERS, LayerRegistry # noqa: E402 + + +def write_example_layer_config(root) -> str: + """ + Installs the worked-example layer set into a fixture repository. + + Production ships no layer templates: an unconfigured repo gets a single + "Unclassified" bucket, because classifying against an architecture nobody + derived from the code is worse than not classifying at all. Tests that + exercise the rule engine therefore have to say which layer set they mean, + and DEFAULT_LAYERS is the known one the fixture nodes were written against. + """ + return save_layer_config(str(root), LayerRegistry(DEFAULT_LAYERS)) # --------------------------------------------------------------------------- @@ -197,19 +213,20 @@ def __init__(self, root: Path): # -- convenience ------------------------------------------------------- @property def graphify_dir(self) -> Path: - return self.root / "graphify-out" + # graphify's raw export now lives inside the single state directory. + return self.tldrgraph_dir @property def graph_json(self) -> Path: - return self.graphify_dir / "graph.json" + return self.graphify_dir / paths.GRAPHIFY_GRAPH_FILENAME @property def manifest_json(self) -> Path: - return self.graphify_dir / "manifest.json" + return self.graphify_dir / paths.GRAPHIFY_MANIFEST_FILENAME @property def tldrgraph_dir(self) -> Path: - return self.root / ".tldrgraph" + return self.root / paths.STATE_DIRNAME @property def snapshot_path(self) -> Path: @@ -253,6 +270,25 @@ def bump_semantic_hash(self, key: str, new_value: str = "deadbeef" * 4) -> None: self.write_manifest(manifest) +@pytest.fixture(autouse=True) +def baseline_registry(): + """ + Starts every test from the worked-example layer set. + + ``load_layer_config`` installs whatever it finds on disk, so a test that + builds a graph leaves the process-wide registry pointing at that repo's + layers and the next test inherits it. Restoring afterwards is not enough -- + that just propagates whatever the previous test left. Resetting up front is. + + Production starts from :func:`bootstrap_registry` instead (one Unclassified + bucket); that behaviour is asserted directly in test_layer_config. + """ + from tldrgraph.layers import default_registry, set_registry + + set_registry(default_registry()) + return True + + @pytest.fixture(autouse=True) def env_no_llm(monkeypatch): """ @@ -261,6 +297,10 @@ def env_no_llm(monkeypatch): Clears every provider key the enricher looks at and points OLLAMA_HOST at a port nothing listens on, so ``_call_ollama`` fails immediately with connection-refused instead of reaching the network. + + Also switches off the agent-CLI path. A developer machine usually has + ``claude`` or ``gemini`` on PATH, and a test suite must never spend a real + agent's tokens; tests that exercise that path opt back in explicitly. """ for var in ( "GEMINI_API_KEY", @@ -271,6 +311,7 @@ def env_no_llm(monkeypatch): monkeypatch.delenv(var, raising=False) # Port 9 (discard) on loopback: refused instantly, never leaves the host. monkeypatch.setenv("OLLAMA_HOST", "http://127.0.0.1:9") + monkeypatch.setenv("TLDRGRAPH_NO_AGENT", "1") return True @@ -278,8 +319,9 @@ def env_no_llm(monkeypatch): def mini_repo(tmp_path, env_no_llm) -> MiniRepo: """Materialize a hermetic mini-repo with graphify output + real sources.""" root = tmp_path / "minirepo" - graphify = root / "graphify-out" + graphify = root / paths.STATE_DIRNAME graphify.mkdir(parents=True) + write_example_layer_config(root) # 1. Real source files on disk (so the sha256 fallback path is reachable). for rel, content in FILE_CONTENTS.items(): @@ -287,7 +329,7 @@ def mini_repo(tmp_path, env_no_llm) -> MiniRepo: dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(content, encoding="utf-8") - # 2. graphify-out/graph.json + # 2. graphify's raw AST export nodes = [] for key, (label, src, loc, ftype, community, _layer) in NODE_SPECS.items(): nodes.append( @@ -330,9 +372,9 @@ def mini_repo(tmp_path, env_no_llm) -> MiniRepo: "hyperedges": [], "built_at_commit": "0000000", } - (graphify / "graph.json").write_text(json.dumps(graph_doc, indent=2), encoding="utf-8") + (graphify / paths.GRAPHIFY_GRAPH_FILENAME).write_text(json.dumps(graph_doc, indent=2), encoding="utf-8") - # 3. graphify-out/manifest.json -- one entry per source_file. + # 3. graphify's manifest -- one entry per source_file. now = time.time() manifest = {} for rel, content in FILE_CONTENTS.items(): @@ -342,7 +384,7 @@ def mini_repo(tmp_path, env_no_llm) -> MiniRepo: "ast_hash": _hash("ast:" + content), "semantic_hash": _hash("sem:" + content), } - (graphify / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + (graphify / paths.GRAPHIFY_MANIFEST_FILENAME).write_text(json.dumps(manifest, indent=2), encoding="utf-8") return MiniRepo(root) diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 212b5da..0876b1b 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -13,7 +13,7 @@ `query` / `trace` / `layers`. Fixtures are local to this module on purpose; nothing here reads the real repository, -the real ``graphify-out/`` or the real ``.tldrgraph/``, and nothing makes a network call. +the real repository or the real ``.tldrgraph/`` state, and nothing makes a network call. """ import json @@ -23,6 +23,8 @@ import pytest from click.testing import CliRunner +from conftest import write_example_layer_config +from tldrgraph import paths from tldrgraph import cli as cli_module from tldrgraph import installer as installer_module from tldrgraph.cli import ( @@ -78,7 +80,8 @@ def loop_repo(tmp_path) -> Path: """A hermetic mini-repo with graphify output, rooted in tmp_path.""" root = tmp_path / "looprepo" - (root / "graphify-out").mkdir(parents=True) + (root / paths.STATE_DIRNAME).mkdir(parents=True) + write_example_layer_config(root) for rel, body in FILE_BODIES.items(): dest = root / rel @@ -105,7 +108,7 @@ def loop_repo(tmp_path) -> Path: } for src, tgt, relation in LOOP_EDGES ] - (root / "graphify-out" / "graph.json").write_text( + (root / paths.STATE_DIRNAME / paths.GRAPHIFY_GRAPH_FILENAME).write_text( json.dumps({"directed": True, "nodes": nodes, "links": links}, indent=2), encoding="utf-8", ) @@ -382,7 +385,7 @@ def test_queue_orders_hubs_and_seams_before_leaves(run, state): def test_degree_is_recomputed_not_read_from_graphify(run, loop_repo, state): """graphify emits no `degree` key, so the raw value is 0 for every node.""" - raw = json.loads((loop_repo / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + raw = json.loads((loop_repo / paths.STATE_DIRNAME / paths.GRAPHIFY_GRAPH_FILENAME).read_text(encoding="utf-8")) assert all("degree" not in n for n in raw["nodes"]) run("queue-enrichment", "--limit", "0") @@ -587,13 +590,17 @@ def test_dead_code_has_no_delete_capability(run): def test_install_writes_claude_cursor_and_antigravity(tmp_path): written = installer_module.install_agent_rules(str(tmp_path)) + # One body of instructions and one command, per tool. No tool gets bespoke + # prose and no tool gets a third artifact. expected = { "contract": ".tldrgraph/AGENT_CONTRACT.md", - "claude_skill": ".claude/skills/tldrgraph/SKILL.md", - "claude_md": "CLAUDE.md", - "cursor_rule": ".cursor/rules/tldrgraph.mdc", - "antigravity_rule": ".agents/rules/tldrgraph.md", - "antigravity_workflow": ".agents/workflows/tldrgraph.md", + "gitignore": ".gitignore", + "AGENTS.md (instructions, all agents)": "AGENTS.md", + "Claude Code (command)": ".claude/commands/tldrgraph-init.md", + "Cursor (command)": ".cursor/commands/tldrgraph-init.md", + "Antigravity (command)": ".agents/workflows/tldrgraph-init.md", + # Antigravity is the one tool here not known to read AGENTS.md. + "Antigravity (instructions)": ".agents/rules/tldrgraph.md", } assert set(written) == set(expected) for key, rel in expected.items(): @@ -604,21 +611,25 @@ def test_install_writes_claude_cursor_and_antigravity(tmp_path): def test_every_rule_file_points_at_the_contract_and_the_loop(tmp_path): written = installer_module.install_agent_rules(str(tmp_path)) for key, path in written.items(): - if key == "contract": + # .gitignore is not an instruction file, and the contract is what the + # others point AT rather than a pointer itself. + if key in ("contract", "gitignore") or not os.path.isfile(path): continue text = Path(path).read_text(encoding="utf-8") assert "AGENT_CONTRACT.md" in text, key - assert "queue-enrichment" in text, key - assert "apply-enrichment" in text, key + assert "tldrgraph init" in text, key def test_rules_tell_the_agent_to_read_the_source_and_not_invent(tmp_path): written = installer_module.install_agent_rules(str(tmp_path)) - for key in ("claude_skill", "claude_md", "cursor_rule", "antigravity_rule", "contract"): - text = Path(written[key]).read_text(encoding="utf-8").lower() - assert "read the source" in text or "read the actual source" in text, key - assert "invent" in text, key - assert "0.35" in text, key + contract = Path(written["contract"]).read_text(encoding="utf-8").lower() + assert "read the source" in contract or "read the actual source" in contract + assert "invent" in contract + assert "0.35" in contract + + command = Path(written["Claude Code (command)"]).read_text(encoding="utf-8").lower() + assert "open the source file of every node" in command + assert "never invent" in command def test_install_is_idempotent(tmp_path): @@ -639,9 +650,10 @@ def test_existing_claude_md_is_never_clobbered(tmp_path): installer_module.install_agent_rules(str(tmp_path)) text = claude_md.read_text(encoding="utf-8") + # TLDRGraph no longer writes into CLAUDE.md -- AGENTS.md covers Claude Code + # along with every other tool -- but it must not damage what is there. assert "Hand-written house rules." in text - assert installer_module.CLAUDE_MD_BEGIN in text - assert installer_module.CLAUDE_MD_END in text + assert installer_module.CLAUDE_MD_BEGIN not in text def test_claude_md_section_is_replaced_in_place_not_appended(tmp_path): @@ -650,11 +662,11 @@ def test_claude_md_section_is_replaced_in_place_not_appended(tmp_path): installer_module.install_agent_rules(str(tmp_path)) installer_module.install_agent_rules(str(tmp_path)) - text = claude_md.read_text(encoding="utf-8") - assert text.count(installer_module.CLAUDE_MD_BEGIN) == 1 - assert text.count(installer_module.CLAUDE_MD_END) == 1 - assert text.count("keep me") == 1 + agents_md = (tmp_path / "AGENTS.md").read_text(encoding="utf-8") + assert agents_md.count(installer_module.CLAUDE_MD_BEGIN) == 1 + assert agents_md.count(installer_module.CLAUDE_MD_END) == 1 + assert claude_md.read_text(encoding="utf-8").count("keep me") == 1 def test_upsert_preserves_text_after_the_managed_section(): @@ -703,8 +715,8 @@ def test_gitignore_warnings_are_empty_when_nothing_is_hidden(tmp_path): def test_install_command_lists_what_it_wrote(tmp_path): result = CliRunner().invoke(cli, ["install", "--path", str(tmp_path)]) assert result.exit_code == 0 - for key in ("contract", "claude_skill", "claude_md", "cursor_rule", - "antigravity_rule", "antigravity_workflow"): + for key in ("contract", "gitignore", "AGENTS.md", + "Claude Code (command)", "Cursor (command)", "Antigravity (command)"): assert key in result.output @@ -721,7 +733,9 @@ def test_existing_command_names_are_preserved(): def test_scan_still_accepts_rebuild(loop_repo): result = CliRunner().invoke(cli, ["scan", str(loop_repo), "--rebuild"]) assert result.exit_code == 0, result.output - assert "Scan complete" in result.output + # `scan` is now an alias for `init`, which reports a status rather than a + # "complete" banner -- this repo has no layer set, so it asks for one. + assert "status:" in result.output def test_query_still_accepts_top_k_and_path(loop_repo): diff --git a/tests/test_auto_agent.py b/tests/test_auto_agent.py new file mode 100644 index 0000000..8cb69d9 --- /dev/null +++ b/tests/test_auto_agent.py @@ -0,0 +1,863 @@ +""" +Automatic agent path: layer design, in-scan enrichment, and the managed .gitignore. + +Nothing here spawns a real coding agent. Everything above the subprocess boundary +is driven by a fake AgentCLI; the boundary itself is exercised once against +``/bin/echo`` so the argv/parse contract is not just asserted in a mock. +""" + +from __future__ import annotations + +import json + +from pathlib import Path + +import pytest +import yaml +from click.testing import CliRunner + +from tldrgraph import agent_commands, agent_runner, installer as installer_module, paths +from tldrgraph.cli import cli +from tldrgraph.propose_layers import ( + NEEDS_LAYERS, + auto_configure_layers, + propose_layers_with_agent, +) + +# --------------------------------------------------------------------------- # +# Helpers +# --------------------------------------------------------------------------- # + +VALID_LAYER_SET = { + "utility_id": "shared", + "layers": [ + {"id": "entry", "name": "Layer 1: Entry", "order": 1, "description": "entry points", + "rules": [{"file_contains": ["cli.py"]}]}, + {"id": "core", "name": "Layer 2: Core", "order": 2, "description": "core logic", + "rules": [{"file_contains": ["engine"]}]}, + {"id": "shared", "name": "Shared", "order": 3, "description": "catch-all", "rules": []}, + ], +} + + +def fake_agent(name: str = "fake") -> agent_runner.AgentCLI: + """An AgentCLI that is never actually executed (run_agent is monkeypatched).""" + return agent_runner.AgentCLI( + name=name, binary="/nonexistent/" + name, display=f"Fake {name}", + build_args=lambda prompt: ["-p", prompt], + ) + + +@pytest.fixture +def agent_allowed(monkeypatch): + """ + Undo the suite-wide agent kill switch for tests that need the path live. + + Clearing the switch alone is dangerous: a developer machine usually has a + real ``claude`` or ``gemini`` on PATH, and an un-stubbed call would spawn it + for real. So the subprocess boundary is nailed shut here -- a test that + forgets to stub it fails immediately instead of quietly spending tokens. + """ + monkeypatch.delenv(agent_runner.ENV_DISABLE, raising=False) + for marker in agent_runner.NESTED_MARKERS: + monkeypatch.delenv(marker, raising=False) + + def _forbidden(*args, **kwargs): + raise AssertionError( + "a test reached the real agent subprocess: stub agent_runner.run_agent" + ) + + monkeypatch.setattr(agent_runner, "run_agent", _forbidden) + return True + + +@pytest.fixture +def cli_repo(tmp_path) -> Path: + """A minimal Python CLI repo that graphify can extract without network access.""" + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "sample"\n[project.scripts]\nsample = "sample.cli:main"\n', + encoding="utf-8", + ) + pkg = tmp_path / "sample" + pkg.mkdir() + (pkg / "cli.py").write_text( + "from .engine import Engine\n\n\ndef main():\n return Engine().run()\n", + encoding="utf-8", + ) + (pkg / "engine.py").write_text( + "class Engine:\n def run(self):\n return 1\n", encoding="utf-8" + ) + return tmp_path + + +# --------------------------------------------------------------------------- # +# 1. Agent discovery +# --------------------------------------------------------------------------- # + +def test_agent_is_not_used_when_disabled(monkeypatch): + monkeypatch.setenv(agent_runner.ENV_DISABLE, "1") + assert agent_runner.find_agent_cli() is None + assert agent_runner.agent_status()["reason"] == "disabled" + + +def test_agent_is_not_spawned_from_inside_another_agent(monkeypatch, agent_allowed): + """ + A coding agent running `tldrgraph scan` must not cause a second agent to be + spawned: the host already holds the context, so it should be told what to do. + """ + monkeypatch.setenv("CLAUDECODE", "1") + assert agent_runner.nesting_marker() == "CLAUDECODE" + assert agent_runner.find_agent_cli() is None + assert agent_runner.agent_status()["reason"] == "nested" + + +def test_nesting_can_be_opted_into(monkeypatch, agent_allowed, tmp_path): + monkeypatch.setenv("CLAUDECODE", "1") + monkeypatch.setenv(agent_runner.ENV_ALLOW_NESTED, "1") + monkeypatch.setenv(agent_runner.ENV_FORCE_CLI, "/bin/echo") + assert agent_runner.find_agent_cli() is not None + + +def test_missing_binary_reports_not_found(monkeypatch, agent_allowed): + monkeypatch.setattr(agent_runner.shutil, "which", lambda _binary: None) + status = agent_runner.agent_status() + assert status["reason"] == "not_found" + assert status["agent"] is None + + +# --------------------------------------------------------------------------- # +# 2. Output parsing +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("text, expected", [ + ('{"a": 1}', {"a": 1}), + ('```json\n{"a": 1}\n```', {"a": 1}), + ('Sure, here you go:\n```\n[{"id": "x"}]\n```\nHope that helps!', [{"id": "x"}]), + ('Here is the result: [{"id": "x"}]', [{"id": "x"}]), +]) +def test_json_is_recovered_from_however_the_agent_wrapped_it(text, expected): + assert agent_runner.extract_json(text) == expected + + +def test_unparseable_output_raises_rather_than_returning_junk(): + with pytest.raises(agent_runner.AgentError): + agent_runner.extract_json("I could not complete that request.") + with pytest.raises(agent_runner.AgentError): + agent_runner.extract_json("") + + +def test_run_agent_crosses_the_real_subprocess_boundary(tmp_path): + """The one test that actually forks: argv construction and stdout capture.""" + echo = agent_runner.AgentCLI( + name="echo", binary="/bin/echo", display="echo", + build_args=lambda prompt: [prompt], + ) + assert agent_runner.run_agent_json(echo, '{"ok": true}', str(tmp_path)) == {"ok": True} + + +def test_nonzero_exit_becomes_an_agent_error(tmp_path): + failing = agent_runner.AgentCLI( + name="false", binary="/usr/bin/false", display="false", + build_args=lambda prompt: [], + ) + with pytest.raises(agent_runner.AgentError): + agent_runner.run_agent(failing, "anything", str(tmp_path)) + + +def test_claude_error_envelope_is_surfaced(): + envelope = json.dumps({"is_error": True, "result": "rate limited"}) + with pytest.raises(agent_runner.AgentError, match="rate limited"): + agent_runner._claude_parse(envelope) + + +def test_claude_reports_failure_with_exit_code_zero(): + """ + Verified against real `claude -p --output-format json` output: an expired + OAuth token comes back as ``is_error: true`` on a SUCCESSFUL exit status. + Trusting the exit code alone would write the error message into the graph + as if it were an intent. + """ + real_envelope = json.dumps({ + "type": "result", + "subtype": "success", + "is_error": True, + "duration_ms": 1857, + "num_turns": 1, + "result": "Failed to authenticate. API Error: 401 OAuth access token has expired.", + "session_id": "c2a7e5f8-a303-44da-9243-0c3cc3f4d329", + }) + with pytest.raises(agent_runner.AgentError, match="Failed to authenticate"): + agent_runner._claude_parse(real_envelope) + + +def test_claude_success_envelope_is_unwrapped(): + envelope = json.dumps({ + "type": "result", "subtype": "success", "is_error": False, + "result": '{"utility_id": "shared", "layers": []}', + }) + assert agent_runner.extract_json(agent_runner._claude_parse(envelope)) == { + "utility_id": "shared", "layers": [], + } + + +# --------------------------------------------------------------------------- # +# 3. Layer design by agent +# --------------------------------------------------------------------------- # + +def test_agent_layer_proposal_is_validated_and_returned(monkeypatch, cli_repo): + monkeypatch.setattr(agent_runner, "run_agent", lambda *a, **k: json.dumps(VALID_LAYER_SET)) + proposal, detail = propose_layers_with_agent(str(cli_repo), agent=fake_agent()) + assert detail == "fake" + assert [layer["id"] for layer in proposal["layers"]] == ["entry", "core", "shared"] + + +def test_invalid_agent_layer_proposal_is_rejected_not_written(monkeypatch, cli_repo): + """A layer set naming a utility_id that does not exist must never be saved.""" + broken = {"utility_id": "nope", "layers": VALID_LAYER_SET["layers"]} + monkeypatch.setattr(agent_runner, "run_agent", lambda *a, **k: json.dumps(broken)) + proposal, detail = propose_layers_with_agent(str(cli_repo), agent=fake_agent()) + assert proposal is None + assert "invalid layer set" in detail + + +def test_agent_prompt_tells_the_agent_to_read_files(cli_repo): + from tldrgraph.propose_layers import build_agent_layer_prompt, collect_layer_evidence + + prompt = build_agent_layer_prompt(str(cli_repo), collect_layer_evidence(str(cli_repo))) + assert "Read the entry points" in prompt + assert "NOT a substitute for opening the files" in prompt + assert str(cli_repo) in prompt + + +def test_auto_configure_prefers_the_agent_over_the_archetype(monkeypatch, cli_repo): + monkeypatch.setattr(agent_runner, "run_agent", lambda *a, **k: json.dumps(VALID_LAYER_SET)) + reg, cfg_path, source = auto_configure_layers(str(cli_repo), agent=fake_agent(), use_llm=False, use_agent=True) + + assert source == "agent:fake" + assert reg.ids() == ("entry", "core", "shared") + + +def test_no_agent_means_no_layers_and_no_config_file(cli_repo): + """The archetype fallback is gone: nothing writes layers it did not derive.""" + reg, cfg_path, source = auto_configure_layers( + str(cli_repo), enricher=None, use_llm=False, use_agent=False + ) + assert source == NEEDS_LAYERS + assert reg is None and cfg_path is None + assert not (cli_repo / ".tldrgraph" / "layers.config.yaml").exists() + + +def test_an_agent_authored_config_is_never_silently_replaced(monkeypatch, cli_repo): + monkeypatch.setattr(agent_runner, "run_agent", lambda *a, **k: json.dumps(VALID_LAYER_SET)) + auto_configure_layers(str(cli_repo), agent=fake_agent(), use_llm=False, use_agent=True) + + calls = [] + + def _should_not_run(*args, **kwargs): + calls.append(1) + return json.dumps(VALID_LAYER_SET) + + monkeypatch.setattr(agent_runner, "run_agent", _should_not_run) + _, _, source = auto_configure_layers(str(cli_repo), agent=fake_agent(), use_llm=False, use_agent=True) + assert source == "existing_config" + assert calls == [], "a settled config must not cost another agent call" + + +# --------------------------------------------------------------------------- # +# 4. `tldrgraph init` -- one command, resumable +# --------------------------------------------------------------------------- # + +def _is_enrichment_prompt(prompt: str) -> bool: + return "Nodes (" in prompt + + +def _fake_answer(prompt: str) -> str: + """ + One fake agent for the whole run: it designs layers when asked for layers, + and enriches every node id when asked for enrichment. + """ + if not _is_enrichment_prompt(prompt): + return json.dumps(VALID_LAYER_SET) + + start = prompt.index("Nodes (") + nodes = json.loads(prompt[prompt.index("[", start):]) + return json.dumps([ + { + "id": node["id"], + "intent": f"Reads and returns the {node['label']} result.", + "input_fields": ["alpha"], + "output_fields": ["beta"], + "calls": [], + } + for node in nodes + ]) + + +def _answer_layers(repo) -> None: + """Play the agent's part for phase 1, the way the NEXT ACTION block asks.""" + state = repo / ".tldrgraph" + state.mkdir(exist_ok=True) + (state / "propose_layers_response.json").write_text( + json.dumps(VALID_LAYER_SET), encoding="utf-8" + ) + + +def _stub_agent_cli(monkeypatch, answer=_fake_answer): + monkeypatch.setattr(agent_runner, "run_agent", + lambda agent, prompt, cwd, timeout=None, model=None: answer(prompt)) + monkeypatch.setattr(agent_runner, "find_agent_cli", lambda **kw: fake_agent()) + monkeypatch.setattr( + agent_runner, "agent_status", + lambda: {"agent": fake_agent(), "reason": "ready", "detail": "Fake fake"}, + ) + + +def test_init_stops_and_asks_for_layers_first(cli_repo): + """Phase 1: no architecture, no template, so it must stop and ask.""" + res = CliRunner().invoke(cli, ["init", str(cli_repo)]) + assert res.exit_code == 0, res.output + assert "status: needs_layers" in res.output + assert "propose_layers_request.json" in res.output + assert not (cli_repo / ".tldrgraph" / "layers.config.yaml").exists() + + +def test_init_extracts_before_asking_so_the_evidence_has_real_symbols(cli_repo): + """ + The layer request must carry extracted symbols, not just a directory + listing -- two repos with identical file trees can do entirely different + things, and the agent is being asked to name what this one does. + """ + CliRunner().invoke(cli, ["init", str(cli_repo)]) + payload = json.loads( + (cli_repo / ".tldrgraph" / "propose_layers_request.json").read_text(encoding="utf-8") + ) + symbols = payload["evidence"]["extracted_symbols"] + assert symbols["total_symbols"] > 0 + assert any("cli.py" in path for path in symbols["symbols_by_file"]) + + +def test_init_resumes_after_the_agent_answers_the_layers(cli_repo): + """Phase 1 → 2: answering the request and re-running gets past the gate.""" + CliRunner().invoke(cli, ["init", str(cli_repo)]) + _answer_layers(cli_repo) + + res = CliRunner().invoke(cli, ["init", str(cli_repo)]) + assert res.exit_code == 0, res.output + assert "status: needs_layers" not in res.output + assert (cli_repo / ".tldrgraph" / "layers.config.yaml").is_file() + assert (cli_repo / ".tldrgraph" / "graph.json").is_file() + + +def test_init_asks_before_spending_tokens_and_shows_the_estimate(cli_repo): + """Phase 3 gate: the user is told the size of the job before it starts.""" + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo)]) + + assert res.exit_code == 0, res.output + assert "status: needs_confirmation" in res.output + assert "ASK THE USER" in res.output + assert "tldrgraph init --yes" in res.output + + +def test_the_estimate_is_machine_readable(cli_repo): + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--json"]) + assert res.exit_code == 0, res.output + + payload = json.loads(res.output) + assert payload["status"] == "needs_confirmation" + progress = payload["progress"] + assert progress["remaining"] > 0 + assert progress["agent_rounds"] >= 1 + assert progress["total_nodes"] >= progress["remaining"] + + +def test_yes_hands_out_an_enrichment_batch(cli_repo): + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--yes"]) + + assert res.exit_code == 0, res.output + assert "status: needs_enrichment" in res.output + request = cli_repo / ".tldrgraph" / "enrichment_request.yaml" + assert request.is_file() + assert yaml.safe_load(request.read_text(encoding="utf-8"))["nodes"] + + +def test_init_applies_the_agents_enrichment_and_reaches_done(cli_repo): + """The full loop, played out the way an agent would: init, answer, init.""" + _answer_layers(cli_repo) + runner = CliRunner() + runner.invoke(cli, ["init", str(cli_repo), "--yes"]) + + state = cli_repo / ".tldrgraph" + for _ in range(20): + request = yaml.safe_load((state / "enrichment_request.yaml").read_text(encoding="utf-8")) + (state / "enrichment_response.yaml").write_text( + yaml.dump([ + { + "id": node["id"], + "intent": f"Handles {node['label']}.", + "input_fields": [], + "output_fields": [], + "calls": [], + } + for node in request["nodes"] + ]), + encoding="utf-8", + ) + res = runner.invoke(cli, ["init", str(cli_repo), "--yes"]) + assert res.exit_code == 0, res.output + if "status: done" in res.output: + break + else: + raise AssertionError("init never reached status: done") + + snapshot = json.loads((state / "graph.json").read_text(encoding="utf-8")) + assert all( + n.get("enrichment_source") == "agent" + for n in snapshot["nodes"] + if n.get("layer_id") != "shared" + ) + + +def test_an_applied_response_is_not_applied_twice(cli_repo): + """ + The response file is renamed once merged. Left in place, the next `init` + would re-apply the same answers and re-forge the same edges forever. + """ + _answer_layers(cli_repo) + runner = CliRunner() + runner.invoke(cli, ["init", str(cli_repo), "--yes"]) + + state = cli_repo / ".tldrgraph" + request = yaml.safe_load((state / "enrichment_request.yaml").read_text(encoding="utf-8")) + (state / "enrichment_response.yaml").write_text( + yaml.dump([{"id": n["id"], "intent": "Does a thing."} for n in request["nodes"]]), + encoding="utf-8", + ) + runner.invoke(cli, ["init", str(cli_repo), "--yes"]) + + assert not (state / "enrichment_response.yaml").exists() + assert (state / "enrichment_response.applied.yaml").is_file() + + +def test_limit_caps_the_first_pass(cli_repo): + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--yes", "--batch", "2", "--limit", "2"]) + assert res.exit_code == 0, res.output + + request = yaml.safe_load( + (cli_repo / ".tldrgraph" / "enrichment_request.yaml").read_text(encoding="utf-8") + ) + assert len(request["nodes"]) == 2 + + +def test_agent_cli_is_opt_in_not_the_default(monkeypatch, cli_repo, agent_allowed): + """ + Shelling out is off unless asked for. It hung a real user's terminal for ten + minutes with no output, and it does not generalise across agent tools. + """ + calls = [] + monkeypatch.setattr(agent_runner, "find_agent_cli", + lambda **kw: calls.append(1) or fake_agent()) + + _answer_layers(cli_repo) + CliRunner().invoke(cli, ["init", str(cli_repo), "--yes"]) + assert calls == [], "init must not look for an agent CLI without --agent-cli" + + +def test_agent_cli_runs_the_whole_loop_when_asked(monkeypatch, cli_repo, agent_allowed): + _stub_agent_cli(monkeypatch) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--yes", "--agent-cli"]) + + assert res.exit_code == 0, res.output + assert "status: done" in res.output + snapshot = json.loads((cli_repo / ".tldrgraph" / "graph.json").read_text(encoding="utf-8")) + assert any(n.get("enrichment_source") == "agent" for n in snapshot["nodes"]) + + +def test_agent_cli_failure_does_not_lose_the_graph(monkeypatch, cli_repo, agent_allowed): + def _explode(agent, prompt, cwd, timeout=None, model=None): + if _is_enrichment_prompt(prompt): + raise agent_runner.AgentError("boom") + return json.dumps(VALID_LAYER_SET) + + _stub_agent_cli(monkeypatch) + monkeypatch.setattr(agent_runner, "run_agent", _explode) + + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--yes", "--agent-cli"]) + assert res.exit_code == 0, res.output + assert "boom" in res.output + assert (cli_repo / ".tldrgraph" / "graph.json").is_file() + + +def test_empty_intents_do_not_spin_the_loop_forever(monkeypatch, cli_repo, agent_allowed): + """ + An answer that echoes every id but writes no intent applies cleanly and + clears nothing. Measuring progress by ids returned would re-request the same + batch forever; progress is measured by nodes leaving the candidate set. + """ + def _empty_answers(prompt): + if not _is_enrichment_prompt(prompt): + return json.dumps(VALID_LAYER_SET) + start = prompt.index("Nodes (") + nodes = json.loads(prompt[prompt.index("[", start):]) + return json.dumps([{"id": n["id"], "intent": "", "calls": []} for n in nodes]) + + _stub_agent_cli(monkeypatch, answer=_empty_answers) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--yes", "--agent-cli"]) + assert res.exit_code == 0, res.output + assert "cleared no nodes" in res.output + + +# --------------------------------------------------------------------------- # +# 5. One state directory +# --------------------------------------------------------------------------- # + +def test_scan_creates_no_graphify_out_directory(cli_repo): + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo)]) + assert res.exit_code == 0, res.output + + assert not (cli_repo / "graphify-out").exists(), "scanning must add one folder, not two" + assert (cli_repo / ".tldrgraph" / paths.GRAPHIFY_GRAPH_FILENAME).is_file() + assert (cli_repo / ".tldrgraph" / paths.SNAPSHOT_FILENAME).is_file() + + +def test_graphify_export_does_not_overwrite_the_enriched_snapshot(cli_repo): + """ + The two files are different artifacts. Sharing a name inside .tldrgraph/ + would mean the raw AST export silently clobbering enriched intents. + """ + assert paths.GRAPHIFY_GRAPH_FILENAME != paths.SNAPSHOT_FILENAME + + _answer_layers(cli_repo) + CliRunner().invoke(cli, ["init", str(cli_repo)]) + raw = json.loads((cli_repo / ".tldrgraph" / paths.GRAPHIFY_GRAPH_FILENAME).read_text()) + snapshot = json.loads((cli_repo / ".tldrgraph" / paths.SNAPSHOT_FILENAME).read_text()) + + assert "layers" in snapshot or "nodes" in snapshot + assert snapshot.get("nodes") is not None + assert raw.get("nodes") is not None + assert raw is not snapshot + + +# --------------------------------------------------------------------------- # +# 6. Managed .gitignore +# --------------------------------------------------------------------------- # + +def test_gitignore_is_created_with_the_contract_kept(tmp_path): + result = installer_module.ensure_gitignore(str(tmp_path)) + assert result["status"] == "created" + + text = (tmp_path / ".gitignore").read_text(encoding="utf-8") + assert ".tldrgraph/*" in text + assert "!.tldrgraph/AGENT_CONTRACT.md" in text + assert "!.tldrgraph/layers.config.yaml" in text + + +def test_gitignore_uses_star_so_the_negations_can_work(tmp_path): + """ + git never descends into an excluded directory, so `.tldrgraph/` would make + every `!` line below it dead. The managed block must exclude entries instead. + """ + installer_module.ensure_gitignore(str(tmp_path)) + lines = [ + l.strip() for l in (tmp_path / ".gitignore").read_text(encoding="utf-8").splitlines() + ] + assert ".tldrgraph/*" in lines + assert ".tldrgraph/" not in lines + + +def test_an_existing_directory_ignore_is_neutralized(tmp_path): + (tmp_path / ".gitignore").write_text( + "*.pyc\n.tldrgraph/\nbuild/\n", encoding="utf-8" + ) + installer_module.ensure_gitignore(str(tmp_path)) + text = (tmp_path / ".gitignore").read_text(encoding="utf-8") + + assert "# .tldrgraph/" in text, "the old directory ignore must be commented out" + assert "*.pyc" in text and "build/" in text, "unrelated entries must survive" + assert ".tldrgraph/*" in text + + +def test_gitignore_is_idempotent(tmp_path): + installer_module.ensure_gitignore(str(tmp_path)) + first = (tmp_path / ".gitignore").read_text(encoding="utf-8") + second_result = installer_module.ensure_gitignore(str(tmp_path)) + + assert second_result["status"] == "unchanged" + assert (tmp_path / ".gitignore").read_text(encoding="utf-8") == first + assert first.count(installer_module.GITIGNORE_BEGIN) == 1 + + +def test_install_writes_the_gitignore_and_the_one_command(tmp_path): + written = installer_module.install_agent_rules(str(tmp_path)) + + assert Path(written["gitignore"]).is_file() + cmd = Path(written["Claude Code (command)"]) + assert cmd.is_file() + body = cmd.read_text(encoding="utf-8") + assert "tldrgraph init" in body + # Every branch of the state machine must be documented in the command. + for status in ("needs_layers", "needs_confirmation", "needs_enrichment"): + assert status in body, status + + +def test_the_command_lands_in_every_convention_the_repo_uses(tmp_path): + """Adding an agent tool is one row in the table, never new execution code.""" + (tmp_path / ".clinerules").mkdir() + (tmp_path / ".opencode").mkdir() + + written = agent_commands.install_agent_commands(str(tmp_path)) + + assert (tmp_path / ".clinerules" / "workflows" / "tldrgraph-init.md").is_file() + assert (tmp_path / ".opencode" / "command" / "tldrgraph-init.md").is_file() + # Windsurf leaves no marker here, so nothing of its is written. + assert not (tmp_path / ".windsurf").exists() + assert any(agent_commands.AGENTS_MD in key for key in written) + + +def test_all_agents_writes_every_known_tool(tmp_path): + agent_commands.install_agent_commands(str(tmp_path), all_agents=True) + for target in agent_commands.TARGETS: + if target.command_path: + assert (tmp_path / target.command_path).is_file(), target.name + if target.instructions_path: + assert (tmp_path / target.instructions_path).is_file(), target.name + + +def test_every_agent_gets_identical_instructions(tmp_path): + """ + The whole point of the rewrite. Five bespoke rule files with five different + wordings drifted apart and contradicted each other. + """ + agent_commands.install_agent_commands(str(tmp_path), all_agents=True) + + bodies = set() + for target in agent_commands.TARGETS: + if target.instructions_path: + text = (tmp_path / target.instructions_path).read_text(encoding="utf-8") + bodies.add(text.split("---", 2)[-1].strip()) + agents_md = (tmp_path / agent_commands.AGENTS_MD).read_text(encoding="utf-8") + bodies.add( + agents_md.split(agent_commands.BLOCK_BEGIN)[1] + .split(agent_commands.BLOCK_END)[0].strip() + ) + assert len(bodies) == 1, "instruction files have drifted apart" + + +def test_every_agent_gets_an_identical_command(tmp_path): + agent_commands.install_agent_commands(str(tmp_path), all_agents=True) + bodies = { + (tmp_path / t.command_path).read_text(encoding="utf-8").split("---", 2)[-1].strip() + for t in agent_commands.TARGETS if t.command_path + } + assert len(bodies) == 1, "command files have drifted apart" + + +def test_no_tool_gets_a_bespoke_extra_artifact(tmp_path): + """ + Claude used to get a skill AND a CLAUDE.md section AND a command, while + Cursor got one rule file. Every tool now gets the same two artifacts at most. + """ + agent_commands.install_agent_commands(str(tmp_path), all_agents=True) + assert not (tmp_path / ".claude" / "skills").exists() + assert not (tmp_path / "CLAUDE.md").exists() + + claude = next(t for t in agent_commands.TARGETS if t.name == "Claude Code") + # Claude Code reads AGENTS.md, so it needs no instructions file of its own. + assert claude.instructions_path is None + + +def test_superseded_files_are_removed_on_install(tmp_path): + """Two descriptions of the workflow means an agent reads a contradiction.""" + skill = tmp_path / ".claude" / "skills" / "tldrgraph" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text("old workflow", encoding="utf-8") + cursor_rule = tmp_path / ".cursor" / "rules" / "tldrgraph.mdc" + cursor_rule.parent.mkdir(parents=True) + cursor_rule.write_text("old rule", encoding="utf-8") + + installer_module.install_agent_rules(str(tmp_path)) + + assert not skill.exists() + assert not cursor_rule.exists() + + +def test_our_claude_md_block_is_removed_but_user_content_survives(tmp_path): + claude_md = tmp_path / "CLAUDE.md" + claude_md.write_text( + "# My notes\n\nAlways run black.\n\n" + f"{installer_module.CLAUDE_MD_BEGIN}\nold tldrgraph section\n" + f"{installer_module.CLAUDE_MD_END}\n", + encoding="utf-8", + ) + installer_module.install_agent_rules(str(tmp_path)) + + text = claude_md.read_text(encoding="utf-8") + assert "Always run black." in text + assert "old tldrgraph section" not in text + assert installer_module.CLAUDE_MD_BEGIN not in text + + +def test_a_claude_md_we_created_alone_is_deleted(tmp_path): + claude_md = tmp_path / "CLAUDE.md" + claude_md.write_text( + f"{installer_module.CLAUDE_MD_BEGIN}\nonly our block\n" + f"{installer_module.CLAUDE_MD_END}\n", + encoding="utf-8", + ) + installer_module.install_agent_rules(str(tmp_path)) + assert not claude_md.exists(), "a file with nothing but our block should go" + + +def test_agents_md_is_merged_never_clobbered(tmp_path): + (tmp_path / "AGENTS.md").write_text("# House rules\n\nUse tabs.\n", encoding="utf-8") + agent_commands.install_agent_commands(str(tmp_path)) + agent_commands.install_agent_commands(str(tmp_path)) + + text = (tmp_path / "AGENTS.md").read_text(encoding="utf-8") + assert "Use tabs." in text, "the user's own AGENTS.md content must survive" + assert text.count(agent_commands.BLOCK_BEGIN) == 1 + + +def test_init_installs_the_gitignore_block_and_the_agent_command(cli_repo): + """`init` is self-bootstrapping: no separate `install` step to remember.""" + CliRunner().invoke(cli, ["init", str(cli_repo)]) + assert ".tldrgraph/*" in (cli_repo / ".gitignore").read_text(encoding="utf-8") + assert (cli_repo / ".claude" / "commands" / "tldrgraph-init.md").is_file() + assert "TLDRGraph" in (cli_repo / "AGENTS.md").read_text(encoding="utf-8") + + +def test_gitignore_warnings_no_longer_contradict_the_managed_block(tmp_path): + installer_module.ensure_gitignore(str(tmp_path)) + warnings = installer_module.gitignore_warnings(str(tmp_path)) + assert not any(".tldrgraph" in w for w in warnings) + + +def test_invented_ids_are_reported_not_silently_dropped(cli_repo): + """ + An id that is not in the graph contributes nothing. Dropping it in silence + leaves an agent looping and re-inventing the same id every round. + """ + _answer_layers(cli_repo) + runner = CliRunner() + runner.invoke(cli, ["init", str(cli_repo), "--yes"]) + + state = cli_repo / ".tldrgraph" + request = yaml.safe_load((state / "enrichment_request.yaml").read_text(encoding="utf-8")) + real = request["nodes"][0]["id"] + (state / "enrichment_response.yaml").write_text( + yaml.dump([ + {"id": real, "intent": "A real node."}, + {"id": "totally_made_up_node_id", "intent": "Not in the graph."}, + ]), + encoding="utf-8", + ) + + res = runner.invoke(cli, ["init", str(cli_repo), "--yes"]) + assert res.exit_code == 0, res.output + assert "totally_made_up_node_id" in res.output + assert "not in the graph" in res.output + assert "Copy ids verbatim" in res.output + + +def test_model_selection_reaches_the_agent_argv(): + """--agent-model / $TLDRGRAPH_AGENT_MODEL must land on the real command line.""" + claude = next(a for a in agent_runner.KNOWN_AGENTS if a.name == "claude") + gemini = next(a for a in agent_runner.KNOWN_AGENTS if a.name == "gemini") + + assert "--model" not in claude.argv("hi") + assert claude.argv("hi", model="opus")[1:3] == ["--model", "opus"] + # Every CLI spells the flag differently; that lives in the table, not in + # branching at the call site. + assert gemini.argv("hi", model="gemini-2.5-pro")[1:3] == ["-m", "gemini-2.5-pro"] + + +def test_model_can_come_from_the_environment(monkeypatch): + monkeypatch.setenv(agent_runner.ENV_MODEL, "sonnet") + claude = next(a for a in agent_runner.KNOWN_AGENTS if a.name == "claude") + assert claude.argv("hi")[1:3] == ["--model", "sonnet"] + + +def test_init_re_extracts_so_deleted_code_leaves_the_graph(cli_repo): + """ + HARD GATE. `init` used to reuse an existing graphify export and only + re-extract when it was missing. Every later phase then worked from a + snapshot of whatever the code used to be: agents were handed deleted + functions to write intents for, and anything added since was invisible. + """ + _answer_layers(cli_repo) + runner = CliRunner() + runner.invoke(cli, ["init", str(cli_repo)]) + + raw = json.loads( + (cli_repo / ".tldrgraph" / paths.GRAPHIFY_GRAPH_FILENAME).read_text(encoding="utf-8") + ) + assert any(n.get("label", "").startswith("Engine") for n in raw["nodes"]) + + # Delete the class and add a new one, exactly as an edit between runs would. + (cli_repo / "sample" / "engine.py").write_text( + "class Rebuilt:\n def go(self):\n return 2\n", encoding="utf-8" + ) + runner.invoke(cli, ["init", str(cli_repo)]) + + raw = json.loads( + (cli_repo / ".tldrgraph" / paths.GRAPHIFY_GRAPH_FILENAME).read_text(encoding="utf-8") + ) + labels = {n.get("label", "") for n in raw["nodes"]} + assert any(l.startswith("Rebuilt") for l in labels), "new code must appear" + assert not any(l.startswith("Engine") for l in labels), "deleted code must go" + + +def test_prose_nodes_are_never_queued_for_enrichment(): + """ + graphify emits `rationale` nodes whose label IS a sentence of documentation. + Queueing one asks the agent to pay for copying a docstring back onto itself. + On this repository they were 390 of 1419 nodes -- nearly half the backlog. + """ + from tldrgraph.cli import needs_agent_enrichment + + code = {"layer_id": "api", "type": "code", "label": "CasesController"} + prose = {"layer_id": "api", "type": "rationale", + "label": "Handles the pension approval workflow end to end."} + + assert needs_agent_enrichment(code) + assert not needs_agent_enrichment(prose) + + +def test_the_enrichment_queue_and_dead_code_agree_on_what_is_not_code(): + """Both must read the same set, or one will contradict the other.""" + from tldrgraph import cli as cli_module + from tldrgraph.deadcode import NON_CODE_NODE_TYPES as deadcode_set + + assert cli_module.NON_CODE_NODE_TYPES is deadcode_set + + +def test_json_mode_emits_parseable_json_only(cli_repo): + """ + graphify prints progress and warnings to stdout. Under --json those land in + front of the payload and break every parser reading it. + """ + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--json"]) + assert res.exit_code == 0, res.output + + payload = json.loads(res.stdout) + assert payload["status"] in {"needs_confirmation", "needs_enrichment", "done"} + + +def test_enriched_count_does_not_include_excluded_nodes(cli_repo): + """ + "enriched" used to be computed as total-minus-candidates, so utility and + prose nodes -- never eligible in the first place -- were reported as done. + A fresh graph claimed hundreds enriched before a single intent existed. + """ + _answer_layers(cli_repo) + res = CliRunner().invoke(cli, ["init", str(cli_repo), "--json"]) + payload = json.loads(res.stdout) + + assert payload["progress"]["enriched"] == 0, "nothing has been enriched yet" + assert payload["progress"]["excluded"] >= 0 + assert payload["progress"]["remaining"] > 0 diff --git a/tests/test_dynamic_layers.py b/tests/test_dynamic_layers.py index 20c026b..b4ebca2 100644 --- a/tests/test_dynamic_layers.py +++ b/tests/test_dynamic_layers.py @@ -10,7 +10,7 @@ from tldrgraph.cli import cli from tldrgraph.classifier import classify_node from tldrgraph.graph_loader import GraphLoader -from tldrgraph.layer_config import load_layer_config, save_layer_config +from tldrgraph.layer_config import load_layer_config, save_layer_config, validate_layer_config from tldrgraph.layers import LayerRegistry, get_registry, set_registry, default_registry from tldrgraph.propose_layers import ( ARCHETYPE_BACKEND, @@ -18,7 +18,7 @@ ARCHETYPE_FULLSTACK, ARCHETYPE_GENERIC, ARCHETYPE_LIBRARY, - archetype_layer_set, + NEEDS_LAYERS, auto_configure_layers, detect_repository_archetype, propose_layers_with_llm, @@ -91,52 +91,62 @@ def test_detect_archetype_library_repo(tmp_path): # --------------------------------------------------------------------------- # -# 2. Archetype Layer Set Structure & Validation +# 2. Rule matching against a hand-written layer set # --------------------------------------------------------------------------- # -def test_archetype_layer_sets_are_valid(): - for arch in (ARCHETYPE_CLI, ARCHETYPE_FULLSTACK, ARCHETYPE_BACKEND, ARCHETYPE_LIBRARY, ARCHETYPE_GENERIC): - data = archetype_layer_set(arch) - assert "utility_id" in data - assert "layers" in data - assert len(data["layers"]) >= 3 - # Should build into a valid LayerRegistry - reg = LayerRegistry.from_records(data["layers"], utility_id=data["utility_id"]) - assert reg.utility_id == data["utility_id"] - assert len(reg) == len(data["layers"]) +#: A layer set of the shape an agent is asked to produce. Written out here on +#: purpose: TLDRGraph ships no layer templates any more, so the rule engine is +#: tested against a fixture rather than against production defaults. +SAMPLE_LAYER_SET = { + "utility_id": "utility", + "layers": [ + {"id": "cli", "name": "Layer 1: CLI", "order": 1, "description": "commands", + "rules": [{"file_contains": ["cli.py", "/cli/", "commands/"]}]}, + {"id": "engine", "name": "Layer 2: Engine", "order": 2, "description": "logic", + "rules": [{"file_contains": ["flow_engine", "classifier", "extractors"]}]}, + {"id": "storage", "name": "Layer 3: Storage", "order": 3, "description": "state", + "rules": [{"file_contains": ["vector_store", "graph_loader", "hash_gate"]}]}, + {"id": "integrations", "name": "Layer 4: Integrations", "order": 4, + "description": "agents and UI", + "rules": [{"file_contains": ["installer", "visualizer", "llm_enricher"]}]}, + {"id": "utility", "name": "General / Utility", "order": 5, + "description": "catch-all", "rules": []}, + ], +} + + +def test_a_proposed_layer_set_builds_a_registry(): + validate_layer_config(SAMPLE_LAYER_SET) + reg = LayerRegistry.from_records( + SAMPLE_LAYER_SET["layers"], utility_id=SAMPLE_LAYER_SET["utility_id"] + ) + assert reg.utility_id == "utility" + assert len(reg) == len(SAMPLE_LAYER_SET["layers"]) -def test_cli_archetype_classifies_cli_components(tmp_path): - data = archetype_layer_set(ARCHETYPE_CLI) - reg = LayerRegistry.from_records(data["layers"], utility_id=data["utility_id"]) +def test_file_rules_route_symbols_to_their_layer(tmp_path): + reg = LayerRegistry.from_records( + SAMPLE_LAYER_SET["layers"], utility_id=SAMPLE_LAYER_SET["utility_id"] + ) with pytest.MonkeyPatch.context() as mp: mp.setattr("tldrgraph.layers.get_registry", lambda: reg) mp.setattr("tldrgraph.classifier.get_registry", lambda: reg) - # CLI command - layer_cli = classify_node("n1", {"file": "tldrgraph/cli.py", "label": "scan"}) - assert layer_cli.id == "cli" - - # Core engine - layer_eng = classify_node("n2", {"file": "tldrgraph/flow_engine.py", "label": "query_flow"}) - assert layer_eng.id == "engine" - - # Storage - layer_store = classify_node("n3", {"file": "tldrgraph/vector_store.py", "label": "LocalVectorStore"}) - assert layer_store.id == "storage" - - # Agent / Visualizer - layer_int = classify_node("n4", {"file": "tldrgraph/visualizer.py", "label": "generate_html"}) - assert layer_int.id == "integrations" - - # Fallback / Utility - layer_util = classify_node("n5", {"file": "codechakra/helpers.py", "label": "format_date"}) - assert layer_util.id == "utility" + assert classify_node("n1", {"file": "tldrgraph/cli.py", "label": "scan"}).id == "cli" + assert classify_node("n2", {"file": "tldrgraph/flow_engine.py", + "label": "query_flow"}).id == "engine" + assert classify_node("n3", {"file": "tldrgraph/vector_store.py", + "label": "LocalVectorStore"}).id == "storage" + assert classify_node("n4", {"file": "tldrgraph/installer.py", + "label": "install"}).id == "integrations" + # Nothing matches -> the catch-all, never a guess. + assert classify_node("n5", {"file": "tldrgraph/labels.py", + "label": "build"}).id == "utility" # --------------------------------------------------------------------------- # -# 3. LLM Proposal & Auto-Configuration +# 3. Layer synthesis # --------------------------------------------------------------------------- # class MockLLMEnricher: @@ -200,34 +210,40 @@ def test_auto_configure_layers_with_llm(tmp_path): assert reg2.ids() == reg.ids() -def test_auto_configure_layers_offline_fallback(tmp_path): - # Set up a CLI repo +def test_no_layer_source_means_no_layers_not_a_template(tmp_path): + """ + HARD GATE. TLDRGraph used to synthesize a generic archetype layer set here. + That silently became the answer and classified badly. With nothing able to + read the code, the only honest result is "ask the agent". + """ (tmp_path / "pyproject.toml").write_text( '[project.scripts]\nmycmd = "mycmd.cli:main"\n', encoding="utf-8" ) - reg, cfg_path, source = auto_configure_layers(str(tmp_path), enricher=None, use_llm=False) - assert source == "archetype:cli_application" - assert os.path.isfile(cfg_path) - assert "cli" in reg.ids() - assert "engine" in reg.ids() - assert "storage" in reg.ids() + reg, cfg_path, source = auto_configure_layers( + str(tmp_path), enricher=None, use_llm=False, use_agent=False + ) + assert source == NEEDS_LAYERS + assert reg is None and cfg_path is None + assert not (tmp_path / ".tldrgraph" / "layers.config.yaml").exists(), ( + "a layer config must never be written without a real source" + ) # --------------------------------------------------------------------------- # # 4. End-to-End CLI Scan with Dynamic Layers # --------------------------------------------------------------------------- # -def test_cli_propose_layers_auto_command(tmp_path): +def test_cli_propose_layers_falls_through_to_a_request(tmp_path): + """With nothing able to read the code, --auto must queue a request, not a template.""" (tmp_path / "pyproject.toml").write_text( '[project.scripts]\ntool = "tool.cli:main"\n', encoding="utf-8" ) runner = CliRunner() res = runner.invoke(cli, ["propose-layers", "--path", str(tmp_path), "--auto"]) assert res.exit_code == 0 - assert "Automatically configured" in res.output - - cfg_file = tmp_path / ".tldrgraph" / "layers.config.yaml" - assert cfg_file.is_file() + assert "no template to fall back on" in res.output + assert (tmp_path / ".tldrgraph" / "propose_layers_request.json").is_file() + assert not (tmp_path / ".tldrgraph" / "layers.config.yaml").exists() def test_cli_scan_initializes_dynamic_layers_automatically(tmp_path): @@ -250,13 +266,29 @@ def test_cli_scan_initializes_dynamic_layers_automatically(tmp_path): runner = CliRunner() res = runner.invoke(cli, ["scan", str(tmp_path)]) assert res.exit_code == 0 - assert "Configured" in res.output or "Scanning repository" in res.output - - # Verify layers.config.yaml was created with CLI archetype - cfg_file = tmp_path / ".tldrgraph" / "layers.config.yaml" - assert cfg_file.is_file() + # No agent is reachable in tests, so the scan must stop and ask rather than + # classify this repo with layers it never derived. + assert "status: needs_layers" in res.output + assert "tldrgraph init" in res.output + + assert not (tmp_path / ".tldrgraph" / "layers.config.yaml").exists() + + # The request the agent is asked to answer must actually be there, carrying + # the symbols extraction already found -- filenames alone are not evidence. + request = tmp_path / ".tldrgraph" / "propose_layers_request.json" + assert request.is_file() + payload = json.loads(request.read_text(encoding="utf-8")) + assert payload["evidence"]["extracted_symbols"]["total_symbols"] > 0 + + # Answer it the way an agent would, and the next run gets all the way through. + (tmp_path / ".tldrgraph" / "propose_layers_response.json").write_text( + json.dumps(SAMPLE_LAYER_SET), encoding="utf-8" + ) + res2 = runner.invoke(cli, ["init", str(tmp_path)]) + assert res2.exit_code == 0, res2.output + assert "status: needs_layers" not in res2.output + assert (tmp_path / ".tldrgraph" / "layers.config.yaml").is_file() - # Query command should work with the dynamic layers res_layers = runner.invoke(cli, ["layers", "--path", str(tmp_path)]) assert res_layers.exit_code == 0 assert "TLDRGraph Multi-Layer Architecture Summary" in res_layers.output diff --git a/tests/test_extractors.py b/tests/test_extractors.py index bf55995..7d52cc4 100644 --- a/tests/test_extractors.py +++ b/tests/test_extractors.py @@ -14,6 +14,8 @@ import networkx as nx import pytest +from conftest import write_example_layer_config +from tldrgraph import paths from tldrgraph import extractors as ex from tldrgraph.classifier import LayerType, classify_node from tldrgraph.deadcode import ( @@ -644,8 +646,9 @@ def seam_loader(seam_repo, monkeypatch): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("OLLAMA_HOST", "http://127.0.0.1:9") - graphify = seam_repo / "graphify-out" + graphify = seam_repo / paths.STATE_DIRNAME graphify.mkdir(parents=True, exist_ok=True) + write_example_layer_config(seam_repo) nodes = [ {"id": "ui_page", "label": "ApplicationsPage()", "file_type": "code", "source_file": "frontend/src/app/applications/page.tsx", "source_location": "L3"}, @@ -664,8 +667,8 @@ def seam_loader(seam_repo, monkeypatch): "links": [{"source": "ctl", "target": "svc", "relation": "calls", "confidence_score": 1.0}], "hyperedges": []} - (graphify / "graph.json").write_text(json.dumps(graph_doc), encoding="utf-8") - (graphify / "manifest.json").write_text("{}", encoding="utf-8") + (graphify / paths.GRAPHIFY_GRAPH_FILENAME).write_text(json.dumps(graph_doc), encoding="utf-8") + (graphify / paths.GRAPHIFY_MANIFEST_FILENAME).write_text("{}", encoding="utf-8") return GraphLoader(str(seam_repo)) diff --git a/tests/test_flow_engine.py b/tests/test_flow_engine.py index 8b61548..546e893 100644 --- a/tests/test_flow_engine.py +++ b/tests/test_flow_engine.py @@ -3,7 +3,7 @@ Everything here is hermetic: a small synthetic graph plus a real ``LocalVectorStore`` written under pytest's ``tmp_path``. Nothing reads the real -repository, the real ``graphify-out/`` or the real ``.tldrgraph/`` state, and +repository or the real ``.tldrgraph/`` state, and nothing makes a network call. Fixtures are defined locally in this file on purpose -- the shared ``conftest.py`` is owned elsewhere. diff --git a/tests/test_hash_gate.py b/tests/test_hash_gate.py index 8e7dbb4..76b3f32 100644 --- a/tests/test_hash_gate.py +++ b/tests/test_hash_gate.py @@ -4,7 +4,7 @@ The gate hashes ``label + file_path``. Neither changes when someone edits the body of a function, so a node is never marked dirty and never re-enriched. graphify already publishes per-file content hashes in -``graphify-out/manifest.json``; those must drive the signature instead. +graphify's manifest; those must drive the signature instead. Separately, older databases are full of generated placeholder summaries ("Layer 3: Foo located at bar.ts") that look enriched to ``check_node``, so @@ -28,7 +28,7 @@ def test_loader_loads_the_graphify_manifest(loader, mini_repo): assert hasattr(loader, "file_hashes"), ( - "GraphLoader must load graphify-out/manifest.json into self.file_hashes" + "GraphLoader must load graphify's manifest into self.file_hashes" ) manifest = mini_repo.read_manifest() for rel, entry in manifest.items(): @@ -129,7 +129,7 @@ def test_node_goes_dirty_when_semantic_hash_changes(mini_repo, no_network): is_dirty, _ = l2.hash_gate.check_node(nid, l2.node_signature(attrs)) assert is_dirty, ( - "a changed semantic_hash in graphify-out/manifest.json must mark the " + "a changed semantic_hash in graphify's manifest must mark the " "node dirty -- the gate is still hashing label+path" ) diff --git a/tests/test_hierarchy.py b/tests/test_hierarchy.py index 1e9c09e..9306fca 100644 --- a/tests/test_hierarchy.py +++ b/tests/test_hierarchy.py @@ -19,6 +19,8 @@ import pytest +from conftest import write_example_layer_config +from tldrgraph import paths from tldrgraph import extractors as ex from tldrgraph import labels as lb from tldrgraph.graph_loader import GraphLoader @@ -136,8 +138,9 @@ def repo(tmp_path, monkeypatch): dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(content, encoding="utf-8") - graphify = root / "graphify-out" + graphify = root / paths.STATE_DIRNAME graphify.mkdir(parents=True, exist_ok=True) + write_example_layer_config(root) graph_doc = { "directed": True, "multigraph": False, @@ -153,8 +156,8 @@ def repo(tmp_path, monkeypatch): ], "hyperedges": [], } - (graphify / "graph.json").write_text(json.dumps(graph_doc), encoding="utf-8") - (graphify / "manifest.json").write_text("{}", encoding="utf-8") + (graphify / paths.GRAPHIFY_GRAPH_FILENAME).write_text(json.dumps(graph_doc), encoding="utf-8") + (graphify / paths.GRAPHIFY_MANIFEST_FILENAME).write_text("{}", encoding="utf-8") return root @@ -508,7 +511,7 @@ def test_parent_and_child_links_are_symmetric(hierarchy): def test_reading_the_snapshot_neither_doubles_seams_nor_buries_endpoints(repo): """ ``build_multilayer_hierarchy`` prefers ``.tldrgraph/graph.json`` over - ``graphify-out/graph.json`` when it exists, and that snapshot carries both + graphify's raw export when it exists, and that snapshot carries both the synthesized endpoint nodes and the re-derivable seam edges. Reading it must not emit the seams twice, and must not fold endpoints back into the generic symbol tier. diff --git a/tests/test_index_write_through.py b/tests/test_index_write_through.py index a51e175..2a8a447 100644 --- a/tests/test_index_write_through.py +++ b/tests/test_index_write_through.py @@ -128,7 +128,11 @@ def test_bridge_is_created_on_a_completely_fresh_scan(mini_repo, stub_enricher, target_label = mini_repo.label("svc_pension") # "PensionCalculatorService" stub_enricher(lambda n: [target_label] if n["id"] == ui else []) - assert not mini_repo.tldrgraph_dir.exists(), "fixture directory must be fresh" + # The state directory now also holds graphify's raw export, so its mere + # existence proves nothing. What must be absent is TLDRGraph's own state. + assert not mini_repo.snapshot_path.exists(), "snapshot must be fresh" + assert not mini_repo.index_path.exists(), "vector index must be fresh" + assert not mini_repo.db_path.exists(), "hash-gate cache must be fresh" loader = GraphLoader(str(mini_repo.root)) graph = loader.load_or_extract(enrich_llm=True) diff --git a/tests/test_layer_config.py b/tests/test_layer_config.py index 9010681..0d0c3a1 100644 --- a/tests/test_layer_config.py +++ b/tests/test_layer_config.py @@ -343,3 +343,24 @@ def test_enrichment_survives_layer_set_change(mini_repo): assert g3.nodes[case_view]["fields"] == ["caseId", "pensionerName"] assert g3.has_edge(case_view, pension_calc) assert g3[case_view][pension_calc]["relation"] == "cross_layer_link" + + +def test_an_unconfigured_repo_gets_one_honest_bucket_not_six_guesses(tmp_path): + """ + HARD GATE. `load_layer_config` used to hand back a six-layer + UI/API/Service/Data/Async/DevOps default when no config existed, so every + node in an unscanned repo was confidently filed into an architecture nobody + had derived from it. A map that is wrong everywhere it looks right is worse + than no map. + """ + registry, _ = load_layer_config(str(tmp_path)) + + assert len(registry) == 1 + assert registry.ordered()[0].name == "Unclassified" + assert registry.utility_id == registry.ordered()[0].id + assert registry.ordered()[0].rules == () + + +def test_the_example_layer_set_is_never_written_to_disk(tmp_path): + load_layer_config(str(tmp_path)) + assert not (tmp_path / ".tldrgraph" / "layers.config.yaml").exists() diff --git a/tests/test_layers.py b/tests/test_layers.py index dbc1aa3..db5cac8 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -16,6 +16,7 @@ from tldrgraph.deadcode import compute_enrichment_coverage from tldrgraph.flow_engine import FlowEngine from tldrgraph.graph_loader import GraphLoader +from tldrgraph.layer_config import save_layer_config from tldrgraph.layers import ( LAYER_API, LAYER_ASYNC, @@ -133,8 +134,9 @@ def test_registry_rejects_duplicate_ids_and_names(): def test_layer_id_of_prefers_the_id_and_falls_back_to_the_display_name(): assert layer_id_of({"layer_id": LAYER_API, "layer": "anything at all"}) == LAYER_API - # Legacy record: display name only. - assert layer_id_of({"layer": "Layer 2: API Gateway"}) == LAYER_API + # Legacy record: display name only, resolved against the active registry. + with use_registry(default_registry()): + assert layer_id_of({"layer": "Layer 2: API Gateway"}) == LAYER_API assert layer_id_of({}) == "" assert layer_id_of({"layer": "a layer nobody registered"}) == "" @@ -286,10 +288,15 @@ def test_renaming_does_not_change_the_classification_of_a_real_repo(mini_repo): enrich_llm=False).nodes(data=True) } - with use_registry(_renamed_registry()) as renamed_registry: - graph = GraphLoader(str(mini_repo.root)).load_or_extract(enrich_llm=False, rebuild=True) - after = {nid: data["layer_id"] for nid, data in graph.nodes(data=True)} - display = {data["layer"] for _, data in graph.nodes(data=True)} + # The rename has to go through the config file, not an ambient registry: + # a layer set on disk is the source of truth and `load_or_extract` reloads + # it, which is exactly the behaviour worth pinning down here. + renamed_registry = _renamed_registry() + save_layer_config(str(mini_repo.root), renamed_registry) + + graph = GraphLoader(str(mini_repo.root)).load_or_extract(enrich_llm=False, rebuild=True) + after = {nid: data["layer_id"] for nid, data in graph.nodes(data=True)} + display = {data["layer"] for _, data in graph.nodes(data=True)} assert after == before # The display names really did change -- the test above is not vacuous. diff --git a/tests/test_scan_smoke.py b/tests/test_scan_smoke.py index 9bf0681..417bcec 100644 --- a/tests/test_scan_smoke.py +++ b/tests/test_scan_smoke.py @@ -10,6 +10,7 @@ import yaml +from tldrgraph import paths from tldrgraph.classifier import LayerType @@ -82,16 +83,16 @@ def test_ast_edges_from_graphify_survive_the_scan(loader, mini_repo, no_network) def test_auto_graphify_run_when_graph_json_missing(tmp_path, no_network): - """When graphify-out/graph.json is missing, GraphLoader invokes graphify automatically.""" + """When graphify's raw export is missing, GraphLoader invokes graphify automatically.""" from tldrgraph.graph_loader import GraphLoader src_file = tmp_path / "sample.py" src_file.write_text("def hello():\n return 42\n\ndef caller():\n return hello()\n", encoding="utf-8") loader = GraphLoader(str(tmp_path)) - assert not (tmp_path / "graphify-out" / "graph.json").exists() + assert not (tmp_path / paths.STATE_DIRNAME / paths.GRAPHIFY_GRAPH_FILENAME).exists() graph = loader.load_or_extract(enrich_llm=False) - assert (tmp_path / "graphify-out" / "graph.json").exists() + assert (tmp_path / paths.STATE_DIRNAME / paths.GRAPHIFY_GRAPH_FILENAME).exists() assert graph.number_of_nodes() > 0 assert any("hello" in n or "caller" in n for n in graph.nodes) diff --git a/tests/test_visualizer.py b/tests/test_visualizer.py index a718ebb..e6f274e 100644 --- a/tests/test_visualizer.py +++ b/tests/test_visualizer.py @@ -8,8 +8,11 @@ def test_build_layers_config_default_registry(): - cfg = build_layers_config() - assert len(cfg) == 6 # 6 non-utility layers in default registry + # An unconfigured repo now has a single "Unclassified" bucket, so the layer + # set under test has to be named explicitly. + with use_registry(default_registry()): + cfg = build_layers_config() + assert len(cfg) == 6 # 6 non-utility layers in the example registry assert cfg[0]["id"] == "ui" assert "color" in cfg[0] assert "border" in cfg[0] diff --git a/tldrgraph/__init__.py b/tldrgraph/__init__.py index 4b5dac7..ad32020 100644 --- a/tldrgraph/__init__.py +++ b/tldrgraph/__init__.py @@ -3,3 +3,8 @@ """ __version__ = "0.1.0" + +# Imported for its import-time side effect: it pins graphify's output directory +# inside .tldrgraph/ before graphify can be imported and read the default. See +# tldrgraph.paths.pin_graphify_output_dir. +from . import paths as paths # noqa: E402,F401 diff --git a/tldrgraph/agent_commands.py b/tldrgraph/agent_commands.py new file mode 100644 index 0000000..81e17c7 --- /dev/null +++ b/tldrgraph/agent_commands.py @@ -0,0 +1,391 @@ +""" +Agent integration: one body of instructions, written the same way for every tool. + +TLDRGraph is driven **by** the agent through a file handshake, never the reverse. +Every tool can read a file, read source, and run a shell command, so that is the +entire integration surface -- no per-tool flags, auth, or headless quirks. + +Two artifacts, and only two: + +* **instructions** -- always-loaded context. ``AGENTS.md`` is the cross-tool + standard and is always written. A tool gets its own copy *only* if it does not + read AGENTS.md, and that copy is byte-identical apart from frontmatter. +* **command** -- the ``/tldrgraph-init`` workflow, in each tool's command + directory. + +No tool gets bespoke prose and no tool gets extra artifacts. There used to be a +Claude-only skill file plus a CLAUDE.md section plus a Cursor rule plus an +Antigravity rule, each with different wording and different lengths; they drifted +apart immediately and contradicted each other. Adding a tool is one row in +TARGETS -- paths only. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +#: Delimiters for the managed region inside a user-owned instructions file. +BLOCK_BEGIN = "" +BLOCK_END = "" + +#: The command name installed everywhere. Deliberately not bare `tldrgraph`: +#: that would collide with the package's own CLI name in some shells. +COMMAND_NAME = "tldrgraph-init" + +#: The cross-tool instructions standard. Read by Claude Code, Cursor, opencode, +#: Codex, Gemini CLI, Zed and Copilot, so those tools need no file of their own. +AGENTS_MD = "AGENTS.md" + + +@dataclass(frozen=True) +class AgentTarget: + """One coding tool's file conventions. Paths only -- never behaviour.""" + + #: Display name used in installer output. + name: str + #: Where this tool looks for a project command, relative to the repo root. + #: ``None`` for tools with no command concept. + command_path: Optional[str] = None + #: Where this tool looks for always-loaded instructions, for tools that do + #: **not** read AGENTS.md. Leaving this ``None`` is the norm and is what + #: keeps a repo from accumulating five copies of the same paragraph. + instructions_path: Optional[str] = None + #: Directory whose presence means this tool is actually used here. ``None`` + #: installs unconditionally. + marker: Optional[str] = None + #: True when the tool expects YAML frontmatter on these files. + frontmatter: bool = True + + +#: Every tool TLDRGraph knows, and nothing about how to run any of them. +#: +#: Tools listed without an ``instructions_path`` read AGENTS.md, which is always +#: written. Confidence in the per-tool paths varies and that is fine: these are +#: small markdown files a tool ignores if it does not know the path, and +#: AGENTS.md carries the instructions regardless. +TARGETS: Tuple[AgentTarget, ...] = ( + AgentTarget("Claude Code", command_path=f".claude/commands/{COMMAND_NAME}.md"), + AgentTarget("Cursor", command_path=f".cursor/commands/{COMMAND_NAME}.md"), + AgentTarget("opencode", command_path=f".opencode/command/{COMMAND_NAME}.md", + marker=".opencode"), + AgentTarget("Antigravity", + command_path=f".agents/workflows/{COMMAND_NAME}.md", + instructions_path=".agents/rules/tldrgraph.md"), + AgentTarget("Cline", + command_path=f".clinerules/workflows/{COMMAND_NAME}.md", + instructions_path=".clinerules/tldrgraph.md", + marker=".clinerules"), + AgentTarget("Windsurf", + command_path=f".windsurf/workflows/{COMMAND_NAME}.md", + instructions_path=".windsurf/rules/tldrgraph.md", + marker=".windsurf"), + AgentTarget("Roo Code", command_path=f".roo/commands/{COMMAND_NAME}.md", + marker=".roo"), + AgentTarget("Kilo Code", command_path=f".kilocode/workflows/{COMMAND_NAME}.md", + marker=".kilocode"), + AgentTarget("Goose", command_path=f".goosehints/{COMMAND_NAME}.md", + marker=".goosehints", frontmatter=False), +) + +#: Files earlier versions installed. Removed on install so a repo does not carry +#: two contradictory descriptions of the workflow. +SUPERSEDED = ( + ".claude/skills/tldrgraph/SKILL.md", + ".claude/skills/codechakra/SKILL.md", + ".claude/commands/tldrgraph-layers.md", + ".claude/commands/tldrgraph-enrich.md", + ".cursor/rules/tldrgraph.mdc", + ".agents/workflows/tldrgraph.md", +) + + +# --------------------------------------------------------------------------- # +# The single source of content +# --------------------------------------------------------------------------- # + +#: Always-loaded context. Short on purpose: it competes with everything else in +#: the agent's window, so it says what TLDRGraph is for and where the detail is. +INSTRUCTIONS_BODY = f"""## TLDRGraph + +This repository is mapped into architectural layers designed from its own source, +with per-symbol intents you can search and trace. + +**Before planning or implementing a feature**, trace it instead of grepping: + +```bash +tldrgraph query "" # semantic search + end-to-end flow +tldrgraph trace "" "" # exact path between two symbols +tldrgraph layers # node counts per layer +tldrgraph dead-code # review candidates, never a delete list +``` + +Those are read-only and never trigger enrichment. + +**To build or continue the graph**, run `tldrgraph init`, do what the `NEXT ACTION` +block prints, and run it again -- repeat until `status: done`. It has no template +fallback: if this repository has no architecture yet, it will stop and ask you to +design one from the code. Do not skip reading the files. + +Full workflow: `.claude/commands/{COMMAND_NAME}.md` (identical copies live in every +other agent directory). Schema: `.tldrgraph/AGENT_CONTRACT.md`. + +`tldrgraph dead-code` lists **review candidates, not confirmed dead code**. +`unreviewed` means "not enough evidence to conclude" and is never removable. +""" + +#: The full workflow. Every branch of the `init` state machine, spelled out. +COMMAND_BODY = """# TLDRGraph: build this repository's architecture graph + +One command, run repeatedly until it says DONE. `tldrgraph init` never guesses: +it stops and tells you exactly what it needs. + +```bash +tldrgraph init +``` + +Read the `NEXT ACTION` block it prints, do what it says, then run `tldrgraph init` +again. Repeat until the output says `status: done`. There are only three things +it can ask for. + +## 1. `status: needs_layers` + +TLDRGraph ships **no layer templates** and will not invent an architecture. +Design one from this repository. + +1. Read `.tldrgraph/propose_layers_request.json`. It carries the symbols and + files extraction already found -- a starting point, not a substitute for + opening the code. +2. **Open real source files**: entry points first, then a representative file + from each cluster in the evidence. Work out what this codebase actually does + and where responsibility changes hands. +3. Write `.tldrgraph/propose_layers_response.json`: + +```json +{ + "utility_id": "", + "layers": [ + { + "id": "short_machine_id", + "name": "Layer 1: Human Friendly Name", + "order": 1, + "description": "One sentence on what lives here", + "rules": [ + {"file_contains": ["substring"], "exclude_file": ["optional"]}, + {"label_contains": ["SymbolNamePart"]} + ] + } + ] +} +``` + +4. Run `tldrgraph init` again. + +### What a layer set looks like + +Sketches from other codebases, to show the *shape* of an answer. They are not a +menu and none of them will fit this repository -- read the code and name what you +actually find. + +- A web app might split presentation from request handling from domain logic + from persistence, with background jobs and deployment config as their own tiers. +- A CLI tool might split the command surface from the processing engine from + local state, with adapters to outside systems separate again. +- A library might split its public API from the core implementation from its + data types, with backend adapters separate. +- A data pipeline might split ingestion from transformation from model training + from serving. + +The useful question is not "which of these is it?" but "where does responsibility +change hands in *this* code, and what would a new engineer need named?" + +### Rules that hold for any answer + +- 3 to 6 layers, plus exactly one catch-all whose `id` equals `utility_id` and + whose `rules` are `[]`. +- Unique `id` and `name` per layer; sequential integer `order` from 1. +- Rule keys: `file_contains`, `exclude_file`, `path_regex`, `label_contains`, + `exclude_label`, `label_ends_with`, `type_in`, `id_prefix`. Values are lists of + strings. Rules are evaluated in `order` and the first match wins. +- Derive rules from paths and symbol names you actually saw. A rule matching + nothing is worse than no rule; a rule matching everything collapses the map. + +## 2. `status: needs_confirmation` + +The output shows how many nodes need enrichment and how many agent round-trips +that implies. **Ask the user whether to proceed, and show them that estimate.** +Do not decide for them. + +- They agree: `tldrgraph init --yes` +- Smaller first pass: `tldrgraph init --yes --limit 100` +- They decline: stop. The graph is already built and queryable. + +## 3. `status: needs_enrichment` + +1. Read `.tldrgraph/enrichment_request.yaml`. +2. **Open the source file of every node in it.** This is the entire point: an + intent paraphrased from a symbol name poisons semantic search with + confident-sounding noise. +3. Write `.tldrgraph/enrichment_response.yaml` -- a *different* file from the + request, which is regenerated on every run: + +```yaml +- id: "" + intent: | + What this symbol does, why it exists, and its execution logic. + input_fields: [caseId, remarks] + output_fields: [status, disposition] + calls: [ApplicationsService, pension_cases] +``` + +4. Run `tldrgraph init --yes` again. It applies the response and hands you the + next batch, until there is nothing left. + +**Copy every `id` verbatim.** A constructed id matches nothing, is dropped, and +gets reported back to you -- but the work is wasted. + +**Never invent `fields` or `calls`.** Omit what you cannot verify in the code: an +empty list is a correct answer, a wrong `calls` entry becomes a real wrong edge. + +## Once it says DONE + +```bash +tldrgraph query "" +tldrgraph trace "" "" +tldrgraph layers +tldrgraph ui --serve +``` + +Read-only, and they never trigger enrichment. Full schema: +`.tldrgraph/AGENT_CONTRACT.md`. +""" + + +def _frontmatter(name: str, description: str) -> str: + return f"---\nname: {name}\ndescription: {description}\n---\n\n" + + +def command_text(target: Optional[AgentTarget] = None) -> str: + """The command file body, with frontmatter when the target expects it.""" + if target is not None and not target.frontmatter: + return COMMAND_BODY + return _frontmatter( + COMMAND_NAME, + "Build or continue this repository's TLDRGraph architecture graph " + "(layers, extraction, enrichment)", + ) + COMMAND_BODY + + +def instructions_text(target: Optional[AgentTarget] = None) -> str: + """ + The instructions body. Identical everywhere -- that is the whole point. + + AGENTS.md gets it bare, since it is merged into a user-owned file. A per-tool + copy gets frontmatter if that tool expects it. + """ + if target is None or not target.frontmatter: + return INSTRUCTIONS_BODY + return _frontmatter( + "tldrgraph", "TLDRGraph architecture graph: trace before you edit" + ) + INSTRUCTIONS_BODY + + +# --------------------------------------------------------------------------- # +# Installation +# --------------------------------------------------------------------------- # + +def _write_if_changed(path: str, content: str) -> bool: + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + if os.path.isfile(path): + try: + with open(path, "r", encoding="utf-8") as f: + if f.read() == content: + return False + except OSError: + pass + with open(path, "w", encoding="utf-8") as f: + f.write(content) + return True + + +def active_targets(root_dir: str, all_agents: bool = False) -> List[AgentTarget]: + """ + Targets to write for this repository. + + Unconditional tools always; the rest only when the repo shows the tool is in + use, so a project does not accumulate dotfiles for agents it has never + opened. ``all_agents`` writes them all. + """ + root = os.path.abspath(root_dir) + return [ + t for t in TARGETS + if all_agents or t.marker is None or os.path.isdir(os.path.join(root, t.marker)) + ] + + +def remove_superseded(root_dir: str = ".") -> List[str]: + """ + Deletes files earlier versions installed, returning what was removed. + + Left in place they describe a workflow that no longer exists, and an agent + reading both gets contradictory instructions. + """ + root = os.path.abspath(root_dir) + removed: List[str] = [] + for rel in SUPERSEDED: + path = os.path.join(root, rel) + if os.path.isfile(path): + try: + os.remove(path) + removed.append(rel) + except OSError: + continue + # Clean up a directory we created and just emptied. + parent = os.path.dirname(path) + try: + if parent != root and not os.listdir(parent): + os.rmdir(parent) + except OSError: + pass + return removed + + +def install_agent_commands(root_dir: str = ".", all_agents: bool = False) -> Dict[str, str]: + """ + Writes the instructions and the ``tldrgraph-init`` command for every active + tool, plus the AGENTS.md block that covers everything else. + + Returns ``{display_name: path}``. + """ + from .installer import upsert_block # local import: installer imports us back + + root = os.path.abspath(root_dir) + written: Dict[str, str] = {} + + # AGENTS.md first: it is the standard, and it is merged, never clobbered. + agents_md = os.path.join(root, AGENTS_MD) + existing: Optional[str] = None + if os.path.isfile(agents_md): + try: + with open(agents_md, "r", encoding="utf-8") as f: + existing = f.read() + except OSError: + existing = None + _write_if_changed( + agents_md, + upsert_block(existing or "", INSTRUCTIONS_BODY, BLOCK_BEGIN, BLOCK_END), + ) + written[f"{AGENTS_MD} (instructions, all agents)"] = agents_md + + for target in active_targets(root, all_agents=all_agents): + if target.command_path: + path = os.path.join(root, target.command_path) + _write_if_changed(path, command_text(target)) + written[f"{target.name} (command)"] = path + if target.instructions_path: + path = os.path.join(root, target.instructions_path) + _write_if_changed(path, instructions_text(target)) + written[f"{target.name} (instructions)"] = path + + return written diff --git a/tldrgraph/agent_runner.py b/tldrgraph/agent_runner.py new file mode 100644 index 0000000..72bc979 --- /dev/null +++ b/tldrgraph/agent_runner.py @@ -0,0 +1,328 @@ +""" +Host coding-agent runner for TLDRGraph. + +**This is the secondary path.** TLDRGraph is normally driven *by* an agent +through the file handshake in ``tldrgraph init``, which works with every tool +because it only needs "read a file, read source, run a command". Shelling out is +opt-in (``--agent-cli``) and exists for a plain terminal with no agent attached. + +It is not the default for good reasons, all observed in the field: every CLI has +different flags, auth and headless semantics; a blocking ``subprocess.run`` shows +the user nothing while the agent thinks; and some IDEs ship no usable CLI at all +(Antigravity's ``agy`` is a broken symlink on a stock install). + +Nesting is refused. When TLDRGraph is itself being run by a coding agent +(``CLAUDECODE``, ``CURSOR_AGENT``, ...), spawning a second agent burns tokens to +duplicate context the host already has, so the host is told what to do instead. +``TLDRGRAPH_AGENT_NESTED=1`` overrides. + +Environment: + TLDRGRAPH_NO_AGENT=1 never shell out to an agent CLI + TLDRGRAPH_AGENT_CLI= force a specific agent ("claude", "gemini", ...) + TLDRGRAPH_AGENT_NESTED=1 allow spawning even inside an agent session + TLDRGRAPH_AGENT_TIMEOUT= per-call timeout in seconds (default 600) + TLDRGRAPH_AGENT_MODEL= model to pass to the agent CLI + +Model choice only applies to the CLI shell-out. When your own agent drives +TLDRGraph through the file handshake -- the normal path -- the model is whatever +that session is already using, chosen in your IDE. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Sequence + +#: Opt-out and override knobs, documented in the module docstring. +ENV_DISABLE = "TLDRGRAPH_NO_AGENT" +ENV_FORCE_CLI = "TLDRGRAPH_AGENT_CLI" +ENV_ALLOW_NESTED = "TLDRGRAPH_AGENT_NESTED" +ENV_TIMEOUT = "TLDRGRAPH_AGENT_TIMEOUT" +ENV_MODEL = "TLDRGRAPH_AGENT_MODEL" + +DEFAULT_TIMEOUT = 600 + +#: Environment markers meaning "a coding agent is already driving this process". +#: Spawning another one here would pay twice for context the host already holds. +NESTED_MARKERS = ( + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CURSOR_AGENT", + "CURSOR_TRACE_ID", + "GEMINI_CLI", + "AI_AGENT", +) + + +@dataclass(frozen=True) +class AgentCLI: + """One headless coding-agent CLI TLDRGraph knows how to drive.""" + + #: Stable short name, also accepted by $TLDRGRAPH_AGENT_CLI. + name: str + #: Executable to look for on PATH. + binary: str + #: Human-facing label used in CLI output. + display: str + #: Builds the argv (minus the binary) for a one-shot prompt. + build_args: Callable[[str], List[str]] + #: Pulls the assistant's text out of the process stdout. + parse_stdout: Callable[[str], str] = field(default=lambda text: text) + #: This CLI's flag for selecting a model, if it has one. + model_flag: Optional[str] = "--model" + + def argv(self, prompt: str, model: Optional[str] = None) -> List[str]: + args = list(self.build_args(prompt)) + model = model or os.environ.get(ENV_MODEL, "").strip() or None + if model and self.model_flag: + args = [self.model_flag, model] + args + return [self.binary] + args + + +def _claude_args(prompt: str) -> List[str]: + # Read-only toolset: the agent must inspect the repo, never modify it. + # TLDRGraph owns every write, so the agent's only job is to answer. + return [ + "-p", prompt, + "--output-format", "json", + "--tools", "Read,Grep,Glob", + "--permission-mode", "default", + ] + + +def _claude_parse(text: str) -> str: + """`--output-format json` wraps the answer in a result envelope.""" + try: + payload = json.loads(text) + except (ValueError, TypeError): + return text + if isinstance(payload, dict): + if payload.get("is_error"): + raise AgentError(str(payload.get("result") or "claude reported an error")) + result = payload.get("result") + if isinstance(result, str): + return result + return text + + +def _cursor_parse(text: str) -> str: + """cursor-agent's json envelope also carries the answer under `result`.""" + try: + payload = json.loads(text) + except (ValueError, TypeError): + return text + if isinstance(payload, dict): + for key in ("result", "response", "text"): + value = payload.get(key) + if isinstance(value, str): + return value + return text + + +#: Known agents, in preference order. +KNOWN_AGENTS: Sequence[AgentCLI] = ( + AgentCLI( + name="claude", + binary="claude", + display="Claude Code", + build_args=_claude_args, + parse_stdout=_claude_parse, + ), + AgentCLI( + name="cursor-agent", + binary="cursor-agent", + display="Cursor Agent", + build_args=lambda prompt: ["-p", prompt, "--output-format", "json"], + parse_stdout=_cursor_parse, + ), + AgentCLI( + name="gemini", + binary="gemini", + display="Gemini CLI", + build_args=lambda prompt: ["-p", prompt], + model_flag="-m", + ), +) + + +class AgentError(RuntimeError): + """Raised when an agent CLI fails, times out, or returns unusable output.""" + + +# --------------------------------------------------------------------------- # +# Discovery +# --------------------------------------------------------------------------- # + +def nesting_marker() -> Optional[str]: + """The env var proving a coding agent is already driving us, if any.""" + for marker in NESTED_MARKERS: + if os.environ.get(marker): + return marker + return None + + +def agent_disabled() -> bool: + return os.environ.get(ENV_DISABLE, "").strip().lower() in ("1", "true", "yes", "on") + + +def _nesting_allowed() -> bool: + return os.environ.get(ENV_ALLOW_NESTED, "").strip().lower() in ("1", "true", "yes", "on") + + +def find_agent_cli(respect_nesting: bool = True) -> Optional[AgentCLI]: + """ + The agent CLI TLDRGraph may shell out to right now, or None. + + Returns None when disabled, when no known binary is on PATH, or when we are + already running inside an agent session (unless nesting is opted into) -- + in that last case the caller should print instructions for the host agent. + """ + if agent_disabled(): + return None + if respect_nesting and nesting_marker() and not _nesting_allowed(): + return None + + forced = os.environ.get(ENV_FORCE_CLI, "").strip() + if forced: + for agent in KNOWN_AGENTS: + if agent.name == forced or agent.binary == forced: + return agent if shutil.which(agent.binary) else None + # An unknown name is treated as a claude-compatible binary path. + if shutil.which(forced): + return AgentCLI( + name=forced, + binary=forced, + display=forced, + build_args=_claude_args, + parse_stdout=_claude_parse, + ) + return None + + for agent in KNOWN_AGENTS: + if shutil.which(agent.binary): + return agent + return None + + +def agent_status() -> Dict[str, Any]: + """ + Why TLDRGraph will or will not shell out, in a form the CLI can print. + + ``reason`` is one of: ``ready``, ``disabled``, ``nested``, ``not_found``. + """ + if agent_disabled(): + return {"agent": None, "reason": "disabled", "detail": f"${ENV_DISABLE} is set"} + + marker = nesting_marker() + installed = [a for a in KNOWN_AGENTS if shutil.which(a.binary)] + + if marker and not _nesting_allowed(): + return { + "agent": None, + "reason": "nested", + "detail": f"already running inside a coding agent (${marker})", + "installed": [a.name for a in installed], + } + + agent = find_agent_cli() + if agent is None: + return { + "agent": None, + "reason": "not_found", + "detail": "no agent CLI on PATH (" + + ", ".join(a.binary for a in KNOWN_AGENTS) + ")", + } + return {"agent": agent, "reason": "ready", "detail": agent.display} + + +# --------------------------------------------------------------------------- # +# Execution +# --------------------------------------------------------------------------- # + +_FENCE_RE = re.compile(r"```(?:json|yaml|yml)?\s*(.*?)```", re.DOTALL) + + +def extract_json(text: str) -> Any: + """ + Pulls a JSON object/array out of an agent's answer. + + Agents wrap JSON in prose and code fences no matter how firmly asked not to, + so this tries the whole string, then fenced blocks, then the widest + balanced ``{...}`` / ``[...]`` span. + """ + if not text or not text.strip(): + raise AgentError("agent returned empty output") + + candidates: List[str] = [text.strip()] + candidates.extend(block.strip() for block in _FENCE_RE.findall(text)) + + # Widest balanced span per delimiter, tried outermost-first. Ordering by the + # opener's position matters: in `Here you go: [{"id": "x"}]` the first `{` is + # *inside* the array, so trying braces first would return one element and + # silently drop the rest of the batch. + spans = [] + for opener, closer in (("{", "}"), ("[", "]")): + start = text.find(opener) + end = text.rfind(closer) + if start != -1 and end > start: + spans.append((start, text[start:end + 1])) + candidates.extend(span for _, span in sorted(spans)) + + for candidate in candidates: + if not candidate: + continue + try: + return json.loads(candidate) + except ValueError: + continue + + preview = text.strip().replace("\n", " ")[:200] + raise AgentError(f"could not parse JSON from agent output: {preview}") + + +def call_timeout() -> int: + raw = os.environ.get(ENV_TIMEOUT, "").strip() + if raw.isdigit() and int(raw) > 0: + return int(raw) + return DEFAULT_TIMEOUT + + +def run_agent(agent: AgentCLI, prompt: str, cwd: str, timeout: Optional[int] = None, + model: Optional[str] = None) -> str: + """ + Runs the agent headlessly in ``cwd`` and returns its answer text. + + Raises AgentError on non-zero exit, timeout, or an empty answer. + """ + try: + proc = subprocess.run( + agent.argv(prompt, model=model), + cwd=os.path.abspath(cwd), + capture_output=True, + text=True, + timeout=timeout or call_timeout(), + ) + except subprocess.TimeoutExpired as err: + raise AgentError( + f"{agent.display} timed out after {timeout or call_timeout()}s " + f"(raise ${ENV_TIMEOUT} to allow longer)" + ) from err + except OSError as err: + raise AgentError(f"could not run {agent.binary}: {err}") from err + + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout or "").strip().replace("\n", " ")[:300] + raise AgentError(f"{agent.display} exited {proc.returncode}: {detail or 'no output'}") + + return agent.parse_stdout(proc.stdout or "") + + +def run_agent_json(agent: AgentCLI, prompt: str, cwd: str, timeout: Optional[int] = None, + model: Optional[str] = None) -> Any: + """``run_agent`` plus JSON extraction. Raises AgentError on any failure.""" + return extract_json(run_agent(agent, prompt, cwd, timeout=timeout, model=model)) diff --git a/tldrgraph/cli.py b/tldrgraph/cli.py index 3fa2ad3..f4400e1 100644 --- a/tldrgraph/cli.py +++ b/tldrgraph/cli.py @@ -2,12 +2,13 @@ CLI Entry Point for TLDRGraph: Multi-layer code flow & hybrid semantic search engine. """ +import contextlib import os import json import yaml import click from datetime import datetime, timezone -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple from .graph_loader import ( GraphLoader, @@ -16,16 +17,17 @@ resolve_call_target, ) from .flow_engine import FlowEngine -from .installer import install_agent_rules, gitignore_warnings +from .installer import ensure_gitignore, install_agent_rules, gitignore_warnings from .visualizer import generate_visualizer_html from .layers import get_registry, layer_id_of +from .layer_config import config_path from .propose_layers import ( + RESPONSE_FILENAME as PROPOSE_RESPONSE_FILENAME, auto_configure_layers, generate_propose_request, apply_proposed_layers, - detect_repository_archetype, - propose_layers_with_llm, ) +from . import agent_runner, paths from . import vector_store as vs_mod #: Shared option for the retrieval-backend policy. ``off`` (default) is pure @@ -39,9 +41,10 @@ ) try: # provenance tag written by the offline template enricher - from .deadcode import HEURISTIC_ENRICHMENT_SOURCE + from .deadcode import HEURISTIC_ENRICHMENT_SOURCE, NON_CODE_NODE_TYPES except ImportError: # pragma: no cover - deadcode module is optional HEURISTIC_ENRICHMENT_SOURCE = "heuristic" + NON_CODE_NODE_TYPES = {"rationale", "concept", "doc", "documentation"} #: Provenance stamped on nodes enriched through the host-agent loop. AGENT_ENRICHMENT_SOURCE = "agent" @@ -193,9 +196,18 @@ def needs_agent_enrichment(data: Dict[str, Any]) -> bool: An intent produced by the offline template heuristic does not count: it is derived from the label and layer alone, without reading a single line of source, which is exactly the gap this loop exists to close. + + Two kinds of node are excluded outright. The utility bucket is a catch-all + rather than an architectural layer. And graphify's prose nodes -- ``rationale`` + and friends, whose *label is already a sentence of documentation* -- are not + symbols at all: asking an agent to describe one means paying it to copy a + docstring back onto itself. ``deadcode`` already calls these "not source + code"; the same set is reused here so the two cannot disagree. """ if layer_id_of(data) == get_registry().utility_id: return False + if str(data.get("type") or "").lower() in NON_CODE_NODE_TYPES: + return False source = data.get("enrichment_source") or "" if source == AGENT_ENRICHMENT_SOURCE: return False @@ -231,6 +243,381 @@ def _enrichment_candidates(loader: GraphLoader, degrees: Dict[str, Tuple[int, in return candidates +#: Instructions embedded in every enrichment request. Shared by the file-based +#: handshake and the headless-agent prompt so both describe the same contract. +def _enrichment_instructions() -> List[str]: + return [ + "Open and READ each node's source file before writing its intent - you have the repo.", + "Write 'intent' in Markdown (single-line summary or rich multiline markdown). Add as much context as needed.", + "In 'input_fields', list input arguments, parameters, payload attributes, or request body fields.", + "In 'output_fields', list return values, response schemas, emitted event names, or mutated state fields.", + "In 'calls', specify exact downstream targets: node ID, file:symbol (e.g. 'src/services/calc.ts:calculate'), file path, or symbol name.", + "When multiple methods share the same name across files, use 'file_path:method' or exact node 'id' to ensure precise linking.", + "Never invent fields or calls - omit what you cannot verify in the source. An empty list is a correct answer.", + "Copy each 'id' verbatim.", + ] + + +def build_enrichment_batch( + path: str, + loader: GraphLoader, + degrees: Dict[str, Tuple[int, int]], + limit: int, + requeue: bool = False, + reset: bool = False, + skip_cursor: bool = False, +) -> Dict[str, Any]: + """ + Selects the next batch of nodes deserving enrichment and builds the request payload. + + ``skip_cursor`` ignores queue bookkeeping entirely, which is what the + automatic in-scan loop wants: it applies each batch before asking for the + next, so the graph itself is the only progress record it needs. + + Returns a dict with the request ``payload``, the ``batch``, and progress counts. + """ + cursor = {"queued": [], "applied": []} if reset else _read_cursor(path) + + skip: set = set() + if not skip_cursor: + # Anything already answered in a response file counts as done, even if + # apply-enrichment has not run yet. + answered = set(cursor["applied"]) + for filename in (RESPONSE_FILENAME, LEGACY_RESPONSE_FILENAME, "pending_enrichment.yaml", LEGACY_FILENAME): + for item in coerce_enrichment_items(_read_payload(_state_path(path, filename))): + if item.get("id"): + answered.add(str(item["id"])) + skip = set(answered) + if not requeue: + skip |= set(cursor["queued"]) + + candidates = _enrichment_candidates(loader, degrees) + total_candidates = len(candidates) + pending = [c for c in candidates if c["id"] not in skip] + + batch = pending if limit <= 0 else pending[:limit] + for rank, node in enumerate(batch, 1): + node["rank"] = rank + + already_enriched = sum( + 1 for _, d in loader.graph.nodes(data=True) + if layer_id_of(d) != get_registry().utility_id and not needs_agent_enrichment(d) + ) + remaining_after = max(len(pending) - len(batch), 0) + + payload = { + "schema": "codechakra/enrichment-request@1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "response_file": os.path.join(STATE_DIR, RESPONSE_FILENAME), + "contract": os.path.join(STATE_DIR, "AGENT_CONTRACT.md"), + "instructions": _enrichment_instructions() + [ + f"Write a YAML list of {{id, intent, input_fields, output_fields, calls}} to " + f"{os.path.join(STATE_DIR, RESPONSE_FILENAME)} (NOT back into this request file).", + "Then run: tldrgraph apply-enrichment", + ], + "ordering": "cross_layer_degree desc, then degree (in+out) desc, then id asc", + "progress": { + "total_candidates": total_candidates, + "already_enriched": already_enriched, + "queued_now": len(batch), + "remaining_after": remaining_after, + }, + "nodes": batch, + } + + return { + "payload": payload, + "batch": batch, + "cursor": cursor, + "total_candidates": total_candidates, + "already_enriched": already_enriched, + "remaining_after": remaining_after, + } + + +def apply_enrichment_items( + loader: GraphLoader, + path: str, + items: List[Dict[str, Any]], + source_label: str, +) -> Dict[str, Any]: + """ + Merges enrichment objects into the graph, hash-gate cache and vector index. + + This is the one implementation behind both `apply-enrichment` (file handshake) + and the automatic in-scan agent loop, so a bridge edge forged automatically is + resolved exactly the same way as one applied by hand. + + Returns stats: applied_ids, unknown_ids, bridges, unresolved, snapshot_path. + """ + floor = bridge_score_floor(loader.vector_store) + + applied_ids: List[str] = [] + unknown_ids: List[str] = [] + unresolved: List[str] = [] + bridges = 0 + + for item in items: + nid = item.get("id") + if not nid: + continue + nid = str(nid) + if not loader.graph.has_node(nid): + unknown_ids.append(nid) + continue + + node_data = loader.graph.nodes[nid] + intent = item.get("intent", "") + input_fields = item.get("input_fields", []) or [] + output_fields = item.get("output_fields", []) or [] + legacy_fields = item.get("fields", []) or [] + calls = item.get("calls", []) or [] + + # Optional layer_id override from agent + override_lid = item.get("layer_id") + if override_lid and override_lid in get_registry(): + node_data["layer_id"] = override_lid + node_data["layer"] = get_registry().name(override_lid) + node_data["layer_source"] = AGENT_ENRICHMENT_SOURCE + + if intent: + node_data["intent"] = intent + first_line = intent.strip().split("\n")[0].lstrip("#- *").strip() + node_data["summary"] = f"{node_data['layer']}: {node_data['label']} - {first_line or intent}" + node_data["enrichment_source"] = AGENT_ENRICHMENT_SOURCE + + if input_fields or output_fields: + node_data["input_fields"] = input_fields + node_data["output_fields"] = output_fields + node_data["fields"] = list(input_fields) + list(output_fields) + elif legacy_fields: + node_data["input_fields"] = legacy_fields + node_data["output_fields"] = [] + node_data["fields"] = legacy_fields + + fields_dict = { + "input_fields": node_data.get("input_fields", []), + "output_fields": node_data.get("output_fields", []), + "fields": node_data.get("fields", []) + } + loader.hash_gate.update_node( + node_id=nid, + file_path=node_data.get("file", ""), + content=loader.node_signature(node_data), + layer=node_data.get("layer", ""), + summary=node_data.get("summary", ""), + fields_json=json.dumps(fields_dict), + intent=node_data.get("intent", "") + ) + + for call_target in calls: + tgt_id, score = resolve_call_target( + loader.graph, loader.vector_store, call_target, nid, floor + ) + if tgt_id: + loader.graph.add_edge( + nid, tgt_id, + relation="cross_layer_link", + confidence=float(score) + ) + bridges += 1 + else: + unresolved.append(str(call_target)) + + applied_ids.append(nid) + + # Re-index so the applied intents/fields are actually searchable, then persist. + loader.vector_store.add_documents(loader.docs_to_index) + _stamp_degrees(loader) + snapshot_path = loader.save_graph() + loader.export_yaml() + + cursor = _read_cursor(path) + remaining_queued = [i for i in cursor["queued"] if i not in set(applied_ids)] + _write_cursor(path, remaining_queued, cursor["applied"] + applied_ids) + + _append_enrichment_audit(path, items, applied_ids, bridges, unresolved, source_label) + + return { + "applied_ids": applied_ids, + "unknown_ids": unknown_ids, + "bridges": bridges, + "unresolved": unresolved, + "floor": floor, + "snapshot_path": snapshot_path, + } + + +def _append_enrichment_audit( + path: str, + items: List[Dict[str, Any]], + applied_ids: List[str], + bridges: int, + unresolved: List[str], + source_label: str, +) -> None: + """Appends one batch to .tldrgraph/enrichment_audit.log. Never raises.""" + audit_path = _state_path(path, AUDIT_LOG_FILENAME) + timestamp = datetime.now(timezone.utc).isoformat() + applied = set(applied_ids) + try: + os.makedirs(os.path.dirname(audit_path) or ".", exist_ok=True) + with open(audit_path, "a", encoding="utf-8") as f: + f.write(f"\n--- Enrichment Batch Applied: {timestamp} ---\n") + f.write(f"Source file: {source_label}\n") + f.write(f"Applied nodes ({len(applied_ids)}): {', '.join(applied_ids)}\n") + f.write(f"Bridge edges created: {bridges}\n") + for item in items: + nid = str(item.get("id") or "") + if nid in applied: + f.write(f" • [{nid}] intent:\n {item.get('intent', '')}\n") + if item.get("input_fields"): + f.write(f" input_fields: {item.get('input_fields')}\n") + if item.get("output_fields"): + f.write(f" output_fields: {item.get('output_fields')}\n") + elif item.get("fields"): + f.write(f" fields: {item.get('fields')}\n") + if item.get("calls"): + f.write(f" calls: {item.get('calls')}\n") + if unresolved: + f.write(f"Unresolved call targets: {', '.join(unresolved)}\n") + except Exception: + pass + + +# --------------------------------------------------------------------------- # +# Automatic agent enrichment +# --------------------------------------------------------------------------- # + +#: Prompt handed to a headless agent CLI for one enrichment batch. +AGENT_ENRICH_PROMPT = """You are enriching a code architecture graph for the repository you currently have open. + +For each node listed below, OPEN its source file at the given path and read the +actual implementation before writing anything. An intent paraphrased from the +symbol name is worse than none: it poisons semantic search with confident noise. + +{instructions} + +Return ONLY a JSON array, no prose and no markdown fence, of this shape: + +[ + {{ + "id": "", + "intent": "What this symbol does, why it exists, and its execution logic. Markdown allowed.", + "input_fields": ["argument", "payloadField"], + "output_fields": ["returnedField", "emittedEvent"], + "calls": ["DownstreamService", "src/services/calc.ts:calculate", "some_table"] + }} +] + +Include every id exactly once. If a file is unreadable or the symbol is trivial, +still return the id with a short honest intent and empty field/call lists. + +Repository root: {root} + +Nodes ({count}): +{nodes} +""" + + +def build_agent_enrichment_prompt(root: str, batch: List[Dict[str, Any]]) -> str: + """Renders the headless-agent prompt for one enrichment batch.""" + return AGENT_ENRICH_PROMPT.format( + instructions="\n".join(f"- {line}" for line in _enrichment_instructions()), + root=root, + count=len(batch), + nodes=json.dumps(batch, indent=2, default=str), + ) + + +def run_agent_enrichment( + path: str, + loader: GraphLoader, + agent: Any, + batch_size: int = 25, + max_nodes: int = 0, + model: Optional[str] = None, +) -> Dict[str, Any]: + """ + Drives the full enrich loop against a headless agent CLI until the backlog + is empty, applying each batch before requesting the next. + + Progress is durable: every batch is merged, re-indexed and saved as it lands, + so an interrupted run keeps everything it already earned. + """ + root = os.path.abspath(path) + totals = {"applied": 0, "bridges": 0, "unresolved": 0, "batches": 0, "failed_batches": 0} + errors: List[str] = [] + processed = 0 + + while True: + degrees = compute_degrees(loader.graph) + limit = batch_size + if max_nodes: + remaining_budget = max_nodes - processed + if remaining_budget <= 0: + break + limit = min(batch_size, remaining_budget) + + request = build_enrichment_batch(path, loader, degrees, limit=limit, skip_cursor=True) + batch = request["batch"] + if not batch: + break + + # Keep the request file current so an interrupted run leaves the host + # agent a usable handoff instead of a stale batch. + _write_payload(_state_path(path, REQUEST_FILENAME), request["payload"]) + + click.echo(f" 🤖 Batch {totals['batches'] + 1}: {len(batch)} node(s) " + f"({request['remaining_after']} left after this)...") + try: + raw = agent_runner.run_agent_json( + agent, build_agent_enrichment_prompt(root, batch), root, model=model + ) + except agent_runner.AgentError as err: + totals["failed_batches"] += 1 + errors.append(str(err)) + click.echo(f" ⚠️ {err}") + break + + items = coerce_enrichment_items(raw) + if not items: + totals["failed_batches"] += 1 + errors.append("agent returned no enrichment objects") + click.echo(" ⚠️ Agent returned no enrichment objects; stopping the loop.") + break + + batch_ids = {node["id"] for node in batch} + stats = apply_enrichment_items(loader, path, items, f"agent:{agent.name}") + totals["batches"] += 1 + totals["applied"] += len(stats["applied_ids"]) + totals["bridges"] += stats["bridges"] + totals["unresolved"] += len(stats["unresolved"]) + processed += len(batch) + + if not stats["applied_ids"]: + # Nothing landed: the agent is answering with ids we do not have. + # Looping again would just repeat the same batch forever. + errors.append("agent returned ids that are not in the graph") + click.echo(" ⚠️ None of the returned ids matched the graph; stopping the loop.") + break + + # Progress is measured by nodes leaving the candidate set, not by ids + # echoed back. An answer with empty intents applies cleanly yet clears + # nothing, and would otherwise re-request the same batch forever. + cleared = sum( + 1 for nid in batch_ids + if loader.graph.has_node(nid) and not needs_agent_enrichment(loader.graph.nodes[nid]) + ) + if not cleared: + errors.append("agent answers left every node still un-enriched") + click.echo(" ⚠️ That batch cleared no nodes (empty intents?); stopping the loop.") + break + + totals["errors"] = errors + return totals + + def _snapshot_or_graph_nodes(root: str) -> Tuple[List[Dict[str, Any]], str]: """ Node records for read-only commands. Prefers the persisted snapshot (a pure read); @@ -254,39 +641,390 @@ def cli(): pass -@cli.command() -@click.argument("path", default=".", type=click.Path(exists=True)) -@click.option("--rebuild", is_flag=True, help="Discard the persisted snapshot and rebuild enrichment from scratch") -@click.option("--propose-layers", is_flag=True, help="Automatically synthesize and configure new dynamic layers using LLM/archetype") -@embeddings_option -def scan(path, rebuild, propose_layers, embeddings): - """Scan repository, classify dynamic architectural layers, and build local vector index.""" - click.echo(f"🔄 [TLDRGraph] Scanning repository at {os.path.abspath(path)}...") +# --------------------------------------------------------------------------- # +# `tldrgraph init` -- the one command +# --------------------------------------------------------------------------- # - # Auto-configure dynamic layers if no configuration exists or explicitly requested - cfg_file = os.path.join(os.path.abspath(path), ".tldrgraph", "layers.config.yaml") - if propose_layers or not os.path.isfile(cfg_file): - reg, cfg_path, source = auto_configure_layers(path, force=propose_layers) - click.echo(f"🏗️ Configured {len(reg)} dynamic architectural layers ({source}) in {cfg_path}") +#: Statuses `init` can end on. Stable strings: agent command files and any +#: `--json` consumer branch on these. +STATUS_DONE = "done" +STATUS_NEEDS_LAYERS = "needs_layers" +STATUS_NEEDS_CONFIRMATION = "needs_confirmation" +STATUS_NEEDS_ENRICHMENT = "needs_enrichment" + +#: Written by `init` after it merges a response, so the same answers are never +#: applied twice on the next run. +APPLIED_RESPONSE_FILENAME = "enrichment_response.applied.yaml" + + +@contextlib.contextmanager +def _stdout_to_stderr_if(active: bool): + """Redirects stdout to stderr while ``active``, so --json stays parseable.""" + if not active: + yield + return + import sys + with contextlib.redirect_stdout(sys.stderr): + yield + + +def _stdin_is_interactive() -> bool: + """True when there is a real terminal to prompt on.""" + try: + import sys + return bool(sys.stdin and sys.stdin.isatty()) + except Exception: + return False + + +def _emit_status(status: str, phase: str, lines: List[str], + progress: Optional[Dict[str, Any]] = None, + as_json: bool = False) -> None: + """ + Prints the block an agent reads to decide what to do next. + The format is deliberately dull and greppable: every agent tool, whatever + its prompt conventions, can find `status:` and follow the numbered steps. + """ + if as_json: + click.echo(json.dumps({ + "status": status, + "phase": phase, + "next_action": lines, + "progress": progress or {}, + }, indent=2)) + return + + rule = "─" * 68 + click.echo(f"\n{rule}") + if status == STATUS_DONE: + click.echo("TLDRGRAPH INIT — COMPLETE") + else: + click.echo("TLDRGRAPH INIT — NEXT ACTION REQUIRED") + click.echo(f"status: {status}") + click.echo(rule) + for line in lines: + click.echo(line) + click.echo(rule + "\n") + + +def _apply_pending_layer_response(path: str) -> Optional[str]: + """ + Applies .tldrgraph/propose_layers_response.json if the agent has written one. + + Part of the one-command promise: the agent writes the file and runs `init` + again, rather than having to remember `tldrgraph apply-layers`. + """ + for filename in (PROPOSE_RESPONSE_FILENAME, "propose_layers_response.yaml"): + candidate = _state_path(path, filename) + if os.path.isfile(candidate): + return apply_proposed_layers(path, candidate) + return None + + +def _apply_pending_enrichment_response(path: str, loader: GraphLoader) -> Optional[Dict[str, Any]]: + """ + Merges .tldrgraph/enrichment_response.yaml if the agent has written one, + then renames it so a later `init` cannot apply the same answers twice. + + Returns the apply stats, or None when there was nothing to apply. + """ + source = None + for filename in (RESPONSE_FILENAME, LEGACY_RESPONSE_FILENAME, + "pending_enrichment.yaml", LEGACY_FILENAME): + candidate = _state_path(path, filename) + if os.path.isfile(candidate): + source = candidate + break + if not source: + return None + + items = coerce_enrichment_items(_read_payload(source)) + if not items: + return None + + stats = apply_enrichment_items(loader, path, items, source) + try: + os.replace(source, _state_path(path, APPLIED_RESPONSE_FILENAME)) + except OSError: + pass + return stats + + +def _init_pipeline(path: str, assume_yes: bool, batch_size: int, max_nodes: int, + rebuild: bool, relayer: bool, agent_cli: bool, agent_model: Optional[str], + embeddings: Optional[str], as_json: bool) -> str: + """ + Layers, extraction and enrichment in one resumable pass. + + Every deterministic step runs here; the moment judgement is needed that only + an agent can supply, this stops and prints what to do. Re-running picks up + exactly where it left off, so the whole workflow is `init`, act, `init`, + act, `init`. Returns the terminal status. + """ + root = os.path.abspath(path) + + if not as_json: + click.echo(f"🔄 [TLDRGraph] {root}") + + # 0. Make the repo agent-ready. Idempotent: unchanged files are not touched. + ensure_gitignore(path) + install_agent_rules(path) + + # 1. Extraction first, so the layer evidence carries real symbols rather + # than a directory listing -- and so we can size the job up front. + # + # This runs EVERY time, not just when the export is missing. Reusing an + # existing export means every later phase works from whatever the code + # looked like when it was written: an agent gets handed deleted functions + # to describe, and code added since is invisible. graphify caches per file + # by content hash, so a re-run with nothing changed is nearly free. loader = GraphLoader(path, embeddings=embeddings) + if not as_json: + click.echo("📦 Extracting AST with graphify...") + # graphify writes progress and warnings to stdout. Under --json that would + # sit in front of the payload and make it unparseable, so it goes to stderr, + # where a human still sees it and a parser does not. + with _stdout_to_stderr_if(as_json): + loader._run_graphify() + loader.file_hashes = loader._load_file_hashes() + + # 2. Layers. No template, no fallback: either they exist, or we ask. + applied_cfg = None + if relayer or not config_path(root): + applied_cfg = _apply_pending_layer_response(path) + if applied_cfg and not as_json: + click.echo(f"🏗️ Applied the agent's layer design → {applied_cfg}") + + notes: List[str] = [] + registry, cfg_path, source = auto_configure_layers( + path, force=relayer and not applied_cfg, use_agent=agent_cli, + agent_model=agent_model, notes=notes, + ) + for note in notes: + if not as_json: + click.echo(f" ℹ️ {note}") + + if registry is None: + request_path = generate_propose_request(path) + _emit_status(STATUS_NEEDS_LAYERS, "layers", [ + "This repository has no architectural layer set, and TLDRGraph will", + "not invent one from a template. Design it from the code:", + "", + f" 1. Read {os.path.relpath(request_path, root)}", + " 2. OPEN real source files -- entry points first, then one file from", + " each cluster in the evidence. Do not skip this step.", + f" 3. Write {os.path.join(STATE_DIR, PROPOSE_RESPONSE_FILENAME)} with", + ' {"utility_id": "...", "layers": [{id, name, order, description, rules}]}', + " 3-6 layers plus one catch-all whose id equals utility_id and whose", + " rules are []. Name them after THIS repository's concepts.", + " 4. Run: tldrgraph init", + ], as_json=as_json) + return STATUS_NEEDS_LAYERS + + if not as_json: + if applied_cfg: + label = "designed by your agent" + elif source == "existing_config": + label = "already configured" + else: + label = source + click.echo(f"🏛️ {len(registry)} architectural layers ({label})") + + # 3. Full build: classify, index, persist. graph = loader.load_or_extract(rebuild=rebuild) _stamp_degrees(loader) snapshot_path = loader.save_graph() - yaml_path = loader.export_yaml() - - click.echo(f"✅ Ingested {graph.number_of_nodes()} nodes and {graph.number_of_edges()} relationships.") - diag = loader.vector_store.diagnostics() - click.echo(f"🔎 Retrieval backend: {diag['backend']} " - f"(bridge score floor {diag['score_floor']}) — `tldrgraph doctor` for detail") - click.echo(f"💾 Graph snapshot persisted at: {snapshot_path}") - click.echo(f"📊 Layer breakdown exported to YAML at: {yaml_path}") - html_path = generate_visualizer_html(path) - click.echo(f"🌐 Interactive Visualizer generated at: {html_path}\n") + loader.export_yaml() - for layer, nodes in loader.nodes_by_layer.items(): - click.echo(f" • {layer.ljust(35)} : {len(nodes)} nodes") - click.echo("\n✨ Scan complete! Open .tldrgraph/TLDRGRAPH_VISUALIZER.html to explore the architectural layers visually.") + if not as_json: + click.echo(f"✅ {graph.number_of_nodes()} nodes, {graph.number_of_edges()} relationships") + diag = loader.vector_store.diagnostics() + click.echo(f"🔎 Retrieval: {diag['backend']} (floor {diag['score_floor']})") + click.echo(f"💾 {snapshot_path}") + + # 4. Enrichment. Apply anything the agent already answered, then either + # finish, ask permission, or hand out the next batch. + applied = _apply_pending_enrichment_response(path, loader) + if applied and not as_json: + click.echo(f"🧠 Applied {len(applied['applied_ids'])} enrichment(s), " + f"{applied['bridges']} bridge edge(s)") + # An id that is not in the graph is silently worthless, and an agent + # that invented one will keep inventing it. Say so, with examples. + if applied["unknown_ids"]: + preview = ", ".join(applied["unknown_ids"][:4]) + click.echo(f" ⚠️ {len(applied['unknown_ids'])} id(s) are not in the graph " + f"and were dropped: {preview}") + click.echo(" Copy ids verbatim from the request; do not construct them.") + if applied["unresolved"]: + preview = ", ".join(sorted(set(applied["unresolved"]))[:4]) + click.echo(f" ⚠️ {len(applied['unresolved'])} call target(s) matched nothing " + f"above the score floor: {preview}") + + generate_visualizer_html(path) + + candidates = _enrichment_candidates(loader, compute_degrees(loader.graph)) + total = loader.graph.number_of_nodes() + # Counting `total - candidates` as "enriched" would fold in every node that + # was never eligible -- the utility bucket and graphify's prose nodes -- and + # report a large number before a single intent had been written. + enriched = sum( + 1 for _, d in loader.graph.nodes(data=True) + if (d.get("enrichment_source") or "") == AGENT_ENRICHMENT_SOURCE + ) + excluded = total - enriched - len(candidates) + + if not candidates: + _emit_status(STATUS_DONE, "enrichment", [ + f"{total} nodes across {len(registry)} layers. {enriched} enriched from " + f"source; {excluded} not eligible (utility bucket and prose nodes).", + "", + ' tldrgraph query ""', + ' tldrgraph trace "" ""', + " tldrgraph layers", + " tldrgraph ui --serve", + ], progress={"total_nodes": total, "enriched": enriched, "remaining": 0}, + as_json=as_json) + return STATUS_DONE + + planned = min(len(candidates), max_nodes) if max_nodes else len(candidates) + rounds = (planned + batch_size - 1) // batch_size + progress = { + "total_nodes": total, + "enriched": enriched, + "excluded": excluded, + "remaining": len(candidates), + "planned_this_run": planned, + "batch_size": batch_size, + "agent_rounds": rounds, + } + + if not assume_yes: + if _stdin_is_interactive(): + click.echo(f"\n🧠 {len(candidates)} node(s) need an intent read from the source " + f"({rounds} batch(es) of {batch_size}).") + if click.confirm(" Enrich now?", default=True): + assume_yes = True + else: + click.echo(" Skipped. Run `tldrgraph init` again when ready.") + return STATUS_NEEDS_CONFIRMATION + else: + _emit_status(STATUS_NEEDS_CONFIRMATION, "enrichment", [ + f"The graph is built and queryable: {total} nodes, {enriched} enriched " + f"from source, {excluded} not eligible (utility bucket, prose nodes).", + "", + f"{len(candidates)} node(s) still carry generated summaries rather than", + "an intent read from the source. Enriching them means roughly", + f"{rounds} round(s) of {batch_size} nodes, and each round costs tokens.", + "", + "ASK THE USER whether to proceed, showing them that estimate. Then:", + "", + " they agree → tldrgraph init --yes", + " smaller first pass → tldrgraph init --yes --limit 100", + " they decline → stop here; the graph is already usable", + ], progress=progress, as_json=as_json) + return STATUS_NEEDS_CONFIRMATION + + # 5. Enrich: drive an agent CLI if one was opted into, else hand off. + if agent_cli: + agent = agent_runner.find_agent_cli() + if agent is not None: + if not as_json: + click.echo(f"\n🤖 Enriching via {agent.display}...") + totals = run_agent_enrichment(path, loader, agent, batch_size=batch_size, + max_nodes=max_nodes, model=agent_model) + remaining = len(_enrichment_candidates(loader, compute_degrees(loader.graph))) + status = STATUS_DONE if not remaining else STATUS_NEEDS_ENRICHMENT + _emit_status(status, "enrichment", [ + f"Enriched {totals['applied']} node(s) in {totals['batches']} batch(es); " + f"{totals['bridges']} bridge edge(s).", + f"{remaining} still un-enriched." + if remaining else "Nothing left to enrich.", + ] + (["Run `tldrgraph init --yes` to continue."] if remaining else []), + progress={**progress, "remaining": remaining}, as_json=as_json) + return status + if not as_json: + click.echo(f" ℹ️ No agent CLI available " + f"({agent_runner.agent_status()['detail']}); handing off instead.") + + request = build_enrichment_batch( + path, loader, compute_degrees(loader.graph), + limit=batch_size if not max_nodes else min(batch_size, max_nodes), + skip_cursor=True, + ) + request_path = _write_payload(_state_path(path, REQUEST_FILENAME), request["payload"]) + + _emit_status(STATUS_NEEDS_ENRICHMENT, "enrichment", [ + f"{len(request['batch'])} node(s) queued, {len(candidates)} remaining overall.", + "", + f" 1. Read {os.path.relpath(request_path, root)}", + " 2. OPEN the source file of every node in it. An intent guessed from a", + " symbol name is worse than none -- it poisons semantic search.", + f" 3. Write {os.path.join(STATE_DIR, RESPONSE_FILENAME)} as a YAML list of", + " {id, intent, input_fields, output_fields, calls}. Never invent", + " fields or calls; omit what you cannot verify.", + " 4. Run: tldrgraph init --yes", + "", + "Repeat until this says status: done.", + ], progress=progress, as_json=as_json) + return STATUS_NEEDS_ENRICHMENT + + +_init_options = [ + click.argument("path", default=".", type=click.Path(exists=True)), + click.option("--yes", "-y", "assume_yes", is_flag=True, + help="Proceed with enrichment without asking (agents: only after the user agrees)"), + click.option("--batch", "batch_size", default=25, show_default=True, + help="Nodes handed to the agent per round"), + click.option("--limit", "max_nodes", default=0, show_default=True, + help="Cap on nodes to enrich this run. 0 enriches every candidate."), + click.option("--rebuild", is_flag=True, help="Re-extract and rebuild enrichment from scratch"), + click.option("--relayer", is_flag=True, help="Discard the layer set and design it again"), + click.option("--agent-cli", is_flag=True, + help="Shell out to an agent CLI (claude/cursor-agent/gemini) instead of " + "handing off. Off by default: agent CLIs differ per tool and can hang."), + click.option("--agent-model", default=None, + help="Model for --agent-cli (e.g. opus, sonnet, gemini-2.5-pro). Defaults " + "to $TLDRGRAPH_AGENT_MODEL. Ignored on the handshake path, where your " + "own agent session picks the model."), + click.option("--json", "as_json", is_flag=True, help="Emit machine-readable status"), + embeddings_option, +] + + +def _with_init_options(fn): + for option in reversed(_init_options): + fn = option(fn) + return fn + + +@cli.command() +@_with_init_options +def init(path, assume_yes, batch_size, max_nodes, rebuild, relayer, agent_cli, agent_model, as_json, embeddings): + """ + Build this repository's graph: layers, extraction, and enrichment, in one command. + + Resumable. Run it, do whatever the NEXT ACTION block says, run it again. + Layers are always designed from your code -- there is no template fallback. + """ + _init_pipeline(path, assume_yes, batch_size, max_nodes, rebuild, relayer, + agent_cli, agent_model, embeddings, as_json) + + +@cli.command() +@_with_init_options +def scan(path, assume_yes, batch_size, max_nodes, rebuild, relayer, agent_cli, agent_model, as_json, embeddings): + """Alias for `init`, kept for existing scripts and agent rules.""" + _init_pipeline(path, assume_yes, batch_size, max_nodes, rebuild, relayer, + agent_cli, agent_model, embeddings, as_json) + + +@cli.command() +@_with_init_options +def enrich(path, assume_yes, batch_size, max_nodes, rebuild, relayer, agent_cli, agent_model, as_json, embeddings): + """Alias for `init`, which already resumes enrichment where it left off.""" + _init_pipeline(path, assume_yes, batch_size, max_nodes, rebuild, relayer, + agent_cli, agent_model, embeddings, as_json) @cli.command(name="ui") @@ -434,63 +1172,18 @@ def queue_enrichment(path, limit, requeue, reset): degrees = _stamp_degrees(loader) loader.save_graph() - cursor = _read_cursor(path) - if reset: - cursor = {"queued": [], "applied": []} - - # Anything already answered in a response file counts as done, even if - # apply-enrichment has not run yet. - answered = set(cursor["applied"]) - for filename in (RESPONSE_FILENAME, LEGACY_RESPONSE_FILENAME, "pending_enrichment.yaml", LEGACY_FILENAME): - for item in coerce_enrichment_items(_read_payload(_state_path(path, filename))): - if item.get("id"): - answered.add(str(item["id"])) - - skip = set(answered) - if not requeue: - skip |= set(cursor["queued"]) - - candidates = _enrichment_candidates(loader, degrees) - total_candidates = len(candidates) - pending = [c for c in candidates if c["id"] not in skip] - - batch = pending if limit <= 0 else pending[:limit] - for rank, node in enumerate(batch, 1): - node["rank"] = rank - - already_enriched = sum( - 1 for _, d in loader.graph.nodes(data=True) - if layer_id_of(d) != get_registry().utility_id and not needs_agent_enrichment(d) + request = build_enrichment_batch( + path, loader, degrees, limit=limit, requeue=requeue, reset=reset ) - remaining_after = max(len(pending) - len(batch), 0) + batch = request["batch"] + cursor = request["cursor"] + total_candidates = request["total_candidates"] + already_enriched = request["already_enriched"] + remaining_after = request["remaining_after"] request_path = _state_path(path, REQUEST_FILENAME) response_path = _state_path(path, RESPONSE_FILENAME) - _write_payload(request_path, { - "schema": "codechakra/enrichment-request@1", - "generated_at": datetime.now(timezone.utc).isoformat(), - "response_file": os.path.join(STATE_DIR, RESPONSE_FILENAME), - "contract": os.path.join(STATE_DIR, "AGENT_CONTRACT.md"), - "instructions": [ - "Open and READ each node's source file before writing its intent - you have the repo.", - "Write 'intent' in Markdown (single-line summary or rich multiline markdown). Add as much context as needed.", - "In 'input_fields', list input arguments, parameters, payload attributes, or request body fields.", - "In 'output_fields', list return values, response schemas, emitted event names, or mutated state fields.", - "In 'calls', specify exact downstream targets: node ID, file:symbol (e.g. 'src/services/calc.ts:calculate'), file path, or symbol name.", - "When multiple methods share the same name across files, use 'file_path:method' or exact node 'id' to ensure precise linking.", - "Copy each 'id' verbatim.", - f"Write a YAML list of {{id, intent, input_fields, output_fields, calls}} to {os.path.join(STATE_DIR, RESPONSE_FILENAME)} (NOT back into this request file).", - "Then run: tldrgraph apply-enrichment", - ], - "ordering": "cross_layer_degree desc, then degree (in+out) desc, then id asc", - "progress": { - "total_candidates": total_candidates, - "already_enriched": already_enriched, - "queued_now": len(batch), - "remaining_after": remaining_after, - }, - "nodes": batch, - }) + _write_payload(request_path, request["payload"]) cursor_path = _write_cursor(path, cursor["queued"] + [n["id"] for n in batch], cursor["applied"]) @@ -557,130 +1250,23 @@ def apply_enrichment(enrichment_file, path): loader = GraphLoader(path) loader.load_or_extract(enrich_llm=False) - floor = bridge_score_floor(loader.vector_store) - - applied_ids: List[str] = [] - unknown_ids: List[str] = [] - bridges = 0 - unresolved: List[str] = [] - - for item in items: - nid = item.get("id") - if not nid: - continue - nid = str(nid) - if not loader.graph.has_node(nid): - unknown_ids.append(nid) - continue - - node_data = loader.graph.nodes[nid] - intent = item.get("intent", "") - input_fields = item.get("input_fields", []) or [] - output_fields = item.get("output_fields", []) or [] - legacy_fields = item.get("fields", []) or [] - calls = item.get("calls", []) or [] - - # Optional layer_id override from agent - override_lid = item.get("layer_id") - if override_lid and override_lid in get_registry(): - node_data["layer_id"] = override_lid - node_data["layer"] = get_registry().name(override_lid) - node_data["layer_source"] = AGENT_ENRICHMENT_SOURCE - - if intent: - node_data["intent"] = intent - first_line = intent.strip().split("\n")[0].lstrip("#- *").strip() - node_data["summary"] = f"{node_data['layer']}: {node_data['label']} - {first_line or intent}" - node_data["enrichment_source"] = AGENT_ENRICHMENT_SOURCE - - if input_fields or output_fields: - node_data["input_fields"] = input_fields - node_data["output_fields"] = output_fields - node_data["fields"] = list(input_fields) + list(output_fields) - elif legacy_fields: - node_data["input_fields"] = legacy_fields - node_data["output_fields"] = [] - node_data["fields"] = legacy_fields - - fields_dict = { - "input_fields": node_data.get("input_fields", []), - "output_fields": node_data.get("output_fields", []), - "fields": node_data.get("fields", []) - } - loader.hash_gate.update_node( - node_id=nid, - file_path=node_data.get("file", ""), - content=loader.node_signature(node_data), - layer=node_data.get("layer", ""), - summary=node_data.get("summary", ""), - fields_json=json.dumps(fields_dict), - intent=node_data.get("intent", "") - ) - - for call_target in calls: - tgt_id, score = resolve_call_target( - loader.graph, loader.vector_store, call_target, nid, floor - ) - if tgt_id: - loader.graph.add_edge( - nid, tgt_id, - relation="cross_layer_link", - confidence=float(score) - ) - bridges += 1 - else: - unresolved.append(str(call_target)) - - applied_ids.append(nid) - - # Re-index so the applied intents/fields are actually searchable, then persist. - loader.vector_store.add_documents(loader.docs_to_index) - _stamp_degrees(loader) - snapshot_path = loader.save_graph() - loader.export_yaml() - - cursor = _read_cursor(path) - remaining_queued = [i for i in cursor["queued"] if i not in set(applied_ids)] - _write_cursor(path, remaining_queued, cursor["applied"] + applied_ids) + stats = apply_enrichment_items(loader, path, items, enrichment_file) + applied_ids = stats["applied_ids"] + unknown_ids = stats["unknown_ids"] + unresolved = stats["unresolved"] still_pending = sum(1 for _, d in loader.graph.nodes(data=True) if needs_agent_enrichment(d)) - # Append to audit log - audit_path = _state_path(path, AUDIT_LOG_FILENAME) - timestamp = datetime.now(timezone.utc).isoformat() - try: - with open(audit_path, "a", encoding="utf-8") as f: - f.write(f"\n--- Enrichment Batch Applied: {timestamp} ---\n") - f.write(f"Source file: {enrichment_file}\n") - f.write(f"Applied nodes ({len(applied_ids)}): {', '.join(applied_ids)}\n") - f.write(f"Bridge edges created: {bridges}\n") - for item in items: - nid = str(item.get("id") or "") - if nid in applied_ids: - f.write(f" • [{nid}] intent:\n {item.get('intent', '')}\n") - if item.get("input_fields"): - f.write(f" input_fields: {item.get('input_fields')}\n") - if item.get("output_fields"): - f.write(f" output_fields: {item.get('output_fields')}\n") - elif item.get("fields"): - f.write(f" fields: {item.get('fields')}\n") - if item.get("calls"): - f.write(f" calls: {item.get('calls')}\n") - if unresolved: - f.write(f"Unresolved call targets: {', '.join(unresolved)}\n") - except Exception: - pass - click.echo(f"✅ Applied {len(applied_ids)} enrichment(s) from {enrichment_file}") - click.echo(f"🔗 Created {bridges} cross-layer bridge edge(s) " - f"(score floor {floor}, backend {loader.vector_store.backend})") + click.echo(f"🔗 Created {stats['bridges']} cross-layer bridge edge(s) " + f"(score floor {stats['floor']}, backend {loader.vector_store.backend})") if unresolved: preview = ", ".join(sorted(set(unresolved))[:6]) click.echo(f"⚠️ {len(unresolved)} call target(s) below the score floor / unmatched: {preview}") if unknown_ids: preview = ", ".join(unknown_ids[:3]) click.echo(f"⚠️ {len(unknown_ids)} id(s) not in the graph, skipped: {preview}") - click.echo(f"💾 Graph snapshot updated at: {snapshot_path}") + click.echo(f"💾 Graph snapshot updated at: {stats['snapshot_path']}") click.echo(f"📊 {still_pending} candidate(s) still un-enriched. Run `tldrgraph queue-enrichment` for the next batch.") @@ -880,12 +1466,28 @@ def doctor(path, embeddings, as_json): @cli.command() @click.option("--path", default=".", help="Repository root path") -def install(path): - """Install TLDRGraph agent rules for Claude Code, Cursor and Antigravity.""" - res = install_agent_rules(path) +@click.option("--all-agents", is_flag=True, + help="Write the /tldrgraph-init command for every agent tool TLDRGraph " + "knows, not just the ones this repo shows signs of using.") +def install(path, all_agents): + """ + Install TLDRGraph agent rules for Claude Code, Cursor and Antigravity. + + Also adds a managed .gitignore block: generated state under .tldrgraph/ is + ignored, while AGENT_CONTRACT.md and layers.config.yaml stay committable so + the whole team shares one architecture map. + """ + gitignore = ensure_gitignore(path) + res = install_agent_rules(path, all_agents=all_agents) click.echo("✅ TLDRGraph agent skills & rules installed successfully:") for k, v in res.items(): + # Reported separately below, with its status. + if k == "gitignore": + continue click.echo(f" • {k}: {v}") + click.echo(f" • gitignore: {gitignore['path']} ({gitignore['status']})") + click.echo("\n💡 Your agent can now run /tldrgraph-init (or just `tldrgraph init`) " + "to build the whole graph.") for warning in gitignore_warnings(path): click.echo(f"⚠️ {warning}") @@ -893,20 +1495,29 @@ def install(path): @cli.command("propose-layers") @click.option("--path", default=".", help="Repository root path") -@click.option("--auto", is_flag=True, help="Automatically synthesize and apply dynamic layer config via LLM or archetype") -@click.option("--force", is_flag=True, help="Force overwrite existing layers.config.yaml when using --auto") +@click.option("--auto", is_flag=True, + help="Try to synthesize the layer set now via an agent CLI or a configured LLM") +@click.option("--force", is_flag=True, help="Force overwrite an existing layers.config.yaml") def propose_layers_cmd(path, auto, force): - """Sample repository architecture evidence and synthesize/queue dynamic layer proposal.""" + """ + Write the layer-proposal request for the agent, or try to synthesize it now. + + `tldrgraph init` calls this for you. Reach for it directly only to re-open the + architecture question without rebuilding anything else. + """ if auto: - reg, out_path, source = auto_configure_layers(path, force=force) - click.echo(f"✅ Automatically configured {len(reg)} architectural layers ({source}) in {out_path}") - click.echo("🔄 Run `tldrgraph scan .` to reclassify nodes with the new layer set.") - return + reg, out_path, source = auto_configure_layers(path, force=force, use_agent=True) + if reg is not None: + click.echo(f"✅ Configured {len(reg)} architectural layers ({source}) in {out_path}") + click.echo("🔄 Run `tldrgraph init` to reclassify nodes with the new layer set.") + return + click.echo("ℹ️ Nothing could design the layers automatically, and TLDRGraph " + "has no template to fall back on.") req_path = generate_propose_request(path) click.echo(f"📋 Queued layer proposal request in {req_path}") - resp_rel = os.path.join(STATE_DIR, "propose_layers_response.json") - click.echo(f"👉 Write the response to {resp_rel}, then run `tldrgraph apply-layers`.") + resp_rel = os.path.join(STATE_DIR, PROPOSE_RESPONSE_FILENAME) + click.echo(f"👉 Read it, READ THE SOURCE, write {resp_rel}, then run `tldrgraph init`.") @cli.command("apply-layers") diff --git a/tldrgraph/graph_loader.py b/tldrgraph/graph_loader.py index b98ab85..b38d62c 100644 --- a/tldrgraph/graph_loader.py +++ b/tldrgraph/graph_loader.py @@ -10,7 +10,7 @@ import networkx as nx from datetime import datetime, timezone from typing import Dict, Any, List, Tuple, Optional, Set -from . import __version__, extractors +from . import __version__, extractors, paths from .classifier import classify_node, classify_node_with_source from .layer_config import load_layer_config, compute_registry_hash from .layers import ( @@ -66,8 +66,16 @@ def bridge_score_floor(vector_store: LocalVectorStore) -> float: #: Schema version of .tldrgraph/graph.json SNAPSHOT_SCHEMA_VERSION = 1 -#: Filename of the persisted TLDRGraph graph snapshot (inside .tldrgraph/). -SNAPSHOT_FILENAME = "graph.json" +#: Canonical on-disk layout, re-exported so existing importers keep working. +#: :mod:`tldrgraph.paths` is the single source of truth -- see its docstring for +#: why graphify's export is renamed rather than dropped in as ``graph.json``. +SNAPSHOT_FILENAME = paths.SNAPSHOT_FILENAME +STATE_DIRNAME = paths.STATE_DIRNAME +GRAPHIFY_GRAPH_FILENAME = paths.GRAPHIFY_GRAPH_FILENAME +GRAPHIFY_MANIFEST_FILENAME = paths.GRAPHIFY_MANIFEST_FILENAME +LEGACY_GRAPHIFY_DIRNAME = paths.LEGACY_GRAPHIFY_DIRNAME +graphify_graph_path = paths.graphify_graph_path +graphify_manifest_path = paths.graphify_manifest_path def placeholder_summary(layer: str, label: str, file_path: str) -> str: @@ -259,10 +267,10 @@ def _reset_layer_buckets(self) -> None: def _load_file_hashes(self) -> Dict[str, str]: """ - Loads graphify-out/manifest.json into {repo_relative_path: content_hash}. + Loads graphify's file manifest into {repo_relative_path: content_hash}. Prefers semantic_hash and falls back to ast_hash. """ - manifest_path = os.path.join(self.root_dir, "graphify-out", "manifest.json") + manifest_path = graphify_manifest_path(self.root_dir) if not os.path.exists(manifest_path): return {} try: @@ -533,7 +541,7 @@ def _carry_forward_snapshot(self) -> Tuple[int, int]: def _run_graphify(self) -> str: """ - Runs graphify AST extraction and builds graphify-out/graph.json. + Runs graphify AST extraction into the .tldrgraph state directory. TLDRGraph relies directly on graphify as the core extraction engine. """ from pathlib import Path @@ -544,9 +552,9 @@ def _run_graphify(self) -> str: from graphify.export import to_json root_path = Path(self.root_dir).resolve() - out_dir = root_path / "graphify-out" + out_dir = root_path / STATE_DIRNAME out_dir.mkdir(parents=True, exist_ok=True) - graph_json_path = out_dir / "graph.json" + graph_json_path = out_dir / GRAPHIFY_GRAPH_FILENAME det = detect(root_path) code_files = [] @@ -565,7 +573,7 @@ def _run_graphify(self) -> str: if det.get("files"): try: - save_manifest(det["files"], str(out_dir / "manifest.json"), root=root_path) + save_manifest(det["files"], str(out_dir / GRAPHIFY_MANIFEST_FILENAME), root=root_path) except Exception: pass @@ -576,10 +584,15 @@ def _run_graphify(self) -> str: # ------------------------------------------------------------------ # def load_or_extract(self, enrich_llm: bool = True, rebuild: bool = False) -> nx.DiGraph: - graph_json_path = os.path.join(self.root_dir, "graphify-out", "graph.json") + graph_json_path = graphify_graph_path(self.root_dir) if not os.path.exists(graph_json_path): self._run_graphify() + # The manifest only exists after that first extraction. Without this + # reload the first scan signs every node with a raw sha256 while the + # second signs it with graphify's semantic_hash -- which would mark + # the entire graph dirty on scan #2 and re-enrich all of it. + self.file_hashes = self._load_file_hashes() # Reload latest layer config from disk / environment self.registry, self.layers_config_hash = load_layer_config(self.root_dir) diff --git a/tldrgraph/hierarchy.py b/tldrgraph/hierarchy.py index 69eaed4..58f35e9 100644 --- a/tldrgraph/hierarchy.py +++ b/tldrgraph/hierarchy.py @@ -73,7 +73,7 @@ get_registry, layer_name, ) -from . import extractors +from . import extractors, paths #: Version of the structure returned by :func:`build_multilayer_hierarchy`. #: Phase 4's visualizer is written against this contract. @@ -200,7 +200,7 @@ def build_multilayer_hierarchy(root_dir: str = ".") -> Dict[str, Any]: # Load base AST graph if available graph_json_path = os.path.join(root_dir, ".tldrgraph", "graph.json") if not os.path.exists(graph_json_path): - graph_json_path = os.path.join(root_dir, "graphify-out", "graph.json") + graph_json_path = paths.graphify_graph_path(root_dir) ast_nodes: List[Dict[str, Any]] = [] ast_edges: List[Dict[str, Any]] = [] diff --git a/tldrgraph/installer.py b/tldrgraph/installer.py index e20589d..48efd60 100644 --- a/tldrgraph/installer.py +++ b/tldrgraph/installer.py @@ -1,18 +1,21 @@ """ Agent Installer for TLDRGraph. -Writes host-agent rules for the three coding agents that actually drive the -enrichment loop, plus the agent contract they all point at: - - /.tldrgraph/AGENT_CONTRACT.md -- the request/response schema - /.claude/skills/codechakra/SKILL.md -- Claude Code skill - /CLAUDE.md -- delimited TLDRGraph section - /.cursor/rules/tldrgraph.mdc -- Cursor project rule - /.agents/rules/tldrgraph.md -- Antigravity rule - /.agents/workflows/tldrgraph.md -- Antigravity workflow - -Every write is idempotent: unchanged files are left alone, and CLAUDE.md is -*never* clobbered -- only the region between the TLDRGraph markers is replaced. +Writes the two things a repository needs to be agent-ready: + + /.tldrgraph/AGENT_CONTRACT.md -- the request/response schema + /.gitignore -- managed block for generated state + +plus the per-tool instruction and command files, which live in +:mod:`.agent_commands` so that every agent gets byte-identical content. + +This module used to generate five differently-worded rule files -- a Claude +skill, a CLAUDE.md section, a Cursor rule, an Antigravity rule and an Antigravity +workflow. They drifted apart immediately. There is now one body of instructions +and one command, written to whatever paths each tool uses. + +Every write is idempotent: unchanged files are left alone, and user-owned files +are *never* clobbered -- only the region between the TLDRGraph markers changes. """ from __future__ import annotations @@ -20,6 +23,7 @@ import os from typing import Dict, List, Optional +from .agent_commands import install_agent_commands, remove_superseded from .layer_config import load_layer_config from .layers import LayerRegistry, get_registry @@ -27,6 +31,29 @@ CLAUDE_MD_BEGIN = "" CLAUDE_MD_END = "" +#: Delimiters for the managed region inside a user-owned .gitignore. +GITIGNORE_BEGIN = "# BEGIN TLDRGRAPH" +GITIGNORE_END = "# END TLDRGRAPH" + +#: Files inside .tldrgraph/ that are worth committing: the contract every agent +#: reads, and the layer map, so a teammate's scan classifies the code the same +#: way instead of re-deriving its own. +GITIGNORE_KEEP = ("AGENT_CONTRACT.md", "layers.config.yaml") + +#: The managed .gitignore body. +#: +#: `.tldrgraph/*` rather than `.tldrgraph/` is load-bearing: git never descends +#: into an excluded *directory*, so a trailing-slash ignore makes the negations +#: below unreachable. Excluding the directory's *entries* keeps them working. +GITIGNORE_BLOCK = "\n".join( + [ + "# TLDRGraph analysis state. Generated artifacts are ignored; the agent", + "# contract and layer map are committed so the whole team shares them.", + ".tldrgraph/*", + ] + + [f"!.tldrgraph/{name}" for name in GITIGNORE_KEEP] +) + #: Where the contract is installed inside a target repository. CONTRACT_REL_PATH = os.path.join(".tldrgraph", "AGENT_CONTRACT.md") @@ -53,12 +80,27 @@ def generate_layers_prose(registry: Optional[LayerRegistry] = None) -> str: return "\n".join(lines) -_LOOP = """```bash +_LOOP = """One command does everything -- layers, extraction, enrichment -- and is resumable: + +```bash +tldrgraph init # prints a NEXT ACTION block whenever it needs you +# do exactly what that block says (design layers, or read source and write intents) +tldrgraph init --yes # run it again; repeat until it prints status: done +``` + +`init` never guesses. It has no template to fall back on, so if this repository has no +architecture yet it stops and asks you to design one from the code. When it needs +enrichment it hands you a batch, and you OPEN THE SOURCE FILES before writing anything. + +The underlying steps stay available for scripting: + +```bash tldrgraph queue-enrichment --limit 50 # writes .tldrgraph/enrichment_request.yaml -# read the request, OPEN THE SOURCE FILES, write .tldrgraph/enrichment_response.yaml tldrgraph apply-enrichment # merges intents + bridge edges into the graph -tldrgraph queue-enrichment --limit 50 # repeat; the queue advances by itself -```""" +``` + +The `/tldrgraph-init` command installed in your agent's command directory is this loop +written out in full.""" _RULES_SHORT = """- **Read the source file before writing an intent.** You have the repo open -- that is the entire reason this path exists. An intent paraphrased from the symbol name is worse @@ -175,193 +217,9 @@ def _contract_text(registry: Optional[LayerRegistry] = None) -> str: # Per-agent payloads # --------------------------------------------------------------------------- # -def make_claude_skill(registry: Optional[LayerRegistry] = None) -> str: - layers_prose = generate_layers_prose(registry) - count = len(registry.ordered()) - 1 if registry else 6 - return f"""--- -name: codechakra -description: >- - Trace and enrich this repository's {count}-layer architecture graph. Use when asked how a - feature flows end to end, which layer a symbol belongs to, what reaches a DB table or - endpoint, or when asked to enrich / describe un-enriched TLDRGraph nodes. ---- - -# TLDRGraph: {count}-Layer Code Flow & Semantic Navigation - -{layers_prose} - -## Before planning or changing a feature - -```bash -tldrgraph query "" # semantic search + end-to-end flow -tldrgraph trace "" "" # exact path between two symbols -tldrgraph layers # node counts per layer -``` - -`query`, `trace`, `layers` and `dead-code` are read-only: they never fire enrichment. -`.tldrgraph/layers.yaml` and `.tldrgraph/flows.yaml` hold the exported context. - -## Enrichment loop -- this is your job, not an API's - -The hosted-LLM path only ever sees a label and a file path. You have the repo open, so -you are the primary enrichment path. - -{_LOOP} - -Response format (`.tldrgraph/enrichment_response.json`), a bare JSON array: - -{_RESPONSE_SCHEMA} - -{_RULES_SHORT} - -Full schema: `{CONTRACT_REL_PATH}`. - -## dead-code - -```bash -tldrgraph dead-code # defaults to --status candidate -tldrgraph dead-code --status unreviewed --json -``` - -{_DEAD_CODE_NOTE} -""" - - -def make_claude_md_section(registry: Optional[LayerRegistry] = None) -> str: - count = len(registry.ordered()) - 1 if registry else 6 - names = " -> ".join(l.name.split(":")[-1].strip() for l in (registry or get_registry()).ordered() if l.id != (registry or get_registry()).utility_id) - return f"""## TLDRGraph ({count}-layer code flow engine) - -This repo is mapped by TLDRGraph into {count} architectural layers ({names}). Snapshot: `.tldrgraph/graph.json`, summaries: -`.tldrgraph/layers.yaml`. - -**Before planning or implementing a feature**, trace it rather than grepping: - -```bash -tldrgraph query "" -tldrgraph trace "" "" -``` - -**When nodes are un-enriched, enrich them yourself** -- you can open the files, the API -path cannot: - -{_LOOP} - -Write a bare JSON array of `{{id, intent, fields, calls}}` to the response file. -{_RULES_SHORT} - -Full contract: `{CONTRACT_REL_PATH}`. - -`tldrgraph dead-code` lists **review candidates, not confirmed dead code**; `unreviewed` -means "not enough evidence to conclude" and is never removable. Confirm in the source. -""" - - -def make_cursor_rule(registry: Optional[LayerRegistry] = None) -> str: - layers_prose = generate_layers_prose(registry) - count = len(registry.ordered()) - 1 if registry else 6 - return f"""--- -description: TLDRGraph {count}-layer code flow, semantic navigation, and agent enrichment loop -globs: ["**/*"] -alwaysApply: true ---- - -# TLDRGraph Multi-Layer Code Flow Rules - -{layers_prose} - -## Navigate before you edit - -- Run `tldrgraph query ""` or `tldrgraph trace "" ""` to get - the real end-to-end path across layers before planning a change. -- Read `.tldrgraph/layers.yaml` and `.tldrgraph/flows.yaml` for architectural context. -- `query`, `trace`, `layers`, `dead-code` are read-only and never trigger enrichment. - -## Enrichment loop (you are the primary enrichment path) - -{_LOOP} - -Response file is a bare JSON array: - -{_RESPONSE_SCHEMA} - -{_RULES_SHORT} - -Full schema: `{CONTRACT_REL_PATH}`. - -## dead-code - -{_DEAD_CODE_NOTE} -""" - - -def make_antigravity_rule(registry: Optional[LayerRegistry] = None) -> str: - layers_prose = generate_layers_prose(registry) - count = len(registry.ordered()) - 1 if registry else 6 - return f"""--- -description: TLDRGraph {count}-Layer Code Flow & Semantic Navigation Engine -globs: **/* ---- - -# TLDRGraph Multi-Layer Code Flow Rules - -This project uses TLDRGraph to map full-stack code into {count} layers: - -{layers_prose} - -## Rules for Antigravity - -- Before planning or implementing any feature, run `tldrgraph query ""` or - `tldrgraph trace "" ""` to trace exact end-to-end execution paths. -- Read `.tldrgraph/layers.yaml` and `.tldrgraph/flows.yaml` for architectural context. -- Un-enriched nodes are enriched by **you**, locally, with no third-party API key: - -{_LOOP} - -Response file is a bare JSON array: - -{_RESPONSE_SCHEMA} - -{_RULES_SHORT} - -Full schema: `{CONTRACT_REL_PATH}`. - -## dead-code - -{_DEAD_CODE_NOTE} -""" - - -def make_antigravity_workflow(registry: Optional[LayerRegistry] = None) -> str: - count = len(registry.ordered()) - 1 if registry else 6 - return f"""--- -name: codechakra -description: Multi-layer code flow tracing, semantic search, and agent enrichment ({count} Layers) ---- - -# Workflow: TLDRGraph Flow Engine - -1. `tldrgraph scan .` -- refresh AST, layers, and the local vector index. -2. `tldrgraph query ""` -- inspect the full {count}-layer execution path. -3. `tldrgraph layers` -- architectural layer summary. -4. Enrichment loop, highest-value nodes first: - -{_LOOP} - -5. `tldrgraph dead-code` -- list review candidates (never a delete list). - -Full schema for step 4: `{CONTRACT_REL_PATH}`. -""" - - -# Legacy module-level aliases for backwards compatibility +# Legacy module-level aliases for backwards compatibility. _LAYERS = generate_layers_prose() AGENT_CONTRACT_FALLBACK = make_agent_contract_fallback() -CLAUDE_SKILL = make_claude_skill() -CLAUDE_MD_SECTION = make_claude_md_section() -CURSOR_RULE = make_cursor_rule() -ANTIGRAVITY_RULE = make_antigravity_rule() -ANTIGRAVITY_WORKFLOW = make_antigravity_workflow() # --------------------------------------------------------------------------- # @@ -383,71 +241,164 @@ def _write_if_changed(path: str, content: str) -> bool: return True -def upsert_delimited_section(existing: Optional[str], section_body: str) -> str: +def upsert_block(existing: Optional[str], body: str, begin: str, end: str) -> str: """ - Returns ``existing`` with the TLDRGraph-delimited region replaced by - ``section_body``, appending the region if it is not present yet. + Returns ``existing`` with the region between ``begin`` and ``end`` replaced + by ``body``, appending the region if it is not present yet. """ - block = f"{CLAUDE_MD_BEGIN}\n{section_body.strip()}\n{CLAUDE_MD_END}\n" + block = f"{begin}\n{body.strip()}\n{end}\n" if not existing or not existing.strip(): return block - start = existing.find(CLAUDE_MD_BEGIN) - end = existing.find(CLAUDE_MD_END) - if start != -1 and end != -1 and end > start: + start = existing.find(begin) + stop = existing.find(end) + if start != -1 and stop != -1 and stop > start: head = existing[:start] - tail = existing[end + len(CLAUDE_MD_END):] - tail = tail.lstrip("\n") + tail = existing[stop + len(end):].lstrip("\n") return f"{head}{block}{tail}" separator = "" if existing.endswith("\n\n") else ("\n" if existing.endswith("\n") else "\n\n") return f"{existing}{separator}{block}" -def install_agent_rules(root_dir: str = ".") -> Dict[str, str]: +def strip_block(existing: str, begin: str, end: str) -> str: + """Returns ``existing`` with the managed region and its markers removed.""" + start = existing.find(begin) + stop = existing.find(end) + if start == -1 or stop == -1 or stop < start: + return existing + return (existing[:start].rstrip("\n") + "\n" + existing[stop + len(end):].lstrip("\n")).lstrip("\n") + + +def upsert_delimited_section(existing: Optional[str], section_body: str) -> str: + """ + Returns ``existing`` with the TLDRGraph-delimited region replaced by + ``section_body``, appending the region if it is not present yet. + """ + return upsert_block(existing, section_body, CLAUDE_MD_BEGIN, CLAUDE_MD_END) + + +def _neutralize_directory_ignores(text: str) -> str: + """ + Comments out any hand-written ``.tldrgraph`` / ``.tldrgraph/`` line outside + the managed block. + + Such a line ignores the *directory*, which stops git descending into it, so + the managed block's ``!.tldrgraph/...`` negations could never take effect. + The line is commented rather than deleted so the edit is visible and + reversible in a diff. + """ + if not text: + return text + + out: List[str] = [] + inside_block = False + for raw in text.splitlines(): + stripped = raw.strip() + if stripped == GITIGNORE_BEGIN: + inside_block = True + elif stripped == GITIGNORE_END: + inside_block = False + elif not inside_block and stripped.lstrip("/").rstrip("/") == ".tldrgraph" \ + and not stripped.startswith("#"): + out.append(f"# {raw} # superseded by the TLDRGraph block below") + continue + out.append(raw) + return "\n".join(out) + ("\n" if text.endswith("\n") else "") + + +def ensure_gitignore(root_dir: str = ".") -> Dict[str, str]: + """ + Adds (or refreshes) TLDRGraph's managed block in the repository's .gitignore. + + Ignores everything generated under ``.tldrgraph/`` while keeping the agent + contract and layer map committable. Creates the file if it does not exist. + Idempotent: an already-correct .gitignore is left byte-identical. + + Returns ``{"path": ..., "status": created|updated|unchanged}``. + """ + path = os.path.join(os.path.abspath(root_dir), ".gitignore") + + existing: Optional[str] = None + if os.path.isfile(path): + try: + with open(path, "r", encoding="utf-8") as f: + existing = f.read() + except OSError: + existing = None + + updated = upsert_block( + _neutralize_directory_ignores(existing or ""), GITIGNORE_BLOCK, + GITIGNORE_BEGIN, GITIGNORE_END, + ) + + if existing == updated: + return {"path": path, "status": "unchanged"} + + try: + with open(path, "w", encoding="utf-8") as f: + f.write(updated) + except OSError as err: + return {"path": path, "status": f"failed: {err}"} + + return {"path": path, "status": "created" if existing is None else "updated"} + + +def install_agent_rules(root_dir: str = ".", all_agents: bool = False) -> Dict[str, str]: """ - Installs TLDRGraph rules for Claude Code, Cursor and Antigravity, plus the shared - agent contract. + Makes a repository agent-ready: the .gitignore block, the shared contract, + and one body of instructions plus one command written to whatever paths each + tool uses. No tool gets bespoke prose and no tool gets extra artifacts. + + ``all_agents`` writes for every tool TLDRGraph knows, not just the ones this + repository shows signs of using. """ root = os.path.abspath(root_dir) registry, _ = load_layer_config(root) written: Dict[str, str] = {} + # 0a. Keep generated state out of git, but leave the contract and layer map + # committable so the team shares one architecture definition. + written["gitignore"] = ensure_gitignore(root)["path"] + # 0. The contract every rule file points at. contract_path = os.path.join(root, CONTRACT_REL_PATH) _write_if_changed(contract_path, _contract_text(registry)) written["contract"] = contract_path - # 1. Claude Code -- skill + a delimited section in CLAUDE.md. - skill_path = os.path.join(root, ".claude", "skills", "tldrgraph", "SKILL.md") - _write_if_changed(skill_path, make_claude_skill(registry)) - written["claude_skill"] = skill_path + # 1. The instructions and the command, identical for every tool, at + # whatever paths each one uses. See agent_commands.TARGETS. + written.update(install_agent_commands(root, all_agents=all_agents)) + + # 2. Remove what earlier versions installed. Left behind, those files + # describe a workflow that no longer exists. + removed = remove_superseded(root) + # 3. Our managed block inside a user-owned CLAUDE.md is superseded by + # AGENTS.md, which Claude Code also reads. Drop the block, keep whatever + # the user wrote around it, and delete the file only if we created it and + # nothing else is left. claude_md_path = os.path.join(root, "CLAUDE.md") - existing = None if os.path.isfile(claude_md_path): try: with open(claude_md_path, "r", encoding="utf-8") as f: existing = f.read() except OSError: existing = None - _write_if_changed(claude_md_path, upsert_delimited_section(existing, make_claude_md_section(registry))) - written["claude_md"] = claude_md_path - - # 2. Cursor project rule. - cursor_path = os.path.join(root, ".cursor", "rules", "tldrgraph.mdc") - _write_if_changed(cursor_path, make_cursor_rule(registry)) - written["cursor_rule"] = cursor_path - - # 3. Antigravity rule + workflow. - antigravity_rule = os.path.join(root, ".agents", "rules", "tldrgraph.md") - _write_if_changed(antigravity_rule, make_antigravity_rule(registry)) - written["antigravity_rule"] = antigravity_rule - - antigravity_workflow = os.path.join(root, ".agents", "workflows", "tldrgraph.md") - _write_if_changed(antigravity_workflow, make_antigravity_workflow(registry)) - written["antigravity_workflow"] = antigravity_workflow + if existing and CLAUDE_MD_BEGIN in existing: + remainder = strip_block(existing, CLAUDE_MD_BEGIN, CLAUDE_MD_END) + if remainder.strip(): + _write_if_changed(claude_md_path, remainder) + else: + try: + os.remove(claude_md_path) + removed.append("CLAUDE.md") + except OSError: + pass + + if removed: + written["superseded (removed)"] = ", ".join(removed) return written @@ -463,8 +414,11 @@ def gitignore_warnings(root_dir: str = ".") -> List[str]: except OSError: return [] + # `.tldrgraph` is deliberately absent: ensure_gitignore now manages it with + # negations that keep the contract and layer map committable, so warning + # about it would contradict what this installer just wrote. watched = {".agents": ".agents/", ".claude": ".claude/", ".cursor": ".cursor/", - ".tldrgraph": ".tldrgraph/", "CLAUDE.md": "CLAUDE.md"} + "CLAUDE.md": "CLAUDE.md"} warnings: List[str] = [] for lineno, raw in enumerate(lines, 1): entry = raw.strip() diff --git a/tldrgraph/layer_config.py b/tldrgraph/layer_config.py index 864d2e9..2af7b24 100644 --- a/tldrgraph/layer_config.py +++ b/tldrgraph/layer_config.py @@ -14,10 +14,9 @@ import yaml from .layers import ( - DEFAULT_LAYERS, LAYER_UTILITY, LayerRegistry, - default_registry, + bootstrap_registry, set_registry, ) from .rules import Rule @@ -121,12 +120,18 @@ def load_layer_config(root_dir: str = ".") -> Tuple[LayerRegistry, str]: Loads the layer configuration from .tldrgraph/layers.config.yaml (or .json). If found, validates and sets the active registry. - If absent, falls back to the default registry. + + If absent, returns the single-bucket bootstrap registry. It deliberately does + NOT fall back to a plausible six-layer default: classifying a repository + against an architecture nobody derived from it produces a map that is wrong + everywhere it looks right. `tldrgraph init` stops and asks instead. + Returns (registry, config_hash). """ c_path = config_path(root_dir) if not c_path: - reg = default_registry() + reg = bootstrap_registry() + set_registry(reg) return reg, compute_registry_hash(reg) try: @@ -150,13 +155,17 @@ def save_layer_config(root_dir: str, registry: LayerRegistry) -> str: """ Persists a LayerRegistry into .tldrgraph/layers.config.yaml. + Every config that reaches this function was derived from the repository's + own source -- there is no template path any more -- so nothing here needs to + mark a config as second-class. + Returns the written file path. """ out_dir = os.path.join(os.path.abspath(root_dir), ".tldrgraph") os.makedirs(out_dir, exist_ok=True) out_path = os.path.join(out_dir, CONFIG_FILENAME_YAML) - payload = { + payload: Dict[str, Any] = { "version": CONFIG_SCHEMA_VERSION, "utility_id": registry.utility_id, "layers": [layer.as_record() for layer in registry.ordered()], diff --git a/tldrgraph/layers.py b/tldrgraph/layers.py index df222c1..44e2a25 100644 --- a/tldrgraph/layers.py +++ b/tldrgraph/layers.py @@ -29,6 +29,8 @@ "LAYER_DEVOPS", "LAYER_UTILITY", "DEFAULT_LAYERS", + "BOOTSTRAP_LAYERS", + "bootstrap_registry", "default_registry", "get_registry", "set_registry", @@ -156,7 +158,13 @@ def as_record(self) -> Dict[str, Any]: Rule(file_contains=_DEVOPS_KEYWORDS), ) -#: The built-in default layer set. +#: A worked example of a layer set, for a full-stack web app. +#: +#: **Nothing applies this to a repository.** TLDRGraph classifies only against a +#: layer set an agent designed from the code in front of it; a generic set looks +#: plausible, classifies badly, and quietly becomes the answer. This is kept as +#: a reference for the shape of a `Layer` and as documentation of the well-known +#: role ids below, nothing more. DEFAULT_LAYERS: Tuple[Layer, ...] = ( Layer(LAYER_UI, "Layer 1: UI Trigger", 1, "React components, forms, buttons", _DEFAULT_UI_RULES), @@ -304,12 +312,37 @@ def id_of(self, node_data: Mapping[str, Any]) -> str: return self.id_for_name(str(node_data.get("layer") or "")) +#: The layer every node lands in before an agent has designed anything. +#: +#: One honest bucket, not six confident guesses: a node here is visibly +#: unclassified, which is what it actually is. +BOOTSTRAP_LAYERS: Tuple[Layer, ...] = ( + Layer(LAYER_UTILITY, "Unclassified", 1, + "No architecture designed yet -- run `tldrgraph init`", ()), +) + + +def bootstrap_registry() -> LayerRegistry: + """ + The single-bucket registry used until a real layer set exists. + + Everything in the codebase needs *a* registry to read, so there has to be + one before `init` has run. What there must not be is a plausible-looking one. + """ + return LayerRegistry(BOOTSTRAP_LAYERS) + + def default_registry() -> LayerRegistry: - """A fresh registry over the built-in default layer set.""" + """ + A fresh registry over :data:`DEFAULT_LAYERS`, the worked example. + + Retained for tests and external callers that reference the example set. It + is not what an unconfigured repository gets -- see :func:`bootstrap_registry`. + """ return LayerRegistry(DEFAULT_LAYERS) -_ACTIVE_REGISTRY: LayerRegistry = default_registry() +_ACTIVE_REGISTRY: LayerRegistry = bootstrap_registry() def get_registry() -> LayerRegistry: diff --git a/tldrgraph/paths.py b/tldrgraph/paths.py new file mode 100644 index 0000000..3477404 --- /dev/null +++ b/tldrgraph/paths.py @@ -0,0 +1,89 @@ +""" +Canonical on-disk locations for everything TLDRGraph writes. + +One state directory, ``.tldrgraph/``, holds all of it -- including graphify's +raw AST export, which used to live in a second top-level ``graphify-out/`` +folder. Scanning a repository therefore adds one directory, not two. + +graphify's export is renamed on the way in (``graphify_graph.json``) because +``graph.json`` inside ``.tldrgraph/`` is already taken by TLDRGraph's *enriched* +snapshot, which is a different artifact with a different schema. + +This module deliberately imports nothing from the package so that every other +module can depend on it without creating a cycle. +""" + +from __future__ import annotations + +import os + +#: The single state directory, relative to the repository root. +STATE_DIRNAME = ".tldrgraph" + +#: TLDRGraph's own enriched graph snapshot. +SNAPSHOT_FILENAME = "graph.json" + +#: graphify's raw AST export and file manifest, inside the state directory. +GRAPHIFY_GRAPH_FILENAME = "graphify_graph.json" +GRAPHIFY_MANIFEST_FILENAME = "graphify_manifest.json" + +#: Pre-consolidation location. Never read and never written any more -- kept so +#: `scan` can point out that a leftover folder is now dead weight. +LEGACY_GRAPHIFY_DIRNAME = "graphify-out" + +#: Where graphify keeps its own AST cache, relative to the scanned root. +#: Forward slash on purpose: graphify does ``Path(root) / Path(value)``, which +#: splits this correctly on every platform, and a backslash would not. +GRAPHIFY_WORK_SUBDIR = f"{STATE_DIRNAME}/graphify" + +#: graphify's own override for where it writes, read once when it is imported. +GRAPHIFY_OUT_ENV = "GRAPHIFY_OUT" + + +def pin_graphify_output_dir() -> None: + """ + Points graphify's cache inside our state directory, before it is imported. + + graphify defaults to a top-level ``graphify-out/`` and reads + ``$GRAPHIFY_OUT`` once at import time, so relocating it has to happen this + early -- otherwise scanning a repository leaves two directories behind no + matter where TLDRGraph puts its own files. + + An explicit ``$GRAPHIFY_OUT`` from the environment always wins: that is the + user pointing graphify somewhere on purpose. This only ever changes the + current process, so a separate ``graphify`` CLI run is unaffected. + """ + os.environ.setdefault(GRAPHIFY_OUT_ENV, GRAPHIFY_WORK_SUBDIR) + + +pin_graphify_output_dir() + + +def state_dir(root_dir: str) -> str: + """The .tldrgraph directory for ``root_dir``.""" + return os.path.join(root_dir, STATE_DIRNAME) + + +def state_path(root_dir: str, filename: str) -> str: + """A file inside the state directory.""" + return os.path.join(root_dir, STATE_DIRNAME, filename) + + +def snapshot_path(root_dir: str) -> str: + """TLDRGraph's enriched graph snapshot.""" + return state_path(root_dir, SNAPSHOT_FILENAME) + + +def graphify_graph_path(root_dir: str) -> str: + """graphify's raw AST export.""" + return state_path(root_dir, GRAPHIFY_GRAPH_FILENAME) + + +def graphify_manifest_path(root_dir: str) -> str: + """graphify's file manifest, source of the semantic hashes the gate uses.""" + return state_path(root_dir, GRAPHIFY_MANIFEST_FILENAME) + + +def legacy_graphify_dir(root_dir: str) -> str: + """The obsolete graphify-out/ directory, for cleanup hints only.""" + return os.path.join(root_dir, LEGACY_GRAPHIFY_DIRNAME) diff --git a/tldrgraph/propose_layers.py b/tldrgraph/propose_layers.py index c950d0b..895998e 100644 --- a/tldrgraph/propose_layers.py +++ b/tldrgraph/propose_layers.py @@ -1,11 +1,14 @@ """ Propose Layers: Dynamic Multi-Layer Discovery & Configuration Generator. -Samples repository evidence (directory tree, framework markers, sample paths, -dependencies, and entry points) and determines the optimal architectural layer set: -1. Via LLM (Gemini, OpenAI, Ollama) if available or through the coding agent loop. -2. Via intelligent repository archetype detection (CLI app, Library, Full-stack Web, - Backend API, Data/ML Pipeline, Generic Modular) as an offline, zero-token fallback. +Samples repository evidence (directory tree, framework markers, extracted +symbols, dependencies, entry points) and hands it to an agent that reads the +actual source before deciding what the layers are. + +**There is no template fallback, by design.** A generic archetype layer set +looks plausible, classifies badly, and -- worst of all -- silently becomes the +answer. TLDRGraph would rather stop and ask. If no layer set can be obtained, +``needs_layers`` is returned and the caller tells the agent what to write. """ from __future__ import annotations @@ -18,6 +21,7 @@ import yaml +from . import agent_runner, paths from .layer_config import ( CONFIG_FILENAME_YAML, config_path, @@ -246,289 +250,199 @@ def detect_repository_archetype(root_dir: str) -> str: return ARCHETYPE_GENERIC -def archetype_layer_set(archetype: str, evidence: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: +#: Sketches of how codebases *can* divide, to show the shape of an answer. +#: +#: Deliberately prose, not JSON: a copyable template gets copied. The point is +#: to convey that a layer is "a place where responsibility changes hands", then +#: get out of the way so the agent names what it actually found. +LAYER_SET_IDEAS = [ + "A web app might split presentation from request handling from domain logic " + "from persistence, with background jobs and deployment config as their own tiers.", + "A CLI tool might split the command surface from the processing engine from " + "local state, with adapters to outside systems separate again.", + "A library might split its public API from the core implementation from its " + "data types, with backend adapters separate.", + "A data pipeline might split ingestion from transformation from model " + "training from serving.", + "None of these will fit this repository. The useful question is not 'which " + "one is it?' but 'where does responsibility change hands in THIS code, and " + "what would a new engineer need named?'", +] + + +#: Prompt for a coding-agent CLI running headless inside the repository. +#: +#: The difference from the hosted-LLM prompt is the first instruction: the agent +#: has the repo on disk, so it is told to READ files rather than infer a layer +#: set from a directory listing. That is the entire reason this path exists. +AGENT_LAYERS_PROMPT = """You are designing the architectural layer map for the repository you currently have open. + +Do this before answering: +1. Read the entry points and a representative sample of real source files across + the main directories. The evidence bundle below lists candidates; it is a + starting point, NOT a substitute for opening the files. +2. Work out what this codebase actually does and how responsibility is split. + Name the layers after THIS repository's concepts, not a generic template. + For the shape of an answer only: +{ideas} +3. Derive matching rules from real paths and real symbol names you saw. + +Then return ONLY a JSON object, no prose and no markdown fence, of this shape: + +{{ + "utility_id": "", + "layers": [ + {{ + "id": "short_machine_id", + "name": "Layer 1: Human Friendly Name", + "order": 1, + "description": "One sentence on what lives here", + "rules": [ + {{"file_contains": ["substring", "another"], "exclude_file": ["optional"]}}, + {{"label_contains": ["SymbolNamePart"]}} + ] + }} + ] +}} + +Hard requirements: +- 3 to 6 layers plus exactly one catch-all utility layer. +- Every layer needs a unique `id`, a unique `name`, and a sequential integer + `order` starting at 1. +- Exactly one layer's `id` equals `utility_id`, and that layer has `rules: []`. +- Supported rule keys: file_contains, exclude_file, path_regex, label_contains, + exclude_label, label_ends_with, type_in, id_prefix. Each value is a list of strings. +- Rules must match paths that really exist here. A rule that matches nothing is + worse than no rule, and a rule that matches everything collapses the map. + +Repository root: {root} + +Evidence bundle (sampled automatically, verify against the real files): +{evidence} +""" + + +def build_agent_layer_prompt(root: str, evidence: Dict[str, Any]) -> str: + """Renders the headless-agent prompt for a layer proposal.""" + return AGENT_LAYERS_PROMPT.format( + root=root, + ideas="\n".join(f" - {idea}" for idea in LAYER_SET_IDEAS), + evidence=json.dumps(evidence, indent=2, default=str), + ) + + +#: How many of the busiest files, and symbols within them, to put in front of the +#: agent. Enough to show the shape of the codebase, small enough to stay cheap. +_EVIDENCE_TOP_FILES = 40 +_EVIDENCE_SYMBOLS_PER_FILE = 8 + + +def _extracted_symbol_evidence(root_dir: str) -> Dict[str, Any]: """ - Returns a customized, multi-layer architectural definition tailored to the given archetype. + Real symbols per file, taken from graphify's raw AST export. + + Filenames alone under-describe a codebase: two repos with identical + directory listings can be doing entirely different things. ``init`` runs + extraction *before* asking for layers precisely so this is available, and + files are ranked by symbol count so the busiest ones lead. + + Returns ``{}`` when nothing has been extracted yet -- the caller degrades to + filename evidence rather than failing. """ - if archetype == ARCHETYPE_CLI: - return { - "utility_id": "utility", - "layers": [ - { - "id": "cli", - "name": "Layer 1: CLI & Commands", - "order": 1, - "description": "Command line interface, argument parsing, options, and commands", - "rules": [ - {"file_contains": ["cli.py", "/cli/", "commands/", "cmd/", "bin/"]}, - {"file_contains": ["cli"], "exclude_file": ["flow_engine", "vector_store", "graph_loader", "classifier"]}, - {"label_contains": ["cli_command", "cli_main", "main_cli", "subcommand"]} - ] - }, - { - "id": "engine", - "name": "Layer 2: Core Flow Engine & Logic", - "order": 2, - "description": "Core algorithms, flow traversal, analysis, classification, and extractors", - "rules": [ - {"file_contains": ["flow_engine", "classifier", "extractors", "deadcode", "propose_layers", "hierarchy", "engine"]}, - {"label_contains": ["engine", "flow", "classify", "extract", "traverse", "walk", "rule"]} - ] - }, - { - "id": "storage", - "name": "Layer 3: Graph Loader, Storage & Index", - "order": 3, - "description": "Graph ingestion, SQLite caching, hash gating, vector store, and configuration", - "rules": [ - {"file_contains": ["graph_loader", "hash_gate", "vector_store", "layer_config", "layers", "rules"]}, - {"label_contains": ["loader", "graph", "store", "cache", "gate", "config", "registry"]} - ] - }, - { - "id": "integrations", - "name": "Layer 4: Agent Loop & Visualizer", - "order": 4, - "description": "Host-agent rules installer, LLM enrichment providers, and HTML visualizer", - "rules": [ - {"file_contains": ["installer", "llm_enricher", "visualizer"]}, - {"label_contains": ["installer", "visualizer", "enricher", "agent"]} - ] - }, - { - "id": "utility", - "name": "General / Utility", - "order": 5, - "description": "Shared utility helpers, formatting, and common routines", - "rules": [] - } - ] - } - - if archetype == ARCHETYPE_FULLSTACK: - return { - "utility_id": "utility", - "layers": [ - { - "id": "ui", - "name": "Layer 1: Presentation & UI", - "order": 1, - "description": "User interface components, pages, views, and forms", - "rules": [ - {"file_contains": ["frontend/", "src/app", "src/components", "pages/"]}, - {"label_contains": ["View", "Page", "Component", "Button", "Form"]} - ] - }, - { - "id": "api", - "name": "Layer 2: API Gateway & Routing", - "order": 2, - "description": "Controllers, endpoints, route handlers, and guards", - "rules": [ - {"file_contains": ["controller", "route", "api/"]}, - {"label_contains": ["Controller", "Route", "Endpoint", "Guard"]} - ] - }, - { - "id": "service", - "name": "Layer 3: Domain & Business Services", - "order": 3, - "description": "Business logic, workflows, calculation engines, and service layer", - "rules": [ - {"file_contains": ["service", "calc", "workflow", "domain/"]}, - {"label_contains": ["Service", "Calculator", "Workflow", "Manager"]} - ] - }, - { - "id": "data", - "name": "Layer 4: Data & Persistence", - "order": 4, - "description": "Database models, repositories, schemas, and entities", - "rules": [ - {"file_contains": ["prisma", "repository", "entities", "schema.prisma", "database"]}, - {"label_contains": ["Repository", "Entity", "Model", "Prisma"]} - ] - }, - { - "id": "async", - "name": "Layer 5: Async & Background Tasks", - "order": 5, - "description": "Cron jobs, queue workers, polling, and scheduled tasks", - "rules": [ - {"file_contains": ["cron", "queue", "worker", "polling", "tasks"]}, - {"label_contains": ["Job", "Worker", "Cron", "Polling"]} - ] - }, - { - "id": "devops", - "name": "Layer 6: DevOps & Infrastructure", - "order": 6, - "description": "Docker, Kubernetes, CI/CD pipelines, and cloud deployment configs", - "rules": [ - {"file_contains": ["docker", "k8s", "helm", ".github/workflows", "Dockerfile"]} - ] - }, - { - "id": "utility", - "name": "General / Utility", - "order": 7, - "description": "Shared helpers and catch-all", - "rules": [] - } - ] - } - - if archetype == ARCHETYPE_BACKEND: - return { - "utility_id": "utility", - "layers": [ - { - "id": "api", - "name": "Layer 1: API & Handlers", - "order": 1, - "description": "HTTP/gRPC endpoints, routing, controllers, and middleware", - "rules": [ - {"file_contains": ["controller", "router", "endpoint", "handler", "api/"]}, - {"label_contains": ["Controller", "Handler", "Router", "Endpoint"]} - ] - }, - { - "id": "service", - "name": "Layer 2: Domain Services", - "order": 2, - "description": "Core business logic, domain calculations, and application services", - "rules": [ - {"file_contains": ["service", "domain", "logic", "usecase"]}, - {"label_contains": ["Service", "Logic", "Manager", "Workflow"]} - ] - }, - { - "id": "data", - "name": "Layer 3: Persistence & Repositories", - "order": 3, - "description": "Database models, schemas, repositories, and persistence layer", - "rules": [ - {"file_contains": ["repository", "model", "entity", "schema", "db"]}, - {"label_contains": ["Repository", "Entity", "Model", "Table"]} - ] - }, - { - "id": "async", - "name": "Layer 4: Async & Worker Tasks", - "order": 4, - "description": "Background queues, event consumers, and cron schedules", - "rules": [ - {"file_contains": ["worker", "task", "job", "queue", "consumer"]}, - {"label_contains": ["Worker", "Consumer", "Job", "Queue"]} - ] - }, - { - "id": "utility", - "name": "General / Utility", - "order": 5, - "description": "Shared helpers and cross-cutting utilities", - "rules": [] - } - ] - } - - if archetype == ARCHETYPE_LIBRARY: - return { - "utility_id": "utility", - "layers": [ - { - "id": "public_api", - "name": "Layer 1: Public API & Interfaces", - "order": 1, - "description": "Entry points, public facade, client classes, and exported functions", - "rules": [ - {"file_contains": ["__init__.py", "api", "client", "index.ts", "public/"]}, - {"label_contains": ["Client", "API", "Facade", "Interface"]} - ] - }, - { - "id": "core_engine", - "name": "Layer 2: Core Processing & Engine", - "order": 2, - "description": "Core algorithms, parsing, processing, and computational logic", - "rules": [ - {"file_contains": ["core", "engine", "processor", "parser", "builder"]}, - {"label_contains": ["Engine", "Processor", "Parser", "Builder", "Transformer"]} - ] - }, - { - "id": "types_models", - "name": "Layer 3: Types & Models", - "order": 3, - "description": "Data structures, types, interfaces, schemas, and entities", - "rules": [ - {"file_contains": ["types", "models", "schema", "interfaces"]}, - {"label_contains": ["Type", "Model", "Schema", "Config"]} - ] - }, - { - "id": "adapters", - "name": "Layer 4: Adapters & Backends", - "order": 4, - "description": "Backend adapters, transports, network connectors, and storage drivers", - "rules": [ - {"file_contains": ["adapter", "transport", "driver", "connector", "backend"]}, - {"label_contains": ["Adapter", "Transport", "Driver", "Connector"]} - ] - }, - { - "id": "utility", - "name": "General / Utility", - "order": 5, - "description": "Internal utilities and helpers", - "rules": [] - } - ] - } - - # Generic Modular Fallback + graph_path = paths.graphify_graph_path(os.path.abspath(root_dir)) + if not os.path.isfile(graph_path): + return {} + + try: + with open(graph_path, "r", encoding="utf-8") as f: + raw = json.load(f) + except (OSError, ValueError): + return {} + + by_file: Dict[str, List[str]] = {} + for node in raw.get("nodes", []): + if not isinstance(node, dict): + continue + # graphify also emits `rationale` nodes whose label is a sentence of + # docstring prose. Those are ~a third of the export and would fill the + # evidence with English instead of the symbol names the rules match on. + if node.get("file_type") != "code": + continue + src = node.get("source_file") or node.get("file") or node.get("path") + label = node.get("label") or node.get("id") + if not src or not label: + continue + by_file.setdefault(str(src), []).append(str(label)) + + if not by_file: + return {} + + busiest = sorted(by_file.items(), key=lambda kv: (-len(kv[1]), kv[0])) return { - "utility_id": "utility", - "layers": [ - { - "id": "interface", - "name": "Layer 1: Entry Points & Interface", - "order": 1, - "description": "Entry points, CLI, HTTP routing, or user interfaces", - "rules": [ - {"file_contains": ["main", "app", "cli", "entry", "interface"]}, - {"label_contains": ["main", "app", "cli", "Controller", "View"]} - ] - }, - { - "id": "domain", - "name": "Layer 2: Core Domain Logic", - "order": 2, - "description": "Business logic, algorithms, calculation, and core operations", - "rules": [ - {"file_contains": ["service", "domain", "core", "logic", "engine"]}, - {"label_contains": ["Service", "Logic", "Engine", "Manager"]} - ] - }, - { - "id": "storage", - "name": "Layer 3: Storage & Persistence", - "order": 3, - "description": "Data storage, repositories, models, cache, and state", - "rules": [ - {"file_contains": ["data", "db", "storage", "repository", "model", "cache"]}, - {"label_contains": ["Repository", "Model", "Store", "Cache"]} - ] - }, - { - "id": "utility", - "name": "General / Utility", - "order": 4, - "description": "Shared helpers and catch-all utilities", - "rules": [] - } - ] + "total_files": len(by_file), + "total_symbols": sum(len(v) for v in by_file.values()), + "symbols_by_file": { + path: sorted(set(labels))[:_EVIDENCE_SYMBOLS_PER_FILE] + for path, labels in busiest[:_EVIDENCE_TOP_FILES] + }, + } + + +def collect_layer_evidence(root_dir: str = ".") -> Dict[str, Any]: + """The sampled repository evidence handed to an agent or LLM.""" + root = os.path.abspath(root_dir) + evidence: Dict[str, Any] = { + "framework_markers": _collect_framework_markers(root), + "sampled_directory_clusters": _sample_repo_files(root), + "detected_archetype": detect_repository_archetype(root), } + extracted = _extracted_symbol_evidence(root) + if extracted: + evidence["extracted_symbols"] = extracted + return evidence + + +def propose_layers_with_agent( + root_dir: str = ".", + agent: Optional[Any] = None, + model: Optional[str] = None, +) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + Asks a headless coding-agent CLI to read the repo and design its layer set. + + Returns ``(config, agent_name)`` on success, or ``(None, reason)`` where + reason is a short human-readable string explaining why nothing came back. + """ + if agent is None: + agent = agent_runner.find_agent_cli() + if agent is None: + status = agent_runner.agent_status() + return None, str(status.get("detail") or status.get("reason") or "no agent available") + + root = os.path.abspath(root_dir) + prompt = build_agent_layer_prompt(root, collect_layer_evidence(root)) + + try: + proposal = agent_runner.run_agent_json(agent, prompt, root, model=model) + except agent_runner.AgentError as err: + return None, str(err) + + if isinstance(proposal, dict) and "layers" not in proposal: + for key in ("proposal", "layer_set", "config"): + nested = proposal.get(key) + if isinstance(nested, dict) and "layers" in nested: + proposal = nested + break + + if not isinstance(proposal, dict) or "layers" not in proposal: + return None, f"{agent.display} did not return a layer set" + + try: + validate_layer_config(proposal) + except ValueError as err: + return None, f"{agent.display} returned an invalid layer set: {err}" + + return proposal, agent.name def propose_layers_with_llm(root_dir: str = ".", enricher: Optional[Any] = None) -> Optional[Dict[str, Any]]: @@ -544,11 +458,7 @@ def propose_layers_with_llm(root_dir: str = ".", enricher: Optional[Any] = None) return None root = os.path.abspath(root_dir) - evidence = { - "framework_markers": _collect_framework_markers(root), - "sampled_directory_clusters": _sample_repo_files(root), - "detected_archetype": detect_repository_archetype(root), - } + evidence = collect_layer_evidence(root) try: proposal = enricher.propose_layers(evidence) @@ -566,28 +476,65 @@ def propose_layers_with_llm(root_dir: str = ".", enricher: Optional[Any] = None) return None +#: Returned as the ``source`` when no layer set could be obtained. The caller +#: turns this into a NEXT ACTION for the agent instead of inventing layers. +NEEDS_LAYERS = "needs_layers" + + def auto_configure_layers( root_dir: str = ".", enricher: Optional[Any] = None, force: bool = False, - use_llm: bool = True -) -> Tuple[LayerRegistry, str, str]: + use_llm: bool = True, + use_agent: bool = False, + agent: Optional[Any] = None, + agent_model: Optional[str] = None, + notes: Optional[List[str]] = None, +) -> Tuple[Optional[LayerRegistry], Optional[str], str]: """ - Ensures .tldrgraph/layers.config.yaml exists by: - 1. Loading existing configuration if present and not force. - 2. Prompting LLM if use_llm is True and LLM is configured. - 3. Auto-detecting archetype and generating a tailored archetype layer set. + Resolves this repository's architectural layers, or reports that it cannot. + + In order: + + 1. An existing configuration, unless ``force``. + 2. A headless coding-agent CLI -- **opt-in** (``use_agent``), because every + agent has different flags, auth and headless semantics and some ship no + working CLI at all. The file handshake is the path that generalises. + 3. A hosted LLM, if an API key is configured. Sees the evidence bundle only. + + If none of those produces a layer set, this returns ``(None, None, + NEEDS_LAYERS)``. It never falls back to a template: layers that were not + derived from this repository are worse than no layers, because they are + wrong in a way that looks right. - Returns (LayerRegistry, saved_config_path, source_description). + ``notes`` collects human-readable explanations of every path tried and + skipped, so the caller can tell the user what happened. + + Returns ``(LayerRegistry | None, saved_config_path | None, source)``. """ root = os.path.abspath(root_dir) existing_path = config_path(root) + log = notes if notes is not None else [] if existing_path and not force: reg, _ = load_layer_config(root) return reg, existing_path, "existing_config" - # Try LLM proposal + # 1. Coding agent CLI, opt-in. + if use_agent: + if agent is None: + agent = agent_runner.find_agent_cli() + proposal, detail = propose_layers_with_agent(root, agent=agent, model=agent_model) + if proposal: + registry = LayerRegistry.from_records( + proposal["layers"], utility_id=str(proposal["utility_id"]) + ) + out_path = save_layer_config(root, registry) + return registry, out_path, f"agent:{detail}" + if detail: + log.append(f"Agent CLI layer proposal unavailable: {detail}") + + # 2. Hosted LLM, evidence-only. if use_llm: proposal = propose_layers_with_llm(root, enricher=enricher) if proposal: @@ -596,14 +543,7 @@ def auto_configure_layers( out_path = save_layer_config(root, registry) return registry, out_path, "llm_synthesis" - # Fallback to smart archetype detection - archetype = detect_repository_archetype(root) - layer_data = archetype_layer_set(archetype) - validate_layer_config(layer_data) - utility_id = str(layer_data["utility_id"]) - registry = LayerRegistry.from_records(layer_data["layers"], utility_id=utility_id) - out_path = save_layer_config(root, registry) - return registry, out_path, f"archetype:{archetype}" + return None, None, NEEDS_LAYERS def generate_propose_request(root_dir: str = ".") -> str: @@ -617,16 +557,11 @@ def generate_propose_request(root_dir: str = ".") -> str: os.makedirs(state_dir, exist_ok=True) req_path = os.path.join(state_dir, REQUEST_FILENAME) - archetype = detect_repository_archetype(root) - archetype_example = archetype_layer_set(archetype) - - evidence = { - "detected_archetype": archetype, - "framework_markers": _collect_framework_markers(root), - "sampled_directory_clusters": _sample_repo_files(root), - "active_layers": [layer.as_record() for layer in get_registry().ordered()], - "active_utility_id": get_registry().utility_id, - } + evidence = collect_layer_evidence(root) + existing = config_path(root) + if existing: + evidence["active_layers"] = [layer.as_record() for layer in get_registry().ordered()] + evidence["active_utility_id"] = get_registry().utility_id payload = { "schema": "codechakra/propose-layers-request@1", @@ -637,10 +572,13 @@ def generate_propose_request(root_dir: str = ".") -> str: "Analyze the repository evidence and propose a dynamic architectural layer set tailored to this codebase.", "Each layer must have: id (unique machine string), name (display string), order (1..N), description, and rules.", "Rules support: file_contains, exclude_file, path_regex, label_contains, exclude_label, label_ends_with, type_in, id_prefix.", - "Must designate a utility_id matching one of the proposed layer ids (the fallback catch-all bucket).", - f"Write the JSON or YAML response to .tldrgraph/{RESPONSE_FILENAME}, then run `tldrgraph apply-layers`.", + "Must designate a utility_id matching one of the proposed layer ids (the catch-all bucket, with empty rules).", + "Name the layers after THIS repository's concepts. TLDRGraph ships no layer templates and none will be applied for you.", + "A rule that matches nothing is worse than no rule; a rule that matches everything collapses the map.", + "See 'layer_set_ideas' below for the SHAPE of an answer. They are sketches from other codebases, not a menu -- none of them fits this repository.", + f"Write the JSON or YAML response to .tldrgraph/{RESPONSE_FILENAME}, then run `tldrgraph init`.", ], - "suggested_archetype_layers": archetype_example, + "layer_set_ideas": LAYER_SET_IDEAS, "evidence": evidence, } From cc9fae71879753411afef5e3a6ae2cfc19bc0d2a Mon Sep 17 00:00:00 2001 From: Vikrant Date: Fri, 21 Aug 2026 14:15:11 +0530 Subject: [PATCH 2/3] Better code quality --- .agents/rules/tldrgraph.md | 31 - .../tldrgraph-init/SKILL.md} | 0 .githooks/pre-commit | 17 + .tldrgraph/AGENT_CONTRACT.md | 268 ++++ .tldrgraph/layers.config.yaml | 75 + AGENTS.md | 5 + README.md | 6 +- scripts/check_code_health.py | 129 ++ tests/test_agent_loop.py | 4 +- tests/test_code_health.py | 13 + tldrgraph/agent_commands.py | 34 +- tldrgraph/call_resolver.py | 139 ++ tldrgraph/cli.py | 1398 ++--------------- tldrgraph/cli_agent_loop.py | 150 ++ tldrgraph/cli_commands.py | 289 ++++ tldrgraph/cli_enrichment.py | 382 +++++ tldrgraph/cli_pipeline.py | 355 +++++ tldrgraph/deadcode.py | 384 ++--- tldrgraph/dense_embedder.py | 163 ++ tldrgraph/extractors.py | 908 +---------- tldrgraph/extractors_client.py | 123 ++ tldrgraph/extractors_prisma.py | 251 +++ tldrgraph/extractors_route.py | 309 ++++ tldrgraph/flow_engine.py | 433 +---- tldrgraph/flow_traversal.py | 220 +++ tldrgraph/graph_loader.py | 1222 +++----------- tldrgraph/hierarchy.py | 798 +++------- tldrgraph/hierarchy_builder.py | 247 +++ tldrgraph/installer.py | 309 +--- tldrgraph/installer_contract.py | 172 ++ tldrgraph/labels.py | 122 +- tldrgraph/layer_config.py | 87 +- tldrgraph/layer_evidence.py | 231 +++ tldrgraph/llm_enricher.py | 130 +- tldrgraph/node_registrar.py | 253 +++ tldrgraph/propose_layers.py | 562 ++----- tldrgraph/rules.py | 17 +- tldrgraph/snapshot_sync.py | 273 ++++ tldrgraph/vector_store.py | 730 ++------- tldrgraph/vector_tfidf.py | 130 ++ tldrgraph/visualizer/data.py | 323 ++-- tldrgraph/visualizer/source.py | 51 +- 42 files changed, 5679 insertions(+), 6064 deletions(-) delete mode 100644 .agents/rules/tldrgraph.md rename .agents/{workflows/tldrgraph-init.md => skills/tldrgraph-init/SKILL.md} (100%) create mode 100755 .githooks/pre-commit create mode 100644 .tldrgraph/AGENT_CONTRACT.md create mode 100644 .tldrgraph/layers.config.yaml create mode 100755 scripts/check_code_health.py create mode 100644 tests/test_code_health.py create mode 100644 tldrgraph/call_resolver.py create mode 100644 tldrgraph/cli_agent_loop.py create mode 100644 tldrgraph/cli_commands.py create mode 100644 tldrgraph/cli_enrichment.py create mode 100644 tldrgraph/cli_pipeline.py create mode 100644 tldrgraph/dense_embedder.py create mode 100644 tldrgraph/extractors_client.py create mode 100644 tldrgraph/extractors_prisma.py create mode 100644 tldrgraph/extractors_route.py create mode 100644 tldrgraph/flow_traversal.py create mode 100644 tldrgraph/hierarchy_builder.py create mode 100644 tldrgraph/installer_contract.py create mode 100644 tldrgraph/layer_evidence.py create mode 100644 tldrgraph/node_registrar.py create mode 100644 tldrgraph/snapshot_sync.py create mode 100644 tldrgraph/vector_tfidf.py diff --git a/.agents/rules/tldrgraph.md b/.agents/rules/tldrgraph.md deleted file mode 100644 index 131dc80..0000000 --- a/.agents/rules/tldrgraph.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: tldrgraph -description: TLDRGraph architecture graph: trace before you edit ---- - -## TLDRGraph - -This repository is mapped into architectural layers designed from its own source, -with per-symbol intents you can search and trace. - -**Before planning or implementing a feature**, trace it instead of grepping: - -```bash -tldrgraph query "" # semantic search + end-to-end flow -tldrgraph trace "" "" # exact path between two symbols -tldrgraph layers # node counts per layer -tldrgraph dead-code # review candidates, never a delete list -``` - -Those are read-only and never trigger enrichment. - -**To build or continue the graph**, run `tldrgraph init`, do what the `NEXT ACTION` -block prints, and run it again -- repeat until `status: done`. It has no template -fallback: if this repository has no architecture yet, it will stop and ask you to -design one from the code. Do not skip reading the files. - -Full workflow: `.claude/commands/tldrgraph-init.md` (identical copies live in every -other agent directory). Schema: `.tldrgraph/AGENT_CONTRACT.md`. - -`tldrgraph dead-code` lists **review candidates, not confirmed dead code**. -`unreviewed` means "not enough evidence to conclude" and is never removable. diff --git a/.agents/workflows/tldrgraph-init.md b/.agents/skills/tldrgraph-init/SKILL.md similarity index 100% rename from .agents/workflows/tldrgraph-init.md rename to .agents/skills/tldrgraph-init/SKILL.md diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..45c8b4f --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Git pre-commit hook to verify code health rules and test integrity. + +set -e + +echo "🔍 Running TLDRGraph pre-commit code health check..." + +# Run code health linter +python3 scripts/check_code_health.py --target-dir tldrgraph + +if [ $? -ne 0 ]; then + echo "❌ Pre-commit check failed: Code quality limits exceeded." + echo "Please ensure all files are < 400 lines and functions have complexity <= 15." + exit 1 +fi + +echo "✅ Code health check passed." diff --git a/.tldrgraph/AGENT_CONTRACT.md b/.tldrgraph/AGENT_CONTRACT.md new file mode 100644 index 0000000..d9c48ba --- /dev/null +++ b/.tldrgraph/AGENT_CONTRACT.md @@ -0,0 +1,268 @@ +# TLDRGraph Agent Contract + +**Audience: the coding agent with this repository open** (Claude Code, Cursor, Antigravity). + +TLDRGraph builds an architectural graph from the graphify AST export. The layer set +itself is designed by you, reading this repository. TLDRGraph ships no layer templates. +Structure *within* a layer, and the high-volume deterministic seams between layers, are +extracted automatically. What cannot be extracted automatically is: + +- indirect dispatch, queue / event hops, dynamically-built routes; +- the natural-language **intent** that makes semantic search work at all. + +That is your job. You are not a fallback for a hosted model — you are the primary +enrichment path, because **you can open the files**. The API path only ever sees a label +and a path (`snippet` is never populated), so it guesses. You do not have to guess. + +--- + +## Start here: `tldrgraph init` + +One command does everything, and it is resumable: + +```bash +tldrgraph init +``` + +It runs every deterministic step — extraction, classification, indexing, applying whatever +you last wrote — then stops with a `NEXT ACTION` block the moment it needs judgement only +you can supply. Do what the block says and run it again. Repeat until `status: done`. + +| status | what it wants | +| --- | --- | +| `needs_layers` | Read the code and design this repository's architecture. **TLDRGraph ships no layer templates**; nothing will be applied for you. The request carries sketches of how other kinds of codebase divide — for shape only, never to copy. | +| `needs_confirmation` | Enrichment costs the user tokens. Show them the estimate and ask. Then `tldrgraph init --yes`. | +| `needs_enrichment` | A batch to open, read and describe, per the schema below. | +| `done` | Nothing left. Use `query` / `trace` / `layers`. | + +`--json` gives you the same thing machine-readably. The sections below document the file +formats `init` reads and writes; the underlying `queue-enrichment` / `apply-enrichment` +commands remain available for scripting. + +**Copy every `id` verbatim from the request.** A constructed id matches nothing, is +dropped, and will be reported back to you — but the work is wasted. + +--- + +## The loop + +```bash +tldrgraph queue-enrichment --limit 50 # 1. writes .tldrgraph/enrichment_request.yaml +# 2. you read it, read the SOURCE, and write +# .tldrgraph/enrichment_response.yaml +tldrgraph apply-enrichment # 3. merges into the graph, cache and index +tldrgraph queue-enrichment --limit 50 # 4. repeat -- the queue advances automatically +``` + +Request and response are **separate files**. Never write your answer back into +`enrichment_request.yaml`; it is regenerated on every run and your work would be lost. + +| File | Written by | Read by | +| --- | --- | --- | +| `.tldrgraph/enrichment_request.yaml` (or `enrichment_request.json`) | `queue-enrichment` | you | +| `.tldrgraph/enrichment_response.yaml` (or `enrichment_response.json`) | **you** | `apply-enrichment` | +| `.tldrgraph/enrichment_cursor.json` | both commands | both commands | +| `.tldrgraph/pending_enrichment.json` | *(legacy)* | `apply-enrichment`, only if no response file exists | + +--- + +## Request schema (`enrichment_request.yaml`) + +```yaml +schema: codechakra/enrichment-request@1 +generated_at: "2026-08-19T00:00:00+00:00" +response_file: .tldrgraph/enrichment_response.yaml +contract: .tldrgraph/AGENT_CONTRACT.md +progress: + total_candidates: 1873 # un-enriched, non-utility nodes + already_enriched: 12 # nodes that already carry an intent + queued_now: 50 # entries in "nodes" below + remaining_after: 1823 # still waiting after this batch is applied +nodes: + - id: backend_src_applications_applications_controller_applicationscontroller + label: ApplicationsController + layer_id: api + layer: "Layer 2: API Gateway" + file: backend/src/applications/applications.controller.ts + source_location: L31 + degree: 41 # in + out edges in the AST graph + cross_layer_degree: 17 # of those, how many cross a layer boundary + rank: 1 # 1 = highest priority in this batch + existing_intent_source: heuristic # "" when the node has no intent at all +``` + +`file` is repo-relative. `source_location` is graphify's line hint and may be `null`. +`layer_id` is the stable machine key (e.g. `cli`, `engine`, `storage`, `api`, `ui`). + +`existing_intent_source` is `"heuristic"` when the node already carries an intent written +by the offline template enricher. That text was generated from the label and layer alone +— it has not read a line of source — so the node is still a candidate and your answer +should overwrite it. Applied answers are stamped `"agent"` and are never re-queued. + +--- + +## Response schema (`enrichment_response.yaml` or `enrichment_response.json`) + +A **YAML list** (preferred) or **JSON array** of objects: + +```yaml +- id: backend_src_applications_applications_controller_applicationscontroller + intent: | + ### Pension Application Lifecycle Gateway + REST gateway for the pension application lifecycle. Authorizes DEO/AAO/AO/DAG roles, + dispatches cases to ApplicationsService and records status transitions. + input_fields: + - caseId + - transitionPayload + - remarks + - sanctionOrderNo + output_fields: + - applicationStatus + - disposition + calls: + - ApplicationsService + - JwtAuthGuard + - RolesGuard + - pension_cases +``` + +Equivalent JSON format (also accepted from `.tldrgraph/enrichment_response.json` or `.tldrgraph/pending_enrichment.json`): +```json +[ + { + "id": "backend_src_applications_applications_controller_applicationscontroller", + "intent": "### Pension Application Lifecycle Gateway\nREST gateway for the pension application lifecycle.", + "input_fields": ["caseId", "transitionPayload", "remarks", "sanctionOrderNo"], + "output_fields": ["applicationStatus", "disposition"], + "calls": ["ApplicationsService", "JwtAuthGuard", "RolesGuard", "pension_cases"] + } +] +``` + +| Key | Type | Meaning | +| --- | --- | --- | +| `id` | string, **required** | The node id, copied **verbatim** from the request. An id that is not in the graph is skipped silently. | +| `intent` | string (Markdown) | Markdown formatted explanation: what this symbol does, its role, and why it exists. AI decides how much depth is needed. This is the text semantic search matches against. | +| `input_fields` | array of strings | Input parameters, arguments, request body payload attributes, query filters. | +| `output_fields` | array of strings | Return types, response models, emitted event names, or mutated state attributes. | +| `fields` | array of strings (legacy) | Supported for backwards compatibility (maps to input fields). | +| `calls` | array of strings or objects | Downstream symbols, files (`file:symbol`), or node IDs this symbol calls. Cross-layer bridges are created with 100% confidence. | +| `layer_id` | string (optional) | Explicitly reassign the architectural layer ID if the AST classification miscategorized it. | + +`input_fields`, `output_fields`, and `calls` may be omitted or empty. An object with only `id` and `intent` is +valid and useful. + +--- + +## Hard rules + +1. **Open and read the actual source file before writing an intent.** You have the repo + checked out; that is the entire reason this path exists. Read `file` (use + `source_location` to find the symbol), and read enough of its imports and callees to + describe what it really does. An intent paraphrased from the label is worse than no + intent, because it poisons search with confident-sounding noise. + +2. **Do not invent fields or calls. Omit what you cannot verify in the code.** If you + read the file and it handles three params, list three. Do not pad the list with what a + symbol of that name "usually" has. `"fields": []` is a correct, honest answer. + A wrong `calls` entry creates a real, wrong edge in the graph that later queries will + follow. + +3. **`calls` entries are resolved with 2-tier high precision.** + - **Tier 1 (Exact Match, 100% confidence):** Exact symbol names (`ApplicationsService`), + function names, node IDs, file paths (`calc.ts`), or database table names (`pension_cases`). + - **Tier 2 (Vector Fallback):** Semantic search with a calibrated 0.35 score floor. + + | Good | Bad | + | --- | --- | + | `ApplicationsService` | `the application service` | + | `calc.ts` | `some calculation helper` | + | `pension_cases` | `the database` | + | `JwtAuthGuard` | `auth stuff` | + + Prefer the exact symbol name, file name, or table/model name as it appears in the source. + +4. **Copy `id` verbatim.** Do not normalize, shorten or re-case it. + +5. **Answer only the nodes in the request.** Extra ids are ignored; missing ids just come + back in a later batch. + +--- + +## Priority order in the queue + +The queue is not arbitrary — a node that many things depend on is worth more of your +attention than a leaf. A node is a **candidate** when it sits outside `General / Utility` +and either has no intent at all, or has one that came from the offline template heuristic +(`enrichment_source: "heuristic"`, i.e. nobody read the source). Candidates are sorted by: + +1. **`cross_layer_degree` descending** — neighbours that sit in a *different* layer. + These are the seams TLDRGraph exists to describe, and they are exactly where the AST + alone is weakest. +2. **`degree` descending** — total in + out edges. Hub nodes first. +3. **node id ascending** — only to make the ordering deterministic. + +Both degrees are computed from the live graph. (The `degree` key that graphify emits is +absent, so anything reading `node["degree"]` from the raw export sees `0`; TLDRGraph +recomputes it and stamps it back into `.tldrgraph/graph.json`.) + +--- + +## Paging and progress + +`queue-enrichment` remembers what it has handed out in `.tldrgraph/enrichment_cursor.json`: + +- `applied` — ids successfully merged by `apply-enrichment`. Never re-queued. +- `queued` — ids handed out but not yet applied ("in flight"). Skipped by default. + +So running `queue-enrichment` twice in a row **advances** to the next batch instead of +repeating. Two escape hatches: + +- `--requeue` — also hand out in-flight ids again (use when a batch was abandoned). +- `--reset` — clear all progress and start again from the highest-priority node. +- `--limit 0` — no cap; queue every remaining candidate at once. + +--- + +## What `apply-enrichment` does with your answer + +For each object it can match to a node: + +1. sets `intent`, rewrites `summary` to `":