From 8d758a992e6e14e71a90c6f3b7679757be1688e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:16:26 +0000 Subject: [PATCH 001/130] perf(binders): group defs once and intersect with import targets The four name binders (resolveCallEdges, buildCallerIndex, buildSymbolGraph, resolveRelations) filtered every same-name definition per call site and built a `${from}|${to}` string per candidate to test import corroboration. That is quadratic in homonyms: the TypeScript repo's test fixtures declare `C` 5,335 times, so 236k call sites cost 6.8M candidate checks. Definitions are now grouped by (name, language family) with a per-file index, the import pairs are regrouped into per-file target sets, and the corroborated candidates are an intersection walked from the smaller side. The call binders also resolve each name once per file, and the proximity score walks characters instead of splitting both paths per candidate. Output is byte-identical (callers, recall callers, relations, call edges, hierarchy and symbol graph compared on flask, gin and microsoft/TypeScript). On the TypeScript repo: resolveRelations 1.35 s -> 0.13 s, buildCallerIndex 1.65 s -> 0.5 s, resolveCallEdges 0.7 s -> 0.27 s, buildTypeHierarchy 1.7 s -> 0.57 s, buildSymbolGraph 4.5 s -> 2.3 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/callers.ts | 81 +++++++++---------- src/calls.ts | 130 ++++++++++++++++++++++++++---- src/relations.ts | 39 +++++---- src/symbolgraph.ts | 46 ++++++----- tests/binder.test.ts | 187 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 380 insertions(+), 103 deletions(-) create mode 100644 tests/binder.test.ts diff --git a/src/callers.ts b/src/callers.ts index 20074ad..fe8bd91 100644 --- a/src/callers.ts +++ b/src/callers.ts @@ -13,7 +13,7 @@ // graph.json stays byte-compatible with ultraindex. import type { CodeSymbol } from "./types.js"; import type { RepoScan } from "./scan.js"; -import { familyOf, pickCandidate } from "./calls.js"; +import { addDef, defsOutside, familyOf, importTargets, importedDefs, pickCandidate, type DefTable } from "./calls.js"; import { importPairsFor } from "./derived.js"; import { byStr } from "./sort.js"; @@ -73,34 +73,17 @@ export function buildCallerIndex( const pairs = importPairs ?? importPairsFor(scan); const recall = opts.recall === true; - // name → def sites (first symbol per (name, file) wins, like resolveCallEdges). - const defs = new Map(); + // name → family → def sites (first symbol per (name, file) wins, like + // resolveCallEdges). CodeSymbol structurally contains Cand's file/lang + // fields, and pickCandidate returns the selected object unchanged. + const defs: DefTable = new Map(); for (const f of scan.files) { - const seen = new Set(); for (const s of f.symbols) { if (!s.exported || REFERENCE_KINDS.has(s.kind)) continue; - if (seen.has(s.name)) continue; - seen.add(s.name); - let arr = defs.get(s.name); - if (!arr) defs.set(s.name, (arr = [])); - arr.push(s); + addDef(defs, s.name, s); } } - // The hot loop below resolves every call. Group definitions by language - // family once so it does not allocate a filtered + mapped candidate list for - // each site. CodeSymbol structurally contains Cand's file/lang fields, and - // pickCandidate returns the selected object unchanged. - const defsByFamily = new Map>(); - for (const [name, sites] of defs) { - const families = new Map(); - for (const site of sites) { - const family = familyOf(site.lang); - let grouped = families.get(family); - if (!grouped) families.set(family, (grouped = [])); - grouped.push(site); - } - defsByFamily.set(name, families); - } + const targetsOf = importTargets(pairs); // Same-file binding also needs non-exported defs (a private helper shadows // an exported symbol of the same name elsewhere). const localDefs = new Map>(); @@ -123,6 +106,11 @@ export function buildCallerIndex( if (!f.calls?.length) continue; const family = familyOf(f.lang); const own = localDefs.get(f.rel)!; + const targets = targetsOf.get(f.rel); + // Every cross-file call of one name in one file binds the same way — the + // choice depends on (file, name) only — so resolve each name once per file. + // null = resolved to "no binding". + const bound = new Map(); for (const c of f.calls) { const local = own.get(c.name); if (local) { @@ -132,27 +120,34 @@ export function buildCallerIndex( record(local, recall ? { file: f.rel, line: c.line, confidence: "corroborated" } : { file: f.rel, line: c.line }); continue; } - const cands = (defsByFamily.get(c.name)?.get(family) ?? []).filter((d) => d.file !== f.rel); - if (!cands.length) continue; - const imported = cands.filter((d) => pairs.has(`${f.rel}|${d.file}`)); - const chosen = - family === "js" - ? imported.length - ? pickCandidate(f.rel, imported) - : // JS/TS gate: no corroborating import → no binding. Recall mode - // relaxes this to a unique-repo-wide name match (issue #7). - recall && cands.length === 1 - ? cands[0] - : undefined - : imported.length - ? pickCandidate(f.rel, imported) - : pickCandidate(f.rel, cands); - if (!chosen) continue; - const def = chosen as CodeSymbol; + let hit = bound.get(c.name); + if (hit === undefined) { + hit = null; + const group = defs.get(c.name)?.get(family); + const cands = group ? defsOutside(group, f.rel) : []; + if (group && cands.length) { + const imported = importedDefs(group, targets); + const chosen = + family === "js" + ? imported.length + ? pickCandidate(f.rel, imported) + : // JS/TS gate: no corroborating import → no binding. Recall mode + // relaxes this to a unique-repo-wide name match (issue #7). + recall && cands.length === 1 + ? cands[0] + : undefined + : imported.length + ? pickCandidate(f.rel, imported) + : pickCandidate(f.rel, cands); + if (chosen) hit = { def: chosen, corroborated: imported.length > 0 }; + } + bound.set(c.name, hit); + } + if (!hit) continue; record( - def, + hit.def, recall - ? { file: f.rel, line: c.line, confidence: imported.length ? "corroborated" : "unique-name" } + ? { file: f.rel, line: c.line, confidence: hit.corroborated ? "corroborated" : "unique-name" } : { file: f.rel, line: c.line }, ); } diff --git a/src/calls.ts b/src/calls.ts index c9899f4..1e20b0d 100644 --- a/src/calls.ts +++ b/src/calls.ts @@ -23,12 +23,24 @@ export function familyOf(lang: string): string { // Leading path segments two repo-relative paths share (the filename never counts, // as it always differs between distinct files). Higher = closer in the tree. +// +// Walks the characters instead of splitting both paths: pickCandidate scores +// EVERY candidate of an ambiguous name, and splitting allocated two arrays per +// candidate. Same count as comparing `a.split("/")` with `b.split("/")` +// pairwise: each "/" passed while the strings agree closes an equal segment, +// and the segment in progress where they stop is equal only if both end there. function sharedSegments(a: string, b: string): number { - const as = a.split("/"); - const bs = b.split("/"); + const len = Math.min(a.length, b.length); let n = 0; - while (n < as.length && n < bs.length && as[n] === bs[n]) n++; - return n; + let i = 0; + for (; i < len; i++) { + const c = a.charCodeAt(i); + if (c !== b.charCodeAt(i)) break; + if (c === 47 /* "/" */) n++; + } + const endA = i === a.length || a.charCodeAt(i) === 47; + const endB = i === b.length || b.charCodeAt(i) === 47; + return endA && endB ? n + 1 : n; } export interface Cand { @@ -39,10 +51,14 @@ export interface Cand { // Pick a single candidate for a call: the sole candidate, else the one sharing // the strictly-most leading path segments with the caller. A tie at the maximum // (or an empty list) is unresolvable — return undefined so the caller skips it. -export function pickCandidate(callerRel: string, cands: Cand[]): Cand | undefined { +// The answer depends on the candidate SET, never its order: a strictly higher +// score clears `tied`, so it ends true exactly when the maximum occurs twice. +// Returns the chosen object itself, so a binder passing richer records (a +// CodeSymbol, a type def) gets its own record back. +export function pickCandidate(callerRel: string, cands: readonly T[]): T | undefined { if (cands.length === 1) return cands[0]; if (cands.length === 0) return undefined; - let best: Cand | undefined; + let best: T | undefined; let bestScore = -1; let tied = false; for (const c of cands) { @@ -58,6 +74,88 @@ export function pickCandidate(callerRel: string, cands: Cand[]): Cand | undefine return tied ? undefined : best; } +// --- Shared binder plumbing ------------------------------------------------- +// resolveCallEdges below, buildCallerIndex (callers.ts), buildSymbolGraph +// (symbolgraph.ts) and resolveRelations (relations.ts) all bind a NAME to one +// of its definitions by asking the same questions: which defs share the +// caller's language family, which of those an import corroborates, which is +// closest. Each used to answer them per call site by filtering every same-name +// def and building a `${from}|${to}` key per candidate — quadratic in homonyms, +// and the TypeScript repo's test fixtures declare `C` 5,335 times and `A` +// 2,449 times. These helpers group the defs ONCE and intersect them with the +// caller's (few) import targets instead. Same answers; only the work changed. + +/** One name's definitions within ONE language family. */ +export interface DefGroup { + list: T[]; // registration order — the proximity fallback's pool + byFile: Map; // for intersecting with a caller's import targets +} + +/** name → language family → definitions. Filled with addDef. */ +export type DefTable = Map>>; + +// Register `def` as a definition of `name`. The first def per (name, file) +// wins — the dedup each binder applied with its own `${name} ${file}` set — so +// a later one returns false and is dropped, whatever its family. +export function addDef(table: DefTable, name: string, def: T): boolean { + let families = table.get(name); + if (!families) table.set(name, (families = new Map())); + for (const group of families.values()) if (group.byFile.has(def.file)) return false; + const family = familyOf(def.lang); + let group = families.get(family); + if (!group) families.set(family, (group = { list: [], byFile: new Map() })); + group.list.push(def); + group.byFile.set(def.file, def); + return true; +} + +// `${from}|${to}` import pairs regrouped as from → the files it imports, so a +// binder asks "does this file import that one" without building a string per +// candidate. Self-pairs are dropped: the call binders never weigh a same-file +// def here and relations admit one unconditionally, so neither consulted them. +// A path may itself contain "|"; such a pair is registered under EVERY split +// point, which keeps `targets.get(a)?.has(b)` exactly `pairs.has(`${a}|${b}`)` +// without having to know which split is the real one. +export function importTargets(pairs: Iterable): Map> { + const out = new Map>(); + for (const pair of pairs) { + for (let i = pair.indexOf("|"); i !== -1; i = pair.indexOf("|", i + 1)) { + const from = pair.slice(0, i); + const to = pair.slice(i + 1); + if (from === to) continue; + let set = out.get(from); + if (!set) out.set(from, (set = new Set())); + set.add(to); + } + } + return out; +} + +// The defs of `group` an import corroborates: those living in one of the +// caller's import targets. Walks whichever side is smaller — a file imports a +// handful of others, a fixture name can have thousands of definitions. The +// order follows the walked side, which no binder observes (pickCandidate is +// order-independent; everything else only counts). Always a fresh array the +// caller may extend; never the caller's own file (importTargets drops self-pairs). +export function importedDefs(group: DefGroup, targets: ReadonlySet | undefined): T[] { + if (!targets?.size) return []; + if (targets.size < group.list.length) { + const out: T[] = []; + for (const file of targets) { + const d = group.byFile.get(file); + if (d) out.push(d); + } + return out; + } + return group.list.filter((d) => targets.has(d.file)); +} + +// The group's defs outside `rel`: a call never binds cross-file to its own +// file. Allocates only when `rel` itself declares the name. +export function defsOutside(group: DefGroup, rel: string): readonly T[] { + return group.byFile.has(rel) ? group.list.filter((d) => d.file !== rel) : group.list; +} + // Resolve every collected call site to a cross-file `call` edge in a global second // pass. An import between the two files promotes the edge to `extracted`; a unique // repo-wide name match with no import yields `inferred`. JS/TS is import-gated (no @@ -65,35 +163,33 @@ export function pickCandidate(callerRel: string, cands: Cand[]): Cand | undefine // other languages fall back to a unique-name inference. Deterministic: the emitted // array is sorted and never depends on Map iteration order. export function resolveCallEdges(scan: RepoScan, importPairs: Set): Edge[] { - // name → distinct def sites (deduped per file; overloads collapse to one file). - const defs = new Map(); - const seen = new Set(); + // name → family → distinct def sites (deduped per file; overloads collapse to one file). + const defs: DefTable = new Map(); for (const f of scan.files) { for (const s of f.symbols) { if (!s.exported || REFERENCE_KINDS.has(s.kind)) continue; - const dedup = `${s.name} ${s.file}`; - if (seen.has(dedup)) continue; - seen.add(dedup); - let arr = defs.get(s.name); - if (!arr) defs.set(s.name, (arr = [])); - arr.push({ file: s.file, lang: s.lang }); + addDef(defs, s.name, { file: s.file, lang: s.lang }); } } + const targetsOf = importTargets(importPairs); // (from|to) → aggregated edge. Strongest confidence wins; counts sum. const agg = new Map(); for (const f of scan.files) { if (!f.calls?.length) continue; const family = familyOf(f.lang); + const targets = targetsOf.get(f.rel); const ownNames = new Set(f.symbols.map((s) => s.name)); const counts = new Map(); for (const c of f.calls) counts.set(c.name, (counts.get(c.name) ?? 0) + 1); for (const [name, count] of counts) { if (ownNames.has(name)) continue; // same-file call — not a cross-file edge - const cands = (defs.get(name) ?? []).filter((d) => familyOf(d.lang) === family && d.file !== f.rel); + const group = defs.get(name)?.get(family); + if (!group) continue; + const cands = defsOutside(group, f.rel); if (!cands.length) continue; - const imported = cands.filter((d) => importPairs.has(`${f.rel}|${d.file}`)); + const imported = importedDefs(group, targets); let chosen: Cand | undefined; let confidence: "extracted" | "inferred"; diff --git a/src/relations.ts b/src/relations.ts index 806a3d9..308418c 100644 --- a/src/relations.ts +++ b/src/relations.ts @@ -17,7 +17,7 @@ // looked like. import type { Edge, RawRelation } from "./types.js"; import type { RepoScan } from "./scan.js"; -import { familyOf, pickCandidate, type Cand } from "./calls.js"; +import { addDef, familyOf, importTargets, importedDefs, pickCandidate, type DefTable } from "./calls.js"; import { byStr } from "./sort.js"; // Internal Map-key separator. Written as an ESCAPE, never as a literal NUL: a @@ -68,19 +68,13 @@ interface TypeDef { line: number; } -// name → every type-ish definition of it, deduped per file. -function typeDefs(scan: RepoScan): Map { - const defs = new Map(); - const seen = new Set(); +// name → family → every type-ish definition of it, deduped per file. +function typeDefs(scan: RepoScan): DefTable { + const defs: DefTable = new Map(); for (const f of scan.files) { for (const s of f.symbols) { if (!TYPE_KINDS.has(s.kind)) continue; - const dedup = `${s.name} ${s.file}`; - if (seen.has(dedup)) continue; - seen.add(dedup); - let arr = defs.get(s.name); - if (!arr) defs.set(s.name, (arr = [])); - arr.push({ name: s.name, file: s.file, kind: s.kind, lang: s.lang, line: s.line }); + addDef(defs, s.name, { name: s.name, file: s.file, kind: s.kind, lang: s.lang, line: s.line }); } } return defs; @@ -96,19 +90,22 @@ function typeDefs(scan: RepoScan): Map { */ export function resolveRelations(scan: RepoScan, importPairs: Set): ResolvedRelation[] { const defs = typeDefs(scan); + const targetsOf = importTargets(importPairs); const out: ResolvedRelation[] = []; for (const f of scan.files) { if (!f.relations?.length) continue; const family = familyOf(f.lang); + const targets = targetsOf.get(f.rel); for (const r of f.relations) { - const cands = (defs.get(r.to) ?? []).filter((d) => familyOf(d.lang) === family); - if (!cands.length) continue; - // Prefer a candidate the file actually imports; fall back to proximity. - const imported = cands.filter((d) => importPairs.has(`${f.rel}|${d.file}`) || d.file === f.rel); - const pool = imported.length ? imported : cands; - const chosen = pickCandidate(f.rel, pool.map((d): Cand => ({ file: d.file, lang: d.lang }))); - if (!chosen) continue; - const target = pool.find((d) => d.file === chosen.file)!; + const group = defs.get(r.to)?.get(family); + if (!group) continue; + // Prefer a candidate the file actually imports (or declares itself); + // fall back to proximity. + const imported = importedDefs(group, targets); + const local = group.byFile.get(f.rel); + if (local) imported.push(local); + const target = pickCandidate(f.rel, imported.length ? imported : group.list); + if (!target) continue; out.push({ kind: CONTRACT_KINDS.has(target.kind) ? "implements" : r.kind, from: r.from, @@ -179,8 +176,8 @@ export function buildTypeHierarchy(scan: RepoScan, importPairs: Set): Ma // Which declaration a (name, file) pair refers to. const entries = new Map(); const keyOf = (name: string, file: string): string => `${name}${SEP}${file}`; - for (const arr of defs.values()) { - for (const d of arr) { + for (const families of defs.values()) { + for (const d of [...families.values()].flatMap((g) => g.list)) { entries.set(keyOf(d.name, d.file), { name: d.name, file: d.file, diff --git a/src/symbolgraph.ts b/src/symbolgraph.ts index aaf3dde..6cd98c1 100644 --- a/src/symbolgraph.ts +++ b/src/symbolgraph.ts @@ -13,7 +13,7 @@ // the graph is built on demand from the scan, so no artifact or schema grows. import type { CodeSymbol } from "./types.js"; import type { RepoScan } from "./scan.js"; -import { familyOf, pickCandidate, type Cand } from "./calls.js"; +import { addDef, defsOutside, familyOf, importTargets, importedDefs, pickCandidate, type DefTable } from "./calls.js"; import { enclosingAmong } from "./callers.js"; import { resolveRelations } from "./relations.js"; import { byStr } from "./sort.js"; @@ -94,9 +94,8 @@ export function buildSymbolGraph(scan: RepoScan, importPairs: Set): Symb // Per-file symbol lists, filtered once and reused for every call site in that // file (the reason enclosingAmong is factored out of enclosingSymbol). const perFile = new Map(); - // Callable definitions by name, deduped per (name, file). - const defs = new Map(); - const defSeen = new Set(); + // Callable definitions by name and family, deduped per (name, file). + const defs: DefTable = new Map(); for (const f of scan.files) { const usable: CodeSymbol[] = []; @@ -104,16 +103,11 @@ export function buildSymbolGraph(scan: RepoScan, importPairs: Set): Symb if (REFERENCE_KINDS.has(s.kind)) continue; usable.push(s); nodes.set(symbolId(s), toNode(s)); - if (!s.exported) continue; - const key = `${s.name} ${s.file}`; - if (defSeen.has(key)) continue; - defSeen.add(key); - let arr = defs.get(s.name); - if (!arr) defs.set(s.name, (arr = [])); - arr.push(s); + if (s.exported) addDef(defs, s.name, s); } perFile.set(f.rel, usable); } + const targetsOf = importTargets(importPairs); const agg = new Map(); const add = (from: string, to: string, kind: SymbolEdgeKind): void => { @@ -131,6 +125,10 @@ export function buildSymbolGraph(scan: RepoScan, importPairs: Set): Symb const own = perFile.get(f.rel) ?? []; const localByName = new Map(); for (const s of own) if (!localByName.has(s.name)) localByName.set(s.name, s); + const targets = targetsOf.get(f.rel); + // The callee of a cross-file call depends on (file, name) only, so each + // name is bound once per file (null = no binding) — see buildCallerIndex. + const bound = new Map(); for (const c of f.calls) { const caller = enclosingAmong(own, c.line); @@ -142,17 +140,21 @@ export function buildSymbolGraph(scan: RepoScan, importPairs: Set): Symb if (local.line !== c.line) add(symbolId(caller), symbolId(local), "calls"); continue; } - const cands = (defs.get(c.name) ?? []).filter((d) => familyOf(d.lang) === family && d.file !== f.rel); - if (!cands.length) continue; - const imported = cands.filter((d) => importPairs.has(`${f.rel}|${d.file}`)); - // JS/TS keeps its import gate: a bare identifier is too ambiguous to bind - // on name alone, and a wrong edge here misleads an impact analysis. - const pool = imported.length ? imported : family === "js" ? [] : cands; - if (!pool.length) continue; - const chosen = pickCandidate(f.rel, pool.map((d): Cand => ({ file: d.file, lang: d.lang }))); - if (!chosen) continue; - const target = pool.find((d) => d.file === chosen.file)!; - add(symbolId(caller), symbolId(target), "calls"); + let callee = bound.get(c.name); + if (callee === undefined) { + callee = null; + const group = defs.get(c.name)?.get(family); + if (group) { + const imported = importedDefs(group, targets); + // JS/TS keeps its import gate: a bare identifier is too ambiguous to + // bind on name alone, and a wrong edge here misleads an impact analysis. + const pool = imported.length ? imported : family === "js" ? [] : defsOutside(group, f.rel); + const target = pickCandidate(f.rel, pool); + if (target) callee = symbolId(target); + } + bound.set(c.name, callee); + } + if (callee) add(symbolId(caller), callee, "calls"); } } diff --git a/tests/binder.test.ts b/tests/binder.test.ts new file mode 100644 index 0000000..2a9ad09 --- /dev/null +++ b/tests/binder.test.ts @@ -0,0 +1,187 @@ +import { describe, it, expect } from "vitest"; +import { addDef, importTargets, importedDefs, pickCandidate, resolveCallEdges, type Cand, type DefTable } from "../src/calls.js"; +import { buildCallerIndex } from "../src/callers.js"; +import { resolveRelations } from "../src/relations.js"; +import { buildSymbolGraph } from "../src/symbolgraph.js"; +import type { RepoScan } from "../src/scan.js"; +import type { CodeSymbol, FileRecord, RawRelation } from "../src/types.js"; + +// The shared binder plumbing in calls.ts (grouped defs, import targets, the +// split-free proximity score). Every binder used to filter all same-name defs +// per call site and build a `${from}|${to}` string per candidate; these tests +// pin that the grouped path answers exactly what that did — at the scale that +// made it slow (thousands of homonyms) and on the path shapes a character walk +// could get wrong. + +function sym(name: string, file: string, o: Partial = {}): CodeSymbol { + return { name, kind: o.kind ?? "function", file, line: o.line ?? 1, exported: o.exported ?? true, lang: o.lang ?? "typescript" }; +} + +function file( + rel: string, + o: { lang?: string; symbols?: CodeSymbol[]; calls?: { name: string; line: number }[]; relations?: RawRelation[] } = {}, +): FileRecord { + return { + rel, + ext: rel.slice(rel.lastIndexOf(".")), + size: 0, + lines: 1, + hash: "h", + kind: "code", + lang: o.lang ?? "typescript", + headings: [], + symbols: o.symbols ?? [], + refs: [], + ...(o.calls ? { calls: o.calls } : {}), + ...(o.relations ? { relations: o.relations } : {}), + }; +} + +function scanOf(files: FileRecord[]): RepoScan { + return { root: "/repo", files, languages: {}, docText: new Map(), mtimes: new Map(), capped: false, excluded: 0, contentUnchanged: false, cacheDirty: true }; +} + +// The segment count the binders historically computed, kept here as the oracle. +function splitShared(a: string, b: string): number { + const as = a.split("/"); + const bs = b.split("/"); + let n = 0; + while (n < as.length && n < bs.length && as[n] === bs[n]) n++; + return n; +} + +describe("pickCandidate", () => { + const cand = (f: string): Cand => ({ file: f, lang: "python" }); + + it("scores proximity exactly like comparing split segments, on awkward paths", () => { + const paths = ["", "a", "ab", "a/b", "a/b/c", "a/bc", "a/b/", "a//b", "/a", "/a/b", "a/b/c/d.py", "a/x/d.py", "ab/c", "a/", "x/y/z.py"]; + // Two candidates: the scorer decides between them, so any disagreement with + // the split oracle on either score flips or ties the answer. + for (const caller of paths) { + for (const x of paths) { + for (const y of paths) { + if (x === y) continue; + const sx = splitShared(caller, x); + const sy = splitShared(caller, y); + const want = sx === sy ? undefined : sx > sy ? x : y; + expect(pickCandidate(caller, [cand(x), cand(y)])?.file, `${caller} vs ${x} | ${y}`).toBe(want); + } + } + } + }); + + it("depends on the candidate set, never its order", () => { + const cands = ["pkg/a/x.py", "pkg/b/x.py", "pkg/a/sub/x.py", "other/x.py"].map(cand); + const forward = pickCandidate("pkg/a/sub/caller.py", cands); + expect(forward?.file).toBe("pkg/a/sub/x.py"); + expect(pickCandidate("pkg/a/sub/caller.py", [...cands].reverse())).toBe(forward); + // A tie at the maximum stays a tie whichever candidate comes first. + expect(pickCandidate("pkg/c.py", ["pkg/a/x.py", "pkg/b/x.py", "z.py"].map(cand))).toBeUndefined(); + expect(pickCandidate("pkg/c.py", ["z.py", "pkg/b/x.py", "pkg/a/x.py"].map(cand))).toBeUndefined(); + }); + + it("returns the candidate object itself", () => { + const def = sym("f", "a/b.py", { lang: "python" }); + expect(pickCandidate("a/c.py", [def, sym("f", "z/b.py", { lang: "python" })])).toBe(def); + }); +}); + +describe("importTargets", () => { + it("answers exactly what `pairs.has(`${from}|${to}`)` answered, even for paths containing '|'", () => { + const pairs = new Set(["a.ts|b.ts", "a|b.ts|c.ts", "x.ts|x.ts"]); + const targets = importTargets(pairs); + const has = (from: string, to: string): boolean => targets.get(from)?.has(to) === true; + for (const [from, to] of [ + ["a.ts", "b.ts"], + ["a", "b.ts|c.ts"], + ["a|b.ts", "c.ts"], + ["a", "b.ts"], + ["b.ts", "c.ts"], + ] as const) { + const self = from === to; + expect(has(from, to), `${from} -> ${to}`).toBe(!self && pairs.has(`${from}|${to}`)); + } + // A self-pair is never a target: no binder ever weighed one. + expect(has("x.ts", "x.ts")).toBe(false); + }); +}); + +describe("addDef / importedDefs", () => { + it("keeps the first def per (name, file) and groups by language family", () => { + const table: DefTable = new Map(); + const first = sym("f", "a.ts"); + expect(addDef(table, "f", first)).toBe(true); + expect(addDef(table, "f", sym("f", "a.ts", { line: 9 }))).toBe(false); + expect(addDef(table, "f", sym("f", "b.js", { lang: "javascript" }))).toBe(true); + expect(addDef(table, "f", sym("f", "c.py", { lang: "python" }))).toBe(true); + expect(table.get("f")!.get("js")!.list.map((d) => d.file)).toEqual(["a.ts", "b.js"]); + expect(table.get("f")!.get("js")!.byFile.get("a.ts")).toBe(first); + expect(table.get("f")!.get("python")!.list.map((d) => d.file)).toEqual(["c.py"]); + }); + + it("intersects with the import targets from whichever side is smaller", () => { + const table: DefTable = new Map(); + for (let i = 0; i < 50; i++) addDef(table, "C", { file: `t/${i}.ts`, lang: "typescript" }); + const group = table.get("C")!.get("js")!; + // Few targets (walks the targets) and many targets (walks the defs) agree. + const few = new Set(["t/7.ts", "elsewhere.ts"]); + const many = new Set([...Array(80).keys()].map((i) => `t/${i + 7}.ts`)); + expect(importedDefs(group, few).map((d) => d.file)).toEqual(["t/7.ts"]); + expect(importedDefs(group, many).map((d) => d.file).sort()).toEqual( + group.list.filter((d) => many.has(d.file)).map((d) => d.file).sort(), + ); + expect(importedDefs(group, undefined)).toEqual([]); + // A fresh array each time: resolveRelations appends the same-file def to it. + expect(importedDefs(group, few)).not.toBe(importedDefs(group, few)); + }); +}); + +// A repo shaped like the TypeScript compiler's test fixtures: one name declared +// in thousands of files, of which the caller imports exactly one. +describe("binding among thousands of homonyms", () => { + const N = 3000; + const defs = [...Array(N).keys()].map((i) => file(`tests/cases/c${i}.ts`, { symbols: [sym("C", `tests/cases/c${i}.ts`, { kind: "class" })] })); + const caller = file("src/app/main.ts", { + symbols: [sym("run", "src/app/main.ts", { line: 1 })], + calls: [ + { name: "C", line: 2 }, + { name: "C", line: 3 }, + ], + relations: [{ kind: "extends", from: "Sub", to: "C", line: 4 }], + }); + const scan = scanOf([caller, ...defs]); + const pairs = new Set(["src/app/main.ts|tests/cases/c1234.ts"]); + + it("binds every binder to the imported definition", () => { + expect(resolveCallEdges(scan, pairs)).toEqual([ + { from: "src/app/main.ts", to: "tests/cases/c1234.ts", kind: "call", weight: 2, confidence: "extracted" }, + ]); + const index = buildCallerIndex(scan, pairs); + expect(index.get("C")?.def.file).toBe("tests/cases/c1234.ts"); + expect(index.get("C")?.callers).toEqual([ + { file: "src/app/main.ts", line: 2 }, + { file: "src/app/main.ts", line: 3 }, + ]); + expect(resolveRelations(scan, pairs).map((r) => r.toFile)).toEqual(["tests/cases/c1234.ts"]); + const graph = buildSymbolGraph(scan, pairs); + expect(graph.out.get("src/app/main.ts#run")).toEqual([ + { from: "src/app/main.ts#run", to: "tests/cases/c1234.ts#C", kind: "calls", weight: 2 }, + ]); + }); + + it("keeps the JS/TS gate without an import, and a proximity tie unbound for relations", () => { + expect(resolveCallEdges(scan, new Set())).toEqual([]); + expect(buildCallerIndex(scan, new Set()).size).toBe(0); + // Every fixture shares zero segments with src/app/main.ts: a tie, no edge. + expect(resolveRelations(scan, new Set())).toEqual([]); + }); + + it("labels recall confidence per site even though each name is bound once per file", () => { + const withImport = buildCallerIndex(scan, pairs, { recall: true }).get("C")!; + expect(withImport.callers.map((c) => c.confidence)).toEqual(["corroborated", "corroborated"]); + const oneDef = scanOf([caller, defs[5]!]); + const unique = buildCallerIndex(oneDef, new Set(), { recall: true }).get("C")!; + expect(unique.def.file).toBe("tests/cases/c5.ts"); + expect(unique.callers.map((c) => c.confidence)).toEqual(["unique-name", "unique-name"]); + }); +}); From a79e1ef9fc604310de19fe959c3d63d50cc419a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:22:06 +0000 Subject: [PATCH 002/130] fix(resolve): fix Python roots and TS, Go and doc-link resolution gaps Python: only the dirs holding top-level packages (plus the repo root and pyproject/setup dirs and their src/) are import roots now. Treating every package dir as a root bound stdlib `import typing` to src/flask/typing.py, sent `import flask` to a 4-line test fixture and never reached src-layout packages; on django it also made resolution ~40x slower. Absolute imports try the importer's enclosing roots first and are memoized per directory. TypeScript/JS: a baseUrl declared without paths now resolves bare names (tried after paths, only a hit counts); paths follow tsc precedence (exact alias, then longest prefix) instead of object order; `extends` can load a config from an in-repo workspace package; `${configDir}` is substituted with the extending config's dir; package.json `imports` (#subpath) resolve in the nearest package.json; bundler `?query` suffixes are stripped before probing. The asset-extension short-circuit is JS-family only, so Python `.map`, Java `Map` and C# `*.Svg` resolve. A Go package's representative file is its first non-test file. Markdown links starting with "/" resolve from the repo root. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 11 +- src/resolve.ts | 419 +++++++++++++++++++++++++++++++----------- tests/resolve.test.ts | 318 ++++++++++++++++++++++++++++++++ 3 files changed, 637 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 17af6f6..f5029b7 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,15 @@ compares](#how-it-compares). interface members, class fields, enum members, every `declare`/`.d.ts` declaration, Rust trait method signatures, Go interface method sets, record components and constructor `val` parameters. -- **Resolve imports** across languages: tsconfig paths, package `exports`, - go.mod, Cargo, Java packages, PSR-4, C# namespaces. +- **Resolve imports** across languages: tsconfig `paths` (tsc's precedence: + exact alias, then longest prefix) and `baseUrl`, `extends` chains into + workspace packages and `${configDir}`, package `exports` and `imports` + (`#subpath`), bundler `?query` suffixes, Python import roots found the way + mypy finds them (the dir holding each top-level package, so src layouts + resolve and a package's own `typing.py` does not shadow the stdlib), go.mod + (a package's representative file is never a `_test.go` when it has other + files), Cargo, Java packages, PSR-4, C# namespaces. A markdown link starting + with `/` is repo-root-relative, as GitHub renders it. - **Build a typed link-graph**: `import` / `call` / `extends` / `implements` / `use` / `doc-link` / `mention` edges at file and module level, plus Louvain communities, PageRank/betweenness centrality, a tests→code map, and diff --git a/src/resolve.ts b/src/resolve.ts index 49023eb..6e98e55 100644 --- a/src/resolve.ts +++ b/src/resolve.ts @@ -24,7 +24,11 @@ interface TsPath { interface TsConfigScope { dir: string; // the config's own directory (posix, "" = repo root) — scope test baseUrl: string; // repo-relative posix dir the `targets` resolve against - paths: TsPath[]; + // True only when the config (or its extends chain) DECLARES baseUrl: tsc then + // also resolves a bare name (`components/Card`) against it. `paths` without + // baseUrl (TS 4.1+) anchors the targets but gives bare names no such base. + baseUrlSet: boolean; + paths: TsPath[]; // tsc precedence order: exact aliases, then longest prefix } // One subpath of a package.json `exports` map, conditions already flattened into @@ -40,6 +44,14 @@ interface WorkspacePackage { dir: string; // posix dir of its package.json, "" for root exportEntries: ExportEntry[]; // empty when the package declares no `exports` mainCandidates: string[]; // source/main/module/types fields, priority order + tsconfig?: string; // the `tsconfig` field — what `"extends": ""` loads +} + +// One package.json's scope for `#subpath` imports. Node resolves `#x` against +// the NEAREST package.json only (named or not), never a further-up one. +interface PackageScope { + dir: string; // posix dir of the package.json, "" for root + importEntries: ExportEntry[]; // its `imports` map; empty when it declares none } interface GoModule { @@ -63,8 +75,9 @@ export interface ResolveContext { goModules: GoModule[]; // every in-repo go.mod, deepest dir first rustCrates: RustCrate[]; // every in-repo Cargo.toml [package], deepest dir first javaRoots: string[]; // dirs that java package paths resolve against - pyRoots: string[]; // posix dirs that are python import roots ("" allowed) + pyRoots: string[]; // python import roots (sys.path entries), shortest first ("" allowed) workspacePackages: WorkspacePackage[]; // monorepo pkg name -> its dir + entry points + packageScopes: PackageScope[]; // every package.json with its `imports` map, deepest dir first cIncludeRoots: string[]; // dirs a C/C++ `#include "x"` resolves against (besides the file's dir) rubyLibRoots: string[]; // dirs a Ruby bare `require` resolves against phpPsr4: { prefix: string; dir: string }[]; // composer PSR-4 namespace prefix -> dir, longest first @@ -76,6 +89,9 @@ export interface ResolveContext { // import the same handful of specifiers over and over. Each hit skips ~20 // normalize+probe rounds. Optional so a hand-built context still works. jsMemo?: Map; + // The same memo for Python, same key: resolvePython also only reads the + // importer's directory (relative base, enclosing roots). + pyMemo?: Map; // Per-directory sorted file lists by extension ("\0"), for the // Go-package / Java-wildcard "first file in the directory" rule. dirFilesMemo?: Map; @@ -221,34 +237,70 @@ function tolerantJsonParse(text: string): unknown { // a package specifier (a node_modules base we don't index) or a missing file. // A bare non-relative target ("base.json", written without the "./" prefix — // tsc accepts it) is probed relative to the config's own directory too; only -// when no such file exists is it treated as a package base (external), so a -// true package extends like "@tsconfig/node18" stays external. -function resolveExtends(fileSet: Set, fromDir: string, ext: string): string | undefined { +// when no such file exists is it treated as a package base, so a true package +// extends like "@tsconfig/node18" stays external. +// +// A package base can still be IN the repo: Turborepo's layout is a +// `packages/typescript-config` workspace named `@repo/typescript-config` that +// apps extend as "@repo/typescript-config/base.json". tsc resolves that like a +// module, so we do too against the workspace packages: `` loads the +// package's `tsconfig` field or its tsconfig.json, `/` loads +// (.json) or /tsconfig.json inside it. +function resolveExtends( + fileSet: Set, + fromDir: string, + ext: string, + packages: WorkspacePackage[], +): string | undefined { const base = norm(posix.join(fromDir, ext)); const cands = ext.endsWith(".json") ? [base] : [base + ".json", posix.join(base, "tsconfig.json")]; for (const c of cands) if (fileSet.has(c)) return c; + if (ext.startsWith(".") || ext.startsWith("/")) return undefined; + for (const pkg of packages) { + let pkgCands: string[]; + if (ext === pkg.name) { + pkgCands = [...(pkg.tsconfig ? [pkg.tsconfig] : []), "tsconfig.json"]; + } else if (ext.startsWith(pkg.name + "/")) { + const sub = ext.slice(pkg.name.length + 1); + pkgCands = sub.endsWith(".json") ? [sub] : [sub + ".json", posix.join(sub, "tsconfig.json")]; + } else continue; + for (const c of pkgCands) { + const p = norm(posix.join(pkg.dir, c)); + if (!p.startsWith("..") && fileSet.has(p)) return p; + } + return undefined; // the longest matching package name owns the specifier + } return undefined; } interface TsEffective { - baseUrl?: string; // as written, relative to baseUrlDir + baseUrl?: string; // as written, relative to baseUrlDir (a `${configDir}` prefix kept verbatim) baseUrlDir: string; // dir of the config that DECLARED baseUrl - paths?: Record; + paths?: Record; // targets as written, `${configDir}` kept verbatim pathsDir: string; // dir of the config that DECLARED paths } -// Read a tsconfig/jsconfig and fold in its `extends` chain (in-repo, relative -// bases only), child overriding base — so a monorepo package whose baseUrl/paths -// live in a shared base (the dominant Nx/Turborepo/lerna layout) still -// contributes its aliases. baseUrl and paths are each tracked with the dir of the -// config that DECLARED them (TS resolves them relative to that file). Cycles are -// broken via `seen`; a missing relative base is surfaced as a warning. +// The TS 5.5 `${configDir}` template: a path starting with it is relative to +// the config being COMPILED — the leaf that extends a shared base — not to the +// base that wrote it. That is its whole point: one shared base serving many +// projects. So it is carried verbatim through the extends fold and substituted +// per leaf. +const CONFIG_DIR = "${configDir}"; + +// Read a tsconfig/jsconfig and fold in its `extends` chain (in-repo bases only: +// relative files or workspace packages), child overriding base — so a monorepo +// package whose baseUrl/paths live in a shared base (the dominant +// Nx/Turborepo/lerna layout) still contributes its aliases. baseUrl and paths +// are each tracked with the dir of the config that DECLARED them (TS resolves +// them relative to that file, `${configDir}` aside). Cycles are broken via +// `seen`; a missing relative base is surfaced as a warning. function readTsConfig( root: string, fileSet: Set, rel: string, warnings: string[], seen: Set, + packages: WorkspacePackage[], ): TsEffective | undefined { if (seen.has(rel)) return undefined; seen.add(rel); @@ -264,12 +316,12 @@ function readTsConfig( const exts = cfg.extends === undefined ? [] : Array.isArray(cfg.extends) ? cfg.extends : [cfg.extends]; for (const ext of exts) { if (typeof ext !== "string") continue; - const baseRel = resolveExtends(fileSet, dir, ext); + const baseRel = resolveExtends(fileSet, dir, ext, packages); if (!baseRel) { if (/^\.\.?\//.test(ext)) warnings.push(`${rel} extends "${ext}" which is missing — its path aliases were ignored`); continue; // bare specifiers (node_modules tooling bases) carry no repo paths } - const inherited = readTsConfig(root, fileSet, baseRel, warnings, seen); + const inherited = readTsConfig(root, fileSet, baseRel, warnings, seen, packages); if (inherited?.baseUrl !== undefined) { eff.baseUrl = inherited.baseUrl; eff.baseUrlDir = inherited.baseUrlDir; @@ -280,7 +332,7 @@ function readTsConfig( } } const co = cfg.compilerOptions; - if (co?.baseUrl !== undefined) { + if (typeof co?.baseUrl === "string") { eff.baseUrl = co.baseUrl; eff.baseUrlDir = dir; } @@ -344,6 +396,41 @@ function parseExportEntries(exportsField: unknown): ExportEntry[] { return entries; } +// Parse a package.json `imports` field (`"#internal/*": "./src/lib/*.ts"`) into +// the same ordered entries as `exports` — same value grammar, same precedence; +// only the keys differ (`#…` instead of `./…`). +function parseImportEntries(importsField: unknown): ExportEntry[] { + if (importsField === null || typeof importsField !== "object" || Array.isArray(importsField)) return []; + const entries: ExportEntry[] = []; + for (const [key, value] of Object.entries(importsField)) { + if (!key.startsWith("#")) continue; + const targets: string[] = []; + flattenExportTargets(value, targets); + if (targets.length) entries.push({ key, star: key.includes("*"), targets }); + } + entries.sort((a, b) => Number(a.star) - Number(b.star) || b.key.length - a.key.length || byStr(a.key, b.key)); + return entries; +} + +// The targets of the first entry matching `key` (entries are already in +// precedence order), with a wildcard entry's `*` filled in; undefined when no +// entry matches. Shared by `exports` subpaths and `imports` specifiers. +function matchEntryTargets(entries: ExportEntry[], key: string): string[] | undefined { + for (const entry of entries) { + if (!entry.star) { + if (entry.key === key) return entry.targets; + continue; + } + const starAt = entry.key.indexOf("*"); + const pre = entry.key.slice(0, starAt); + const post = entry.key.slice(starAt + 1); + if (!key.startsWith(pre) || !key.endsWith(post) || key.length < pre.length + post.length) continue; + const fill = key.slice(pre.length, key.length - post.length); + return entry.targets.map((t) => t.replace(/\*/g, fill)); + } + return undefined; +} + // Parse a go.mod's `replace` directives (single-line and block form), keeping // only relative targets that stay inside the repo — those are the directives // that rewire one in-repo module onto another's source tree. @@ -386,12 +473,61 @@ export function buildResolveContext(scan: RepoScan): ResolveContext { } } + // Workspace packages: map each in-repo package.json `name` to its directory so + // bare cross-package imports (`@scope/pkg`) resolve to in-repo source, not + // "external". Longest name first so `@scope/a-b` wins over `@scope/a`. Also + // keep its `exports` map and main-ish fields — modern monorepo packages route + // subpath imports (`@scope/pkg/utils`) through `exports`, and probing those + // declared entry points beats guessing `src/index`. Parsed BEFORE tsconfigs: + // a tsconfig may `extends` a config shipped by one of these packages. + const warnings: string[] = []; + const pkgWarnings: string[] = []; // appended after the tsconfig ones (their historic order) + const workspacePackages: WorkspacePackage[] = []; + const packageScopes: PackageScope[] = []; + for (const rel of fileSet) { + if (rel !== "package.json" && !rel.endsWith("/package.json")) continue; + // Parse with the same JSONC-tolerant path as tsconfig (some package.json carry + // comments/trailing commas); a truly unparseable one is surfaced, not silently + // dropped — losing it erases every cross-package edge for that workspace. + const pkg = tolerantJsonParse(readText(join(scan.root, rel))) as + | { + name?: string; + exports?: unknown; + imports?: unknown; + source?: unknown; + main?: unknown; + module?: unknown; + types?: unknown; + tsconfig?: unknown; + } + | undefined; + if (pkg === undefined) { + pkgWarnings.push(`unparseable ${rel} — skipped for workspace resolution`); + continue; + } + const dir = rel.includes("/") ? posix.dirname(rel) : ""; + // Every package.json bounds a `#subpath` scope, named or not. + packageScopes.push({ dir, importEntries: parseImportEntries(pkg.imports) }); + if (typeof pkg.name !== "string") continue; + const mainCandidates = [pkg.source, pkg.main, pkg.module, pkg.types].filter( + (v): v is string => typeof v === "string", + ); + workspacePackages.push({ + name: pkg.name, + dir, + exportEntries: parseExportEntries(pkg.exports), + mainCandidates, + ...(typeof pkg.tsconfig === "string" ? { tsconfig: pkg.tsconfig } : {}), + }); + } + workspacePackages.sort((a, b) => b.name.length - a.name.length); + packageScopes.sort((a, b) => b.dir.length - a.dir.length || byStr(a.dir, b.dir)); + // tsconfig/jsconfig path aliases. Collect EVERY config, not just the root one // (each monorepo package declares its own baseUrl/paths), and fold in each // config's `extends` chain so aliases declared in a shared base config still // resolve. An import resolves against the nearest enclosing config. Unparseable // or missing configs surface as build warnings rather than vanishing silently. - const warnings: string[] = []; const tsConfigs: TsConfigScope[] = []; for (const rel of fileSet) { const base = rel.slice(rel.lastIndexOf("/") + 1); @@ -402,25 +538,47 @@ export function buildResolveContext(scan: RepoScan): ResolveContext { const isRootBase = rel === "tsconfig.base.json"; if (base !== "tsconfig.json" && base !== "jsconfig.json" && !isRootBase) continue; const dir = rel.includes("/") ? posix.dirname(rel) : ""; - const eff = readTsConfig(scan.root, fileSet, rel, warnings, new Set()); - if (!eff?.paths) continue; // no aliases to contribute + const eff = readTsConfig(scan.root, fileSet, rel, warnings, new Set(), workspacePackages); + // A config contributes when it declares aliases OR a baseUrl: CRA/Next.js + // style `"baseUrl": "src"` with no `paths` is how `import "components/Card"` + // resolves, and dropping it lost every such import without a trace. + if (!eff || (!eff.paths && eff.baseUrl === undefined)) continue; + // `paths` resolve against baseUrl when set (relative to the config that + // declared baseUrl), else relative to the config that declared `paths`. + const baseUrl = + eff.baseUrl === undefined + ? eff.pathsDir + : eff.baseUrl.startsWith(CONFIG_DIR) + ? norm(posix.join(dir, eff.baseUrl.slice(CONFIG_DIR.length))).replace(/^\.$/, "") + : norm(posix.join(eff.baseUrlDir, eff.baseUrl)).replace(/^\.$/, ""); + // A `${configDir}` target is this leaf's dir, re-expressed relative to + // baseUrl so resolveJs keeps joining every target the same way. + const fromBase = posix.relative(baseUrl, dir) || "."; const tsPaths: TsPath[] = []; - for (const [alias, targets] of Object.entries(eff.paths)) { + for (const [alias, targets] of Object.entries(eff.paths ?? {})) { if (!Array.isArray(targets)) continue; const star = alias.endsWith("*"); - tsPaths.push({ prefix: star ? alias.slice(0, -1) : alias, star, targets }); + tsPaths.push({ + prefix: star ? alias.slice(0, -1) : alias, + star, + targets: targets + .filter((t): t is string => typeof t === "string") + .map((t) => (t.startsWith(CONFIG_DIR) ? posix.join(fromBase, t.slice(CONFIG_DIR.length)) : t)), + }); } - if (!tsPaths.length) continue; // only path-alias configs affect resolution - // `paths` resolve against baseUrl when set (relative to the config that - // declared baseUrl), else relative to the config that declared `paths`. - const baseUrl = - eff.baseUrl !== undefined - ? norm(posix.join(eff.baseUrlDir, eff.baseUrl)).replace(/^\.$/, "") - : eff.pathsDir; - tsConfigs.push({ dir, baseUrl, paths: tsPaths }); + // tsc's precedence, not object order: an exact alias beats any pattern, and + // among patterns the longest prefix wins (findBestPatternMatch). First-match + // in declaration order sent "@/components/Button" to "@/*" whenever that + // broader pattern happened to be listed first. + tsPaths.sort( + (a, b) => Number(a.star) - Number(b.star) || b.prefix.length - a.prefix.length || byStr(a.prefix, b.prefix), + ); + if (!tsPaths.length && eff.baseUrl === undefined) continue; // nothing affects resolution + tsConfigs.push({ dir, baseUrl, baseUrlSet: eff.baseUrl !== undefined, paths: tsPaths }); } // Nearest-enclosing first: deepest dir wins; the root ("") is the fallback. tsConfigs.sort((a, b) => b.dir.length - a.dir.length); + warnings.push(...pkgWarnings); // Every go.mod, not just the one nearest the root — multi-module repos (a Go // service beside a Go CLI) are normal. Deepest dir first so the module @@ -466,46 +624,41 @@ export function buildResolveContext(scan: RepoScan): ResolveContext { else if (dir.endsWith("/" + pkgPath)) javaRoots.add(dir.slice(0, -pkgPath.length - 1)); } - // Python roots: dirs containing __init__.py / pyproject.toml / setup.py, plus root. - const pyRoots = new Set([""]); + // Python import roots: the sys.path entries an absolute `import a.b` resolves + // against. A package dir is NOT one — Python 3 has no implicit relative + // imports, and treating `src/flask/` as a root bound the stdlib's + // `import typing` to src/flask/typing.py (and every other stdlib name to a + // same-named package module), while never adding the `src/` that `import + // flask` actually needs. The roots are, as mypy's crawl-up finds them: + // - the repo root; + // - the dir holding a TOP-LEVEL package: a dir with __init__.py whose parent + // has none and sits inside no regular package (`src/` in a src layout, + // `lib/` in ansible's); + // - every pyproject.toml / setup.py / setup.cfg dir, and its `src/` if any + // (src-layout single modules and namespace packages have no __init__.py to + // anchor on). + // Known gap, shared with mypy: a PEP 420 namespace dir (`google/cloud/`, no + // __init__.py) looks like a root too, so a stdlib-named package under it + // (`google/cloud/logging/`) still captures `import logging`. + const pyPkgDirs = new Set(); + const pyRootSet = new Set([""]); for (const rel of fileSet) { - const base = rel.split("/").pop()!; - if (base === "__init__.py" || base === "pyproject.toml" || base === "setup.py") { - pyRoots.add(rel.includes("/") ? posix.dirname(rel) : ""); + const base = rel.slice(rel.lastIndexOf("/") + 1); + const dir = rel.includes("/") ? posix.dirname(rel) : ""; + if (base === "__init__.py" || base === "__init__.pyi") pyPkgDirs.add(dir); + else if (base === "pyproject.toml" || base === "setup.py" || base === "setup.cfg") { + pyRootSet.add(dir); + const src = dir ? dir + "/src" : "src"; + if (dirSet.has(src)) pyRootSet.add(src); } } - - // Workspace packages: map each in-repo package.json `name` to its directory so - // bare cross-package imports (`@scope/pkg`) resolve to in-repo source, not - // "external". Longest name first so `@scope/a-b` wins over `@scope/a`. Also - // keep its `exports` map and main-ish fields — modern monorepo packages route - // subpath imports (`@scope/pkg/utils`) through `exports`, and probing those - // declared entry points beats guessing `src/index`. - const workspacePackages: WorkspacePackage[] = []; - for (const rel of fileSet) { - if (rel !== "package.json" && !rel.endsWith("/package.json")) continue; - // Parse with the same JSONC-tolerant path as tsconfig (some package.json carry - // comments/trailing commas); a truly unparseable one is surfaced, not silently - // dropped — losing it erases every cross-package edge for that workspace. - const pkg = tolerantJsonParse(readText(join(scan.root, rel))) as - | { name?: string; exports?: unknown; source?: unknown; main?: unknown; module?: unknown; types?: unknown } - | undefined; - if (pkg === undefined) { - warnings.push(`unparseable ${rel} — skipped for workspace resolution`); - continue; - } - if (typeof pkg.name !== "string") continue; - const mainCandidates = [pkg.source, pkg.main, pkg.module, pkg.types].filter( - (v): v is string => typeof v === "string", - ); - workspacePackages.push({ - name: pkg.name, - dir: rel.includes("/") ? posix.dirname(rel) : "", - exportEntries: parseExportEntries(pkg.exports), - mainCandidates, - }); + for (const pkgDir of pyPkgDirs) { + if (!pkgDir) continue; // a package AT the repo root: nothing above it to hold it + const parent = pkgDir.includes("/") ? posix.dirname(pkgDir) : ""; + let inPackage = false; + for (let d = parent; d && !inPackage; d = d.includes("/") ? posix.dirname(d) : "") inPackage = pyPkgDirs.has(d); + if (!inPackage) pyRootSet.add(parent); } - workspacePackages.sort((a, b) => b.name.length - a.name.length); // C/C++ include roots: dirs literally named include/inc, plus the repo root, so // `#include "a/b.h"` resolves whether written relative to the file or to a root. @@ -564,8 +717,9 @@ export function buildResolveContext(scan: RepoScan): ResolveContext { goModules, rustCrates, javaRoots: [...javaRoots].sort(byLen), - pyRoots: [...pyRoots], + pyRoots: [...pyRootSet].sort(byLen), workspacePackages, + packageScopes, cIncludeRoots: [...cIncludeRoots].sort(byLen), rubyLibRoots: [...rubyLibRoots].sort(byLen), phpPsr4, @@ -588,8 +742,12 @@ export function resolveDocLink(fromRel: string, spec: string, ctx: ResolveContex let target = spec.split("#")[0]!.split("?")[0]!; if (!target) return { kind: "external" }; // pure in-page anchor if (target.startsWith("//") || /^[a-z][a-z0-9+.-]*:/i.test(target)) return { kind: "external" }; - const base = fromRel.includes("/") ? posix.dirname(fromRel) : ""; - const p = norm(posix.join(base, target)); + // A leading "/" is repo-root-relative — how GitHub/GitLab render + // `[bench](/BENCHMARKS.md)` — not relative to the linking file's dir. + const rooted = target.startsWith("/"); + const base = rooted || !fromRel.includes("/") ? "" : posix.dirname(fromRel); + let p = norm(posix.join(base, rooted ? target.slice(1) : target)); + if (p === ".") p = ""; // the repo root itself (`/`, or `./` from a root doc) if (p.startsWith("..")) return { kind: "dangling", reason: "escapes-repo-root" }; const hit = firstExisting(ctx, [ p, p + ".md", p + ".mdx", @@ -599,11 +757,11 @@ export function resolveDocLink(fromRel: string, spec: string, ctx: ResolveContex if (hit) return { kind: "resolved", target: hit }; // A link to a real directory (even one without a README/index) is valid — it's // just not a file-node edge. Don't cry "broken link". - if (ctx.dirSet.has(p)) return { kind: "external" }; + if (!p || ctx.dirSet.has(p)) return { kind: "external" }; return { kind: "dangling", reason: "missing-target" }; } -function resolveJs(fromRel: string, spec: string, ctx: ResolveContext): Resolution { +function resolveJs(fromRel: string, rawSpec: string, ctx: ResolveContext): Resolution { const probe = (p: string): string | undefined => firstExisting(ctx, [...JS_EXT_PROBES.map((e) => p + e), ...JS_INDEX.map((i) => posix.join(p, i))]); // TS/NodeNext style writes `import "./x.js"` for a source file `x.ts` — so if a @@ -614,6 +772,23 @@ function resolveJs(fromRel: string, spec: string, ctx: ResolveContext): Resoluti const noJs = p.replace(/\.(js|jsx|mjs|cjs)$/, ""); return noJs !== p ? probe(noJs) : undefined; }; + // Probe a package-dir-relative entry-point path, then dist→src remaps of it: + // exports/imports maps usually point at compiled output (`./dist/esm/index.js`) + // while only the source tree is committed — peel build dirs and retry under `src/`. + const probeEntry = (pkgDir: string, entry: string): string | undefined => { + for (const cand of [entry, ...distToSrcCandidates(entry)]) { + const hit = tryResolve(norm(posix.join(pkgDir, cand))); + if (hit) return hit; + } + return undefined; + }; + + // A bundler resource query (Vite `?worker`/`?inline`/`?url`/`?raw`, webpack + // `?raw`) picks HOW a file loads, not WHICH file: probe without it. The raw + // specifier still names a dangling edge (graph.ts writes `ref.spec`). + const q = rawSpec.indexOf("?"); + const spec = q === -1 ? rawSpec : rawSpec.slice(0, q); + if (!spec) return { kind: "external" }; if (spec.startsWith(".")) { const base = fromRel.includes("/") ? posix.dirname(fromRel) : ""; @@ -625,8 +800,10 @@ function resolveJs(fromRel: string, spec: string, ctx: ResolveContext): Resoluti // tsconfig path aliases (e.g. "@/x" -> "src/x"), nearest enclosing config first. let aliasFallback: Resolution | undefined; + let bareBase: string | undefined; // baseUrl of the nearest in-scope config declaring one for (const cfg of ctx.tsConfigs) { if (cfg.dir && fromRel !== cfg.dir && !fromRel.startsWith(cfg.dir + "/")) continue; // out of scope + if (bareBase === undefined && cfg.baseUrlSet) bareBase = cfg.baseUrl; let matched = false; for (const tp of cfg.paths) { if (!(tp.star ? spec.startsWith(tp.prefix) : spec === tp.prefix)) continue; @@ -652,42 +829,44 @@ function resolveJs(fromRel: string, spec: string, ctx: ResolveContext): Resoluti if (matched) break; // the nearest matching config wins; stop scanning broader ones } + // baseUrl: tsc resolves any bare name against it after `paths` — even after a + // pattern matched and missed. Only a hit counts: third-party names come + // through here too, and a miss must stay external, never dangling. A rooted + // "/x" is not a bare name (tsc never applies baseUrl to it). + if (bareBase !== undefined && !spec.startsWith("/")) { + const hit = tryResolve(norm(posix.join(bareBase, spec))); + if (hit) return { kind: "resolved", target: hit }; + } + + // package.json `imports` (`#internal/x`): matched in the nearest package.json + // only — Node never consults a further-up one — with the same target + // flattening and dist→src probing as `exports`. A bare-package target, or one + // resolving nowhere, stays external: a '#' name is never a workspace package. + if (spec.startsWith("#")) { + const scope = ctx.packageScopes.find((s) => !s.dir || fromRel.startsWith(s.dir + "/")); + const targets = scope ? matchEntryTargets(scope.importEntries, spec) : undefined; + for (const t of targets ?? []) { + const hit = scope && t.startsWith("./") ? probeEntry(scope.dir, t) : undefined; + if (hit) return { kind: "resolved", target: hit }; + } + return aliasFallback ?? { kind: "external" }; + } + // Monorepo workspace package: resolve `@scope/pkg`(`/subpath`) to in-repo source. for (const pkg of ctx.workspacePackages) { if (spec !== pkg.name && !spec.startsWith(pkg.name + "/")) continue; const sub = spec.slice(pkg.name.length).replace(/^\//, ""); - // Probe a pkg-relative entry-point path, then dist→src remaps of it: exports - // maps usually point at compiled output (`./dist/esm/index.js`) while only - // the source tree is committed — peel build dirs and retry under `src/`. - const probeEntry = (entry: string): string | undefined => { - for (const cand of [entry, ...distToSrcCandidates(entry)]) { - const hit = tryResolve(norm(posix.join(pkg.dir, cand))); - if (hit) return hit; - } - return undefined; - }; // 1) The declared `exports` map — first matching key wins (Node precedence: // exact before wildcard, longest first — already sorted at parse time). - const subKey = sub ? "./" + sub : "."; - for (const entry of pkg.exportEntries) { - let fill: string | undefined; - if (entry.star) { - const starAt = entry.key.indexOf("*"); - const pre = entry.key.slice(0, starAt); - const post = entry.key.slice(starAt + 1); - if (!subKey.startsWith(pre) || !subKey.endsWith(post) || subKey.length < pre.length + post.length) continue; - fill = subKey.slice(pre.length, subKey.length - post.length); - } else if (entry.key !== subKey) continue; - for (const t of entry.targets) { - const hit = probeEntry(fill === undefined ? t : t.replace(/\*/g, fill)); - if (hit) return { kind: "resolved", target: hit }; - } - break; // the matching key resolved nowhere — fall through to the heuristics + // A matching key that resolves nowhere falls through to the heuristics. + for (const t of matchEntryTargets(pkg.exportEntries, sub ? "./" + sub : ".") ?? []) { + const hit = probeEntry(pkg.dir, t); + if (hit) return { kind: "resolved", target: hit }; } // 2) Declared main-ish fields for the bare specifier. if (!sub) { for (const m of pkg.mainCandidates) { - const hit = probeEntry(m); + const hit = probeEntry(pkg.dir, m); if (hit) return { kind: "resolved", target: hit }; } } @@ -727,9 +906,17 @@ function resolvePython(fromRel: string, spec: string, ctx: ResolveContext): Reso return hit ? { kind: "resolved", target: hit } : { kind: "dangling", reason: "missing-module" }; } - // Absolute import: only an edge if it resolves inside the repo (same-package); - // otherwise it's a third-party/stdlib import — external, not dangling. + // Absolute import: only an edge if it resolves inside the repo; otherwise it + // is a third-party/stdlib import — external, not dangling. Roots enclosing the + // importer go first, nearest first, so in a multi-project repo a file's own + // project wins a top-level name two projects share; then every other root, + // shortest first (pyRoots' order). + const enclosing: string[] = []; + const others: string[] = []; for (const root of ctx.pyRoots) { + (!root || fromRel.startsWith(root + "/") ? enclosing : others).push(root); + } + for (const root of [...enclosing.reverse(), ...others]) { const hit = probeModule(root, spec); if (hit) return { kind: "resolved", target: hit }; } @@ -739,13 +926,15 @@ function resolvePython(fromRel: string, spec: string, ctx: ResolveContext): Reso function resolveGo(fromRel: string, spec: string, ctx: ResolveContext): Resolution { if (!ctx.goModules.length) return { kind: "external" }; // Go imports a package (directory); resolve to the lexicographically-first - // .go file in that dir as the representative node. + // NON-test .go file in that dir as the representative node. A `_test.go` file + // is not part of the package an importer links against — picking one (it + // only has to sort first: `api_test.go` < `handler.go`) made production code + // look like it depends on a test. A test-only dir falls back to its first file. const probePkg = (dir: string): Resolution => { const d = norm(dir).replace(/^\.$/, ""); const inDir = filesInDir(ctx, d, ".go"); - return inDir.length - ? { kind: "resolved", target: inDir[0]! } - : { kind: "dangling", reason: "missing-package" }; + const rep = inDir.find((f) => !f.endsWith("_test.go")) ?? inDir[0]; + return rep ? { kind: "resolved", target: rep } : { kind: "dangling", reason: "missing-package" }; }; // The importing file's own module (nearest enclosing; goModules is deepest-first). const home = ctx.goModules.find((g) => !g.dir || fromRel === g.dir || fromRel.startsWith(g.dir + "/")); @@ -949,13 +1138,15 @@ export function resolveImport( spec: string, ctx: ResolveContext, ): Resolution { - // Asset imports (`import logo from './x.svg'`) target files walk() skips on - // purpose — a bundler dependency, not a broken code edge. - const dot = spec.lastIndexOf("."); - if (dot !== -1 && ASSET_EXT.has(spec.slice(dot).toLowerCase().replace(/[?#].*$/, ""))) { - return { kind: "external" }; - } if (JS_TS.has(ext) || SFC_HTML.has(ext)) { + // Asset imports (`import logo from './x.svg'`) target files walk() skips on + // purpose — a bundler dependency, not a broken code edge. JS-family only: + // elsewhere the text after the last dot is a name, not an extension — + // Python `from .map import f`, Java `import com.acme.Map`, C# `using X.Svg`. + const dot = spec.lastIndexOf("."); + if (dot !== -1 && ASSET_EXT.has(spec.slice(dot).toLowerCase().replace(/[?#].*$/, ""))) { + return { kind: "external" }; + } const dir = fromRel.includes("/") ? posix.dirname(fromRel) : ""; const key = dir + "\0" + spec; const memo = (ctx.jsMemo ??= new Map()); @@ -965,7 +1156,17 @@ export function resolveImport( // let one consumer's edit reach every later import of the same spec. return { ...r }; } - if (PY.has(ext)) return resolvePython(fromRel, spec, ctx); + if (PY.has(ext)) { + // Memoized like JS (a package's modules repeat the same imports), which + // also keeps a big repo's absolute stdlib imports from re-probing every + // root per file. + const dir = fromRel.includes("/") ? posix.dirname(fromRel) : ""; + const key = dir + "\0" + spec; + const memo = (ctx.pyMemo ??= new Map()); + let r = memo.get(key); + if (!r) memo.set(key, (r = resolvePython(fromRel, spec, ctx))); + return { ...r }; + } if (ext === ".go") return resolveGo(fromRel, spec, ctx); if (ext === ".rs") return resolveRust(fromRel, spec, ctx); if (ext === ".java") return resolveJava(spec, ctx); diff --git a/tests/resolve.test.ts b/tests/resolve.test.ts index 5099fc5..99a6378 100644 --- a/tests/resolve.test.ts +++ b/tests/resolve.test.ts @@ -241,3 +241,321 @@ describe("resolveImport — Go", () => { expect(resolveImport("gopkg/main.go", ".go", "fmt", c).kind).toBe("external"); }); }); + +describe("resolveImport — Python import roots", () => { + // src layout, a module named like a stdlib one inside the package, and a + // 4-line test fixture that happens to be called flask.py (flask's own tree). + const c = scratchCtx({ + "pyproject.toml": "[project]\nname = \"flask\"\n", + "src/flask/__init__.py": "from .app import Flask\n", + "src/flask/app.py": "import typing as t\nfrom .typing import RouteCallable\n", + "src/flask/typing.py": "RouteCallable = object\n", + "src/flask/json/__init__.py": "import json\n", + "tests/test_basic.py": "import flask\nimport cliapp\n", + "tests/test_apps/cliapp/__init__.py": "", + "tests/test_apps/cliapp/inner1/__init__.py": "", + "tests/test_apps/cliapp/inner1/inner2/__init__.py": "", + "tests/test_apps/cliapp/inner1/inner2/flask.py": "app = None\n", + }); + it("uses the dirs holding top-level packages as roots, never a package dir itself", () => { + expect(c.pyRoots).toEqual(["", "src", "tests/test_apps"]); + }); + it("resolves `import flask` to the src-layout package, not a same-named test fixture", () => { + expect(resolveImport("tests/test_basic.py", ".py", "flask", c)).toEqual({ + kind: "resolved", + target: "src/flask/__init__.py", + }); + expect(resolveImport("src/flask/app.py", ".py", "flask.json", c)).toEqual({ + kind: "resolved", + target: "src/flask/json/__init__.py", + }); + }); + it("keeps stdlib imports external even when the package has a same-named module", () => { + expect(resolveImport("src/flask/app.py", ".py", "typing", c).kind).toBe("external"); + expect(resolveImport("src/flask/json/__init__.py", ".py", "json", c).kind).toBe("external"); + // …while the explicit relative import still binds the package module. + expect(resolveImport("src/flask/app.py", ".py", ".typing", c)).toEqual({ + kind: "resolved", + target: "src/flask/typing.py", + }); + }); + it("reaches a top-level fixture package through its parent root", () => { + expect(resolveImport("tests/test_basic.py", ".py", "cliapp", c)).toEqual({ + kind: "resolved", + target: "tests/test_apps/cliapp/__init__.py", + }); + }); + it("prefers the importer's own project when two projects share a top-level name", () => { + const m = scratchCtx({ + "a/pyproject.toml": "", + "a/src/utils/__init__.py": "", + "a/src/app.py": "import utils\n", + "b/pyproject.toml": "", + "b/src/utils/__init__.py": "", + "b/src/app.py": "import utils\n", + }); + for (const p of ["a", "b"]) { + expect(resolveImport(`${p}/src/app.py`, ".py", "utils", m)).toEqual({ + kind: "resolved", + target: `${p}/src/utils/__init__.py`, + }); + } + }); + it("adds no root inside a regular package, even across a namespace gap", () => { + const m = scratchCtx({ + "pkg/__init__.py": "", + "pkg/data/sub/__init__.py": "", + "pkg/data/sub/json/__init__.py": "", + "pkg/core.py": "import json\n", + }); + expect(m.pyRoots).toEqual([""]); + expect(resolveImport("pkg/core.py", ".py", "json", m).kind).toBe("external"); + expect(resolveImport("pkg/core.py", ".py", "pkg.data.sub", m)).toEqual({ + kind: "resolved", + target: "pkg/data/sub/__init__.py", + }); + }); +}); + +describe("resolveImport — asset extensions are a JS-family rule only", () => { + const c = scratchCtx({ + "src/mypkg/__init__.py": "", + "src/mypkg/map.py": "def mapit(): pass\n", + "src/mypkg/pdf/__init__.py": "", + "src/mypkg/core.py": "from .map import mapit\nfrom .pdf import render\n", + "java/com/acme/core/Map.java": "package com.acme.core;\npublic class Map {}\n", + "java/com/acme/core/App.java": "package com.acme.core;\nimport com.acme.core.Map;\npublic class App {}\n", + "cs/Models/Svg.cs": "namespace Acme.Models.Svg;\npublic class Icon {}\n", + "cs/Program.cs": "using Acme.Models.Svg;\nnamespace Acme;\n", + "web/main.ts": 'import logo from "./logo.svg";\n', + }); + it("resolves Python modules named map/pdf", () => { + expect(resolveImport("src/mypkg/core.py", ".py", ".map", c)).toEqual({ + kind: "resolved", + target: "src/mypkg/map.py", + }); + expect(resolveImport("src/mypkg/core.py", ".py", ".pdf", c)).toEqual({ + kind: "resolved", + target: "src/mypkg/pdf/__init__.py", + }); + }); + it("resolves a Java class Map and a C# namespace ending in .Svg", () => { + expect(resolveImport("java/com/acme/core/App.java", ".java", "com.acme.core.Map", c)).toEqual({ + kind: "resolved", + target: "java/com/acme/core/Map.java", + }); + expect(resolveImport("cs/Program.cs", ".cs", "Acme.Models.Svg", c)).toEqual({ + kind: "resolved", + target: "cs/Models/Svg.cs", + }); + }); + it("still treats a JS asset import as external", () => { + expect(resolveImport("web/main.ts", ".ts", "./logo.svg", c).kind).toBe("external"); + }); +}); + +describe("resolveImport — tsconfig baseUrl and paths precedence", () => { + it("resolves a bare name against a baseUrl declared without paths", () => { + const c = scratchCtx({ + "web/tsconfig.json": '{ "compilerOptions": { "baseUrl": "src" } }', + "web/src/page.ts": 'import { Card } from "components/Card";', + "web/src/components/Card.ts": "export const Card = 1;", + }); + expect(resolveImport("web/src/page.ts", ".ts", "components/Card", c)).toEqual({ + kind: "resolved", + target: "web/src/components/Card.ts", + }); + // Only a hit counts: a third-party name stays external, never dangling. + expect(resolveImport("web/src/page.ts", ".ts", "react", c).kind).toBe("external"); + }); + it("tries baseUrl after a paths pattern that matched and missed", () => { + const c = scratchCtx({ + "tsconfig.json": '{ "compilerOptions": { "baseUrl": ".", "paths": { "lib/*": ["vendor/lib/*"] } } }', + "src/main.ts": 'import "src/lib/helpers";', + "src/lib/helpers.ts": "export {};", + "lib/only-here.ts": "export {};", + }); + expect(resolveImport("src/main.ts", ".ts", "src/lib/helpers", c)).toEqual({ + kind: "resolved", + target: "src/lib/helpers.ts", + }); + expect(resolveImport("src/main.ts", ".ts", "lib/only-here", c)).toEqual({ + kind: "resolved", + target: "lib/only-here.ts", + }); + }); + it("gives bare names no base when only paths is declared", () => { + const c = scratchCtx({ + "tsconfig.json": '{ "compilerOptions": { "paths": { "@/*": ["./src/*"] } } }', + "src/main.ts": 'import "components/Card";', + "components/Card.ts": "export {};", + }); + expect(resolveImport("src/main.ts", ".ts", "components/Card", c).kind).toBe("external"); + }); + it("picks the longest matching prefix, and an exact alias before any pattern", () => { + const c = scratchCtx({ + "tsconfig.json": JSON.stringify({ + compilerOptions: { + baseUrl: ".", + paths: { + "@/*": ["src/*"], + "@/components/*": ["src/ui/components/*"], + "@/components/Button": ["src/exact/Button"], + }, + }, + }), + "src/components/Button.ts": "export {};", + "src/components/Input.ts": "export {};", + "src/ui/components/Button.ts": "export {};", + "src/ui/components/Input.ts": "export {};", + "src/exact/Button.ts": "export {};", + "src/main.ts": "export {};", + }); + expect(resolveImport("src/main.ts", ".ts", "@/components/Input", c)).toEqual({ + kind: "resolved", + target: "src/ui/components/Input.ts", + }); + expect(resolveImport("src/main.ts", ".ts", "@/components/Button", c)).toEqual({ + kind: "resolved", + target: "src/exact/Button.ts", + }); + }); +}); + +describe("tsconfig extends — workspace packages and ${configDir}", () => { + it("follows extends into an in-repo workspace package (Turborepo layout)", () => { + const c = scratchCtx({ + "packages/tsconfig/package.json": '{ "name": "@repo/tsconfig" }', + "packages/tsconfig/base.json": '{ "compilerOptions": { "paths": { "@shared/*": ["src/lib/*"] } } }', + "packages/tsconfig/src/lib/helpers.ts": "export const x = 1;", + "packages/ui/package.json": '{ "name": "@repo/ui", "tsconfig": "./tsconfig.lib.json" }', + "packages/ui/tsconfig.lib.json": '{ "compilerOptions": { "baseUrl": ".", "paths": { "@ui/*": ["src/*"] } } }', + "packages/ui/src/button.ts": "export {};", + "app/tsconfig.json": '{ "extends": "@repo/tsconfig/base.json" }', + "app/main.ts": 'import { x } from "@shared/helpers";', + "site/tsconfig.json": '{ "extends": "@repo/ui" }', + "site/main.ts": 'import "@ui/button";', + }); + expect(resolveImport("app/main.ts", ".ts", "@shared/helpers", c)).toEqual({ + kind: "resolved", + target: "packages/tsconfig/src/lib/helpers.ts", + }); + expect(resolveImport("site/main.ts", ".ts", "@ui/button", c)).toEqual({ + kind: "resolved", + target: "packages/ui/src/button.ts", + }); + expect(c.warnings).toEqual([]); + }); + it("substitutes ${configDir} with the extending config's dir", () => { + const c = scratchCtx({ + "config/tsconfig.shared.json": '{ "compilerOptions": { "paths": { "~/*": ["${configDir}/src/*"] } } }', + "apps/web/tsconfig.json": '{ "extends": "../../config/tsconfig.shared.json" }', + "apps/web/src/lib/a.ts": "export {};", + "apps/web/src/b.ts": 'import "~/lib/a";', + "apps/api/tsconfig.json": + '{ "extends": "../../config/tsconfig.shared.json", "compilerOptions": { "baseUrl": "${configDir}/src" } }', + "apps/api/src/lib/a.ts": "export {};", + "apps/api/src/b.ts": 'import "lib/a";', + }); + expect(resolveImport("apps/web/src/b.ts", ".ts", "~/lib/a", c)).toEqual({ + kind: "resolved", + target: "apps/web/src/lib/a.ts", + }); + expect(resolveImport("apps/api/src/b.ts", ".ts", "~/lib/a", c)).toEqual({ + kind: "resolved", + target: "apps/api/src/lib/a.ts", + }); + expect(resolveImport("apps/api/src/b.ts", ".ts", "lib/a", c)).toEqual({ + kind: "resolved", + target: "apps/api/src/lib/a.ts", + }); + }); +}); + +describe("resolveImport — package.json imports and resource queries", () => { + const c = scratchCtx({ + "package.json": JSON.stringify({ + name: "app", + imports: { + "#internal/*": "./src/lib/*.ts", + "#config": { node: "./src/lib/config.ts" }, + "#dist": "./dist/entry.js", + }, + }), + "src/lib/helpers.ts": "export {};", + "src/lib/config.ts": "export {};", + "src/entry.ts": "export {};", + "src/main.ts": "export {};", + "src/workers/heavy.ts": "export {};", + "src/styles/app.css": "a {}", + "nested/package.json": '{ "name": "nested" }', + "nested/index.ts": 'import "#config";', + }); + it("resolves #subpath imports through the nearest package.json", () => { + expect(resolveImport("src/main.ts", ".ts", "#internal/helpers", c)).toEqual({ + kind: "resolved", + target: "src/lib/helpers.ts", + }); + expect(resolveImport("src/main.ts", ".ts", "#config", c)).toEqual({ + kind: "resolved", + target: "src/lib/config.ts", + }); + expect(resolveImport("src/main.ts", ".ts", "#dist", c)).toEqual({ kind: "resolved", target: "src/entry.ts" }); + expect(resolveImport("src/main.ts", ".ts", "#nope", c).kind).toBe("external"); + // Node never looks past the nearest package.json, which declares no imports. + expect(resolveImport("nested/index.ts", ".ts", "#config", c).kind).toBe("external"); + }); + it("strips a bundler resource query before probing, keeping missing files dangling", () => { + expect(resolveImport("src/main.ts", ".ts", "./workers/heavy.ts?worker", c)).toEqual({ + kind: "resolved", + target: "src/workers/heavy.ts", + }); + expect(resolveImport("src/main.ts", ".ts", "./styles/app.css?inline", c)).toEqual({ + kind: "resolved", + target: "src/styles/app.css", + }); + expect(resolveImport("src/main.ts", ".ts", "./workers/gone.ts?worker", c)).toEqual({ + kind: "dangling", + reason: "missing-module", + }); + }); +}); + +describe("resolveImport — Go package representative", () => { + const c = scratchCtx({ + "go.mod": "module example.com/svc\n", + "pkg/x/api_test.go": "package x\n", + "pkg/x/handler.go": "package x\n", + "pkg/x/x.go": "package x\n", + "pkg/onlytests/a_test.go": "package onlytests\n", + "main.go": "package main\n", + }); + it("never picks a _test.go file when the package has other files", () => { + expect(resolveImport("main.go", ".go", "example.com/svc/pkg/x", c)).toEqual({ + kind: "resolved", + target: "pkg/x/handler.go", + }); + }); + it("falls back to a test file for a test-only directory", () => { + expect(resolveImport("main.go", ".go", "example.com/svc/pkg/onlytests", c)).toEqual({ + kind: "resolved", + target: "pkg/onlytests/a_test.go", + }); + }); +}); + +describe("resolveDocLink — repo-root-relative links", () => { + const c = scratchCtx({ + "README.md": "[bench](/BENCHMARKS.md)\n", + "BENCHMARKS.md": "# bench\n", + "docs/doc.md": "[home](/README.md) [gone](/NOPE.md) [root](/)\n", + }); + it("resolves a leading-slash link against the repo root", () => { + expect(resolveDocLink("README.md", "/BENCHMARKS.md", c)).toEqual({ kind: "resolved", target: "BENCHMARKS.md" }); + expect(resolveDocLink("docs/doc.md", "/README.md", c)).toEqual({ kind: "resolved", target: "README.md" }); + expect(resolveDocLink("docs/doc.md", "/", c)).toEqual({ kind: "resolved", target: "README.md" }); + }); + it("still flags a missing root-relative target as dangling", () => { + expect(resolveDocLink("docs/doc.md", "/NOPE.md", c)).toEqual({ kind: "dangling", reason: "missing-target" }); + }); +}); From cbf64acead37044b47b133d94a39282a3efad9cf Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:22:28 +0000 Subject: [PATCH 003/130] feat(graph): resolve soft refs only when they land on an in-repo file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RawRef gains an optional `soft: true` for speculative refs — Python `from pkg import name`, where `name` may be a submodule or just an attribute of pkg. buildGraph resolves a file's soft refs after its firm ones and turns one into an import edge only when it resolves to another in-repo file the file does not already link to: a miss is dropped silently (no external, no dangling edge) and a duplicate adds no weight. Records without soft refs produce byte-identical graphs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/derived.ts | 2 ++ src/graph.ts | 20 +++++++++++++++- src/types.ts | 2 ++ tests/graph.test.ts | 56 ++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/derived.ts b/src/derived.ts index 9818e8c..981301e 100644 --- a/src/derived.ts +++ b/src/derived.ts @@ -114,6 +114,8 @@ export function importPairsFor(scan: RepoScan): Set { for (const f of scan.files) { for (const ref of f.refs) { if (ref.kind !== "import") continue; + // No `soft` special case needed: a soft ref adds a pair exactly when + // buildGraph gives it an edge — resolved to another in-repo file. const r = resolveImport(f.rel, f.ext, ref.spec, ctx); if (r.kind === "resolved" && r.target !== f.rel) pairs.add(`${f.rel}|${r.target}`); } diff --git a/src/graph.ts b/src/graph.ts index 5390a00..4114684 100644 --- a/src/graph.ts +++ b/src/graph.ts @@ -1,5 +1,5 @@ import { ENGINE_VERSION, SCHEMA_VERSION } from "./types.js"; -import type { Edge, FileNode, Graph, ModuleNode } from "./types.js"; +import type { Edge, FileNode, Graph, ModuleNode, RawRef } from "./types.js"; import type { RepoScan } from "./scan.js"; import type { ModuleInfo } from "./modules.js"; import { resolveDocLink, resolveImport, type ResolveContext } from "./resolve.js"; @@ -81,7 +81,12 @@ export function buildGraph( // doc-link and import edges from each file's raw refs. for (const f of scan.files) { + let soft: RawRef[] | undefined; for (const ref of f.refs) { + if (ref.soft) { + (soft ??= []).push(ref); + continue; + } if (ref.kind === "doc-link") { const r = resolveDocLink(f.rel, ref.spec, ctx); if (r.kind === "external") continue; @@ -101,6 +106,19 @@ export function buildGraph( } } } + // Soft refs are guesses — Python `from pkg import name` may bind a submodule + // (pkg/name.py) or just an attribute of pkg — so one only counts when it + // lands on an in-repo file this file does not already link to: never an + // external, never a dangling edge, never extra weight on an existing one. + // Resolved after every firm ref of the file, so that verdict does not + // depend on where the extractor put the soft ref in the list. + for (const ref of soft ?? []) { + const r = + ref.kind === "doc-link" ? resolveDocLink(f.rel, ref.spec, ctx) : resolveImport(f.rel, f.ext, ref.spec, ctx); + if (r.kind !== "resolved" || r.target === f.rel || fileEdgeMap.has(keyOf(f.rel, r.target, ref.kind))) continue; + collect(fileEdgeMap, { from: f.rel, to: r.target, kind: ref.kind, weight: 1 }); + if (ref.kind === "import") importPairs.add(`${f.rel}|${r.target}`); + } } // Cross-file call edges: a global second pass over every file's collected call diff --git a/src/types.ts b/src/types.ts index f838986..5bae03b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -126,6 +126,8 @@ export interface CodeSymbol { export interface RawRef { kind: "doc-link" | "import"; spec: string; // the target/specifier exactly as written + /** Speculative ref: an edge when it resolves to an in-repo file, otherwise dropped silently (never external, never dangling). */ + soft?: true; } // A literal VALUE as written, kept verbatim. `terms` destroys exactly this: diff --git a/tests/graph.test.ts b/tests/graph.test.ts index e262c74..e12e2c2 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -7,7 +7,7 @@ import { scanRepo } from "../src/scan.js"; import { buildResolveContext } from "../src/resolve.js"; import { buildModules } from "../src/modules.js"; import { buildGraph } from "../src/graph.js"; -import type { Edge } from "../src/types.js"; +import type { Edge, RawRef } from "../src/types.js"; const REPO = fileURLToPath(new URL("./fixtures/mini-repo", import.meta.url)); @@ -135,3 +135,57 @@ describe("buildGraph", () => { })())).toEqual(graph); }); }); + +describe("buildGraph — soft refs", () => { + // The Python extractor turns `from . import util` into the firm ref "." plus a + // soft ".util" (util may be a submodule or just an attribute). The soft ones + // are injected by hand here so this pins the graph side of that contract. + function softGraph(soft: Record) { + const root = mkdtempSync(join(tmpdir(), "ui-soft-")); + mkdirSync(join(root, "app"), { recursive: true }); + writeFileSync(join(root, "app", "__init__.py"), ""); + writeFileSync(join(root, "app", "util.py"), "def fmt():\n return 1\n"); + writeFileSync(join(root, "app", "core.py"), "class Engine:\n pass\n"); + writeFileSync(join(root, "app", "views.py"), "from . import util\nfrom .core import Engine\n"); + const scan = scanRepo(root); + for (const f of scan.files) { + const extra = soft[f.rel]; + if (extra) f.refs = [...extra, ...f.refs]; // soft BEFORE the firm refs: order must not matter + } + const ctx = buildResolveContext(scan); + const { modules, moduleOf } = buildModules(scan); + return buildGraph(scan, ctx, modules, moduleOf); + } + const edgesFrom = (g: ReturnType, from: string) => + g.fileEdges + .filter((e) => e.from === from && e.kind === "import") + .map((e) => `${e.to}:${e.weight}${e.dangling ? ":dangling" : ""}`); + + it("adds an edge when a soft ref resolves to an in-repo file", () => { + const g = softGraph({ "app/views.py": [{ kind: "import", spec: ".util", soft: true }] }); + expect(edgesFrom(g, "app/views.py")).toEqual(["app/__init__.py:1", "app/core.py:1", "app/util.py:1"]); + }); + + it("drops a soft ref that resolves nowhere — no dangling edge, no change at all", () => { + const plain = softGraph({}); + const g = softGraph({ + "app/views.py": [ + { kind: "import", spec: ".Engine", soft: true }, // a class, not a module: would dangle if firm + { kind: "import", spec: "requests.adapters", soft: true }, // third-party: external if firm + ], + }); + expect(edgesFrom(g, "app/views.py")).toEqual(["app/__init__.py:1", "app/core.py:1"]); + expect(JSON.stringify(g)).toBe(JSON.stringify(plain)); + }); + + it("never adds weight to a target the file already imports, nor counts one twice", () => { + const g = softGraph({ + "app/views.py": [ + { kind: "import", spec: ".core", soft: true }, // the firm `.core` ref already links it + { kind: "import", spec: ".util", soft: true }, + { kind: "import", spec: "app.util", soft: true }, // same file, another spelling + ], + }); + expect(edgesFrom(g, "app/views.py")).toEqual(["app/__init__.py:1", "app/core.py:1", "app/util.py:1"]); + }); +}); From 47807d9f84a1488939f60049b70640fa19ccdf43 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:22:32 +0000 Subject: [PATCH 004/130] fix(symbols): keep symbols named __proto__ in symbols.json defs and refs were plain objects, so `defs["__proto__"] = [...]` hit the prototype setter and every symbol of that name vanished from the artifact. Both maps are now prototype-less; every other name serializes byte-identically. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/render/symbols-json.ts | 7 +++++-- tests/graph.test.ts | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/render/symbols-json.ts b/src/render/symbols-json.ts index 763e60b..1a0e7fb 100644 --- a/src/render/symbols-json.ts +++ b/src/render/symbols-json.ts @@ -70,7 +70,10 @@ export function buildSymbolIndex( } } - const defs: SymbolIndex["defs"] = {}; + // Prototype-less maps: on a plain `{}`, `defs["__proto__"] = [...]` hits the + // prototype setter instead of creating a key, and a symbol of that name + // vanished from the artifact. Serializes byte-identically for every other name. + const defs: SymbolIndex["defs"] = Object.create(null); for (const name of [...defsByName.keys()].sort(byStr)) { defs[name] = defsByName .get(name)! @@ -78,7 +81,7 @@ export function buildSymbolIndex( .sort((a, b) => byStr(a.file, b.file) || a.line - b.line || byStr(a.kind, b.kind)); } - const refsOut: SymbolIndex["refs"] = {}; + const refsOut: SymbolIndex["refs"] = Object.create(null); for (const name of [...refs.keys()].sort(byStr)) { const files = [...refs.get(name)!].sort(byStr); if (files.length) refsOut[name] = files; diff --git a/tests/graph.test.ts b/tests/graph.test.ts index e12e2c2..f4a08e7 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -7,6 +7,7 @@ import { scanRepo } from "../src/scan.js"; import { buildResolveContext } from "../src/resolve.js"; import { buildModules } from "../src/modules.js"; import { buildGraph } from "../src/graph.js"; +import { buildSymbolIndex, renderSymbolsJson } from "../src/render/symbols-json.js"; import type { Edge, RawRef } from "../src/types.js"; const REPO = fileURLToPath(new URL("./fixtures/mini-repo", import.meta.url)); @@ -189,3 +190,17 @@ describe("buildGraph — soft refs", () => { expect(edgesFrom(g, "app/views.py")).toEqual(["app/__init__.py:1", "app/core.py:1", "app/util.py:1"]); }); }); + +describe("buildSymbolIndex", () => { + it("keeps symbols named like Object.prototype members, __proto__ included", () => { + const root = mkdtempSync(join(tmpdir(), "ui-proto-")); + writeFileSync( + join(root, "a.ts"), + "export interface Weird { __proto__: object; constructor: Function; toString(): string }\n" + + "export const __proto__ = 2;\n", + ); + const index = JSON.parse(renderSymbolsJson(buildSymbolIndex(scanRepo(root)))); + expect(Object.keys(index.defs)).toEqual(["Weird", "__proto__", "constructor", "toString"]); + expect(index.defs.__proto__.map((d: { kind: string }) => d.kind).sort()).toEqual(["const", "property"]); + }); +}); From aafde9628c37fb23fe5e43694bceabc51f93f79f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:23:30 +0000 Subject: [PATCH 005/130] feat(workspaces): check declared deps, report resolution health, fix detection gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workspace detection missed valid layouts: pnpm flow-style `packages: [...]` lists, multi-line Gradle `include(...)` (and `projects.x` type-safe accessor edges), nested Maven aggregators, and multi-module Go repos without a go.work. All four now resolve; nested go.mod discovery skips vendor/testdata/fixtures. Manifests are read as JSONC like the resolver reads them, and malformed ones reach the `workspaces` JSON as `warnings` (only when non-empty, so clean output is byte-identical). `workspaces --check` (MCP `check: true`) compares each package's declared sibling dependencies with the link-graph's resolved imports: `undeclared` imports exit 1 as a CI gate, `unusedDeclared` is informational. `codeindex resolution` / MCP `resolution_report` aggregates the resolver pass per language (resolved / external / dangling by reason / unsupported, top dangling specs, top external packages, notes) with the config warnings the resolver collected but no surface ever read; `index` now prints those warnings to stderr as well. Mermaid node ids are injective (a colliding readable id takes `_2`, `_3`…; unique ones are unchanged), and a focus target resolves as a slug, a module directory or a file, failing on anything else instead of printing an empty diagram. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 52 ++++++- src/engine-cli.ts | 67 ++++++--- src/mcp.ts | 14 +- src/mcp/tools.ts | 51 ++++++- src/resolution.ts | 192 ++++++++++++++++++++++++++ src/resolve.ts | 11 +- src/viz.ts | 65 +++++++-- src/workspaces.ts | 291 +++++++++++++++++++++++++++++++++------ tests/mcp-output.test.ts | 3 +- tests/mcp.test.ts | 25 +++- tests/phase2.test.ts | 44 +++++- tests/resolution.test.ts | 129 +++++++++++++++++ tests/workspaces.test.ts | 210 ++++++++++++++++++++++++++++ 13 files changed, 1067 insertions(+), 87 deletions(-) create mode 100644 src/resolution.ts create mode 100644 tests/resolution.test.ts create mode 100644 tests/workspaces.test.ts diff --git a/README.md b/README.md index 17af6f6..0bd16c2 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,9 @@ codeindex implementations Runnable --repo . # who implements it, transitively codeindex callgraph buildGraph --repo . --depth 2 codeindex grep 'pattern' --repo . codeindex literals --repo . # values with no single source of truth +codeindex workspaces --repo . --check # monorepo packages; undeclared sibling imports exit 1 +codeindex resolution --repo . # per-language import resolution health +codeindex mermaid src/app --repo . # module diagram around a module, dir or file ``` ## Values with no single source of truth @@ -426,6 +429,45 @@ a *consumer*, not a source of truth, and is reported as a call site. A lookup table (`export const ROUTES = { … }`) genuinely is one, and is reported as a holder. +## Monorepos and import resolution + +`codeindex workspaces` lists the packages of a monorepo with their declared +dependency graph, one cycle if there is one, and a topological build order. It +reads npm/yarn `workspaces`, `pnpm-workspace.yaml` (block or flow list), lerna, +nx, Cargo `[workspace]`, `go.work` (without one, every nested `go.mod` outside +`vendor`, `testdata` and `fixtures` dirs), Maven `` (recursing into +nested aggregators), uv workspaces, Composer path repositories and Gradle +`include` (multi-line forms too; `project(':x')` and `projects.x` type-safe +accessors become edges). Manifests are read as JSONC, like the resolver reads +them; one that still does not parse is named in `warnings` instead of being +dropped silently. + +`--check` compares what each package declares with what its code imports, +using the link-graph's resolved import edges: + +- `undeclared` — a package imports a sibling its manifest does not list. It + works in a hoisted checkout and breaks the isolated install, the publish or + `go mod tidy`. Any entry makes the command exit 1, a CI gate like `rules`. + Nx members are skipped: Nx infers project dependencies from imports. +- `unusedDeclared` — a declared sibling no import uses (npm, pnpm, lerna, + Cargo and Go only; informational, never fails the check). + +`codeindex resolution` says whether the graph can be trusted for a language +before you rely on `impact`, `callers` or `deadcode` there. Per importer +language it counts imports that `resolved` to an in-repo file, went `external` +(third-party or stdlib, by design no edge), `dangling` (a local target that +does not exist, by reason) and `unsupported` (no resolver for that language), +lists the top dangling specifiers with an example importer and the top +external packages (`--limit`, default 10; `--lang` for one language), notes a +language that yields no import edges at all, and repeats the config `warnings` +(an unparseable `tsconfig.json` or `package.json`, a missing `extends` base) +that silently turn resolvable imports external. `index` prints those warnings +to stderr too. Nothing here changes an artifact. + +`codeindex mermaid [target]` focuses the diagram on a module slug, a module +directory or a file (its module), and fails on anything else rather than +printing an empty diagram. + ## Docker `ghcr.io/maxgfr/codeindex` ships the same zero-dependency bundle (`engine.mjs` @@ -740,13 +782,13 @@ Register it in Claude Code with: claude mcp add codeindex -- codeindex mcp ``` -**33 tools**, grouped by what they answer: +**34 tools**, grouped by what they answer: | group | tools | |---|---| | orient | `scan_summary`, `onboard` *(write)*, `repo_map`, `graph`, `mermaid`, `workspaces` | | find | `search`, `explain_search`, `grep`, `find_symbol`, `symbols`, `symbols_overview` | -| impact | `find_references`, `callers`, `call_graph`, `dead_code` | +| impact | `find_references`, `callers`, `call_graph`, `dead_code`, `resolution_report` | | types | `type_hierarchy`, `implementations` | | risk | `hotspots`, `churn`, `coupling`, `complexity`, `check_rules`, `duplicated_literals` | | edit *(write)* | `replace_symbol_body`, `insert_after_symbol`, `insert_before_symbol` | @@ -828,9 +870,9 @@ introduced are only sent to clients that asked for it, so an older client sees exactly what it saw before. From `2025-03-26` every tool carries behaviour annotations — `readOnlyHint` on -the 27 read tools, `destructiveHint`/`idempotentHint` on the six that write — +the 28 read tools, `destructiveHint`/`idempotentHint` on the six that write — which is what lets a host auto-approve reads and confirm only writes. From -`2025-06-18`, the 20 tools whose result is always a JSON object also declare an +`2025-06-18`, the 21 tools whose result is always a JSON object also declare an `outputSchema` and return `structuredContent`, so a client can validate and type the result instead of re-parsing a string. The remaining tools return arrays, argument-dependent shapes or plain text, which cannot yield a conforming @@ -906,7 +948,7 @@ dates in one table, said out loud rather than implied._ | language coverage | 16 regex extractors, 21 tree-sitter grammars | **~40**, generic parser rules | any language with an LSP server | 36 via tree-sitter | **ctags / Serena** | | type-aware references | opt-in LSP tier, annotating the static answer | none | **native** | none | **Serena** | | install footprint | **23.5 MB, zero runtime deps** | single binary | 114.3 MB venv + language servers | 140.1 MB Python venv | **ctags** | -| MCP server | **33 tools**, subsettable by profile | none | yes, LSP-backed | yes | **codeindex** | +| MCP server | **34 tools**, subsettable by profile | none | yes, LSP-backed | yes | **codeindex** | | onboarding brief | `onboard`, one call, persisted as a memory | none | `onboarding` | none | tie | | says when a query matched nothing | **verdict on every search** (`match`/`weak`/`none`) | no | not measured | not measured | — | diff --git a/src/engine-cli.ts b/src/engine-cli.ts index a03a3b3..033fa1e 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -25,7 +25,7 @@ import { buildTypeHierarchy, implementationsOf } from "./relations.js"; import { computeImportPairs } from "./callers.js"; import { buildSymbolGraph, neighborhood } from "./symbolgraph.js"; import { buildCallerIndex, lookupCallerEntry } from "./callers.js"; -import { detectWorkspaces } from "./workspaces.js"; +import { checkWorkspaceDeps, detectWorkspaces, workspaceReport } from "./workspaces.js"; import { gitChurn } from "./git.js"; import { grepRepo } from "./grep.js"; import { changeCoupling, rankHotspots } from "./coupling.js"; @@ -34,6 +34,8 @@ import { findDeadCode } from "./deadcode.js"; import { findLiteralDuplications } from "./literals.js"; import { symbolComplexity, riskHotspots } from "./complexity.js"; import { renderMermaid } from "./viz.js"; +import { resolutionReport } from "./resolution.js"; +import { resolveContextFor } from "./derived.js"; import { impactOf, neighborsOf } from "./traverse.js"; import { deltaFor, formatDeltaPanel } from "./delta.js"; import { explainQuery, searchIndex } from "./bm25.js"; @@ -68,7 +70,10 @@ Commands: hierarchy Type hierarchy: extends/implements, and what extends/implements it implementations Everything implementing/extending a type (transitively) callgraph Bounded symbol-to-symbol neighborhood (--depth, --direction) - workspaces Monorepo packages + dependency graph (JSON) + workspaces Monorepo packages + dependency graph (JSON), with warnings for + malformed manifests. --check compares each package's declared + sibling dependencies with the imports it really makes + (undeclared / unusedDeclared) and exits 1 on an undeclared one churn Per-file git commit counts (JSON; --since to bound) grep Search: cli.mjs grep --repo (JSON hits) search Keyless BM25 lexical search over symbol names, path segments, @@ -132,21 +137,29 @@ Commands: transitively imports/uses/calls it (--depth ; JSON) neighbors Graph neighbours of a file or module, both directions (--depth , --kind import,call,use,doc-link,mention; JSON) - mermaid Mermaid diagram of the module graph; pass a module positional to - focus on one neighborhood + resolution How much of each language's imports resolved: resolved / + external / dangling (by reason) / unsupported counts, the top + dangling specs and external packages, and the config warnings + that silently turn imports external — whether the graph can be + trusted for a language (--lang , --limit per list, + default 10; JSON) + mermaid Mermaid diagram of the module graph; pass a module slug, module + directory or file positional to focus on one neighborhood (an + unknown target is an error) rewrite Map an expensive tree-wide search onto its indexed equivalent: cli.mjs rewrite ''. Prints the replacement command and exits 0, or exits 1 when it has no opinion (run the original). Deliberately conservative — any shell metacharacter or unknown flag refuses the rewrite - mcp Run as an MCP server over stdio (33 tools: scan_summary, graph, + mcp Run as an MCP server over stdio (34 tools: scan_summary, graph, symbols, callers, workspaces, churn, symbols_overview, find_symbol, find_references, lsp_status, onboard, repo_map, hotspots, coupling, dead_code, complexity, mermaid, grep, search, - explain_search, embed_status, check_rules, the memory quartet and - the three symbolic-edit writes). Flags: --repo pins ONE - repository so the per-tool repo argument becomes optional (an - explicit per-call repo still wins); --server-name overrides + explain_search, embed_status, check_rules, resolution_report, the + memory quartet and the three symbolic-edit writes). Flags: + --repo pins ONE repository so the per-tool repo argument + becomes optional (an explicit per-call repo still wins); + --server-name overrides the announced serverInfo; --max-response-bytes caps a single tool response (default 1e6; a response under the cap is byte-identical, one over it is replaced by an actionable notice @@ -188,7 +201,10 @@ Flags (accepted before OR after the subcommand: '--repo X scan' and entirely. Stale/absent/corrupt → a normal cold build --no-index-cache Never reuse a persisted index; always build from scratch --config Rules config for \`rules\` (JSON: [{name, from, to, …}]) - --limit Max results for \`search\` (default 20) + --limit Max results for \`search\` (default 20); entries per top + list for \`resolution\` (default 10) + --lang \`resolution\`: report one language (as \`scan\` names it) + --check \`workspaces\`: check declared vs imported dependencies --no-fuzzy \`search\`: disable trigram fuzzy fallback for query terms with zero document frequency (default: enabled) --exact \`search\`: drop results that carry no verbatim term match @@ -254,6 +270,8 @@ interface CliFlags { direction?: "out" | "in" | "both"; // callgraph: which way to walk rank?: "graph" | "lexical"; // search: structural prior (default lexical) json?: boolean; // delta: emit JSON instead of the human panel + check?: boolean; // workspaces: compare declared deps with real imports (exit 1 on undeclared) + lang?: string; // resolution: one language's row positional?: string; // e.g. the grep pattern or search query } @@ -327,6 +345,8 @@ function parseFlags(args: string[]): CliFlags { flags.direction = v; } else if (a === "--json") flags.json = true; + else if (a === "--check") flags.check = true; + else if (a === "--lang") flags.lang = next(); else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a; else throw new Error(`unknown flag: ${a}`); } @@ -444,6 +464,7 @@ const VALUE_FLAGS = new Set([ "--kind", "--rank", "--direction", + "--lang", ]); // Accept global flags BEFORE the subcommand as well as after, so @@ -523,7 +544,10 @@ export async function runCli(rawArgv: string[]): Promise { // extensions, then handed to the scan via precomputedWalk so the tree is // traversed a single time. --no-ast keeps the regex tier: no walk, no warm — // scanRepo walks itself, exactly as before. - const scans = !SCANLESS_COMMANDS.has(cmd) && !(cmd === "embed" && flags.positional !== "build"); + // `workspaces --check` reads the link-graph, so it scans like any graph command. + const scans = + (!SCANLESS_COMMANDS.has(cmd) || (cmd === "workspaces" && flags.check === true)) && + !(cmd === "embed" && flags.positional !== "build"); let precomputedWalk: WalkResult | undefined; if (scans && !flags.noAst) { precomputedWalk = walk(flags.repo, { @@ -743,6 +767,13 @@ export async function runCli(rawArgv: string[]): Promise { // fail the guard on the next run (safe: it just rebuilds). writeCache({ graphSha1: sha1(graphJson), symbolsSha1: sha1(symbolsJson), embed: embedMeta }); process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`); + // Config the resolver could not use (an unparseable tsconfig, a missing + // `extends` base…) turns resolvable imports into externals without a + // trace in the artifacts. The build just paid for the resolve context, so + // saying so costs nothing; `resolution` reports the same list on demand. + for (const w of [...new Set(resolveContextFor(scan).warnings)].sort()) { + process.stderr.write(`codeindex: warning: ${w}\n`); + } } } else if (cmd === "scan") { // Summary-only: a file count and a language histogram need the walk and the @@ -1043,14 +1074,9 @@ export async function runCli(rawArgv: string[]): Promise { if (errors > 0) process.exitCode = 1; // the CI gate } else if (cmd === "workspaces") { const info = detectWorkspaces(flags.repo); - emit( - JSON.stringify( - { packages: info.packages, cycle: info.cycle ?? null, topoOrder: info.topoOrder }, - null, - 2, - ) + "\n", - flags.out, - ); + const check = flags.check ? checkWorkspaceDeps(info, (await readArtifacts()).graph) : undefined; + emit(JSON.stringify(workspaceReport(info, check), null, 2) + "\n", flags.out); + if (check && !check.ok) process.exitCode = 1; // the CI gate, like `rules` } else if (cmd === "churn") { const { churn, ok } = gitChurn(flags.repo, { since: flags.since }); const sorted: Record = {}; @@ -1104,6 +1130,9 @@ export async function runCli(rawArgv: string[]): Promise { const res = neighborsOf(graph, flags.positional, flags.depth ?? 1, kinds); if (!res) throw new Error(`no such file or module in the index: ${flags.positional}`); emit(JSON.stringify(res, null, 2) + "\n", flags.out); + } else if (cmd === "resolution") { + const report = resolutionReport(await readScan(), { lang: flags.lang, limit: flags.limit }); + emit(JSON.stringify(report, null, 2) + "\n", flags.out); } else if (cmd === "mermaid") { const { graph } = await readArtifacts(); emit(renderMermaid(graph, { module: flags.positional }), flags.out); diff --git a/src/mcp.ts b/src/mcp.ts index 724dfc6..9ace9fb 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -18,7 +18,7 @@ import { buildCallerIndex, lookupCallerEntry } from "./callers.js"; import { callerIndexFor, hierarchyFor, symbolGraphFor } from "./derived.js"; import { implementationsOf } from "./relations.js"; import { neighborhood, type Direction } from "./symbolgraph.js"; -import { detectWorkspaces } from "./workspaces.js"; +import { checkWorkspaceDeps, detectWorkspaces, workspaceReport } from "./workspaces.js"; import { gitChurn } from "./git.js"; import { grepRepo } from "./grep.js"; import { changeCoupling, rankHotspots } from "./coupling.js"; @@ -27,6 +27,7 @@ import { findDeadCode } from "./deadcode.js"; import { findLiteralDuplications } from "./literals.js"; import { symbolComplexity, riskHotspots } from "./complexity.js"; import { renderMermaid } from "./viz.js"; +import { resolutionReport } from "./resolution.js"; import { symbolsOverview, findSymbol, findReferences } from "./query.js"; import { lspStatus, referencesWithLsp, callersWithLsp } from "./lsp/index.js"; import { conciseCaller, conciseReferences, conciseSymbolIndex, symbolLocation } from "./mcp/concise.js"; @@ -166,7 +167,10 @@ async function callTool(name: string, args: Record, defaultRepo // ONE walk feeds both the warm and the scan below — see warmGrammarsForWalk. let walked: WalkResult | undefined; let preparedScan: ReturnType | undefined; - if (!SCANLESS_TOOLS.has(name)) { + // `workspaces` with `check` compares manifests against the link-graph, so it + // needs the scan (and its grammars) like any graph tool. + const scanless = SCANLESS_TOOLS.has(name) && !(name === "workspaces" && args.check === true); + if (!scanless) { // fs.watch is an eager invalidation hint, never a freshness oracle: an // immediate request can beat event delivery. Always perform the normal // walk/stat proof before trusting a warm scan. @@ -226,7 +230,8 @@ async function callTool(name: string, args: Record, defaultRepo } if (name === "workspaces") { const info = detectWorkspaces(repo); - return JSON.stringify({ packages: info.packages, cycle: info.cycle ?? null, topoOrder: info.topoOrder }, null, 2); + const check = args.check === true ? checkWorkspaceDeps(info, readArtifacts().graph) : undefined; + return JSON.stringify(workspaceReport(info, check), null, 2); } if (name === "churn") { const { churn, ok } = gitChurn(repo, { since: str(args.since) }); @@ -529,6 +534,9 @@ async function callTool(name: string, args: Record, defaultRepo const { graph } = readArtifacts(); return JSON.stringify(checkRules(graph, rules), null, 2); } + if (name === "resolution_report") { + return JSON.stringify(resolutionReport(readScan(), { lang: str(args.lang), limit: positiveNum(args.limit) }), null, 2); + } throw new Error(`unknown tool: ${name}`); } diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index af0d9f5..3c3500e 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -60,8 +60,18 @@ export const TOOLS = [ { name: "workspaces", description: - "Detect monorepo packages (npm/pnpm/yarn/lerna/nx/cargo/go.work/maven) with the workspace dependency graph, one cycle if present, and a topological build order.", - inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] }, + "Detect monorepo packages (npm/pnpm/yarn/lerna/nx/cargo/go modules/maven/gradle/uv/composer) with the workspace dependency graph, one cycle if present, a topological build order, and warnings for malformed manifests. `check: true` also compares each package's declared sibling dependencies with the imports it really makes: `undeclared` (imported, not declared — breaks isolated installs and publishing) and `unusedDeclared` (declared, never imported).", + inputSchema: { + type: "object", + properties: { + ...repoProp, + check: { + type: "boolean", + description: "Compare declared workspace dependencies with resolved cross-package imports (builds the link-graph; default false)", + }, + }, + required: ["repo"], + }, }, { name: "churn", @@ -442,6 +452,21 @@ export const TOOLS = [ required: ["repo"], }, }, + { + name: "resolution_report", + description: + "Can the link-graph be trusted for a language? Per importer language: how many imports resolved to in-repo files, went external (third-party/stdlib), dangle (local target missing, by reason) or are unsupported (no resolver for that language), the top dangling specs with an example importer, the top external packages, and the config warnings (unparseable tsconfig/package.json, missing `extends`) that silently turn imports external. Check it before relying on impact, callers or dead_code for a language.", + inputSchema: { + type: "object", + properties: { + ...repoProp, + ...scopeProps, + lang: { type: "string", description: "Only this language (as scan_summary names it)" }, + limit: { type: "number", minimum: 1, description: "Entries per top list (default 10)" }, + }, + required: ["repo"], + }, + }, ] as const; @@ -538,6 +563,16 @@ export const OUTPUT_SCHEMAS: Record> = { packages: { type: "array", items: anyObj }, cycle: { type: ["array", "null"], items: { type: "string" } }, topoOrder: strArr, + warnings: strArr, + check: { + type: "object", + properties: { + ok: { type: "boolean" }, + undeclared: { type: "array", items: anyObj }, + unusedDeclared: { type: "array", items: anyObj }, + }, + required: ["ok", "undeclared", "unusedDeclared"], + }, }, required: ["packages", "topoOrder"], }, @@ -623,6 +658,15 @@ export const OUTPUT_SCHEMAS: Record> = { }, required: ["embedVersion", "mode"], }, + resolution_report: { + type: "object", + properties: { + totals: anyObj, + languages: { type: "array", items: anyObj }, + warnings: strArr, + }, + required: ["totals", "languages", "warnings"], + }, write_memory: { type: "object", properties: { written: { type: "string" } }, @@ -703,6 +747,7 @@ export const TOOL_META: Record = { implementations: { title: "Implementations" }, call_graph: { title: "Call graph neighborhood" }, check_rules: { title: "Check architecture rules" }, + resolution_report: { title: "Import resolution report" }, }; export function annotationsFor(name: string): Record | undefined { @@ -740,7 +785,7 @@ export const TOOL_PROFILES: Record = { // Locate a thing. find: ["search", "explain_search", "grep", "find_symbol", "symbols", "symbols_overview"], // Decide whether changing it is safe. - impact: ["find_references", "callers", "call_graph", "dead_code", "type_hierarchy", "implementations", "lsp_status"], + impact: ["find_references", "callers", "call_graph", "dead_code", "type_hierarchy", "implementations", "lsp_status", "resolution_report"], // Change it. edit: ["find_symbol", "symbols_overview", "replace_symbol_body", "insert_after_symbol", "insert_before_symbol"], // Where the work and the risk concentrate. diff --git a/src/resolution.ts b/src/resolution.ts new file mode 100644 index 0000000..037d20c --- /dev/null +++ b/src/resolution.ts @@ -0,0 +1,192 @@ +// Resolution report: how much of each language's imports (and markdown links) +// the resolver actually turned into edges. +// +// The link-graph is right to drop `external` refs and to keep `dangling` ones +// as edges to nowhere, but it leaves no way to tell "this Kotlin code has no +// in-repo imports" from "nothing here resolves Kotlin imports at all", or to +// notice that a tsconfig the resolver could not parse turned every alias into +// an external package. Whoever is about to trust impact or dead-code answers +// for a language needs that first. This is the same resolveImport / +// resolveDocLink pass buildGraph makes, aggregated per importer language — a +// query over the scan that changes no artifact. +import type { RepoScan } from "./scan.js"; +import { hasImportResolver, resolveDocLink, resolveImport } from "./resolve.js"; +import { resolveContextFor } from "./derived.js"; +import { detectWorkspaces } from "./workspaces.js"; +import { byStr } from "./sort.js"; + +export interface ResolutionOptions { + lang?: string; // only this language's row + limit?: number; // top dangling specs / external packages kept per language (default 10) +} + +export interface LanguageResolution { + lang: string; + files: number; // files of this language in the scan + filesWithRefs: number; + refs: number; // import specifiers (link targets for markdown), as extracted + resolved: number; // an in-repo file: an edge + external: number; // third-party, stdlib, URL: no edge, by design + dangling: number; // a local target that does not exist: an edge to nowhere + unsupported: number; // the importer's extension has no resolver: no edge, ever + danglingByReason: Record; + topDangling: { spec: string; reason: string; count: number; example: string }[]; + topExternal: { name: string; count: number }[]; + note?: string; // set when the counts mean "the graph cannot be trusted here" +} + +export interface ResolutionReport { + totals: { refs: number; resolved: number; external: number; dangling: number; unsupported: number }; + languages: LanguageResolution[]; + // Config the resolver or the workspace detector could not use (an + // unparseable tsconfig or package.json, a missing `extends` base…), each of + // which silently turns resolvable imports into externals. Sorted, deduped. + warnings: string[]; +} + +const DEFAULT_LIMIT = 10; +const JS_FAMILY = new Set(["typescript", "javascript", "vue", "svelte", "astro", "html"]); + +// The package an external specifier belongs to, so `lodash/fp` and +// `lodash/get` count as one dependency. Display grouping only — nothing +// resolves through it. +function externalName(lang: string, kind: string, spec: string): string { + // Markdown extraction already drops URLs, so an external link is a link to a + // directory: its path is its name. + if (kind === "doc-link") return spec; + if (JS_FAMILY.has(lang)) { + if (/^[./]/.test(spec)) return spec; // a relative asset: the path is the name + const segs = spec.split("/"); + return spec.startsWith("@") && segs.length > 1 ? `${segs[0]}/${segs[1]}` : segs[0]!; + } + if (lang === "python") return spec.startsWith(".") ? spec : spec.split(".")[0]!; + if (lang === "go") { + // A module path starts with a domain (github.com/org/repo/...); the + // standard library does not (net/http). + const segs = spec.split("/"); + return segs[0]!.includes(".") ? segs.slice(0, 3).join("/") : segs[0]!; + } + if (lang === "rust") return spec.split("::")[0]!; + if (lang === "java" || lang === "csharp") return spec.split(".").slice(0, 2).join("."); + if (lang === "php") return spec.replace(/^\\+/, "").split("\\")[0]!; + return spec.split("/")[0]!; +} + +interface Acc { + files: number; + withRefs: number; + row: Omit; + reasons: Map; + dangling: Map; // spec\0reason → … + external: Map; + code: boolean; // the language has code files (a row even with zero refs) +} + +const byCountThen = (count: (x: T) => number, key: (x: T) => string) => (a: T, b: T): number => + count(b) - count(a) || byStr(key(a), key(b)); + +export function resolutionReport(scan: RepoScan, opts: ResolutionOptions = {}): ResolutionReport { + const limit = opts.limit ?? DEFAULT_LIMIT; + const ctx = resolveContextFor(scan); + const accs = new Map(); + for (const f of scan.files) { + if (opts.lang !== undefined && f.lang !== opts.lang) continue; + let acc = accs.get(f.lang); + if (!acc) { + acc = { + files: 0, + withRefs: 0, + row: { refs: 0, resolved: 0, external: 0, dangling: 0, unsupported: 0 }, + reasons: new Map(), + dangling: new Map(), + external: new Map(), + code: false, + }; + accs.set(f.lang, acc); + } + acc.files++; + if (f.kind === "code") acc.code = true; + if (f.refs.length) acc.withRefs++; + for (const ref of f.refs) { + acc.row.refs++; + if (ref.kind === "import" && !hasImportResolver(f.ext)) { + acc.row.unsupported++; + continue; + } + const r = ref.kind === "doc-link" ? resolveDocLink(f.rel, ref.spec, ctx) : resolveImport(f.rel, f.ext, ref.spec, ctx); + if (r.kind === "resolved") { + acc.row.resolved++; + } else if (r.kind === "external") { + acc.row.external++; + const name = externalName(f.lang, ref.kind, ref.spec); + acc.external.set(name, (acc.external.get(name) ?? 0) + 1); + } else { + acc.row.dangling++; + acc.reasons.set(r.reason, (acc.reasons.get(r.reason) ?? 0) + 1); + const key = `${ref.spec}\0${r.reason}`; + const hit = acc.dangling.get(key); + if (!hit) acc.dangling.set(key, { reason: r.reason, count: 1, example: f.rel }); + else { + hit.count++; + if (byStr(f.rel, hit.example) < 0) hit.example = f.rel; + } + } + } + } + if (opts.lang !== undefined && !accs.size) { + const known = Object.keys(scan.languages).sort(byStr).join(", "); + throw new Error(`no indexed files in language "${opts.lang}"${known ? ` — one of: ${known}` : ""}`); + } + + const totals = { refs: 0, resolved: 0, external: 0, dangling: 0, unsupported: 0 }; + const languages: LanguageResolution[] = []; + for (const lang of [...accs.keys()].sort(byStr)) { + const acc = accs.get(lang)!; + // Config-only languages (json, yaml…) carry no refs by nature: a row of + // zeros there says nothing. A code language with zero refs does. + if (!acc.code && acc.row.refs === 0 && opts.lang === undefined) continue; + for (const k of Object.keys(totals) as (keyof typeof totals)[]) totals[k] += acc.row[k]; + const danglingByReason: Record = {}; + for (const reason of [...acc.reasons.keys()].sort(byStr)) danglingByReason[reason] = acc.reasons.get(reason)!; + const topDangling = [...acc.dangling.entries()] + .map(([key, v]) => ({ spec: key.slice(0, key.indexOf("\0")), reason: v.reason, count: v.count, example: v.example })) + .sort(byCountThen((d) => d.count, (d) => `${d.spec}\0${d.reason}`)) + .slice(0, limit); + const topExternal = [...acc.external.entries()] + .map(([name, count]) => ({ name, count })) + .sort(byCountThen((e) => e.count, (e) => e.name)) + .slice(0, limit); + const row: LanguageResolution = { + lang, + files: acc.files, + filesWithRefs: acc.withRefs, + ...acc.row, + danglingByReason, + topDangling, + topExternal, + }; + const note = noteFor(row); + if (note) row.note = note; + languages.push(row); + } + + const warnings = [...new Set([...ctx.warnings, ...detectWorkspaces(scan.root).warnings])].sort(byStr); + return { totals, languages, warnings }; +} + +function noteFor(row: LanguageResolution): string | undefined { + // Markdown refs are links, resolved by resolveDocLink; everything else is an import. + const noun = row.lang === "markdown" ? "link" : "import"; + const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? "" : "s"}`; + if (row.refs === 0) return `no ${noun}s extracted from ${plural(row.files, "file")} — this language gets no ${noun} edges`; + if (row.unsupported === row.refs) { + return `no import resolver for this language — its ${plural(row.refs, "import")} never become edges`; + } + if (row.resolved === 0 && row.dangling === 0) return `every ${noun} is external — no in-repo ${noun} edges for this language`; + if (row.dangling > row.resolved) { + return noun === "link" + ? "more links dangle than resolve — broken relative links in the docs" + : "more imports dangle than resolve — check path aliases and `warnings` before trusting impact or dead-code answers"; + } + return undefined; +} diff --git a/src/resolve.ts b/src/resolve.ts index 49023eb..64228cb 100644 --- a/src/resolve.ts +++ b/src/resolve.ts @@ -155,7 +155,7 @@ function byLen(a: string, b: string): number { return a.length - b.length || (a < b ? -1 : a > b ? 1 : 0); } -function tolerantJsonParse(text: string): unknown { +export function tolerantJsonParse(text: string): unknown { // tsconfig.json is JSONC: strip // and /* */ comments and trailing commas. This // MUST be string-aware — tsconfig glob values like "**/*.ts" or // "./src/styled-system/*" contain `/*`, `*/` and `//` that a naive regex @@ -942,6 +942,15 @@ function resolveCsharp(spec: string, ctx: ResolveContext): Resolution { return best ? { kind: "resolved", target: best } : { kind: "external" }; } +// Extensions resolveImport dispatches on. Any other importer falls through to +// `external`, so its imports can never become edges — the resolution report +// calls those `unsupported` rather than letting them pass as third-party. +// Keep in step with the dispatch below. +const RESOLVER_EXTS = new Set([".go", ".rs", ".java", ".rb", ".rake", ".php", ".cs"]); +export function hasImportResolver(ext: string): boolean { + return JS_TS.has(ext) || SFC_HTML.has(ext) || PY.has(ext) || C_CPP.has(ext) || RESOLVER_EXTS.has(ext); +} + // Resolve an import specifier for a file of the given extension. export function resolveImport( fromRel: string, diff --git a/src/viz.ts b/src/viz.ts index e22e1bd..fb46d54 100644 --- a/src/viz.ts +++ b/src/viz.ts @@ -6,18 +6,60 @@ import { byStr } from "./sort.js"; export interface MermaidOptions { // Restrict to one module's neighborhood (the module plus every module it - // touches, either direction). + // touches, either direction). A module slug, a module's directory path, or a + // file (meaning its module); anything else throws rather than rendering an + // empty diagram that reads as "no dependencies". module?: string; maxEdges?: number; // default 80 — keeps diagrams renderable } -const sanitizeId = (slug: string): string => slug.replace(/[^\w]/g, "_"); +// Mermaid node ids must be identifier-safe, and the readable mapping (every +// other character → "_") is not injective: `src/a-b` and `src/a_b` (slugs +// `src-a-b`, `src-a_b`) both became `src_a_b`, silently merging two modules +// into one node. Ids are assigned over EVERY slug of the graph in sorted +// order, so they never depend on which subset a diagram shows: a slug whose +// readable id is unique keeps it (existing diagrams stay byte-identical), and +// within a colliding group the first keeps it and the rest take the first free +// `_2`, `_3`, …. +function mermaidIds(graph: Graph, prefix: string): Map { + const slugs = new Set(graph.modules.map((m) => m.slug)); + for (const e of graph.moduleEdges) slugs.add(e.from).add(e.to); + const sorted = [...slugs].sort(byStr); + const base = new Map(sorted.map((s) => [s, prefix + s.replace(/[^A-Za-z0-9_]/g, "_")])); + const uses = new Map(); + for (const b of base.values()) uses.set(b, (uses.get(b) ?? 0) + 1); + const taken = new Set([...base.values()].filter((b) => uses.get(b) === 1)); + const ids = new Map(); + for (const s of sorted) { + const b = base.get(s)!; + let id = b; + if (uses.get(b)! > 1) { + for (let n = 2; taken.has(id); n++) id = `${b}_${n}`; + taken.add(id); + } + ids.set(s, id); + } + return ids; +} + +// The module slug a focus target names: a slug, a module directory path +// (`src/flask/json`, trailing slash tolerated), or a file rel (its module). +export function moduleSlugFor(graph: Graph, target: string): string | undefined { + if (graph.modules.some((m) => m.slug === target)) return target; + const path = target.replace(/\/+$/, ""); + const byPath = graph.modules.find((m) => m.path === path); + if (byPath) return byPath.slug; + return graph.files.find((f) => f.rel === target)?.module; +} export function renderMermaid(graph: Graph, opts: MermaidOptions = {}): string { const maxEdges = opts.maxEdges ?? 80; + const focus = opts.module ? moduleSlugFor(graph, opts.module) : undefined; + if (opts.module && !focus) throw new Error(`no such file or module in the index: ${opts.module}`); + const idOf = mermaidIds(graph, ""); let edges = [...graph.moduleEdges].filter((e) => !e.dangling); - if (opts.module) { - edges = edges.filter((e) => e.from === opts.module || e.to === opts.module); + if (focus) { + edges = edges.filter((e) => e.from === focus || e.to === focus); } edges.sort((a, b) => b.weight - a.weight || byStr(a.from, b.from) || byStr(a.to, b.to)); const dropped = Math.max(0, edges.length - maxEdges); @@ -28,16 +70,16 @@ export function renderMermaid(graph: Graph, opts: MermaidOptions = {}): string { shown.add(e.from); shown.add(e.to); } - if (opts.module) shown.add(opts.module); + if (focus) shown.add(focus); const lines: string[] = ["graph LR"]; for (const m of [...graph.modules].sort((a, b) => byStr(a.slug, b.slug))) { if (!shown.has(m.slug)) continue; - lines.push(` ${sanitizeId(m.slug)}["${m.slug}${m.tier === 0 ? " (core)" : ""}"]`); + lines.push(` ${idOf.get(m.slug)}["${m.slug}${m.tier === 0 ? " (core)" : ""}"]`); } for (const e of edges) { const label = e.kind === "import" ? "" : `|${e.kind}|`; - lines.push(` ${sanitizeId(e.from)} -->${label} ${sanitizeId(e.to)}`); + lines.push(` ${idOf.get(e.from)} -->${label} ${idOf.get(e.to)}`); } if (dropped) lines.push(` %% ${dropped} lighter edges omitted (maxEdges=${maxEdges})`); return lines.join("\n") + "\n"; @@ -77,18 +119,15 @@ const CLUSTER_MAX_EDGES = 80; const degreeOf = (m: ModuleNode): number => m.degIn + m.degOut; -// Mermaid node ids must be identifier-safe; slugs may contain dashes. Prefixed -// so a slug starting with a digit still yields a valid id. -function clusterNodeId(slug: string): string { - return "m_" + slug.replace(/[^A-Za-z0-9_]/g, "_"); -} - export function renderMermaidClustered( graph: Graph, opts: ClusteredMermaidOptions = {}, ): ClusteredMermaidResult { const maxModules = opts.maxModules ?? CLUSTER_MAX_MODULES; const maxEdges = opts.maxEdges ?? CLUSTER_MAX_EDGES; + // Prefixed so a slug starting with a digit still yields a valid id. + const ids = mermaidIds(graph, "m_"); + const clusterNodeId = (slug: string): string => ids.get(slug)!; const ranked = graph.modules.slice().sort((a, b) => degreeOf(b) - degreeOf(a) || byStr(a.slug, b.slug)); const shown = ranked.slice(0, maxModules); diff --git a/src/workspaces.ts b/src/workspaces.ts index 476979c..e23a0f2 100644 --- a/src/workspaces.ts +++ b/src/workspaces.ts @@ -1,14 +1,18 @@ // Multi-ecosystem workspace/monorepo detection (merged from ultradoc's manifest // probing and reconstruct's superset): npm/yarn workspaces, pnpm, lerna, nx, -// cargo workspaces, go.work, maven modules, uv workspaces (pyproject), Composer -// path repositories, and Gradle settings includes. Returns the package list -// with a workspace-level dependency graph (name edges + path edges), one cycle -// when present, a topological order, malformed-manifest warnings, and a -// longest-prefix packageOf() matcher. Deterministic: packages sorted by dir, +// cargo workspaces, go.work (or, without one, every nested go.mod), nested +// maven modules, uv workspaces (pyproject), Composer path repositories, and +// Gradle settings includes. Returns the package list with a workspace-level +// dependency graph (name edges + path edges), one cycle when present, a +// topological order, malformed-manifest warnings, and a longest-prefix +// packageOf() matcher; checkWorkspaceDeps() then compares those declared edges +// with the link-graph's real imports. Deterministic: packages sorted by dir, // edges and warnings sorted, no wall-clock. import { existsSync, readdirSync, statSync } from "node:fs"; -import { join } from "node:path"; +import { join, posix } from "node:path"; +import type { Graph } from "./types.js"; import { readText } from "./walk.js"; +import { tolerantJsonParse } from "./resolve.js"; import { byStr } from "./sort.js"; import { escapeRegExp } from "./util.js"; @@ -50,21 +54,29 @@ export interface WorkspaceInfo { const WS_SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", "target", "coverage"]); const MAX_RECURSE_DEPTH = 4; +// Manifests are read with the resolver's JSONC tolerance (comments, trailing +// commas): the same package.json must not be valid for import resolution yet +// "malformed" here. Strict JSON.parse runs first only so a genuinely broken +// file is reported with the parser's own reason. function readJson(path: string, label?: string, warnings?: string[]): Record | undefined { const raw = readText(path); if (!raw) return undefined; + let parsed: unknown; try { - const parsed: unknown = JSON.parse(raw); - if (parsed && typeof parsed === "object") return parsed as Record; - if (label && warnings) warnings.push(`malformed ${label}: not a JSON object`); - return undefined; + parsed = JSON.parse(raw); } catch (e) { - if (label && warnings) { - const reason = String(e instanceof Error ? e.message : e).split("\n")[0]; - warnings.push(`malformed ${label}: ${reason}`); + parsed = tolerantJsonParse(raw); + if (parsed === undefined) { + if (label && warnings) { + const reason = String(e instanceof Error ? e.message : e).split("\n")[0]; + warnings.push(`malformed ${label}: ${reason}`); + } + return undefined; } - return undefined; } + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record; + if (label && warnings) warnings.push(`malformed ${label}: not a JSON object`); + return undefined; } function tomlSectionBody(toml: string, section: string): string | null { @@ -379,18 +391,59 @@ function npmFamilyPatterns(root: string, warnings: string[]): { positives: WsPat } else if (ws && typeof ws === "object" && Array.isArray((ws as { packages?: unknown }).packages)) { for (const x of (ws as { packages: unknown[] }).packages) if (typeof x === "string") push(x, "npm"); } - const pnpm = readText(join(root, "pnpm-workspace.yaml")); - let inPackages = false; - for (const line of pnpm.split(/\r?\n/)) { - if (/^\S/.test(line)) { - inPackages = /^packages\s*:/.test(line); + for (const pattern of pnpmPackagePatterns(readText(join(root, "pnpm-workspace.yaml")))) push(pattern, "pnpm"); + return { positives, negations }; +} + +// A YAML comment starts at a `#` that opens the line or follows whitespace, +// outside quotes. +function stripYamlComment(line: string): string { + let quote = ""; + for (let i = 0; i < line.length; i++) { + const c = line[i]!; + if (quote) { + if (c === quote) quote = ""; + } else if (c === '"' || c === "'") { + quote = c; + } else if (c === "#" && (i === 0 || /\s/.test(line[i - 1]!))) { + return line.slice(0, i); + } + } + return line; +} + +const unquoteYaml = (s: string): string => s.trim().replace(/^(["'])(.*)\1$/, "$2").trim(); + +// pnpm-workspace.yaml `packages:` in either YAML sequence style: the block form +// (`- 'packages/*'` lines, indented or at the key's own column) or the flow +// form (`packages: ["packages/*", 'tools/*']`, which may span lines). Reading +// only the block form made a valid flow-style workspace look empty. +function pnpmPackagePatterns(yaml: string): string[] { + const out: string[] = []; + const lines = yaml.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const head = lines[i]!.match(/^packages\s*:(.*)$/); + if (!head) continue; + let flow = stripYamlComment(head[1]!).trim(); + if (flow.startsWith("[")) { + while (!flow.includes("]") && i + 1 < lines.length) flow += " " + stripYamlComment(lines[++i]!); + const close = flow.indexOf("]"); + // Items split on commas outside quotes: a quoted glob may hold one. + for (const m of flow.slice(1, close === -1 ? undefined : close).matchAll(/\s*(?:"([^"]*)"|'([^']*)'|([^,]+))/g)) { + const v = (m[1] ?? m[2] ?? m[3]!).trim(); + if (v) out.push(v); + } continue; } - if (!inPackages) continue; - const m = line.match(/^\s*-\s*['"]?([^'"#]+?)['"]?\s*(?:#.*)?$/); - if (m) push(m[1]!.trim(), "pnpm"); + // Block form: every following line that is indented, blank, or a `- ` + // entry at column 0 belongs to this key. + while (i + 1 < lines.length && /^(\s|-(\s|$)|$)/.test(lines[i + 1]!)) { + const m = stripYamlComment(lines[++i]!).match(/^\s*-\s*(.*)$/); + const v = m ? unquoteYaml(m[1]!) : ""; + if (v) out.push(v); + } } - return { positives, negations }; + return out; } function fallbackNpmPatterns(root: string, warnings: string[]): WsPattern[] { @@ -426,9 +479,43 @@ function detectCargoMembers(root: string, found: Map, } } +// Directories the go tool itself never builds from — vendored copies, testdata +// and `_`-prefixed dirs (dot-dirs are skipped everywhere already) — plus the +// fixture dirs other ecosystems keep test repos in: a Go fixture module inside +// a JS project's tests/fixtures is not a workspace member. +const GO_SKIP_DIRS = new Set(["vendor", "testdata", "fixtures", "__fixtures__"]); + +// Every nested go.mod under `base`, bounded like the glob walker. One readdir +// per directory answers both "is there a go.mod here" and "where next". +function goModDirs(root: string, base: string, depth: number, out: string[]): void { + let entries; + try { + entries = readdirSync(base ? join(root, base) : root, { withFileTypes: true }); + } catch { + return; + } + if (base && entries.some((e) => e.name === "go.mod" && !e.isDirectory())) out.push(base); + if (depth > MAX_RECURSE_DEPTH) return; + const subs = entries + .filter((e) => e.isDirectory() && !/^[._]/.test(e.name) && !WS_SKIP_DIRS.has(e.name) && !GO_SKIP_DIRS.has(e.name)) + .map((e) => e.name) + .sort(byStr); + for (const name of subs) goModDirs(root, base ? `${base}/${name}` : name, depth + 1, out); +} + +// go.work lists the workspace modules explicitly. Without one, a repo can still +// hold several modules side by side (a service beside a CLI beside a shared +// lib) — the import resolver already links across every in-repo go.mod, so the +// workspace view lists each nested module too instead of reporting none. function detectGoWork(root: string, found: Map, warnings: string[]): void { const gowork = readText(join(root, "go.work")); - if (!gowork) return; + if (!gowork) { + if (existsSync(join(root, "go.work"))) return; // an empty go.work still declares the workspace + const dirs: string[] = []; + goModDirs(root, "", 0, dirs); + for (const dir of dirs) addPackage(root, dir, found, "go", warnings); + return; + } const dirs: string[] = []; for (const block of gowork.matchAll(/^use\s*\(([\s\S]*?)\)/gm)) { for (const line of block[1]!.split(/\r?\n/)) { @@ -443,14 +530,29 @@ function detectGoWork(root: string, found: Map, warnin } } +// Maven reactor modules. A module that is itself an aggregator lists its own +// , relative to ITS pom — nested reactors are the norm in large Maven +// builds, and reading only the root pom dropped every leaf module along with +// the dependency edges pointing at them. Every block counts (profiles +// add modules too); XML comments are stripped first, since a commented-out +// is a disabled one. A may name a pom file instead of a dir. function detectMavenModules(root: string, found: Map, warnings: string[]): void { - const pom = readText(join(root, "pom.xml")); - if (!pom) return; - const modules = pom.match(/([\s\S]*?)<\/modules>/)?.[1]; - if (!modules) return; - for (const m of modules.matchAll(/\s*([^<]+?)\s*<\/module>/g)) { - addPackage(root, m[1]!, found, "maven", warnings); - } + const seen = new Set(); + const visit = (dir: string, depth: number): void => { + if (depth > MAX_RECURSE_DEPTH || seen.has(dir)) return; + seen.add(dir); + const pom = readText(join(root, dir, "pom.xml")).replace(//g, ""); + for (const block of pom.matchAll(/([\s\S]*?)<\/modules>/g)) { + for (const m of block[1]!.matchAll(/\s*([^<]+?)\s*<\/module>/g)) { + const spec = m[1]!.endsWith(".xml") ? posix.dirname(m[1]!) : m[1]!; + const child = posix.normalize(posix.join(dir || ".", spec)).replace(/\/+$/, ""); + if (child === "." || child === ".." || child.startsWith("../")) continue; // never leave the repo root + addPackage(root, child, found, "maven", warnings); + visit(child, depth + 1); + } + } + }; + visit("", 0); } // uv workspaces: [tool.uv.workspace] members/exclude in the root pyproject. @@ -484,14 +586,19 @@ function detectComposerPathRepos(root: string, found: Map, warnings: string[]): void { for (const f of ["settings.gradle", "settings.gradle.kts"]) { const text = readText(join(root, f)); if (!text) continue; - for (const line of text.split(/\r?\n/)) { - if (!/^\s*include[\s(]/.test(line)) continue; - for (const m of line.matchAll(/["']([^"']+)["']/g)) { + // Comments out first (a commented include is a disabled one); `://` is a + // URL inside a string, not a comment. + const code = text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(^|[^:])\/\/.*$/gm, "$1"); + for (const call of code.matchAll(/^[ \t]*include\b[ \t]*(\([^)]*\)|(?:[^\n]*,[ \t]*\r?\n)*[^\n]*)/gm)) { + for (const m of call[1]!.matchAll(/["']([^"']+)["']/g)) { const dir = m[1]!.replace(/^:/, "").replace(/:/g, "/"); if (dir) addPackage(root, dir, found, "gradle", warnings); } @@ -616,17 +723,45 @@ function composerEdges(root: string, pkg: WorkspacePackage, byName: Set, return [...edges]; } -function gradleEdges(root: string, pkg: WorkspacePackage, byName: Set, byDir: Map): string[] { +// Gradle's type-safe project accessor for a project dir: `libs/my-core` is +// `projects.libs.myCore` (each path segment camelCased on `-`/`_`). +function gradleAccessor(dir: string): string { + return dir + .split("/") + .map((seg) => seg.replace(/[-_]+([A-Za-z0-9])/g, (_, c: string) => c.toUpperCase())) + .join("."); +} + +function gradleEdges( + root: string, + pkg: WorkspacePackage, + byName: Set, + byDir: Map, + accessors: Map, +): string[] { for (const f of ["build.gradle", "build.gradle.kts"]) { const text = readText(join(root, pkg.dir, f)); if (!text) continue; const edges = new Set(); - // implementation project(':libs:core') — a project path is a dir path. - for (const m of text.matchAll(/project\s*\(\s*["']:?([^"']+)["']\s*\)/g)) { + // implementation project(':libs:core') / project(path: ':libs:core', …) — + // a project path is a dir path. + for (const m of text.matchAll(/project\s*\(\s*(?:path\s*[:=]\s*)?["']:?([^"']+)["']/g)) { const path = m[1]!.replace(/:/g, "/"); const target = byDir.get(path) ?? (byName.has(path) ? path : undefined); if (target && target !== pkg.name) edges.add(target); } + // implementation(projects.libs.core) — Gradle 7+ type-safe accessors. The + // longest dotted prefix naming a project wins, so a trailing property + // (`projects.libs.core.dependencyProject`) does not hide the edge. + for (const m of text.matchAll(/\bprojects((?:\.[A-Za-z_]\w*)+)/g)) { + const segs = m[1]!.slice(1).split("."); + for (let n = segs.length; n > 0; n--) { + const target = accessors.get(segs.slice(0, n).join(".")); + if (!target) continue; + if (target !== pkg.name) edges.add(target); + break; + } + } return [...edges]; } return []; @@ -637,6 +772,7 @@ function edgesFor( pkg: WorkspacePackage, byName: Set, byDir: Map, + accessors: Map, warnings: string[], ): string[] { switch (pkg.kind) { @@ -651,7 +787,7 @@ function edgesFor( case "composer": return composerEdges(root, pkg, byName, warnings); case "gradle": - return gradleEdges(root, pkg, byName, byDir); + return gradleEdges(root, pkg, byName, byDir, accessors); default: return npmEdges(root, pkg, byName, warnings); } @@ -732,8 +868,9 @@ export function detectWorkspaces(root: string): WorkspaceInfo { const byName = new Set(packages.map((p) => p.name)); const byDir = new Map(packages.map((p) => [p.dir, p.name])); + const accessors = new Map(packages.map((p) => [gradleAccessor(p.dir), p.name])); for (const pkg of packages) { - const edges = edgesFor(root, pkg, byName, byDir, warnings); + const edges = edgesFor(root, pkg, byName, byDir, accessors, warnings); if (edges.length) pkg.dependsOn = edges.sort(byStr); } @@ -746,3 +883,77 @@ export function detectWorkspaces(root: string): WorkspaceInfo { packageOf: (rel: string) => byDepth.find((p) => rel === p.dir || rel.startsWith(p.dir + "/")), }; } + +// --- declared vs actual dependencies ---------------------------------------- + +export interface UndeclaredDependency { + from: string; // importing package + to: string; // sibling package it imports without declaring it + files: number; // distinct importing files + example: string; // the first of them, sorted +} + +export interface WorkspaceCheck { + ok: boolean; // no undeclared cross-package import + undeclared: UndeclaredDependency[]; + // Declared sibling dependencies no resolved import uses. Informational: a + // package can be a real dependency without being imported (a CLI, a shared + // config, a compiled-only entry the resolver cannot map), so it never fails + // the check. + unusedDeclared: { from: string; to: string }[]; +} + +// Nx derives project dependencies from imports (tsconfig paths) instead of +// declaring them, so an import with no manifest entry is how it is supposed to +// work — its members are left out of the check entirely. +const INFERRED_DEPS = new Set(["nx"]); +// Kinds whose declared sibling dependencies exist to be imported by code. A +// Maven/Gradle/uv/Composer declaration also carries runtime-only and plugin +// wiring that an import graph cannot see, so "unused" would mostly be noise. +const CODE_LEVEL_DEPS = new Set(["npm", "pnpm", "lerna", "cargo", "go"]); + +// Compare the manifests' declared workspace dependencies with the resolved +// import edges of the link-graph. An import of a sibling package that the +// importer's manifest does not declare works in a hoisted local checkout and +// breaks the isolated install, the publish, or `go mod tidy` — nothing else in +// the toolchain says so before that. Deterministic: sorted by (from, to). +export function checkWorkspaceDeps(info: WorkspaceInfo, graph: Pick): WorkspaceCheck { + // "from\0to" package names → the pair and its importing files. + const imported = new Map }>(); + for (const e of graph.fileEdges) { + if (e.kind !== "import" || e.dangling) continue; + const from = info.packageOf(e.from); + const to = info.packageOf(e.to); + if (!from || !to || from === to) continue; + const key = `${from.name}\0${to.name}`; + let pair = imported.get(key); + if (!pair) imported.set(key, (pair = { from, to, files: new Set() })); + pair.files.add(e.from); + } + const undeclared: UndeclaredDependency[] = []; + for (const { from, to, files } of imported.values()) { + if (INFERRED_DEPS.has(from.kind) || from.dependsOn?.includes(to.name)) continue; + const sorted = [...files].sort(byStr); + undeclared.push({ from: from.name, to: to.name, files: sorted.length, example: sorted[0]! }); + } + const unusedDeclared: { from: string; to: string }[] = []; + for (const pkg of info.packages) { + if (!CODE_LEVEL_DEPS.has(pkg.kind)) continue; + for (const dep of pkg.dependsOn ?? []) { + if (!imported.has(`${pkg.name}\0${dep}`)) unusedDeclared.push({ from: pkg.name, to: dep }); + } + } + const byPair = (a: { from: string; to: string }, b: { from: string; to: string }): number => + byStr(a.from, b.from) || byStr(a.to, b.to); + return { ok: undeclared.length === 0, undeclared: undeclared.sort(byPair), unusedDeclared: unusedDeclared.sort(byPair) }; +} + +// The JSON the `workspaces` CLI command and MCP tool print. `warnings` and +// `check` appear only when there is something to say, so the output of a +// clean, unchecked workspace stays byte-identical to earlier releases. +export function workspaceReport(info: WorkspaceInfo, check?: WorkspaceCheck): Record { + const out: Record = { packages: info.packages, cycle: info.cycle ?? null, topoOrder: info.topoOrder }; + if (info.warnings.length) out.warnings = info.warnings; + if (check) out.check = check; + return out; +} diff --git a/tests/mcp-output.test.ts b/tests/mcp-output.test.ts index 6f52de1..5935a82 100644 --- a/tests/mcp-output.test.ts +++ b/tests/mcp-output.test.ts @@ -110,7 +110,7 @@ const CASES: Record> = { graph: {}, symbols: {}, callers: {}, - workspaces: {}, + workspaces: { check: true }, churn: {}, find_references: { name: "HttpClient" }, explain_search: { query: "http client retry" }, @@ -121,6 +121,7 @@ const CASES: Record> = { coupling: {}, duplicated_literals: {}, embed_status: {}, + resolution_report: {}, write_memory: { name: "schema-probe", content: "x" }, delete_memory: { name: "schema-probe" }, // The three symbolic edits share a schema and would mutate the fixture, so diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index 0e4e90a..a21829f 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -362,7 +362,7 @@ describe("MCP server", () => { expect(res.get(1)!.result!.serverInfo!.name).toBe("codeindex"); const toolNames = res.get(2)!.result!.tools!.map((t) => t.name); - expect(toolNames).toEqual(["scan_summary", "graph", "symbols", "callers", "workspaces", "churn", "symbols_overview", "find_symbol", "find_references", "lsp_status", "onboard", "repo_map", "hotspots", "coupling", "replace_symbol_body", "insert_after_symbol", "insert_before_symbol", "write_memory", "read_memory", "list_memories", "delete_memory", "dead_code", "duplicated_literals", "complexity", "mermaid", "grep", "search", "explain_search", "embed_status", "type_hierarchy", "implementations", "call_graph", "check_rules"]); + expect(toolNames).toEqual(["scan_summary", "graph", "symbols", "callers", "workspaces", "churn", "symbols_overview", "find_symbol", "find_references", "lsp_status", "onboard", "repo_map", "hotspots", "coupling", "replace_symbol_body", "insert_after_symbol", "insert_before_symbol", "write_memory", "read_memory", "list_memories", "delete_memory", "dead_code", "duplicated_literals", "complexity", "mermaid", "grep", "search", "explain_search", "embed_status", "type_hierarchy", "implementations", "call_graph", "check_rules", "resolution_report"]); const summary = JSON.parse(res.get(3)!.result!.content![0]!.text) as { fileCount: number }; expect(summary.fileCount).toBeGreaterThan(0); @@ -1602,3 +1602,26 @@ describe("tool profiles and onboarding", () => { expect(res.get(3)!.result!.content![0]!.text).toBe(brief.brief); }, 30_000); }); + +describe("workspaces check and resolution_report over MCP", () => { + it("workspaces `check: true` reports an undeclared sibling import; resolution_report answers per language", async () => { + const monorepo = fileURLToPath(new URL("./fixtures/mini-monorepo", import.meta.url)); + const res = await mcpSession([ + { id: 1, method: "initialize", params: { protocolVersion: "2025-06-18", capabilities: {} } }, + { method: "notifications/initialized" }, + { id: 2, method: "tools/call", params: { name: "workspaces", arguments: { repo: monorepo, check: true } } }, + { id: 3, method: "tools/call", params: { name: "workspaces", arguments: { repo: monorepo } } }, + { id: 4, method: "tools/call", params: { name: "resolution_report", arguments: { repo: monorepo, lang: "typescript" } } }, + ]); + const checked = JSON.parse(res.get(2)!.result!.content![0]!.text) as { + check: { ok: boolean; undeclared: { from: string; to: string; example: string }[] }; + }; + expect(checked.check.ok).toBe(false); + expect(checked.check.undeclared).toEqual([{ from: "@scope/b", to: "@scope/a", files: 1, example: "packages/b/src/consumer.ts" }]); + // Unchecked: the historical shape, byte for byte (no `check`, no empty `warnings`). + expect(Object.keys(JSON.parse(res.get(3)!.result!.content![0]!.text) as object)).toEqual(["packages", "cycle", "topoOrder"]); + const report = JSON.parse(res.get(4)!.result!.content![0]!.text) as { languages: { lang: string; resolved: number }[] }; + expect(report.languages.map((l) => l.lang)).toEqual(["typescript"]); + expect(report.languages[0]!.resolved).toBe(1); + }, 30_000); +}); diff --git a/tests/phase2.test.ts b/tests/phase2.test.ts index 7a9507e..2cf5cb8 100644 --- a/tests/phase2.test.ts +++ b/tests/phase2.test.ts @@ -17,7 +17,7 @@ import { writeMemory, readMemory, deleteMemory, listMemories } from "../src/memo import { readText as engineRead } from "../src/walk.js"; import { findDeadCode } from "../src/deadcode.js"; import { symbolComplexity, riskHotspots } from "../src/complexity.js"; -import { renderMermaid } from "../src/viz.js"; +import { renderMermaid, renderMermaidClustered } from "../src/viz.js"; import { buildIndexArtifacts } from "../src/pipeline.js"; import { grepRepo } from "../src/grep.js"; import { extractCode } from "../src/extract/code.js"; @@ -537,4 +537,46 @@ describe("dead code, complexity, mermaid", () => { expect(mmd).toContain("-->"); expect(renderMermaid(graph)).toBe(mmd); }); + + // `src/a-b` and `src/a_b` (slugs src-a-b, src-a_b) used to share the node id + // src_a_b, so the diagram drew one module with a doubled edge. + function collidingRepo(): string { + const root = mkdtempSync(join(tmpdir(), "ci-mmd-ids-")); + for (const dir of ["a-b", "a_b", "c"]) mkdirSync(join(root, "src", dir), { recursive: true }); + writeFileSync(join(root, "src", "a-b", "x.ts"), "export const x = 1;\n"); + writeFileSync(join(root, "src", "a_b", "y.ts"), "export const y = 2;\n"); + writeFileSync(join(root, "src", "c", "z.ts"), 'import { x } from "../a-b/x";\nimport { y } from "../a_b/y";\nexport const z = x + y;\n'); + return root; + } + + it("renderMermaid gives every module its own node id, keeping readable ids where they are unique", () => { + const { graph } = buildIndexArtifacts(collidingRepo()); + expect(renderMermaid(graph)).toBe( + [ + "graph LR", + ' src_a_b["src-a-b"]', + ' src_a_b_2["src-a_b"]', + ' src_c["src-c"]', + " src_c --> src_a_b", + " src_c --> src_a_b_2", + "", + ].join("\n"), + ); + const clustered = renderMermaidClustered(graph).content; + expect(clustered).toContain("m_src_a_b["); + expect(clustered).toContain("m_src_a_b_2["); + expect(clustered).toContain("m_src_c --> m_src_a_b\n"); + expect(clustered).toContain("m_src_c --> m_src_a_b_2\n"); + }); + + it("renderMermaid focuses on a slug, a module directory or a file, and throws on anything else", () => { + const { graph } = buildIndexArtifacts(collidingRepo()); + const bySlug = renderMermaid(graph, { module: "src-a_b" }); + expect(bySlug).toBe(['graph LR', ' src_a_b_2["src-a_b"]', ' src_c["src-c"]', " src_c --> src_a_b_2", ""].join("\n")); + expect(renderMermaid(graph, { module: "src/a_b" })).toBe(bySlug); + expect(renderMermaid(graph, { module: "src/a_b/" })).toBe(bySlug); + expect(renderMermaid(graph, { module: "src/a_b/y.ts" })).toBe(bySlug); + // It used to print a bare "graph LR" — indistinguishable from "no dependencies". + expect(() => renderMermaid(graph, { module: "nonexistent" })).toThrow(/no such file or module in the index: nonexistent/); + }); }); diff --git a/tests/resolution.test.ts b/tests/resolution.test.ts new file mode 100644 index 0000000..65bd15d --- /dev/null +++ b/tests/resolution.test.ts @@ -0,0 +1,129 @@ +// The resolution report (`codeindex resolution`, MCP `resolution_report`): +// per-language resolved/external/dangling/unsupported accounting, top lists, +// notes for languages the graph cannot be trusted on, and the config warnings +// the resolver used to collect without any surface reading them. +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { resolutionReport } from "../src/resolution.js"; +import { scanRepo } from "../src/scan.js"; + +const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); + +function scratchRepo(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "ci-resolution-")); + for (const [rel, body] of Object.entries(files)) { + const abs = join(root, rel); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, body); + } + return root; +} + +const REPO_FILES = { + // Missing closing brace: the resolver skips it, and nothing used to say so. + "tsconfig.json": '{ "compilerOptions": { "paths": { "@/*": ["src/*"] } }\n', + "package.json": JSON.stringify({ name: "demo", workspaces: ["packages/*"] }), + "packages/bad/package.json": '{"name": "bad",, }', + "src/a.ts": [ + 'import { b } from "./b";', + 'import { gone } from "./gone";', + 'import fp from "lodash/fp";', + 'import get from "lodash/get";', + 'import React from "react";', + 'import { x } from "@scope/pkg/deep";', + "export const a = b + gone + fp + get + React + x;", + ].join("\n"), + "src/b.ts": 'import { gone } from "./gone";\nexport const b = gone;\n', + "src/c.ts": 'import { gone } from "./gone";\nexport const c = 1;\n', + "src/Main.kt": "package a\nimport b.Util\nfun main() { Util.x() }\n", + "docs/guide.md": "# Guide\n\nSee [a](../src/a.ts), [missing](./nope.md), [the sources](../src/) and [site](https://example.com/x).\n", + "config.json": '{ "k": 1 }\n', +}; + +describe("resolutionReport", () => { + const root = scratchRepo(REPO_FILES); + const report = resolutionReport(scanRepo(root)); + const row = (lang: string) => report.languages.find((l) => l.lang === lang)!; + + it("accounts for every ref of a language as resolved, external, dangling or unsupported", () => { + const ts = row("typescript"); + expect(ts).toMatchObject({ files: 3, filesWithRefs: 3, refs: 8, resolved: 1, external: 4, dangling: 3, unsupported: 0 }); + expect(ts.danglingByReason).toEqual({ "missing-module": 3 }); + // One spec dangling from three files: counted once per importer, the + // example is the first importer in path order. + expect(ts.topDangling).toEqual([{ spec: "./gone", reason: "missing-module", count: 3, example: "src/a.ts" }]); + // Subpath imports group under their package. + expect(ts.topExternal).toEqual([ + { name: "lodash", count: 2 }, + { name: "@scope/pkg", count: 1 }, + { name: "react", count: 1 }, + ]); + expect(ts.note).toMatch(/more imports dangle than resolve/); + for (const l of report.languages) expect(l.refs).toBe(l.resolved + l.external + l.dangling + l.unsupported); + expect(report.totals.refs).toBe(report.languages.reduce((n, l) => n + l.refs, 0)); + }); + + it("resolves markdown links with the doc-link resolver", () => { + const md = row("markdown"); + // The URL never becomes a ref; a link to a real directory is external. + expect(md).toMatchObject({ refs: 3, resolved: 1, external: 1, dangling: 1 }); + expect(md.topExternal).toEqual([{ name: "../src/", count: 1 }]); + expect(md.topDangling).toEqual([{ spec: "./nope.md", reason: "missing-target", count: 1, example: "docs/guide.md" }]); + }); + + it("says when a code language yields no import edges, and skips config-only languages", () => { + expect(row("kotlin")).toMatchObject({ files: 1, refs: 0, note: "no imports extracted from 1 file — this language gets no import edges" }); + expect(report.languages.map((l) => l.lang)).not.toContain("json"); + }); + + it("surfaces the resolver's and the workspace detector's config warnings", () => { + // The first entry carries JSON.parse's own reason, worded per Node version. + expect(report.warnings).toHaveLength(3); + expect(report.warnings[0]).toMatch(/^malformed packages\/bad\/package\.json: \S/); + expect(report.warnings.slice(1)).toEqual([ + "unparseable packages/bad/package.json — skipped for workspace resolution", + "unparseable tsconfig.json — its path aliases were ignored", + ]); + }); + + it("labels imports from a language with no resolver as unsupported", () => { + const scan = scanRepo(root); + scan.files.find((f) => f.rel === "src/Main.kt")!.refs.push({ kind: "import", spec: "b.Util" }); + const kt = resolutionReport(scan, { lang: "kotlin" }).languages; + expect(kt).toHaveLength(1); + expect(kt[0]).toMatchObject({ refs: 1, unsupported: 1, external: 0, note: "no import resolver for this language — its 1 import never become edges" }); + }); + + it("filters to one language, caps the top lists, and rejects an unknown language", () => { + const only = resolutionReport(scanRepo(root), { lang: "typescript", limit: 1 }); + expect(only.languages.map((l) => l.lang)).toEqual(["typescript"]); + expect(only.languages[0]!.topExternal).toEqual([{ name: "lodash", count: 2 }]); + expect(only.totals.refs).toBe(8); + expect(() => resolutionReport(scanRepo(root), { lang: "cobol" })).toThrow(/no indexed files in language "cobol" — one of: .*typescript/); + }); + + it("is deterministic", () => { + expect(JSON.stringify(resolutionReport(scanRepo(root)))).toBe(JSON.stringify(report)); + }); +}); + +describe("CLI surfaces", () => { + it("`resolution` prints the report; `index` prints the resolver's warnings to stderr", () => { + const root = scratchRepo(REPO_FILES); + const res = spawnSync(process.execPath, [CLI, "resolution", "--repo", root, "--lang", "typescript", "--limit", "2"], { encoding: "utf8" }); + expect(res.status).toBe(0); + const out = JSON.parse(res.stdout) as { languages: { lang: string; topExternal: unknown[] }[]; warnings: string[] }; + expect(out.languages.map((l) => l.lang)).toEqual(["typescript"]); + expect(out.languages[0]!.topExternal).toHaveLength(2); + expect(out.warnings).toContain("unparseable tsconfig.json — its path aliases were ignored"); + + const idx = spawnSync(process.execPath, [CLI, "index", "--repo", root, "--out", join(root, ".codeindex")], { encoding: "utf8" }); + expect(idx.status).toBe(0); + expect(idx.stderr).toContain("codeindex: warning: unparseable tsconfig.json — its path aliases were ignored\n"); + expect(idx.stderr).toContain("codeindex: warning: unparseable packages/bad/package.json — skipped for workspace resolution\n"); + }); +}); diff --git a/tests/workspaces.test.ts b/tests/workspaces.test.ts new file mode 100644 index 0000000..6100655 --- /dev/null +++ b/tests/workspaces.test.ts @@ -0,0 +1,210 @@ +// Workspace detection gaps (pnpm flow lists, multi-line Gradle includes and +// type-safe accessors, nested Maven aggregators, go.work-less multi-module Go +// repos), JSONC manifests, surfaced warnings, and the declared-vs-imported +// dependency check (`workspaces --check`). +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { checkWorkspaceDeps, detectWorkspaces, workspaceReport } from "../src/workspaces.js"; +import { buildIndexArtifacts } from "../src/pipeline.js"; +import type { Edge } from "../src/types.js"; + +const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); +const FIXTURES = fileURLToPath(new URL("./fixtures", import.meta.url)); + +function scratchRepo(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "ci-ws-")); + for (const [rel, body] of Object.entries(files)) { + const abs = join(root, rel); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, body); + } + return root; +} + +const summary = (root: string): string[] => + detectWorkspaces(root).packages.map((p) => `${p.kind}:${p.name}${p.dependsOn ? ` -> ${p.dependsOn.join(",")}` : ""}`); + +describe("pnpm-workspace.yaml sequence styles", () => { + const members = { + "packages/a/package.json": JSON.stringify({ name: "a" }), + "tools/t/package.json": JSON.stringify({ name: "t" }), + "tools/skip/package.json": JSON.stringify({ name: "skip" }), + }; + + it("reads a one-line flow list, quotes and negations included", () => { + const root = scratchRepo({ ...members, "pnpm-workspace.yaml": `packages: ["packages/*", 'tools/*', "!tools/skip"] # flow\n` }); + expect(summary(root)).toEqual(["pnpm:a", "pnpm:t"]); + }); + + it("reads a flow list spanning lines", () => { + const root = scratchRepo({ ...members, "pnpm-workspace.yaml": 'packages: [\n "packages/*", # libs\n "tools/t"\n]\ncatalog:\n x: 1\n' }); + expect(summary(root)).toEqual(["pnpm:a", "pnpm:t"]); + }); + + it("reads block entries at the key's own column, and stops at the next key", () => { + const root = scratchRepo({ ...members, "pnpm-workspace.yaml": "packages:\n- 'packages/*'\n\n- tools/t # the cli\nonlyBuiltDependencies:\n - tools/skip\n" }); + expect(summary(root)).toEqual(["pnpm:a", "pnpm:t"]); + }); +}); + +describe("Gradle settings includes", () => { + it("collects every project of a multi-line include(...) and type-safe accessor edges", () => { + const root = scratchRepo({ + "settings.gradle.kts": 'rootProject.name = "demo"\ninclude(\n ":libs:core",\n ":app", // the app\n)\ninclude(":platform:api")\n/* include(":old") */\nincludeBuild("build-logic")\n', + "libs/core/build.gradle.kts": "plugins { `java-library` }\n", + "platform/api/build.gradle.kts": "dependencies { api(projects.libs.core) }\n", + "app/build.gradle.kts": "dependencies {\n implementation(project(path = \":libs:core\"))\n implementation(projects.platform.api)\n}\n", + "old/build.gradle.kts": "", + "build-logic/build.gradle.kts": "", + }); + expect(summary(root)).toEqual(["gradle:app -> libs/core,platform/api", "gradle:libs/core", "gradle:platform/api -> libs/core"]); + expect(detectWorkspaces(root).topoOrder).toEqual(["libs/core", "platform/api", "app"]); + }); + + it("continues a Groovy include after a trailing comma and camelCases accessor segments", () => { + const root = scratchRepo({ + "settings.gradle": "include ':app',\n ':shared-utils'\n", + "shared-utils/build.gradle": "", + "app/build.gradle": "dependencies { implementation projects.sharedUtils }\n", + }); + expect(summary(root)).toEqual(["gradle:app -> shared-utils", "gradle:shared-utils"]); + }); +}); + +describe("Maven reactor modules", () => { + it("recurses into nested aggregators, relative to the pom that lists them", () => { + const root = scratchRepo({ + "pom.xml": "rootparentsvc", + "parent/pom.xml": "parent-aggchild-a../libs/child-b/pom.xml", + "parent/child-a/pom.xml": "child-a", + "libs/child-b/pom.xml": "child-b", + "gone/pom.xml": "gone", + "svc/pom.xml": + "svcchild-b", + }); + expect(summary(root)).toEqual(["maven:child-b", "maven:parent-agg", "maven:child-a", "maven:svc -> child-b"]); + expect(detectWorkspaces(root).topoOrder).toEqual(["child-a", "child-b", "parent-agg", "svc"]); + }); +}); + +describe("Go modules without go.work", () => { + it("lists every nested go.mod as a module, with replace edges", () => { + const info = detectWorkspaces(join(FIXTURES, "mixed-monorepo")); + expect(info.packages.map((p) => `${p.dir}=${p.name}`)).toEqual([ + "services/api=example.com/api", + "shared-go=example.com/shared", + "tools/cli=example.com/cli", + ]); + expect(info.packages[0]!.dependsOn).toEqual(["example.com/shared"]); + expect(info.topoOrder.indexOf("example.com/shared")).toBeLessThan(info.topoOrder.indexOf("example.com/api")); + }); + + it("skips vendor, testdata, fixtures and _-prefixed dirs, and the root module itself", () => { + const root = scratchRepo({ + "go.mod": "module example.com/root\n", + "svc/go.mod": "module example.com/svc\n", + "vendor/x/go.mod": "module example.com/vendored\n", + "svc/testdata/go.mod": "module example.com/td\n", + "tests/fixtures/repo/go.mod": "module example.com/fixture\n", + "_scratch/go.mod": "module example.com/scratch\n", + }); + expect(summary(root)).toEqual(["go:example.com/svc"]); + }); + + it("leaves module discovery to go.work when there is one", () => { + const root = scratchRepo({ + "go.work": "go 1.22\n\nuse ./a\n", + "a/go.mod": "module example.com/a\n", + "b/go.mod": "module example.com/b\n", + }); + expect(summary(root)).toEqual(["go:example.com/a"]); + }); +}); + +describe("manifest parsing and warnings", () => { + it("accepts JSONC manifests like the resolver does, and still names a broken one", () => { + const root = scratchRepo({ + "package.json": '{\n // monorepo root\n "name": "root",\n "workspaces": ["packages/*"],\n}\n', + "packages/good/package.json": '{ "name": "good", /* ok */ "dependencies": { "bad": "*", }, }', + "packages/bad/package.json": '{"name": "bad",, }', + }); + const info = detectWorkspaces(root); + expect(info.packages.map((p) => p.name)).toEqual(["packages/bad", "good"]); + expect(info.warnings).toHaveLength(1); + expect(info.warnings[0]).toMatch(/^malformed packages\/bad\/package\.json: /); + // The report carries warnings only when there are some, so a clean + // workspace prints the same bytes it always did. + expect(workspaceReport(info).warnings).toEqual(info.warnings); + expect(Object.keys(workspaceReport(detectWorkspaces(join(FIXTURES, "mini-monorepo"))))).toEqual(["packages", "cycle", "topoOrder"]); + }); +}); + +describe("checkWorkspaceDeps", () => { + const edge = (from: string, to: string, kind = "import", dangling?: boolean): Edge => ({ from, to, kind, weight: 1, ...(dangling ? { dangling } : {}) }) as Edge; + + it("flags an import of a sibling the manifest does not declare (mini-monorepo)", () => { + const root = join(FIXTURES, "mini-monorepo"); + const { graph } = buildIndexArtifacts(root); + const check = checkWorkspaceDeps(detectWorkspaces(root), graph); + expect(check).toEqual({ + ok: false, + undeclared: [{ from: "@scope/b", to: "@scope/a", files: 1, example: "packages/b/src/consumer.ts" }], + unusedDeclared: [], + }); + }); + + it("passes declared imports, reports unused declarations, ignores non-import and intra-package edges", () => { + const root = scratchRepo({ + "package.json": JSON.stringify({ name: "root", workspaces: ["packages/*"] }), + "packages/a/package.json": JSON.stringify({ name: "a" }), + "packages/b/package.json": JSON.stringify({ name: "b", dependencies: { a: "*", c: "*" } }), + "packages/c/package.json": JSON.stringify({ name: "c" }), + }); + const info = detectWorkspaces(root); + const check = checkWorkspaceDeps(info, { + fileEdges: [ + edge("packages/b/src/x.ts", "packages/a/src/index.ts"), + edge("packages/b/src/y.ts", "packages/b/src/x.ts"), + edge("packages/c/src/z.ts", "packages/a/src/index.ts", "call"), // not an import + edge("packages/c/src/z.ts", "packages/a/nope", "import", true), // dangling + edge("scripts/tool.ts", "packages/a/src/index.ts"), // outside every package + edge("packages/a/src/index.ts", "packages/c/src/z.ts"), + edge("packages/a/test/t.ts", "packages/c/src/z.ts"), + ], + }); + expect(check).toEqual({ + ok: false, + undeclared: [{ from: "a", to: "c", files: 2, example: "packages/a/src/index.ts" }], + unusedDeclared: [{ from: "b", to: "c" }], + }); + }); + + it("leaves nx members out: nx infers project dependencies from imports", () => { + const root = scratchRepo({ + "nx.json": "{}", + "apps/web/project.json": JSON.stringify({ name: "web" }), + "libs/ui/project.json": JSON.stringify({ name: "ui" }), + }); + const check = checkWorkspaceDeps(detectWorkspaces(root), { fileEdges: [edge("apps/web/main.ts", "libs/ui/index.ts")] }); + expect(check).toEqual({ ok: true, undeclared: [], unusedDeclared: [] }); + }); + + it("CLI: `workspaces --check` prints the check and exits 1 on an undeclared import", () => { + const res = spawnSync(process.execPath, [CLI, "workspaces", "--check", "--no-index-cache", "--repo", join(FIXTURES, "mini-monorepo")], { + encoding: "utf8", + }); + expect(res.status).toBe(1); + const out = JSON.parse(res.stdout) as { check: { ok: boolean; undeclared: { from: string; to: string }[] } }; + expect(out.check.ok).toBe(false); + expect(out.check.undeclared.map((u) => `${u.from}->${u.to}`)).toEqual(["@scope/b->@scope/a"]); + + // Without --check: exit 0, no graph work, no `check` key. + const plain = spawnSync(process.execPath, [CLI, "workspaces", "--repo", join(FIXTURES, "mini-monorepo")], { encoding: "utf8" }); + expect(plain.status).toBe(0); + expect(JSON.parse(plain.stdout).check).toBeUndefined(); + }); +}); From 814af1fb5035be4a16568bc7f069d3ddd4d784fa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:25:30 +0000 Subject: [PATCH 006/130] fix(index): make persisted-index reuse honour everything it depends on The persisted index (.codeindex/cache.json + artifacts) was reused, or ignored, in ways its freshness keys never saw: - An absolute --index was joined under the repo (path.join does not reset on an absolute segment), so every read command silently rebuilt cold next to a fresh index (TS repo search: 55s -> 15s). - Read commands scanned an in-repo custom --index dir, so search returned idx/graph.json and the scan never matched the index to reuse it. The read path now excludes the index dir exactly as `index` excludes --out. - Records were reused whatever tier or call cap produced them: after --no-ast, a grammar pull, or another --max-calls, `index` reported "unchanged" and reads served the old records until each file changed. cache.json now records an extraction profile (AST grammars, max-calls); mismatching code entries are dropped per grammar key and re-extracted. - The npm layout ships only core wasms next to the bundle, and the pulled cache was never searched behind them, so extended grammars (Kotlin, Elixir, Zig, ...) never loaded. Lower tiers are now per-key fallbacks; `grammars status` adds extendedPullNeeded. - `index` ignored --no-index-cache and had no --full-hash, the escape hatches for a same-size edit under a restored mtime. Both work now. - `index --out .` (or an ancestor of --repo) excluded every file and wrote a 0-file graph with exit 0; only the artifacts are excluded there now, and an empty index warns. - --ignore-dir re-exposed .codeindex, which is now structural like .git. - graph.json's commit came from `rev-parse --short`, whose length follows the object count, so identical trees rendered different bytes; it is now a fixed 7-char prefix of HEAD. - A binary changing size under an equal (decoded-text) hash kept a stale record size, so cache.json was rewritten on every run. - Artifacts were truncated and rewritten in place; they are now written to a temp file and renamed, so readers never see a torn file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 33 ++++- src/ast/loader.ts | 64 +++++--- src/cache.ts | 73 +++++++++ src/engine-cli.ts | 149 +++++++++++++------ src/git.ts | 13 +- src/preload.ts | 79 ++++++++-- src/scan.ts | 52 +++++-- src/walk.ts | 12 +- tests/grammars-pull.test.ts | 30 ++++ tests/index-cache.test.ts | 287 ++++++++++++++++++++++++++++++++++++ tests/preload.test.ts | 79 +++++++++- tests/scan.test.ts | 63 ++++++++ tests/walk-nested.test.ts | 11 ++ 13 files changed, 842 insertions(+), 103 deletions(-) create mode 100644 tests/index-cache.test.ts diff --git a/README.md b/README.md index 17af6f6..c92438d 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,9 @@ compares](#how-it-compares). - **Walk** a repo deterministically: ignore lists, `.gitignore` and `.git/info/exclude`, binary/lockfile skips, a size cap, symlink-cycle guard. Nested repositories (a subdirectory with its own `.git` — linked worktrees, - vendored clones, submodules) are skipped like git does, and `.git` itself is - never walked even when `--ignore-dir` replaces the default ignore list. No + vendored clones, submodules) are skipped like git does, and `.git` itself — + like the engine's own `.codeindex` — is never walked even when + `--ignore-dir` replaces the default ignore list. No file-count cap unless you ask for one (`--max-files`), and asking sets the `capped` flag — never a silent truncation. - **Scan** every file into a `FileRecord`: classification, language, symbols, @@ -217,8 +218,9 @@ checkout for a benefit only some can use. It ships inside the per-release `grammars-.tar.gz` asset instead. Without a pull those grammars are simply absent and the engine falls back to the regex tier, exactly as it does for a language it has no grammar for at all — `codeindex grammars status` reports -resolved-vs-missing per tier so a Kotlin repo quietly indexed by regex is visible -rather than guesswork. +resolved-vs-missing per tier (and `extendedPullNeeded` while any extended +grammar is missing) so a Kotlin repo quietly indexed by regex is visible rather +than guesswork. *Not included, and why:* **Swift** publishes no prebuilt wasm at all, and **Dart**'s does not load under web-tree-sitter 0.26 — shipping it would be dead @@ -235,9 +237,13 @@ codeindex grammars status # active tier (adjacent/env/cache/none) + whether a codeindex grammars pull # fetch the per-release grammars asset, sha256-verified, into the cache ``` -Resolution is **adjacent > env > cache > regex**: a bundle-adjacent `grammars/` -still wins if present (offline setups are untouched), then -`CODEINDEX_GRAMMARS_DIR`, then the pulled cache. `pull` fetches the official +Resolution is **adjacent > env > cache > regex**, per grammar: a +bundle-adjacent `grammars/` still wins if present (offline setups are +untouched), then `CODEINDEX_GRAMMARS_DIR`, then the pulled cache — and a +grammar the winner lacks is looked up in the tiers below it. That is what lets +the npm package (which ships only the core wasms) pick up the extended ones a +pull put in the cache. The legacy `CODEINDEX_GRAMMAR_DIR` still pins one dir +with nothing behind it. `pull` fetches the official `grammars-.tar.gz` release asset (its `.sha256` sidecar is verified before anything is written) and extracts it atomically; the same wasm bytes produce **byte-identical** AST extraction from the cache as from a vendored dir. @@ -378,6 +384,19 @@ codeindex grep 'pattern' --repo . codeindex literals --repo . # values with no single source of truth ``` +`index` keeps a `cache.json` next to the artifacts, and every read command +reuses whatever sits in `--index` (default `.codeindex`; relative to the repo, +or absolute): unchanged files skip extraction, and when nothing changed the +artifacts load instead of being rebuilt. The index dir itself is never scanned, +and `--out .` at the repo root skips only the artifacts it writes. A record is +reused only if it was extracted the way this run would extract it — the same +`--no-ast`/`--max-calls` setting and the same grammar per language — so +switching either, or pulling a grammar, re-extracts exactly the files it +affects. Freshness is keyed on `(size, mtime)`; for an edit that preserves +both, `--full-hash` re-hashes every file and `--no-index-cache` ignores the +cache altogether (for `index` too). Artifacts are replaced atomically (a temp +file renamed over the old one), so a concurrent reader never sees a torn file. + ## Values with no single source of truth `codeindex literals` reports the defect a compiler cannot: **one value written diff --git a/src/ast/loader.ts b/src/ast/loader.ts index 373eced..08bf79a 100644 --- a/src/ast/loader.ts +++ b/src/ast/loader.ts @@ -58,11 +58,18 @@ export interface GrammarsTier { tier: GrammarsTierName; dir?: string; // undefined only when tier === "none" cacheDir: string; // where a `grammars pull` would extract, regardless of tier - // Every directory a grammar may be loaded from, in precedence order. Usually - // just `dir`; in a DEV checkout it also includes the sibling - // `grammars-extended/` that `fetch-grammars.mjs --extended` writes, because - // locally the two tiers live in two dirs while the release asset extracts both - // into one. A consumer that only ever pulls sees a single dir here. + // Every directory a grammar may be loaded from, in precedence order, each + // probed per key (ensureGrammars takes the first dir holding `.wasm`). + // `dir` comes first, then the sibling `grammars-extended/` that + // `fetch-grammars.mjs --extended` writes in a DEV checkout, then — unless + // the legacy CODEINDEX_GRAMMAR_DIR pins one dir outright — the lower tiers + // that exist (CODEINDEX_GRAMMARS_DIR, the pulled shared cache). The chain is + // what lets the npm layout work: it ships only the CORE wasms next to the + // bundle, so with `dirs` stopping at the adjacent dir the EXTENDED wasms a + // `grammars pull` put in the cache were never searched and Kotlin, Elixir, + // Zig, Solidity, HCL and Terraform stayed on the regex tier for good. The + // bytes of a key are the same in every tier of one ENGINE_VERSION, so which + // dir supplies it never shows in the output. dirs: string[]; } @@ -85,7 +92,8 @@ export function sharedGrammarsCacheDir(): string { // Resolve the grammars dir AND record which tier supplied it, IN ORDER: // 1. an explicit CODEINDEX_GRAMMAR_DIR / ULTRAINDEX_GRAMMAR_DIR override // (legacy, singular; kept winning outright so vendored/test setups that -// pin it behave exactly as before) — reported as the "env" tier; +// pin it behave exactly as before, with no fallback behind it) — +// reported as the "env" tier; // 2. (a) the bundle-adjacent grammars/ dir — the shipped default: works from // the tsup bundle (scripts/engine.mjs → scripts/grammars), a consumer's // vendored copy (src/vendor → ../../scripts/grammars) or source under @@ -94,31 +102,53 @@ export function sharedGrammarsCacheDir(): string { // 3. (b) CODEINDEX_GRAMMARS_DIR — an explicit shared/custom dir override; // 4. (c) the shared version-scoped cache a `grammars pull` populates; // 5. (d) nothing resolvable → tier "none", dir undefined → the regex tier. +// The winner names the tier and `dir`; the tiers below it that exist stay in +// `dirs` as per-key fallbacks (see GrammarsTier.dirs). // Never touches the network and never throws. `moduleDir` overrides the // module-relative base of the bundle-adjacent probe (tests/tooling only). export function resolveGrammarsTier(opts: { moduleDir?: string } = {}): GrammarsTier { const cacheDir = sharedGrammarsCacheDir(); - const withDirs = (tier: GrammarsTierName, dir: string): GrammarsTier => ({ - tier, - dir, - cacheDir, - dirs: [dir, ...(existsSync(join(dir, "..", EXTENDED_DIR)) ? [join(dir, "..", EXTENDED_DIR)] : [])], - }); + const env = process.env.CODEINDEX_GRAMMARS_DIR; + const envDir = env && env.trim() && existsSync(env) ? env : undefined; + const cached = existsSync(cacheDir) ? cacheDir : undefined; + const withDirs = (tier: GrammarsTierName, dir: string, fallbacks: (string | undefined)[]): GrammarsTier => { + const sibling = join(dir, "..", EXTENDED_DIR); + const dirs: string[] = []; + for (const d of [dir, existsSync(sibling) ? sibling : undefined, ...fallbacks]) { + if (d !== undefined && !dirs.includes(d)) dirs.push(d); + } + return { tier, dir, cacheDir, dirs }; + }; + // The legacy override keeps its "one pinned dir" contract: vendored and test + // setups point it at a deliberately partial set, and a user's pulled cache + // leaking in behind it would silently change what those setups measure. const legacy = process.env.CODEINDEX_GRAMMAR_DIR ?? process.env.ULTRAINDEX_GRAMMAR_DIR; - if (legacy && legacy.trim() && existsSync(legacy)) return withDirs("env", legacy); + if (legacy && legacy.trim() && existsSync(legacy)) return withDirs("env", legacy, []); const here = opts.moduleDir ?? dirname(fileURLToPath(import.meta.url)); const adjacent = [ join(here, "grammars"), // bundle: <...>/scripts/grammars join(here, "..", "..", "scripts", "grammars"), // dev: src/ast → /scripts/grammars join(here, "..", "scripts", "grammars"), ]; - for (const c of adjacent) if (existsSync(c)) return withDirs("adjacent", c); - const env = process.env.CODEINDEX_GRAMMARS_DIR; - if (env && env.trim() && existsSync(env)) return withDirs("env", env); - if (existsSync(cacheDir)) return withDirs("cache", cacheDir); + for (const c of adjacent) if (existsSync(c)) return withDirs("adjacent", c, [envDir, cached]); + if (envDir) return withDirs("env", envDir, [cached]); + if (cached) return withDirs("cache", cached, []); return { tier: "none", cacheDir, dirs: [] }; } +// The grammar keys ensureGrammars WOULD load right now — the runtime wasm and +// `.wasm` present somewhere in the resolved dirs — known from existsSync +// alone, without instantiating any wasm. Lets a caller predict the extraction +// tier before deciding whether to warm at all (see src/preload.ts). A present +// but broken wasm is predicted ready and then fails to load; callers that go on +// to warm re-check with grammarReady. +export function resolvableGrammarKeys(): Set { + const { dirs } = resolveGrammarsTier(); + const present = (name: string): boolean => dirs.some((d) => existsSync(join(d, name))); + if (!present("web-tree-sitter.wasm")) return new Set(); + return new Set(allGrammarKeys().filter((key) => present(`${key}.wasm`))); +} + // The chosen grammars dir, or undefined when nothing is resolvable anywhere // (the caller then stays on the regex tier). Additive companion to // resolveGrammarsTier — same resolution, dir only. diff --git a/src/cache.ts b/src/cache.ts index 2dc3b5f..3b5885b 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -3,6 +3,7 @@ // assertions alone do not make a stat-matching record safe to reuse. import { EXTRACTOR_VERSION, SCHEMA_VERSION } from "./types.js"; import type { FileRecord } from "./types.js"; +import { grammarKeyForExt, grammarKeysForExts } from "./ast/loader.js"; export type PersistedCacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number }; export type PersistedCacheMap = Map; @@ -82,3 +83,75 @@ export function parseCacheEntries(value: unknown): PersistedCacheMap | undefined } return cache; } + +// How a persisted cache's CODE records were extracted: the part of their +// provenance that (schemaVersion, extractorVersion) does not capture. Those pin +// the extractor's code, not the configuration it ran under — and the same +// engine gives a code file different symbols, calls and relations at the AST +// tier than at the regex tier, and a different call list under another +// --max-calls. Neither shows in the (size, mtime) / content-hash freshness +// keys, so an index built with --no-ast, before a grammar was pulled, or with +// another --max-calls kept serving those records (and "unchanged — artifacts +// reused") until each file happened to be edited. Persisted as cache.json's +// additive `extraction` meta by `codeindex index`. +export interface ExtractionProfile { + // Grammar keys whose code files were extracted at the AST tier, sorted. + grammars: string[]; + // ScanOptions.maxCallsPerFile exactly as given; absent = the extractor + // default. Not normalized on purpose: an explicit value equal to the default + // costs one re-extraction, never a wrong record. + maxCallsPerFile?: number; +} + +// The profile a scan's records were extracted under. `ast` answers, per +// grammar key, whether this run extracted at the AST tier — grammarReady when +// the scan has already run, since extractAst gates on exactly that. +export function extractionProfile( + files: readonly FileRecord[], + maxCallsPerFile: number | undefined, + ast: (key: string) => boolean, +): ExtractionProfile { + const grammars = grammarKeysForExts(files.filter((f) => f.kind === "code").map((f) => f.ext)).filter(ast); + return maxCallsPerFile === undefined ? { grammars } : { grammars, maxCallsPerFile }; +} + +export function sameExtractionProfile(a: ExtractionProfile | undefined, b: ExtractionProfile): boolean { + return a !== undefined && a.maxCallsPerFile === b.maxCallsPerFile && a.grammars.join(",") === b.grammars.join(","); +} + +// Lenient where the entries are strict: a malformed profile says nothing about +// any record, so it reads as absent and compatibleEntries drops the code +// entries — the per-file records it does not describe are never trusted. +export function parseExtractionProfile(value: unknown): ExtractionProfile | undefined { + if (!object(value) || !Array.isArray(value.grammars) || !value.grammars.every(string)) return undefined; + const calls = value.maxCallsPerFile; + if (calls !== undefined && !(typeof calls === "number" && Number.isFinite(calls) && calls > 0)) return undefined; + const grammars = [...(value.grammars as string[])]; + return calls === undefined ? { grammars } : { grammars, maxCallsPerFile: calls }; +} + +// The persisted entries a scan extracting under `current` would reproduce. +// Only code records depend on the profile, and only through their own grammar +// key, so a mismatch drops exactly the affected entries: those files are +// re-extracted like new ones, everything else keeps its stat fastpath, and the +// scan's contentUnchanged (hence every artifact fastpath) fails as it must. A +// cache with no profile — written before it was recorded — proves nothing +// about any code record. +export function compatibleEntries( + cache: PersistedCacheMap, + stored: ExtractionProfile | undefined, + current: { maxCallsPerFile?: number; ast: (key: string) => boolean }, +): PersistedCacheMap { + const astBefore = new Set(stored?.grammars); + const callsMatch = stored !== undefined && stored.maxCallsPerFile === current.maxCallsPerFile; + const kept: PersistedCacheMap = new Map(); + for (const [rel, entry] of cache) { + if (entry.record.kind === "code") { + if (!callsMatch) continue; + const key = grammarKeyForExt(entry.record.ext); + if (key !== undefined && astBefore.has(key) !== current.ast(key)) continue; + } + kept.set(rel, entry); + } + return kept; +} diff --git a/src/engine-cli.ts b/src/engine-cli.ts index a03a3b3..dafdae2 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -7,6 +7,7 @@ import { EXTENDED_GRAMMARS, ensureGrammars, grammarKeysForExts, + grammarReady, resolveGrammarsTier, sharedGrammarsCacheDir, } from "./ast/loader.js"; @@ -18,8 +19,8 @@ import { renderSymbolsJson } from "./render/symbols-json.js"; import { renderScip } from "./render/scip.js"; import { scanSummary, type RepoScan } from "./scan.js"; import { scanRepoParallel } from "./pool.js"; -import { preloadSessionLazy, INDEX_DIR } from "./preload.js"; -import { parseCacheEntries } from "./cache.js"; +import { indexDirPath, preloadSessionLazy, readPersistedIndex, INDEX_DIR, type PersistedMeta } from "./preload.js"; +import { compatibleEntries, extractionProfile, sameExtractionProfile } from "./cache.js"; import { walk, type WalkResult } from "./walk.js"; import { buildTypeHierarchy, implementationsOf } from "./relations.js"; import { computeImportPairs } from "./callers.js"; @@ -57,7 +58,10 @@ Usage: codeindex [flags] Commands: index Build graph.json + symbols.json (+ incremental cache.json) into - --out in ONE pass — the fast path for repeated runs + --out in ONE pass — the fast path for repeated runs. Each + artifact is replaced atomically (temp file + rename). An --out + inside the repo is excluded from the scan; at the repo root only + the artifacts are scan Scan summary: file count, language histogram, capped flag graph Full link-graph (graph.json bytes) to stdout or --out symbols Symbol index (symbols.json bytes) to stdout or --out @@ -97,9 +101,13 @@ Commands: grammars Tree-sitter wasm grammars (optional AST tier; regex without them). Two tiers: CORE ships with the bundle; EXTENDED (kotlin, elixir, zig, solidity, hcl/terraform) arrives only via \`grammars pull\`. - Precedence: bundle-adjacent > CODEINDEX_GRAMMARS_DIR > shared cache: + Precedence: bundle-adjacent > CODEINDEX_GRAMMARS_DIR > shared cache, + per grammar — a pulled EXTENDED wasm is found even when the + core ones ship next to the bundle: grammars status Active tier (adjacent/env/cache/none), resolved - dir, pinned ENGINE_VERSION, pull-needed (JSON) + dir, pinned ENGINE_VERSION, pull-needed, and + extendedPullNeeded when an EXTENDED grammar is + still missing (JSON) grammars pull Fetch the per-release grammars-.tar.gz asset into the shared cache (sha256-verified, atomic). Override the source with @@ -171,7 +179,7 @@ Flags (accepted before OR after the subcommand: '--repo X scan' and --no-gitignore Do not honor .gitignore files (default: honored) --ignore-dir Directory names to skip (repeatable) — REPLACES the default ignored-directory set, never merges with it - (\`.git\` stays skipped regardless) + (\`.git\` and \`.codeindex\` stay skipped regardless) --max-files Cap walked files (default: none — the whole tree is indexed; a cap sets the \`capped\` flag) --max-bytes Skip files above this size (default 1 MiB) @@ -182,11 +190,19 @@ Flags (accepted before OR after the subcommand: '--repo X scan' and Also settable with CODEINDEX_WORKERS. Artifacts are byte-identical either way --index Persisted index the READ commands reuse, relative to the - repo (default .codeindex — i.e. what \`index --out\` wrote - there). A fresh index turns the scan into a stat pass and, - when it still matches the worktree, skips the pipeline - entirely. Stale/absent/corrupt → a normal cold build + repo or absolute (default .codeindex — i.e. what + \`index --out\` wrote there). A fresh index turns the scan + into a stat pass and, when it still matches the worktree, + skips the pipeline entirely. Stale/absent/corrupt → a + normal cold build (with a note on stderr when --index was + given). The dir itself is never scanned. Records built + with another --no-ast/--max-calls setting or grammar set + are re-extracted, never reused --no-index-cache Never reuse a persisted index; always build from scratch + (\`index\` too: its cache.json is ignored, then rewritten) + --full-hash Re-read and re-hash every file instead of trusting an + unchanged (size, mtime) — for an edit that kept both. + Unchanged content still reuses its extraction --config Rules config for \`rules\` (JSON: [{name, from, to, …}]) --limit Max results for \`search\` (default 20) --no-fuzzy \`search\`: disable trigram fuzzy fallback for query terms @@ -229,6 +245,7 @@ interface CliFlags { workers?: number; // extraction worker threads (0/1 = sequential) indexDir?: string; // persisted index to read (default .codeindex) noIndexCache?: boolean; // never reuse a persisted index + fullHash?: boolean; // re-read and re-hash every file (no (size, mtime) fastpath) since?: string; ignoreCase?: boolean; maxHits?: number; @@ -294,6 +311,7 @@ function parseFlags(args: string[]): CliFlags { else if (a === "--no-ast") flags.noAst = true; else if (a === "--index") flags.indexDir = next(); else if (a === "--no-index-cache") flags.noIndexCache = true; + else if (a === "--full-hash") flags.fullHash = true; else if (a === "--workers") { // 0 is meaningful here (force sequential), so this cannot use num(). const raw = next(); @@ -338,6 +356,27 @@ function emit(content: string, out?: string): void { else process.stdout.write(content); } +// Replace an index artifact in one step: write a sibling temp file, then +// rename it over the target (atomic on POSIX). graph.json, symbols.json and +// cache.json used to be truncated and rewritten in place, so a reader polling +// them mid-index — CI, an editor plugin, the MCP server's artifact preload — +// saw an empty or half-written file, and a crash mid-write left torn JSON until +// the next index. The temp name is one the self-index guard skips (scan.ts), so +// even an --out at the repo root never indexes a leftover. Where the temp file +// or the rename is refused (a Windows reader holding the target open), fall +// back to the historical in-place write rather than failing the index. +function writeArtifact(path: string, data: string | Uint8Array): void { + const temp = `${path}.tmp-${process.pid}`; + try { + writeFileSync(temp, data); + renameSync(temp, path); + return; + } catch { + rmSync(temp, { force: true }); + } + writeFileSync(path, data); +} + function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexOptions { return { include: flags.include.length ? flags.include : undefined, @@ -348,6 +387,15 @@ function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexO maxFiles: flags.maxFiles, maxBytes: flags.maxBytes, maxCallsPerFile: flags.maxCalls, + fullHash: flags.fullHash, + // The index the read commands consult is excluded from what they scan, + // exactly as `index` excludes its --out (which overrides this there). An + // in-repo custom dir (`index --out idx` + `--index idx`) was otherwise + // scanned as three config files: search answered "graph" with + // idx/graph.json, and the scan never matched the index that `index` built + // without them, so its artifacts were never reused. The default + // .codeindex is pruned by the walk already; this is a no-op there. + out: indexDirPath(flags.repo, flags.indexDir), // The walk performed once in runCli to warm the present-language grammars, // reused here so scanRepo does not traverse the tree a second time. Absent // for --no-ast / scan-less commands: scanRepo walks itself, unchanged. @@ -568,11 +616,19 @@ export async function runCli(rawArgv: string[]): Promise { if (flags.noIndexCache) return undefined; preloadPromise = preloadSessionLazy( flags.repo, - { ...scanOptions(flags, precomputedWalk), workers: flags.workers }, + { ...scanOptions(flags, precomputedWalk), workers: flags.workers, ast: !flags.noAst }, warmPresentGrammars, indexDir, ).then((p) => { if (p) preloaded = { scan: p.scan, arts: p.arts, loadArtifacts: p.loadArtifacts }; + // The default location being empty is the normal first run; an index the + // user NAMED being unusable is a mistake worth one line (a typo'd path + // otherwise just looks like a slow command). + else if (flags.indexDir !== undefined) { + process.stderr.write( + `codeindex: no usable index at ${indexDirPath(flags.repo, indexDir)} (missing, unreadable, or written by an incompatible engine) — building from scratch\n`, + ); + } return preloaded; }); return preloadPromise; @@ -609,41 +665,33 @@ export async function runCli(rawArgv: string[]): Promise { // old caches lacking them simply never take the fastpath below (their // per-file records are still reused). cache.json embeds mtimes, so it was // never cross-machine byte-reproducible — no determinism surface changes. - type CacheMeta = { - engineVersion?: string; - commit?: string; - graphSha1?: string; - symbolsSha1?: string; - embed?: { embedVersion?: number; modelId?: string; sha1?: string }; - }; - let cache: Map | undefined; - let meta: CacheMeta = {}; - try { - const parsed = JSON.parse(readFileSync(cachePath, "utf8")) as { - schemaVersion: number; - extractorVersion: number; - files: Record; - } & CacheMeta; - cache = parseCacheEntries(parsed); - if (cache) { - meta = { - engineVersion: parsed.engineVersion, - commit: parsed.commit, - graphSha1: parsed.graphSha1, - symbolsSha1: parsed.symbolsSha1, - embed: parsed.embed, - }; - } - } catch { - // no cache yet (or unreadable) — cold build - } + type CacheMeta = Pick; + // --no-index-cache is the documented "always build from scratch": it used + // to be read only by the query commands, so `index` kept trusting a + // cache.json whose (size, mtime) keys hid a same-size edit made under a + // restored mtime, with no escape hatch short of deleting the file by hand. + const persisted = flags.noIndexCache ? undefined : readPersistedIndex(flags.repo, outDir); + const meta: CacheMeta = persisted?.meta ?? {}; await warmPresentGrammars(); + // Grammars are loaded (or deliberately not, under --no-ast), so the tier + // this run extracts each language at is known exactly: keep only records + // extracted the same way — see compatibleEntries. + const cache = persisted && compatibleEntries(persisted.cacheMap, persisted.meta.extraction, { + maxCallsPerFile: flags.maxCalls, + ast: grammarReady, + }); const scan = await scanRepoParallel(flags.repo, { ...scanOptions(flags, precomputedWalk), cache, out: outDir, workers: flags.workers, }); + const extraction = extractionProfile(scan.files, flags.maxCalls, grammarReady); + if (scan.files.length === 0) { + process.stderr.write( + `codeindex: warning: no file of ${flags.repo} was indexed — check --scope/--include/--exclude and the ignore rules\n`, + ); + } const modelDir = resolveEmbedModelDir(flags.repo); const model = modelDir ? loadEmbedModel(modelDir) : undefined; @@ -669,7 +717,7 @@ export async function runCli(rawArgv: string[]): Promise { } // Fixed key order; JSON.stringify drops the undefined-valued keys // (commit outside a git worktree, embed without a model) cleanly. - writeFileSync( + writeArtifact( cachePath, JSON.stringify({ schemaVersion: SCHEMA_VERSION, @@ -679,6 +727,7 @@ export async function runCli(rawArgv: string[]): Promise { graphSha1: out.graphSha1, symbolsSha1: out.symbolsSha1, embed: out.embed, + extraction, files, }) + "\n", ); @@ -714,9 +763,11 @@ export async function runCli(rawArgv: string[]): Promise { if (fastpath) { // Artifacts verified byte-identical to what this build would produce — // leave them untouched. Rewrite cache.json only when the scan says its - // bytes would change (e.g. an mtime drifted); the meta is carried - // forward verbatim since the guard just proved it describes the disk. - if (scan.cacheDirty) writeCache(meta); + // bytes would change (e.g. an mtime drifted) or its extraction profile + // would (a cache written before profiles existed, or a --max-calls switch + // on a tree with no code to re-extract); the meta is carried forward + // verbatim since the guard just proved it describes the disk. + if (scan.cacheDirty || !sameExtractionProfile(persisted?.meta.extraction, extraction)) writeCache(meta); process.stderr.write( `codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${scan.capped ? " (capped)" : ""} (unchanged — artifacts reused)\n`, ); @@ -724,8 +775,8 @@ export async function runCli(rawArgv: string[]): Promise { const { graph, symbols } = buildArtifactsFromScan(scan); const graphJson = renderGraphJson(graph); const symbolsJson = renderSymbolsJson(symbols); - writeFileSync(graphPath, graphJson); - writeFileSync(symbolsPath, symbolsJson); + writeArtifact(graphPath, graphJson); + writeArtifact(symbolsPath, symbolsJson); // Deterministic embeddings sidecar: written next to graph.json ONLY when a // model asset is present (opt-in). Silently skipped otherwise — no model, no // embeddings.bin, no impact on the graph/symbols consumers. @@ -734,7 +785,7 @@ export async function runCli(rawArgv: string[]): Promise { if (model) { const index = buildEmbeddingIndex(scan, model); const bytes = serializeEmbeddings(index); - writeFileSync(embedPath, bytes); + writeArtifact(embedPath, bytes); embedMeta = { embedVersion: EMBED_VERSION, modelId: model.modelId, sha1: sha1(bytes) }; embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`; } @@ -945,7 +996,7 @@ export async function runCli(rawArgv: string[]): Promise { mkdirSync(flags.out, { recursive: true }); const scan = await readScan(); const index = buildEmbeddingIndex(scan, model); - writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index)); + writeArtifact(join(flags.out, "embeddings.bin"), serializeEmbeddings(index)); process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`); } else if (sub === "pull") { // Default: the official published asset + its pinned sha256. A user-set @@ -1012,6 +1063,10 @@ export async function runCli(rawArgv: string[]): Promise { cacheDir, runtimePresent, pullNeeded: !runtimePresent, + // The AST tier can be live (pullNeeded false) while the EXTENDED + // grammars are missing — the npm layout ships only the core ones — and + // those languages then run on the regex tier until a pull. + extendedPullNeeded: extended.length < EXTENDED_GRAMMARS.size, core: { resolved: core.length, of: CORE_GRAMMARS.size, missing: [...CORE_GRAMMARS].filter((k) => !core.includes(k)).sort() }, extended: { resolved: extended.length, diff --git a/src/git.ts b/src/git.ts index d3f274a..ae6e545 100644 --- a/src/git.ts +++ b/src/git.ts @@ -3,9 +3,18 @@ import { sh } from "./util.js"; // The short HEAD commit of a working tree, when it is a git repo. Recorded in // the manifest so an index is pinned to an exact revision. Returns undefined // when `git` is absent or the directory isn't a repo — the index still works. +// +// A FIXED-length prefix of the full id, never `rev-parse --short`: git sizes +// that abbreviation from the object count, so the same commit and tree printed +// "3b08cd7" in one clone and "3b08cd72" in another with more objects — a +// shallow CI clone and a full one, or the same clone after a fetch or gc — +// and graph.json's bytes changed with it (forcing a full rebuild). Seven is +// git's default minimum, so small repos keep exactly the bytes they had. +const COMMIT_CHARS = 7; + export function headCommit(dir: string): string | undefined { - const res = sh("git", ["-C", dir, "rev-parse", "--short", "HEAD"]); - return res.ok ? res.stdout.trim() : undefined; + const res = sh("git", ["-C", dir, "rev-parse", "HEAD"]); + return res.ok ? res.stdout.trim().slice(0, COMMIT_CHARS) : undefined; } // --------------------------------------------------------------------------- diff --git a/src/preload.ts b/src/preload.ts index 25b6d2b..babd202 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -20,11 +20,19 @@ // `codeindex search` cost a full tree-sitter pass over the repo every time it // ran — 6.3s on a 7k-file repo with a fresh index sitting right next to it. import { readFileSync } from "node:fs"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { ENGINE_VERSION, SCHEMA_VERSION } from "./types.js"; import type { Graph, SymbolIndex } from "./types.js"; -import { parseCacheEntries, type PersistedCacheEntry, type PersistedCacheMap } from "./cache.js"; -export type { PersistedCacheEntry, PersistedCacheMap } from "./cache.js"; +import { + compatibleEntries, + parseCacheEntries, + parseExtractionProfile, + type ExtractionProfile, + type PersistedCacheEntry, + type PersistedCacheMap, +} from "./cache.js"; +export type { ExtractionProfile, PersistedCacheEntry, PersistedCacheMap } from "./cache.js"; +import { grammarReady, resolvableGrammarKeys } from "./ast/loader.js"; import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js"; import { scanRepoParallel } from "./pool.js"; import type { IndexArtifacts } from "./pipeline.js"; @@ -45,6 +53,18 @@ export interface PersistedMeta { commit?: string; graphSha1?: string; symbolsSha1?: string; + embed?: { embedVersion?: number; modelId?: string; sha1?: string }; + // How the code records were extracted (see compatibleEntries). Absent in + // caches written before it was recorded, and then no code record is reused. + extraction?: ExtractionProfile; +} + +// Where an index dir lives. RELATIVE to the repo by contract, but an absolute +// --index must be honoured as given: path.join does not reset on an absolute +// segment, so `join(repo, "/abs/idx")` probed /abs/idx, found nothing, +// and every read command silently paid a full cold build next to a fresh index. +export function indexDirPath(repo: string, indexDir: string = INDEX_DIR): string { + return resolve(repo, indexDir); } export interface PreloadedSession { @@ -85,6 +105,9 @@ export function needsGrammarWarm( // match this engine — the exact gate the CLI applies before trusting a cache — // otherwise the whole cache is discarded (cold scan). Any read/parse failure (no // index yet, unreadable, malformed) returns undefined: the cold path. +// `indexDir` is repo-relative, or absolute. The map is returned UNFILTERED: +// before seeding a scan with it, narrow it with compatibleEntries against +// meta.extraction, as preloadSession does. export function readPersistedIndex( repo: string, indexDir: string = INDEX_DIR, @@ -93,7 +116,7 @@ export function readPersistedIndex( | ({ schemaVersion?: number; extractorVersion?: number; files?: Record } & PersistedMeta) | undefined; try { - parsed = JSON.parse(readFileSync(join(repo, indexDir, "cache.json"), "utf8")) as typeof parsed; + parsed = JSON.parse(readFileSync(join(indexDirPath(repo, indexDir), "cache.json"), "utf8")) as typeof parsed; } catch { return undefined; } @@ -106,6 +129,8 @@ export function readPersistedIndex( commit: parsed.commit, graphSha1: parsed.graphSha1, symbolsSha1: parsed.symbolsSha1, + embed: parsed.embed, + extraction: parseExtractionProfile(parsed.extraction), }, }; } @@ -137,7 +162,7 @@ export function preloadArtifacts( ) { return undefined; } - const dir = join(repo, indexDir); + const dir = indexDirPath(repo, indexDir); let graphBytes: Buffer; let symbolsBytes: Buffer; try { @@ -178,8 +203,14 @@ export function preloadSession( // and it computes the contentUnchanged the artifact guard reads. When the // on-disk content drifted from cache.json, changed files are re-read/extracted // here exactly as a cold scan would, so the scan stays correct and the guard - // simply fails (arts undefined → rebuild on demand). - const scan = scanRepo(repo, { ...opts, cache: persisted.cacheMap }); + // simply fails (arts undefined → rebuild on demand). A synchronous scan + // extracts at whatever tier is loaded right now, so that is the tier a + // reused record must have been extracted at. + const cache = compatibleEntries(persisted.cacheMap, persisted.meta.extraction, { + maxCallsPerFile: opts.maxCallsPerFile, + ast: grammarReady, + }); + const scan = scanRepo(repo, { ...opts, cache }); return { scan, cacheMap: toCacheMap(scan), arts: preloadArtifacts(repo, scan, persisted.meta, indexDir) }; } @@ -197,31 +228,49 @@ export function preloadSession( // scanRepo consumes the records exactly as the sequential loop would have // built them (pool.ts) — an unchanged index still loads no wasm and spawns // nothing. +// +// `opts.ast: false` declares a caller that extracts at the regex tier (its warm +// loads nothing, e.g. the CLI's --no-ast); by default the warm is expected to +// load every grammar resolvable on disk. export async function preloadSessionLazy( repo: string, - opts: Omit & { workers?: number }, + opts: Omit & { workers?: number; ast?: boolean }, warm: () => Promise, indexDir: string = INDEX_DIR, ): Promise { const persisted = readPersistedIndex(repo, indexDir); if (!persisted) return undefined; - const walked = opts.precomputedWalk ?? walk(repo, { - maxFileBytes: opts.maxBytes, - maxFiles: opts.maxFiles, - gitignore: opts.gitignore, - ignoreDirs: opts.ignoreDirs, + const { ast, workers, ...scanOpts } = opts; + const walked = scanOpts.precomputedWalk ?? walk(repo, { + maxFileBytes: scanOpts.maxBytes, + maxFiles: scanOpts.maxFiles, + gitignore: scanOpts.gitignore, + ignoreDirs: scanOpts.ignoreDirs, }); + // Records extracted at another tier (or call cap) than this run would use + // are dropped before anything else looks at the cache. Nothing is loaded yet, + // so the tier is PREDICTED from what the warm would load; a dropped entry + // then reads as a new code file below, which is exactly what makes the warm + // happen for it. + const resolvable = ast === false ? new Set() : resolvableGrammarKeys(); + const compatible = (tier: (key: string) => boolean): PersistedCacheMap => + compatibleEntries(persisted.cacheMap, persisted.meta.extraction, { maxCallsPerFile: scanOpts.maxCallsPerFile, ast: tier }); + let cache = compatible((key) => grammarReady(key) || resolvable.has(key)); // Decide whether grammars are needed from metadata BEFORE extraction. The old // flow first extracted every changed code file without grammars, discovered // the scan was stale, then warmed and extracted those files again. A new or // stat-changed path may need AST work; deletions, scope-only differences and // an unchanged index do not. fullHash deliberately warms because equal stats // are no longer a freshness proof in that mode. - const needsWarm = needsGrammarWarm(walked, persisted.cacheMap, opts.fullHash); + const needsWarm = needsGrammarWarm(walked, cache, scanOpts.fullHash); if (needsWarm) { await warm(); + // Loaded now: filter again against the tier extraction will really use. It + // differs from the prediction only for a wasm that is present but fails to + // load, whose records the index built at the regex tier. + cache = compatible(grammarReady); } - const scan = await scanRepoParallel(repo, { ...opts, cache: persisted.cacheMap, precomputedWalk: walked }); + const scan = await scanRepoParallel(repo, { ...scanOpts, workers, cache, precomputedWalk: walked }); let artifactsTried = false; let artifacts: IndexArtifacts | undefined; return { diff --git a/src/scan.ts b/src/scan.ts index a5e0dea..86089fc 100644 --- a/src/scan.ts +++ b/src/scan.ts @@ -1,4 +1,4 @@ -import { basename } from "node:path"; +import { basename, isAbsolute, relative, resolve, sep } from "node:path"; import type { FileRecord, FileKind } from "./types.js"; import { walk, readText, type WalkResult, type WalkedFile } from "./walk.js"; import { headCommit } from "./git.js"; @@ -51,9 +51,9 @@ export interface ScanOptions { scope?: string; // Honor .gitignore files (default true — see WalkOptions.gitignore). gitignore?: boolean; - // Directory names to skip — REPLACES the default set, except `.git` which is - // always skipped (see WalkOptions.ignoreDirs; compose with the IGNORE_DIRS - // export to extend it). + // Directory names to skip — REPLACES the default set, except `.git` and + // `.codeindex` which are always skipped (see WalkOptions.ignoreDirs; compose + // with the IGNORE_DIRS export to extend it). ignoreDirs?: string[]; maxBytes?: number; maxFiles?: number; @@ -61,7 +61,10 @@ export interface ScanOptions { // tiers). Raising it trades index size for call-graph recall; dedup/sort // semantics are unchanged. Absent, output is byte-identical to before. maxCallsPerFile?: number; - out?: string; // absolute output dir to exclude from the scan (self-index guard) + // Absolute output dir to exclude from the scan (self-index guard). Inside the + // repo, the whole dir is skipped; at the repo root, only the index artifacts + // written there (INDEX_ARTIFACTS); above the root, nothing. + out?: string; // Previous build's extraction cache (rel → {hash, record, size?, mtimeMs?}). A // file whose (size,mtime) key matches skips read+hash entirely (the stat // fastpath); one whose content hash is unchanged reuses its record and skips @@ -161,10 +164,10 @@ function* keptFiles( }); // Never index our own output (e.g. a committed `docs/ultraindex/`), or builds // would describe the encyclopedia instead of the code. - const outPrefix = opts.out ? opts.out.replace(/\/+$/, "") + "/" : null; + const guard = selfIndexGuard(root, opts.out); for (const f of walked) { - if (outPrefix && (f.abs === opts.out || f.abs.startsWith(outPrefix))) continue; + if (guard && guard(f)) continue; if (include && !include(f.rel)) continue; if (exclude && exclude(f.rel)) continue; yield { f, kind: classify(f.rel, f.ext), lang: extToLang(f.ext) }; @@ -177,6 +180,28 @@ interface WalkTotals { excluded: number; } +// The files `codeindex index` writes into its --out dir, plus the +// `.tmp-` sibling each is staged under before its atomic rename. +const INDEX_ARTIFACTS = ["graph.json", "symbols.json", "cache.json", "embeddings.bin"]; +const isIndexArtifact = (name: string): boolean => + INDEX_ARTIFACTS.some((a) => name === a || name.startsWith(`${a}.tmp-`)); + +// Which walked files an --out dir takes out of the scan. Excluding everything +// under --out is right while --out sits INSIDE the repo, but `index --out .` +// (or --out at any ancestor of --repo) put every file under it: the scan came +// back empty, and the run still exited 0 with a 0-file graph. There the only +// files the index itself contributes are its artifacts — directly in --out, +// which for an ancestor is outside the repo, so nothing is excluded at all. +function selfIndexGuard(root: string, out: string | undefined): ((f: WalkedFile) => boolean) | undefined { + if (!out) return undefined; + const up = relative(resolve(out), resolve(root)); + if (up === "") return (f) => !f.rel.includes("/") && isIndexArtifact(f.rel); + if (up !== ".." && !up.startsWith(`..${sep}`) && !isAbsolute(up)) return undefined; + const dir = out.replace(/\/+$/, ""); + const prefix = dir + "/"; + return (f) => f.abs === dir || f.abs.startsWith(prefix); +} + // The code files this scan would extract, in the same order scanRepo sees them. // The worker pool builds its job list from here so the set it extracts is // exactly the set scanRepo would have extracted — no file gets a worker record @@ -247,8 +272,8 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan { // every build, so a doc is always read regardless. --full-hash disables the // fastpath. The (size,mtime) pair is the heuristic here (not the exact content // hash below): a real editor bumps mtime on every save, and --full-hash / - // --no-cache are the escape hatches for the astronomically-unlikely edit that - // preserves both. + // --no-index-cache are the escape hatches for the astronomically-unlikely + // edit that preserves both. if ( kind !== "doc" && !opts.fullHash && @@ -280,7 +305,14 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan { const content = preUsable ? undefined : readText(f.abs); const hash = preUsable ? preUsable.record.hash : sha1(content!); if (cached && cached.hash === hash) { - files.push(cached.record); + // The hash is over the DECODED text, so it is blind to bytes the decoder + // drops: every binary hashes as sha1(""), a BOM or a UTF-16 odd trailing + // byte vanishes. Such a file can change size under an equal hash, and + // the record kept its stale size for good — the cache entry then missed + // the stat fastpath on every later run and cache.json was rewritten each + // time. Every other field derives from the decoded text, so only the + // size is refreshed. + files.push(cached.record.size === f.size ? cached.record : { ...cached.record, size: f.size }); if (kind === "doc" && content) docText.set(f.rel, content); // Content proven identical, but a (size, mtimeMs) drift — e.g. a bare // touch, or an old cache without stat keys — still rewrites cache bytes. diff --git a/src/walk.ts b/src/walk.ts index bf389f8..59a032e 100644 --- a/src/walk.ts +++ b/src/walk.ts @@ -20,17 +20,24 @@ export const IGNORE_DIRS = new Set([ // The VCS entry that marks a repository root: a directory for a normal clone, // a "gitdir: " FILE for a linked worktree or a submodule. const GIT_ENTRY = ".git"; +// The engine's own index dir (preload.ts's INDEX_DIR, repeated here because +// preload.ts imports this module). +const INDEX_ENTRY = ".codeindex"; function isIgnoredDirectory(name: string, ignoreDirs: Set): boolean { // `.git` is structural, not a preference: VCS internals (objects, packs, // hooks) never carry signal, so it stays ignored even when a caller-supplied // `ignoreDirs` replaces the default set without listing it — `--ignore-dir // foo` used to pull thousands of loose objects into the index. + // `.codeindex` is structural for the same reason: it is this engine's own + // output (artifacts, pulled models, MCP memories). `--ignore-dir + // node_modules` put it back in the scan, so search answered with + // `.codeindex/symbols.json` and the index described itself. // A process killed during an atomic symbolic edit can leave the // `.codeindex-edit-*` directory beside the source. It contains a copy of that // source and must never become a duplicate phantom file in the next index, // even when the consumer repo has no matching .gitignore rule. - return name === GIT_ENTRY || ignoreDirs.has(name) || name.startsWith(".codeindex-edit-"); + return name === GIT_ENTRY || name === INDEX_ENTRY || ignoreDirs.has(name) || name.startsWith(".codeindex-edit-"); } // A gitfile's mandatory opening bytes. Git's parser (read_gitfile_gently) @@ -146,7 +153,8 @@ export interface WalkOptions { // every consumer; pass false to index generated/ignored trees deliberately. gitignore?: boolean; // Directory names to skip, REPLACING the default set entirely (not merging - // with it) — except `.git`, which is skipped whatever the list says. + // with it) — except `.git` and `.codeindex`, which are skipped whatever the + // list says. // IGNORE_DIRS is a public export, so consumers compose // `[...IGNORE_DIRS, "extra"]` — or filter it — themselves; replace is the // simplest contract. Deliberate scope boundary: grep.ts (the ripgrep diff --git a/tests/grammars-pull.test.ts b/tests/grammars-pull.test.ts index 0d8f1c4..c793b22 100644 --- a/tests/grammars-pull.test.ts +++ b/tests/grammars-pull.test.ts @@ -163,6 +163,36 @@ describe("resolveGrammarsTier — resolution order (adjacent > env > cache > non expect(t.dir).toBe(legacy); }); + // The npm layout ships the CORE wasms next to the bundle and nothing else; + // the EXTENDED ones only ever arrive through `grammars pull` into the shared + // cache. `dirs` used to stop at the winning dir, so a pulled kotlin.wasm was + // never searched and Kotlin stayed on the regex tier with pullNeeded false. + it("keeps the lower tiers that exist as per-key fallbacks behind the winner", () => { + const modDir = mk("ci-gr-mod-"); + mkdirSync(join(modDir, "grammars")); + const envDir = mk("ci-gr-env-"); + process.env.CODEINDEX_GRAMMARS_DIR = envDir; + const home = populatedCacheHome(); + process.env.XDG_CACHE_HOME = home; + const cdir = join(home, "codeindex", "grammars", ENGINE_VERSION); + const adjacent = resolveGrammarsTier({ moduleDir: modDir }); + expect(adjacent.tier).toBe("adjacent"); + expect(adjacent.dirs).toEqual([join(modDir, "grammars"), envDir, cdir]); + const env = resolveGrammarsTier({ moduleDir: mk("ci-gr-mod-") }); + expect(env.tier).toBe("env"); + expect(env.dirs).toEqual([envDir, cdir]); + delete process.env.CODEINDEX_GRAMMARS_DIR; + expect(resolveGrammarsTier({ moduleDir: modDir }).dirs).toEqual([join(modDir, "grammars"), cdir]); + }); + + it("the legacy override stays one pinned dir, with no fallback behind it", () => { + const legacy = mk("ci-gr-legacy-"); + process.env.CODEINDEX_GRAMMAR_DIR = legacy; + process.env.CODEINDEX_GRAMMARS_DIR = mk("ci-gr-env-"); + process.env.XDG_CACHE_HOME = populatedCacheHome(); + expect(resolveGrammarsTier({ moduleDir: mk("ci-gr-mod-") }).dirs).toEqual([legacy]); + }); + it("sharedGrammarsCacheDir is version-scoped and honors XDG_CACHE_HOME", () => { process.env.XDG_CACHE_HOME = join(tmpdir(), "xdg-fixed"); expect(sharedGrammarsCacheDir()).toBe(join(tmpdir(), "xdg-fixed", "codeindex", "grammars", ENGINE_VERSION)); diff --git a/tests/index-cache.test.ts b/tests/index-cache.test.ts new file mode 100644 index 0000000..c30173b --- /dev/null +++ b/tests/index-cache.test.ts @@ -0,0 +1,287 @@ +// `codeindex index` and the cache.json it persists: when a record or an +// artifact may be reused, and when it must not be. Every test drives the shipped +// CLI (scripts/cli.mjs) against a scratch copy of the mini-repo fixture, because +// the failures this file pins were all invisible to the library — each was a +// reuse decision the CLI got wrong while every output-equality test passed. +import { afterAll, describe, expect, it } from "vitest"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + copyFileSync, + cpSync, + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, + appendFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { headCommit } from "../src/git.js"; +import { ENGINE_VERSION } from "../src/types.js"; + +const FIXTURE = fileURLToPath(new URL("./fixtures/mini-repo", import.meta.url)); +const SCRIPTS = fileURLToPath(new URL("../scripts", import.meta.url)); +const CLI = join(SCRIPTS, "cli.mjs"); +// A dev shell's embedding model must never turn the embed leg of the index +// fastpath on for these runs. +const ENV = { ...process.env, CODEINDEX_EMBED_DIR: "" }; + +const dirs: string[] = []; +afterAll(() => { + for (const d of dirs) rmSync(d, { recursive: true, force: true }); +}); +function scratch(prefix = "ci-index-cache-"): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + dirs.push(dir); + return dir; +} +function freshRepo(): string { + const repo = join(scratch(), "repo"); + cpSync(FIXTURE, repo, { recursive: true }); + return repo; +} + +function cli(args: string[], env: NodeJS.ProcessEnv = ENV): { stdout: string; stderr: string } { + const res = spawnSync(process.execPath, [CLI, ...args], { encoding: "utf8", env, maxBuffer: 256 * 1024 * 1024 }); + if (res.status !== 0) throw new Error(`cli ${args.join(" ")} exited ${res.status}: ${res.stderr}`); + return { stdout: res.stdout, stderr: res.stderr }; +} +// Runs `index` and reports whether it took the "unchanged" fastpath. +function index(repo: string, out: string, ...flags: string[]): boolean { + return cli(["index", "--repo", repo, "--out", out, ...flags]).stderr.includes("(unchanged — artifacts reused)"); +} +const read = (dir: string, name: string): string => readFileSync(join(dir, name), "utf8"); +function sameArtifacts(a: string, b: string): void { + expect(read(a, "graph.json")).toBe(read(b, "graph.json")); + expect(read(a, "symbols.json")).toBe(read(b, "symbols.json")); +} + +// (schemaVersion, extractorVersion) pin the extractor's code, not the setting +// it ran under. The tier (--no-ast, a grammar arriving later) and --max-calls +// both change code records without touching any freshness key, so switching +// either used to print "unchanged — artifacts reused" over the old records. +describe("the extraction profile gates record reuse", { timeout: 120_000 }, () => { + it("--no-ast, then a default index: rebuilt at the AST tier, byte-identical to cold", () => { + const repo = freshRepo(); + const out = join(scratch(), "out"); + const cold = join(scratch(), "cold"); + index(repo, cold); + index(repo, out, "--no-ast"); + expect(read(out, "symbols.json")).not.toBe(read(cold, "symbols.json")); // the tiers really differ here + expect(index(repo, out)).toBe(false); + sameArtifacts(out, cold); + expect(index(repo, out)).toBe(true); // and it settles + }); + + it("a default index, then --no-ast: rebuilt at the regex tier, byte-identical to cold", () => { + const repo = freshRepo(); + const out = join(scratch(), "out"); + const cold = join(scratch(), "cold"); + index(repo, cold, "--no-ast"); + index(repo, out); + expect(index(repo, out, "--no-ast")).toBe(false); + sameArtifacts(out, cold); + expect(index(repo, out, "--no-ast")).toBe(true); + }); + + it("--max-calls: a new cap rebuilds, the same cap reuses", () => { + const repo = freshRepo(); + const out = join(scratch(), "out"); + const cold = join(scratch(), "cold"); + index(repo, cold, "--max-calls", "1"); + index(repo, out); + expect(read(out, "graph.json")).not.toBe(read(cold, "graph.json")); + expect(index(repo, out, "--max-calls", "1")).toBe(false); + sameArtifacts(out, cold); + expect(index(repo, out, "--max-calls", "1")).toBe(true); + }); + + it("records the profile in cache.json", () => { + const repo = freshRepo(); + const out = join(scratch(), "out"); + const profile = (): unknown => (JSON.parse(read(out, "cache.json")) as { extraction?: unknown }).extraction; + index(repo, out); + expect(profile()).toEqual({ grammars: expect.arrayContaining(["typescript"]) }); + index(repo, out, "--max-calls", "7"); + expect(profile()).toEqual({ grammars: expect.arrayContaining(["typescript"]), maxCallsPerFile: 7 }); + index(repo, out, "--no-ast"); + expect(profile()).toEqual({ grammars: [] }); + }); + + it("read commands honour --no-ast / --max-calls against an index built without them", () => { + const repo = freshRepo(); + index(repo, join(repo, ".codeindex")); + const q = (...args: string[]): string => cli([...args, "--repo", repo]).stdout; + expect(q("symbols", "--no-ast")).not.toBe(q("symbols")); + expect(q("symbols", "--no-ast")).toBe(q("symbols", "--no-ast", "--no-index-cache")); + expect(q("graph", "--max-calls", "1")).not.toBe(q("graph")); + expect(q("graph", "--max-calls", "1")).toBe(q("graph", "--max-calls", "1", "--no-index-cache")); + expect(q("graph")).toBe(read(join(repo, ".codeindex"), "graph.json")); + }); +}); + +// The (size, mtime) fastpath is a heuristic; the documented escape hatches for +// an edit that keeps both are --full-hash and --no-index-cache. `index` had +// neither: --full-hash was an unknown flag, --no-index-cache was ignored. +describe("index escape hatches for a same-size edit under a restored mtime", { timeout: 60_000 }, () => { + it("--full-hash re-hashes, --no-index-cache rebuilds from scratch", () => { + const repo = freshRepo(); + const out = join(scratch(), "out"); + const file = join(repo, "src", "util.ts"); + // A whole-second mtime, so restoring it is exact (a Date drops sub-ms). + const stamp = new Date(2020, 0, 1); + utimesSync(file, stamp, stamp); + index(repo, out); + const original = readFileSync(file, "utf8"); + const name = /export function (\w+)/.exec(original)![1]!; + const renamed = name.slice(0, -1) + (name.endsWith("Q") ? "R" : "Q"); // same length ⇒ same size + const setContent = (text: string): void => { + writeFileSync(file, text); + utimesSync(file, stamp, stamp); + }; + setContent(original.replace(`function ${name}`, `function ${renamed}`)); + expect(statSync(file).size).toBe(original.length); + + expect(index(repo, out)).toBe(true); // the heuristic, as documented + expect(read(out, "symbols.json")).not.toContain(`"${renamed}"`); + expect(index(repo, out, "--full-hash")).toBe(false); + expect(read(out, "symbols.json")).toContain(`"${renamed}"`); + + setContent(original); + expect(index(repo, out)).toBe(true); + expect(read(out, "symbols.json")).toContain(`"${renamed}"`); // stale again, by the same heuristic + expect(index(repo, out, "--no-index-cache")).toBe(false); + expect(read(out, "symbols.json")).not.toContain(`"${renamed}"`); + }); +}); + +// Every file lives under an --out that IS the repo root (or an ancestor), and +// the self-index guard excluded them all: a 0-file graph, exit 0. +describe("index --out at or above the repo root", { timeout: 60_000 }, () => { + it("--out at the root indexes the repo, skips only its own artifacts, and settles", () => { + const repo = freshRepo(); + const elsewhere = join(scratch(), "out"); + index(repo, elsewhere); + expect(index(repo, repo)).toBe(false); + sameArtifacts(repo, elsewhere); + expect(JSON.parse(read(repo, "graph.json")).fileCount).toBeGreaterThan(0); + expect(index(repo, repo)).toBe(true); + expect(cli(["graph", "--repo", repo, "--index", "."]).stdout).toBe(read(repo, "graph.json")); + }); + + it("--out above the root indexes the whole repo", () => { + const repo = freshRepo(); + const elsewhere = join(scratch(), "out"); + index(join(repo, "src"), elsewhere); + index(join(repo, "src"), repo); + sameArtifacts(repo, elsewhere); + }); + + it("warns when nothing at all was indexed", () => { + const repo = freshRepo(); + const { stderr } = cli(["index", "--repo", repo, "--out", join(scratch(), "out"), "--include", "no-such-dir/**"]); + expect(stderr).toContain("warning: no file"); + }); +}); + +describe("index artifacts are replaced atomically", { timeout: 60_000 }, () => { + // A hard link shares the inode. An in-place rewrite (the old truncate + write) + // changes what the link reads; a rename-based replacement leaves the link on + // the old, complete file — which is exactly what a concurrent reader holding + // the old file sees instead of a torn one. + it("writes a new file and renames it over the old one — never truncates in place", () => { + const repo = freshRepo(); + const out = join(scratch(), "out"); + index(repo, out); + const names = ["graph.json", "symbols.json", "cache.json"]; + const before = names.map((n) => read(out, n)); + for (const n of names) linkSync(join(out, n), join(out, `${n}.link`)); + appendFileSync(join(repo, "src", "util.ts"), "\nexport function atomicProbe(): number {\n return 1;\n}\n"); + expect(index(repo, out)).toBe(false); + names.forEach((n, i) => { + expect(read(out, `${n}.link`)).toBe(before[i]); + expect(read(out, n)).not.toBe(before[i]); + }); + expect(read(out, "symbols.json")).toContain("atomicProbe"); + expect(readdirSync(out).filter((n) => n.includes(".tmp-"))).toEqual([]); + }); +}); + +// `rev-parse --short` sizes the abbreviation from the object count: the same +// commit printed 7 characters in one clone and 8 in another, and graph.json's +// bytes (plus the fastpath) changed with it. +describe("graph.json commit stamp", { timeout: 60_000 }, () => { + it("is a fixed-length prefix of HEAD, whatever git's own abbreviation", () => { + const repo = freshRepo(); + const git = (...args: string[]): string => + execFileSync("git", ["-C", repo, "-c", "user.name=t", "-c", "user.email=t@t", ...args], { encoding: "utf8" }).trim(); + git("init", "-q"); + git("add", "-A"); + git("commit", "-qm", "one"); + git("config", "core.abbrev", "12"); + const full = git("rev-parse", "HEAD"); + expect(git("rev-parse", "--short", "HEAD")).toHaveLength(12); + expect(headCommit(repo)).toBe(full.slice(0, 7)); + const out = join(scratch(), "out"); + index(repo, out); + expect(JSON.parse(read(out, "graph.json")).commit).toBe(full.slice(0, 7)); + git("config", "core.abbrev", "9"); + expect(index(repo, out)).toBe(true); + }); +}); + +// The npm layout: the bundle ships scripts/grammars (CORE) and nothing else; +// `grammars pull` puts CORE + EXTENDED into the shared cache. The adjacent dir +// used to be the only one searched, so pulled Kotlin never loaded and +// `grammars status` still said nothing was missing. +const EXTENDED = join(SCRIPTS, "grammars-extended"); +describe.skipIf(!existsSync(join(EXTENDED, "kotlin.wasm")))("pulled EXTENDED grammars next to a bundled CORE set", { timeout: 60_000 }, () => { + it("loads a pulled grammar and reports what is still missing", () => { + const root = scratch("ci-npm-layout-"); + const pkg = join(root, "pkg", "scripts"); + mkdirSync(pkg, { recursive: true }); + for (const f of ["engine.mjs", "cli.mjs"]) copyFileSync(join(SCRIPTS, f), join(pkg, f)); + cpSync(join(SCRIPTS, "grammars"), join(pkg, "grammars"), { recursive: true }); + const cacheHome = join(root, "xdg"); + const cache = join(cacheHome, "codeindex", "grammars", ENGINE_VERSION); + mkdirSync(cache, { recursive: true }); + copyFileSync(join(EXTENDED, "kotlin.wasm"), join(cache, "kotlin.wasm")); + const repo = join(root, "kt"); + mkdirSync(repo); + writeFileSync( + join(repo, "Main.kt"), + "package demo\n\nobject Registry {\n val items = mutableListOf()\n fun add(x: String) { items.add(x) }\n}\n", + ); + const env = { ...ENV, CODEINDEX_GRAMMAR_DIR: "", ULTRAINDEX_GRAMMAR_DIR: "", CODEINDEX_GRAMMARS_DIR: "", XDG_CACHE_HOME: cacheHome }; + const run = (args: string[]): string => + execFileSync(process.execPath, [join(pkg, "cli.mjs"), ...args], { encoding: "utf8", env }); + + const status = JSON.parse(run(["grammars", "status"])) as { + tier: string; + dirs: string[]; + pullNeeded: boolean; + extendedPullNeeded: boolean; + extended: { missing: string[] }; + }; + expect(status.tier).toBe("adjacent"); + expect(status.dirs).toEqual([join(pkg, "grammars"), cache]); + expect(status.pullNeeded).toBe(false); + expect(status.extended.missing).not.toContain("kotlin"); + expect(status.extendedPullNeeded).toBe(true); // the other five are still missing + + // The AST tier finds the `items` property the regex tier does not, and + // agrees with the dev checkout, whose sibling grammars-extended/ has Kotlin. + const symbols = run(["symbols", "--repo", repo, "--no-index-cache"]); + expect(Object.keys(JSON.parse(symbols).defs)).toContain("items"); + expect(symbols).toBe(cli(["symbols", "--repo", repo, "--no-index-cache"]).stdout); + rmSync(join(cache, "kotlin.wasm")); + expect(Object.keys(JSON.parse(run(["symbols", "--repo", repo, "--no-index-cache"])).defs)).not.toContain("items"); + }); +}); diff --git a/tests/preload.test.ts b/tests/preload.test.ts index 2322490..51d2fdc 100644 --- a/tests/preload.test.ts +++ b/tests/preload.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect } from "vitest"; -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { cpSync, mkdtempSync, rmSync, readFileSync, writeFileSync, appendFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { preloadSessionLazy } from "../src/preload.js"; +import { dirname, join } from "node:path"; +import { preloadSessionLazy, readPersistedIndex } from "../src/preload.js"; import { ensureGrammars, grammarKeysForExts } from "../src/ast/loader.js"; +import { scanRepo } from "../src/scan.js"; import { walk } from "../src/walk.js"; const REPO = fileURLToPath(new URL("./fixtures/mini-repo", import.meta.url)); @@ -110,6 +111,59 @@ describe("persisted-index reuse — output-identical", { timeout: 60_000 }, () = }); }); + // path.join does not reset on an absolute segment: `--index /abs/idx` probed + // /abs/idx, found nothing, and every read paid a full cold build. The + // output was right either way, so only REUSE proves the fix. + it("reuses an ABSOLUTE --index instead of silently rebuilding", async () => { + await withRepoAsync(async (repo) => { + const abs = join(dirname(repo), "elsewhere", "idx"); + run(repo, ["index", "--out", abs]); + expect(readPersistedIndex(repo, abs)?.cacheMap.size).toBeGreaterThan(0); + let warms = 0; + const session = await preloadSessionLazy(repo, {}, async () => { + warms++; + }, abs); + expect(warms).toBe(0); + expect(session?.scan.contentUnchanged).toBe(true); + expect(session?.loadArtifacts?.()?.graph.fileCount).toBe(session?.scan.files.length); + expect(run(repo, ["symbols", "--index", abs])).toBe(run(repo, ["symbols", "--no-index-cache"])); + }); + }); + + // `index` excludes its own --out; the read commands scanned it. An in-repo + // custom index then showed up as three config files (search answered "graph" + // with idx/graph.json) and the read scan never matched the index, so its + // artifacts were never reused. + it("neither scans nor ignores an in-repo custom --index", async () => { + await withRepoAsync(async (repo) => { + run(repo, ["index", "--out", join(repo, "idx")]); + const hits = JSON.parse(run(repo, ["search", "graph symbols cache", "--index", "idx"])) as { file: string }[]; + expect(hits.some((h) => h.file.startsWith("idx/"))).toBe(false); + const count = (args: string[]): number => (JSON.parse(run(repo, ["scan", ...args])) as { fileCount: number }).fileCount; + expect(count(["--index", "idx"])).toBe(scanRepo(repo, { exclude: ["idx/**"] }).files.length); + let warms = 0; + // The options the CLI passes for `--index idx`. + const session = await preloadSessionLazy(repo, { out: join(repo, "idx") }, async () => { + warms++; + }, "idx"); + expect(warms).toBe(0); + expect(session?.scan.contentUnchanged).toBe(true); + expect(session?.loadArtifacts?.()).toBeDefined(); + expect(run(repo, ["graph", "--index", "idx"])).toBe(readFileSync(join(repo, "idx", "graph.json"), "utf8")); + }); + }); + + it("says so on stderr when a NAMED --index is unusable, and still answers", () => { + withRepo((repo) => { + const res = spawnSync(process.execPath, [CLI, "symbols", "--repo", repo, "--index", "no-such-idx"], { encoding: "utf8" }); + expect(res.status).toBe(0); + expect(res.stderr).toContain(`no usable index at ${join(repo, "no-such-idx")}`); + expect(res.stdout).toBe(run(repo, ["symbols", "--no-index-cache"])); + // The default location being empty is the normal first run: no note. + expect(spawnSync(process.execPath, [CLI, "symbols", "--repo", repo], { encoding: "utf8" }).stderr).not.toContain("no usable index"); + }); + }); + it("does not let a scoped read reuse a whole-repo index", () => { withRepo((repo) => { prime(repo); @@ -161,6 +215,25 @@ describe("lazy grammar warm on persisted indexes", { timeout: 30_000 }, () => { }); }); + // An index built at the regex tier (--no-ast, or before a grammar was + // available) passed every freshness key, so its records were served forever + // to a process that extracts at the AST tier. The profile in cache.json makes + // those records misses: warmed, re-extracted, and the stale artifacts refused. + it("re-extracts records an index built at another tier", async () => { + await withRepoAsync(async (repo) => { + run(repo, ["index", "--out", join(repo, ".codeindex"), "--no-ast"]); + expect(readPersistedIndex(repo)?.meta.extraction).toEqual({ grammars: [] }); + let warms = 0; + const session = await preloadSessionLazy(repo, {}, async () => { + warms++; + }); + expect(warms).toBe(1); + expect(session?.scan.contentUnchanged).toBe(false); + expect(session?.loadArtifacts?.()).toBeUndefined(); + expect(session?.scan.files).toEqual(scanRepo(repo).files); + }); + }); + it("does not warm for a deletion that needs no new extraction", async () => { await withRepoAsync(async (repo) => { prime(repo); diff --git a/tests/scan.test.ts b/tests/scan.test.ts index 5878db7..2dfe32c 100644 --- a/tests/scan.test.ts +++ b/tests/scan.test.ts @@ -141,6 +141,69 @@ describe("scanRepo — change-tracking flags", () => { expect(precomputed.capped).toBe(direct.capped); expect(precomputed.excluded).toBe(direct.excluded); }); + + // The content hash is over DECODED text, and every binary decodes to "", so a + // binary that grows keeps an equal hash. Its record kept the stale size: the + // cache entry then missed the stat fastpath on every later run and cache.json + // was rewritten each time (and RepoScan.files[].size was simply wrong). + it("a binary that changes size under an equal hash takes the new size, then settles", () => { + const { root } = setup(); + writeFileSync(join(root, "data.dat"), Buffer.from("a\0bcdef")); + const first = scanRepo(root); + writeFileSync(join(root, "data.dat"), Buffer.from("a\0bcdefghijk")); + const second = scanRepo(root, { cache: cacheOf(first) }); + const before = first.files.find((f) => f.rel === "data.dat")!; + const after = second.files.find((f) => f.rel === "data.dat")!; + expect(after.hash).toBe(before.hash); + expect(after.size).toBe(12); + expect(second.contentUnchanged).toBe(true); + const third = scanRepo(root, { cache: cacheOf(second) }); + expect(third.cacheDirty).toBe(false); + expect(third.contentUnchanged).toBe(true); + }); +}); + +describe("scanRepo — the --out self-index guard", () => { + const dirs: string[] = []; + afterAll(() => { + for (const d of dirs) rmSync(d, { recursive: true, force: true }); + }); + function copy(): string { + const dir = mkdtempSync(join(tmpdir(), "ci-scan-out-")); + dirs.push(dir); + const root = join(dir, "mini-repo"); + cpSync(REPO, root, { recursive: true }); + return root; + } + const rels = (root: string, opts?: Parameters[1]): string[] => scanRepo(root, opts).files.map((f) => f.rel); + + it("an --out inside the repo is excluded whole", () => { + const root = copy(); + writeFileSync(join(root, "src", "graph.json"), "{}\n"); + expect(rels(root, { out: join(root, "src") }).some((r) => r.startsWith("src/"))).toBe(false); + }); + + // `index --out .` used to exclude every file (all of them live under --out), + // so the run wrote a 0-file graph and still exited 0. + it("an --out at the repo root excludes only the index artifacts written there", () => { + const root = copy(); + const artifacts = ["graph.json", "symbols.json", "cache.json", "graph.json.tmp-4242"]; + for (const name of artifacts) writeFileSync(join(root, name), "{}\n"); + writeFileSync(join(root, "src", "graph.json"), "{}\n"); // same name, not at --out: a repo file + const all = rels(root); + const guarded = rels(root, { out: root }); + expect(all).toEqual(expect.arrayContaining(artifacts)); + expect(guarded).toEqual(all.filter((r) => !artifacts.includes(r))); + expect(guarded).toContain("src/graph.json"); + expect(rels(root, { out: `${root}/` })).toEqual(guarded); + }); + + it("an --out above the repo root excludes nothing", () => { + const root = copy(); + const sub = join(root, "src"); + expect(rels(sub, { out: root })).toEqual(rels(sub)); + expect(rels(sub).length).toBeGreaterThan(0); + }); }); describe("extractMarkdown", () => { diff --git a/tests/walk-nested.test.ts b/tests/walk-nested.test.ts index 9f0e065..7d81f01 100644 --- a/tests/walk-nested.test.ts +++ b/tests/walk-nested.test.ts @@ -116,6 +116,17 @@ describe(".git is skipped whatever ignoreDirs says", () => { const root = cloneFixture(); expect(rels(root, { ignoreDirs: [], gitignore: false }).some((r) => r.startsWith(`${GIT}/`))).toBe(false); }); + + // The engine's own index dir is structural too: `--ignore-dir node_modules` + // used to put .codeindex/{graph,symbols,cache}.json and the MCP memories + // back into the scan, so search answered with the index itself. + it("a replacement list does not pull the engine's own .codeindex in", () => { + const root = cloneFixture(); + mkfile(root, ".codeindex/graph.json", "{}\n"); + mkfile(root, ".codeindex/memories/note.md", "# a memory\n"); + expect(rels(root, { ignoreDirs: ["node_modules"] }).some((r) => r.startsWith(".codeindex/"))).toBe(false); + expect(rels(root, { ignoreDirs: [], gitignore: false }).some((r) => r.startsWith(".codeindex/"))).toBe(false); + }); }); // The boundary must be a REPOSITORY, not merely the name `.git`. Git accepts a From bedb4206a7abe7fdaf7be72bd03e581fedf2f0da Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:33:36 +0000 Subject: [PATCH 007/130] fix(extract): recover declarations the AST walk dropped or misread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight shapes the audit measured on real repositories, fixed in the walk's tables and readers: - Doc comments now reach an item across Rust outer attributes and a TypeScript decorator on its own line (memchr: 457 of 1187 symbols had lost theirs; anyhow 14 of 14). - Python definitions under if/elif/else, try/except/finally, with and match are indexed (flask's `if TYPE_CHECKING:` classes); assignments under an `if __name__ == "__main__"` guard stay out. - Visibility is read from the header text before the declared name, so a parameter named `private`/`internal` or a default string no longer flips it, and one `private val` constructor parameter no longer hides a Scala or Kotlin class. Modified `val`/`var` constructor parameters are now indexed too. - Go type aliases (`type B = int`) are indexed; the grammar-vocabulary oracle now matches alias, #define and singleton-class node types. - C/C++: members of `typedef struct {…} T` (parented to T) plus the struct tag, `#define` macros (include guards excluded), function-pointer fields. - C++: out-of-line definitions belong to their class (leveldb: 243 of 993 functions), operators/conversions/destructors are named, and reference-returning functions are no longer named after their return type. A `.h` whose content is C++ is parsed with the C++ grammar (leveldb headers: 142 -> 963+ symbols); its symbols keep lang "c". - C# operators are named by their token and indexers `this[]`. - Generic bases resolve to the base, not the type argument (C#, C++). Quality fixtures gain every shape (all scored 100%); Go's visibility accuracy moves 0.9444 -> 0.9524 only because its denominator grew. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 25 +- site/index.html | 18 +- site/quality.json | 18 +- src/ast/doc.ts | 20 +- src/ast/extract.ts | 78 ++++- src/ast/grammar-coverage.ts | 15 +- src/ast/loader.ts | 32 +- src/ast/node.ts | 59 +++- src/ast/specs.ts | 183 +++++++++- src/ast/tags.ts | 4 +- tests/extraction-shapes.test.ts | 343 +++++++++++++++++++ tests/fixtures/quality/c/expected.json | 10 + tests/fixtures/quality/c/scheduler.h | 18 + tests/fixtures/quality/cpp/expected.json | 27 ++ tests/fixtures/quality/cpp/policy.h | 31 ++ tests/fixtures/quality/cpp/service.cpp | 21 ++ tests/fixtures/quality/csharp/Scheduler.cs | 10 + tests/fixtures/quality/csharp/expected.json | 18 + tests/fixtures/quality/go/expected.json | 15 + tests/fixtures/quality/go/service.go | 9 + tests/fixtures/quality/java/Scheduler.java | 3 + tests/fixtures/quality/java/expected.json | 7 + tests/fixtures/quality/php/Scheduler.php | 5 + tests/fixtures/quality/php/expected.json | 6 + tests/fixtures/quality/python/expected.json | 10 +- tests/fixtures/quality/python/service.py | 21 +- tests/fixtures/quality/rust/expected.json | 5 +- tests/fixtures/quality/rust/service.rs | 16 + tests/fixtures/quality/scala/Scheduler.scala | 3 + tests/fixtures/quality/scala/expected.json | 16 + tests/grammar-coverage.test.ts | 16 + tests/oracles-invariants.test.ts | 12 + tests/oracles/invariants.ts | 20 +- tests/quality/baseline.json | 2 +- 34 files changed, 1022 insertions(+), 74 deletions(-) create mode 100644 tests/extraction-shapes.test.ts create mode 100644 tests/fixtures/quality/cpp/policy.h create mode 100644 tests/fixtures/quality/cpp/service.cpp diff --git a/README.md b/README.md index 17af6f6..34cdff8 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,15 @@ compares](#how-it-compares). not the first physical line), its own **doc comment**, its qualified `parent`, and its line span — including the members a declaration-only walk misses: interface members, class fields, enum members, every `declare`/`.d.ts` - declaration, Rust trait method signatures, Go interface method sets, record - components and constructor `val` parameters. + declaration, Rust trait method signatures, Go interface method sets and type + aliases, record components and constructor `val` parameters, C `#define` + macros and the members of a `typedef struct`, and Python declarations under + `if TYPE_CHECKING:` / `try:` / `with` blocks. A doc comment is found across + Rust attributes and TypeScript decorators; visibility is read from a + declaration's modifiers, never from its parameter names or default values; + an out-of-line C++ definition (`void Widget::draw()`) belongs to its class; + and a `.h` header is parsed as C++ when its content is (a namespace, class + or template), as C otherwise. - **Resolve imports** across languages: tsconfig paths, package `exports`, go.mod, Cargo, Java packages, PSR-4, C# namespaces. - **Build a typed link-graph**: `import` / `call` / `extends` / `implements` / @@ -61,7 +68,7 @@ vocabulary: | **TypeScript compiler index** (`scip-typescript` 0.4.0) | an index built by the real TypeScript compiler — authoritative where every other check here is syntactic | **100%** of its 93 named declarations, against ctags' 94.6% on the same files | | **universal-ctags differential** (Universal Ctags 6.2.1) | an independent, mature indexer covering ~40 languages | reports **2,014** declarations ctags does not over 6 real repositories, and reproduces **61.7%–98.8%** of ctags' names — what is left bucketed by kind, per repo below | | **Official `tags.scm` queries** | the code-navigation patterns each grammar's own authors publish, and GitHub uses | **1** adjudicated difference, over the 14 of 17 languages that publish one | -| **Grammar vocabulary** | each tree-sitter grammar's own declared node types, read at runtime from the parser | 21 grammars audited, **208** declaration-ish node types still unhandled | +| **Grammar vocabulary** | each tree-sitter grammar's own declared node types, read at runtime from the parser | 21 grammars audited, **211** declaration-ish node types still unhandled | ### The one head-to-head @@ -151,12 +158,12 @@ terms live only in prose. | what is scored | score | measured on | |---|---|---| -| symbol precision / recall | **100% / 100%** | 265 labelled declarations in 18 files | -| kind accuracy | **100%** | the same 265 declarations | -| visibility accuracy | **100%** on 16 of 17 languages, 94.4% on Go | the same 265 declarations | -| doc comment attached | **100%** | the 147 declarations labelled with a doc | -| complete signature | **100%** | the 29 declarations labelled with a signature | -| call edges / inheritance (F1) | **100% / 100%** | 47 labelled call sites, 21 relations | +| symbol precision / recall | **100% / 100%** | 307 labelled declarations in 20 files | +| kind accuracy | **100%** | the same 307 declarations | +| visibility accuracy | **100%** on 16 of 17 languages, 95.2% on Go | the same 307 declarations | +| doc comment attached | **100%** | the 174 declarations labelled with a doc | +| complete signature | **100%** | the 32 declarations labelled with a signature | +| call edges / inheritance (F1) | **100% / 100%** | 48 labelled call sites, 22 relations | | search MRR / nDCG@10 / recall@5 | **93.8% / 86.0% / 84.4%** | 16 relevance-judged queries | `pnpm quality:report` reproduces every number; `tests/quality.test.ts` enforces diff --git a/site/index.html b/site/index.html index 0c68030..4cc6d68 100644 --- a/site/index.html +++ b/site/index.html @@ -4465,15 +4465,15 @@

How it compares

a moment, so an unchanged repo must re-render identically. --> ", // 14 + "", // 15 + "", // 18 +].join("\n"); + +describe("sfcParts", () => { + it("keeps the script blocks at their own offsets and blanks the rest", () => { + const parts = sfcParts(".vue", VUE)!; + expect(parts.ext).toBe(".ts"); + expect(parts.script.length).toBe(VUE.length); + const lines = parts.script.split("\n"); + expect(lines.length).toBe(VUE.split("\n").length); + expect(lines[5]).toBe('import { ref } from "vue";'); + expect(lines[11]).toBe(" helper(e);"); + // Markup, tags and style are spaces, not gone. + expect(lines[1]!.trim()).toBe(""); + expect(lines[4]!.trim()).toBe(""); + expect(lines[16]!.trim()).toBe(""); + // The markup is the complement: the template survives, script and style do not. + const markup = parts.markup.split("\n"); + expect(parts.markup.length).toBe(VUE.length); + expect(markup[1]).toContain("formatDate(when)"); + expect(markup[11]!.trim()).toBe(""); + expect(markup[16]!.trim()).toBe(""); + }); + + it("reads the script language from `lang`, TS winning over JS", () => { + expect(sfcParts(".vue", "")!.ext).toBe(".js"); + expect(sfcParts(".vue", '')!.ext).toBe(".tsx"); + expect(sfcParts(".vue", "")!.ext).toBe(".jsx"); + expect(sfcParts(".vue", '\n')!.ext).toBe(".ts"); + expect(sfcParts(".svelte", "")!.ext).toBe(".ts"); + // Astro's frontmatter is TypeScript by definition. + expect(sfcParts(".astro", "---\nconst a = 1;\n---\n

")!.ext).toBe(".ts"); + expect(sfcParts(".ts", "const a = 1;")).toBeUndefined(); + }); + + it("skips data blocks, other languages, commented-out blocks and bodiless tags", () => { + const src = [ + '', + '', + '', + '", + ].join("\n"); + const script = sfcParts(".vue", src)!.script; + expect(script).not.toContain("coffee"); + expect(script).not.toContain("commented"); + expect(script).not.toContain("ld+json"); + expect(script).toContain('import z from "./real";'); + }); + + it("keeps Astro's frontmatter and its "]; + const kept = new Set([1, 2, 6]); + expect(sfcParts(".astro", src.join("\n"))!.script.split("\n")).toEqual( + src.map((line, i) => (kept.has(i) ? line : " ".repeat(line.length))), + ); + }); +}); + +describe("extractCode on single-file components", () => { + it("extracts a Vue component's symbols, imports and calls at their real lines", () => { + const info = extractCode("src/Comp.vue", ".vue", VUE); + expect(info.symbols.map((s) => `${s.kind} ${s.name}@${s.line} ${s.lang}`)).toEqual([ + "const msg@10 vue", + "function onClick@11 vue", + ]); + expect(info.symbols.find((s) => s.name === "msg")!.doc).toBe("The message."); + expect(info.refs.map((r) => r.spec)).toEqual(["vue", "./Child.vue", "./util"]); + // Script calls from the AST, template calls from the markup; CSS is not markup. + expect(info.calls).toEqual([ + { name: "formatDate", line: 2 }, + { name: "helper", line: 12 }, + { name: "ref", line: 10 }, + ]); + expect(info.importedNames).toEqual(["formatDate", "helper", "ref"]); + // Import specifiers are modelled as refs, never as duplicated literals. + expect(info.literals?.some((l) => l.value === "./util")).toBe(false); + }); + + it("gives the same symbols on the regex tier", () => { + const parts = sfcParts(".vue", VUE)!; + expect(extractSymbols("src/Comp.vue", parts.ext, parts.script).map((s) => `${s.name}@${s.line}`)).toEqual([ + "msg@10", + "onClick@11", + ]); + }); + + it("treats Svelte props and Astro frontmatter exports as not exported, a module script's as exported", () => { + const svelte = [ + '", + '", + "", + ].join("\n"); + const exp = (rel: string, ext: string, src: string) => + Object.fromEntries(extractCode(rel, ext, src).symbols.map((s) => [s.name, s.exported])); + expect(exp("B.svelte", ".svelte", svelte)).toEqual({ preload: true, label: false, focus: false }); + // Svelte 5 spells the module script with a bare attribute. + expect(exp("B.svelte", ".svelte", "")).toEqual({ x: true }); + const astro = "---\nexport interface Props { title: string }\nexport async function getStaticPaths() { return []; }\n---\n

"; + expect(exp("P.astro", ".astro", astro)).toMatchObject({ Props: false, getStaticPaths: false }); + // A Vue ")).toEqual({ shared: true }); + }); + + it("warms the JS/TS grammars a component's script needs", () => { + expect(grammarKeysForExts([".vue"])).toEqual(["javascript", "tsx", "typescript"]); + expect(grammarKeysForExts([".astro"])).toEqual(["typescript"]); + }); + + it("binds component calls in the JS family", () => { + for (const lang of ["vue", "svelte", "astro"]) expect(familyOf(lang)).toBe(familyOf("typescript")); + }); +}); + +describe("components in the graph", () => { + function repo(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "ci-sfc-")); + for (const [rel, text] of Object.entries(files)) { + mkdirSync(dirname(join(root, rel)), { recursive: true }); + writeFileSync(join(root, rel), text); + } + return root; + } + + it("draws import and call edges from components, and keeps their helpers alive", () => { + const root = repo({ + "src/util.ts": "export function helper(x?: unknown) {}\nexport function formatDate(d?: unknown) { return d; }\nexport function unused() {}\n", + "src/Comp.vue": VUE, + "src/Child.vue": "\n", + "src/Btn.svelte": '\n\n', + }); + const scan = scanRepo(root); + const { graph } = buildArtifactsFromScan(scan); + expect(graph.fileEdges.map((e) => `${e.from} -${e.kind}-> ${e.to}`)).toEqual([ + "src/Btn.svelte -call-> src/util.ts", + "src/Btn.svelte -import-> src/util.ts", + "src/Comp.vue -import-> src/Child.vue", + "src/Comp.vue -call-> src/util.ts", + "src/Comp.vue -import-> src/util.ts", + ]); + // The helpers a component calls — from its script or its template — are + // not dead; the one nothing calls still is, and a prop never is. + expect(findDeadCode(scan).map((d) => `${d.name} ${d.tier}`)).toEqual(["unused unreferenced"]); + }); +}); From 6f84f1e3be4153052d384e1a4147d7367cbb1e84 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:47:59 +0000 Subject: [PATCH 034/130] feat(analytics): couple indexed files by confidence, rank only changed hotspots coupling mined every path git ever saw: on django 48 of the top 100 pairs named deleted files, 34 were generated pairs (x.po/x.mo) and all 100 had strength 1.0, while --scope/--include were ignored. It now reads the index: pairs are limited to indexed files (so the scope flags and ignore rules apply), each says whether a graph edge already `linked` them, and --hidden keeps the unexplained ones. Pairs named as a pair (same dir, same stem) are left out, and ranking uses the Wilson lower bound (`confidence`) so 12/13 outranks a thin 3/3. The pair counter keys ranked file ids numerically instead of concatenating strings (pair counting 5.5 s -> 0.3 s on a 26k-commit history) and reuses readHistory's pass, whose mass-refactor cut now counts the whole commit. hotspots padded its list with files that never changed in the window, ranked by size alone; they are dropped, test files carry `test: true`, and onboard needs ten commits before it ranks anything. churn, hotspots, risk and coupling (CLI and MCP) now report `error` when there is no history and `shallow`/`note` for a shallow clone; --limit caps hotspots, risk and coupling; coupling takes --min-together, --max-commit-files and --hidden; churn honours --scope/--include/--exclude. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 46 ++++++++++ src/coupling.ts | 178 +++++++++++++++++++++++++++++--------- src/engine-cli.ts | 83 +++++++++++++----- src/mcp.ts | 32 ++++--- src/mcp/tools.ts | 39 ++++++--- src/onboard.ts | 20 +++-- tests/git-history.test.ts | 159 ++++++++++++++++++++++++++++++++-- 7 files changed, 462 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 17af6f6..f8a2db0 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,7 @@ codeindex implementations Runnable --repo . # who implements it, transitively codeindex callgraph buildGraph --repo . --depth 2 codeindex grep 'pattern' --repo . codeindex literals --repo . # values with no single source of truth +codeindex hotspots --repo . --since "6 months ago" # where work concentrates ``` ## Values with no single source of truth @@ -426,6 +427,51 @@ a *consumer*, not a source of truth, and is reported as a call site. A lookup table (`export const ROUTES = { … }`) genuinely is one, and is reported as a holder. +## What git history says + +Four commands read the commit history rather than the code: `churn` (commits +per file), `hotspots` (churn × size: where work and defects concentrate), +`risk` (churn × complexity) and `coupling` (files that change together). + +```sh +codeindex hotspots --repo . --since "6 months ago" --limit 10 +codeindex coupling --repo . --hidden # co-change that no import explains +codeindex churn --repo packages/api # one package of a monorepo +``` + +- **Paths are relative to `--repo`**, which may be any directory inside the git + repository: point it at one package of a monorepo and history is limited to + that package, keyed the way its index is. +- **`--since` takes a ref or a date**: a tag, branch or sha (commits after it), + or `2024-01-01` / `"6 months ago"`. Anything else is an error (exit 2), never + an empty window that reads as "nothing changed". +- **Every answer says what it could read.** Outside a repository, or before the + first commit, `ok`/`churnOk` is `false` and `error` says why. A **shallow + clone** answers with `shallow: true`: counts are lower bounds, and the clone's + boundary commit is left out, because git compares it with an empty tree and + it would count as a change to every file (a depth-1 CI checkout therefore has + no visible history at all). +- **`hotspots` ranks only files that changed** in the window and labels test + files `test: true`. +- **`coupling` works over the index**: pairs are limited to indexed files, so + deleted paths drop out and `--scope`/`--include`/`--exclude` apply. Each pair + says whether a graph edge (import, call, use, inheritance, doc link) already + `linked` the two files; `--hidden` keeps only the pairs with no such edge. + Pairs whose names already declare them (same directory, same name up to the + first dot: `x.po`/`x.mo`, `x.js`/`x.min.js`, `x.ts`/`x.test.ts`) are left + out. Pairs are ranked by `confidence`, the lower bound of the 95% Wilson + interval for `strength`: 12 shared commits out of 13 rank above a thinly + evidenced 3 out of 3. `--min-together` (default 3) and `--max-commit-files` + (default 30) tune the mining. The second one skips mass-refactor commits by + their whole size, including files outside `--repo`. +- **Renames are not followed.** Rename detection is the expensive part of + `git log`, and on a blobless partial clone it downloads blobs. A file's + history before a rename stays under its old path. +- The output does not depend on the user's git config (colour, diff prefixes, + signature display, external diff drivers). One `git log` pass is shared by + all four commands and reused while HEAD stays the same, so an MCP session + asking for `onboard`, `hotspots` and `risk` reads the history once. + ## Docker `ghcr.io/maxgfr/codeindex` ships the same zero-dependency bundle (`engine.mjs` diff --git a/src/coupling.ts b/src/coupling.ts index ce25f98..a3bd980 100644 --- a/src/coupling.ts +++ b/src/coupling.ts @@ -4,16 +4,13 @@ // where most future work and most defects concentrate). Deterministic for a // given HEAD; degrades loudly outside a git repo like gitChurn. import type { RepoScan } from "./scan.js"; -import { sh } from "./util.js"; +import type { Graph } from "./types.js"; +import { readHistory, repoPaths } from "./git.js"; import { byStr } from "./sort.js"; - -// Pair key separator. Written as an escape, never as a literal NUL byte: a -// literal one makes git, grep and file(1) treat this source as binary, and -// makes codeindex drop it from its own index (readText's NUL sniff). -const SEP = "\u0000"; +import { isTestPath } from "./tests-map.js"; export interface ChangeCoupling { - a: string; // repo-relative path (a < b lexicographically) + a: string; // path relative to --repo (a < b lexicographically) b: string; together: number; // commits touching both totalA: number; // commits touching a (within the analysed window) @@ -21,59 +18,147 @@ export interface ChangeCoupling { // together / min(totalA, totalB) — 1.0 means "every change to the less- // churned file also touched the other". The classic logical-coupling ratio. strength: number; + // Lower bound of the 95% Wilson interval for `strength` over those + // min(totalA, totalB) commits: how strong the coupling is WITH confidence. + // The ranking key — 3/3 scores 0.44 and 200/210 scores 0.91, where raw + // strength ranked the former first and filled the top with thinly + // evidenced 1.0s. + confidence: number; + // Set when a graph was given: whether an edge (import, call, use, extends, + // implements, doc-link — either direction) already links the two files. + // false = a hidden dependency, what coupling exists to find. + linked?: boolean; } export interface CouplingOptions { - since?: string; // only mine commits after this ref + since?: string; // only mine commits after this ref, or since this date // Commits touching more than this many files are skipped as mass refactors // (renames, formatting sweeps) that would couple everything to everything. maxCommitFiles?: number; // default 30 minTogether?: number; // drop pairs seen together fewer times (default 3) maxPairs?: number; // cap the result (default 100) + // The index of `dir`. Pairs are then restricted to indexed files — so + // deleted and renamed-away paths, --scope/--include/--exclude and ignore + // rules all apply — and each pair says whether the graph links it. + graph?: Pick; + hidden?: boolean; // with graph: keep only the pairs no edge links +} + +export interface CouplingResult { + ok: boolean; + error?: string; // why ok is false + shallow?: boolean; // a shallow clone: counts are lower bounds + couplings: ChangeCoupling[]; } -export function changeCoupling( - dir: string, - opts: CouplingOptions = {}, -): { ok: boolean; couplings: ChangeCoupling[] } { +export function changeCoupling(dir: string, opts: CouplingOptions = {}): CouplingResult { const maxCommitFiles = opts.maxCommitFiles ?? 30; const minTogether = opts.minTogether ?? 3; const maxPairs = opts.maxPairs ?? 100; - const range = opts.since ? [`${opts.since}..HEAD`] : []; - // \x1e (record separator) delimits commits; --name-only lists the files. - const res = sh("git", ["-C", dir, "-c", "core.quotePath=false", "log", ...range, "--pretty=format:%x1e", "--name-only"]); - if (!res.ok) return { ok: false, couplings: [] }; + const res = readHistory(dir, opts.since); + if (!res.ok) return { ok: false, error: res.error, couplings: [] }; + const { log } = res; + + // Eligible files get a rank in name order, so a pair is two small ints + // (lo < hi ⇔ a < b) and its key a single number. String pair keys cost + // 7.8x on a 20k-commit history: ~6M concatenations for 48k surviving pairs. + const indexed = opts.graph ? new Set(opts.graph.files.map((f) => f.rel)) : undefined; + const rel = repoPaths(log); + const names = rel.filter((p): p is string => p !== undefined && (!indexed || indexed.has(p))).sort(byStr); + const rankOf = new Map(names.map((name, i) => [name, i])); + const rank = rel.map((p) => (p === undefined ? -1 : rankOf.get(p) ?? -1)); + const m = names.length; - const totals = new Map(); - const pairs = new Map(); - for (const block of res.stdout.split("\x1e")) { - const files = block - .split("\n") - .map((l) => l.trim()) - .filter(Boolean); - if (!files.length || files.length > maxCommitFiles) continue; - const unique = [...new Set(files)].sort(byStr); - for (const f of unique) totals.set(f, (totals.get(f) ?? 0) + 1); - for (let i = 0; i < unique.length; i++) { - for (let j = i + 1; j < unique.length; j++) { - const key = `${unique[i]}${SEP}${unique[j]}`; + const totals = new Int32Array(m); + const pairs = new Map(); + const seen = new Int32Array(log.paths.length).fill(-1); // dedupe within a commit + const files: number[] = []; + for (let c = 0; c + 1 < log.starts.length; c++) { + let size = 0; + files.length = 0; + for (let k = log.starts[c]!; k < log.starts[c + 1]!; k++) { + const id = log.ids[k]!; + if (seen[id] === c) continue; + seen[id] = c; + size++; + if (rank[id]! >= 0) files.push(rank[id]!); + } + // The size is the WHOLE commit's, files outside --repo included: a sweep + // across a monorepo is a mass refactor even where it lands on 3 files. + if (size === 0 || size > maxCommitFiles) continue; + files.sort((x, y) => x - y); + for (let i = 0; i < files.length; i++) { + const lo = files[i]!; + totals[lo]!++; + for (let j = i + 1; j < files.length; j++) { + const key = lo * m + files[j]!; pairs.set(key, (pairs.get(key) ?? 0) + 1); } } } + let linked: Set | undefined; + if (opts.graph) { + linked = new Set(); + for (const e of opts.graph.fileEdges) { + // A prose mention is not a dependency; a dangling edge links nothing. + if (e.dangling || e.kind === "mention" || e.kind === "contains") continue; + const x = rankOf.get(e.from); + const y = rankOf.get(e.to); + if (x === undefined || y === undefined || x === y) continue; + linked.add(x < y ? x * m + y : y * m + x); + } + } + const out: ChangeCoupling[] = []; for (const [key, together] of pairs) { if (together < minTogether) continue; - const [a, b] = key.split(SEP) as [string, string]; - const totalA = totals.get(a) ?? together; - const totalB = totals.get(b) ?? together; - out.push({ a, b, together, totalA, totalB, strength: Number((together / Math.min(totalA, totalB)).toFixed(3)) }); + const isLinked = linked?.has(key) ?? false; + if (opts.hidden && isLinked) continue; + const lo = Math.floor(key / m); + const hi = key - lo * m; + if (stem(names[lo]!) === stem(names[hi]!)) continue; + const totalA = totals[lo]!; + const totalB = totals[hi]!; + const n = Math.min(totalA, totalB); + out.push({ + a: names[lo]!, + b: names[hi]!, + together, + totalA, + totalB, + strength: round3(together / n), + confidence: round3(wilsonLower(together, n)), + ...(linked ? { linked: isLinked } : {}), + }); } - out.sort((x, y) => y.strength - x.strength || y.together - x.together || byStr(x.a, y.a) || byStr(x.b, y.b)); - return { ok: true, couplings: out.slice(0, maxPairs) }; + out.sort( + (x, y) => + y.confidence - x.confidence || y.strength - x.strength || y.together - x.together || byStr(x.a, y.a) || byStr(x.b, y.b), + ); + return { ok: true, ...(log.shallow ? { shallow: true } : {}), couplings: out.slice(0, maxPairs) }; +} + +// A path up to the first dot of its file name (a dotfile's own leading dot +// aside). Two files sharing it sit in one directory and are NAMED as a pair — +// x.po/x.mo, x.js/x.min.js, dev.in/dev.txt, x.ts/x.test.ts: a generated +// artifact or a declared companion. Their co-change is what the name already +// says, never the hidden dependency coupling exists to find, and such pairs +// crowded the top of an i18n-heavy history (34 of django's top 100). +function stem(path: string): string { + const dot = path.indexOf(".", path.lastIndexOf("/") + 2); + return dot < 0 ? path : path.slice(0, dot); +} + +// Lower bound of the 95% Wilson score interval for k successes in n trials. +function wilsonLower(k: number, n: number): number { + const z2 = 1.96 * 1.96; + const p = k / n; + return (p + z2 / (2 * n) - Math.sqrt(z2 * (p * (1 - p) + z2 / (4 * n)) / n)) / (1 + z2 / n); } +const round3 = (x: number): number => Number(x.toFixed(3)); + export interface Hotspot { rel: string; lines: number; @@ -82,17 +167,24 @@ export interface Hotspot { // CodeScene insight — effort concentrates in few files — with a size damper // so a churned 5-line config does not outrank a churned 2000-line module. score: number; + // A test file. Kept in the ranking, but labelled: churn in a suite is often + // the tests keeping up with the code rather than risk of its own. + test?: true; } -// Rank the scanned files by churn × size. `churn` comes from gitChurn(); files -// absent from history (new/renamed) count 0 commits and rank by size alone. +// Rank the scanned files by churn × size. `churn` comes from gitChurn(). Only +// files that changed in the window rank: a file with no commits scores 0, and +// padding the list with those — ordered by size alone — presented unchanged +// files as hotspots (16 of gin's top 20 under a short --since). export function rankHotspots(scan: RepoScan, churn: Map, top = 20): Hotspot[] { - const out: Hotspot[] = scan.files - .filter((f) => f.kind === "code") - .map((f) => { - const commits = churn.get(f.rel) ?? 0; - return { rel: f.rel, lines: f.lines, commits, score: Number((commits * Math.log2(f.lines + 1)).toFixed(2)) }; - }); + const out: Hotspot[] = []; + for (const f of scan.files) { + if (f.kind !== "code") continue; + const commits = churn.get(f.rel) ?? 0; + const score = Number((commits * Math.log2(f.lines + 1)).toFixed(2)); + if (score <= 0) continue; + out.push({ rel: f.rel, lines: f.lines, commits, score, ...(isTestPath(f.rel) ? { test: true as const } : {}) }); + } out.sort((a, b) => b.score - a.score || b.lines - a.lines || byStr(a.rel, b.rel)); return out.slice(0, top); } diff --git a/src/engine-cli.ts b/src/engine-cli.ts index a03a3b3..2e0e9fb 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -26,8 +26,9 @@ import { computeImportPairs } from "./callers.js"; import { buildSymbolGraph, neighborhood } from "./symbolgraph.js"; import { buildCallerIndex, lookupCallerEntry } from "./callers.js"; import { detectWorkspaces } from "./workspaces.js"; -import { gitChurn } from "./git.js"; +import { gitChurn, historyStatus } from "./git.js"; import { grepRepo } from "./grep.js"; +import { compileGlobFilter } from "./glob.js"; import { changeCoupling, rankHotspots } from "./coupling.js"; import { renderRepoMap } from "./repomap.js"; import { findDeadCode } from "./deadcode.js"; @@ -69,7 +70,8 @@ Commands: implementations Everything implementing/extending a type (transitively) callgraph Bounded symbol-to-symbol neighborhood (--depth, --direction) workspaces Monorepo packages + dependency graph (JSON) - churn Per-file git commit counts (JSON; --since to bound) + churn Per-file git commit counts (JSON; --since to bound; honours + --scope/--include/--exclude) grep Search: cli.mjs grep --repo (JSON hits) search Keyless BM25 lexical search over symbol names, path segments, markdown headings and summaries: cli.mjs search "" --repo . @@ -108,8 +110,12 @@ Commands: validated against the link-graph: --config ; exits 1 on any error-severity violation (a CI gate) repomap Token-budgeted map of the highest-PageRank files (--budget-tokens) - hotspots Churn × size ranking of the files where work concentrates (JSON) - coupling Change coupling: files that change together (JSON; --since ) + hotspots Churn × size ranking of the files where work concentrates: only + files changed in the window, tests labelled (JSON; --since, --limit) + coupling Change coupling: indexed files that change together, ranked by + confidence, each marked linked when a graph edge already joins + them (JSON; --since, --limit, --min-together, --max-commit-files, + --hidden) literals Values with no single source of truth: one literal written out across many files, in three labeled tiers — 'competing' (two or more exported constants hold it), 'bypassed' (a constant holds @@ -124,7 +130,7 @@ Commands: (referenced — re-export, type position — but never called) complexity Cyclomatic-complexity estimates, most-complex first. Pass a file positional for one file; omit for the repo-wide top - risk Complexity × git-churn ranking (JSON; --since to bound) + risk Complexity × git-churn ranking (JSON; --since to bound, --limit) delta Review panel for the git diff: changed files -> enclosing symbols -> blast radius -> risk score with explained reasons (--base | --staged, --depth , --json) @@ -188,7 +194,19 @@ Flags (accepted before OR after the subcommand: '--repo X scan' and entirely. Stale/absent/corrupt → a normal cold build --no-index-cache Never reuse a persisted index; always build from scratch --config Rules config for \`rules\` (JSON: [{name, from, to, …}]) - --limit Max results for \`search\` (default 20) + --limit Max results for \`search\`, \`hotspots\`, \`risk\` (default 20) + and \`coupling\` (default 100) + --since \`churn\`, \`hotspots\`, \`risk\`, \`coupling\`: only commits + after a ref (tag, branch, sha) or since a date + (2024-01-01, "6 months ago"); anything else is an error. + Paths are relative to --repo, which may be a subdirectory + of the git repository; a shallow clone is reported as + \`shallow: true\` (counts are lower bounds) + --min-together \`coupling\`: commits a pair must share (default 3) + --max-commit-files \`coupling\`: skip commits touching more files, as mass + refactors (default 30) + --hidden \`coupling\`: only pairs no graph edge links — the hidden + dependencies --no-fuzzy \`search\`: disable trigram fuzzy fallback for query terms with zero document frequency (default: enabled) --exact \`search\`: drop results that carry no verbatim term match @@ -229,12 +247,15 @@ interface CliFlags { workers?: number; // extraction worker threads (0/1 = sequential) indexDir?: string; // persisted index to read (default .codeindex) noIndexCache?: boolean; // never reuse a persisted index - since?: string; + since?: string; // churn/hotspots/risk/coupling: a ref or a date + minTogether?: number; // coupling: commits a pair must share + maxCommitFiles?: number; // coupling: mass-refactor cut + hidden?: boolean; // coupling: only pairs no graph edge links ignoreCase?: boolean; maxHits?: number; budgetTokens?: number; config?: string; // rules config path - limit?: number; // search result cap + limit?: number; // search/hotspots/risk/coupling result cap minFiles?: number; // literals: distinct-file floor for a duplication minCount?: number; // literals: total-occurrence floor for a duplication includeTests?: boolean; // literals: count test files too (off by default) @@ -302,6 +323,9 @@ function parseFlags(args: string[]): CliFlags { flags.workers = n; } else if (a === "--since") flags.since = next(); + else if (a === "--min-together") flags.minTogether = num(); + else if (a === "--max-commit-files") flags.maxCommitFiles = num(); + else if (a === "--hidden") flags.hidden = true; else if (a === "--config") flags.config = resolve(next()); else if (a === "--limit") flags.limit = num(); else if (a === "--no-fuzzy") flags.fuzzy = false; @@ -362,7 +386,7 @@ function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexO // excluded by the positional check at the warm site. `grammars` (status/pull) // resolves/downloads the wasms itself and must not warm them. // version/help/mcp return before we get there. -const SCANLESS_COMMANDS = new Set(["grep", "churn", "coupling", "workspaces", "grammars"]); +const SCANLESS_COMMANDS = new Set(["grep", "churn", "workspaces", "grammars"]); // Flags for `codeindex mcp`. Kept separate from parseFlags on purpose (see the // dispatch site). `--repo` is resolved to an absolute path and must exist: a @@ -432,6 +456,8 @@ const VALUE_FLAGS = new Set([ "--min-files", "--min-count", "--since", + "--min-together", + "--max-commit-files", "--config", "--limit", "--server-name", @@ -517,8 +543,8 @@ export async function runCli(rawArgv: string[]): Promise { if (!statSync(flags.repo).isDirectory()) throw new Error(`--repo path is not a directory: ${flags.repo}`); // Warm ONLY the grammars for languages actually present, and only for commands - // that scan the file tree. Scan-less commands (grep, churn, coupling, - // workspaces, embed status|pull|serve) load no grammar at all; version/help/mcp + // that scan the file tree. Scan-less commands (grep, churn, workspaces, + // embed status|pull|serve) load no grammar at all; version/help/mcp // already returned above. The walk is done ONCE here to derive the present // extensions, then handed to the scan via precomputedWalk so the tree is // traversed a single time. --no-ast keeps the regex tier: no walk, no warm — @@ -1052,20 +1078,36 @@ export async function runCli(rawArgv: string[]): Promise { flags.out, ); } else if (cmd === "churn") { - const { churn, ok } = gitChurn(flags.repo, { since: flags.since }); + const res = gitChurn(flags.repo, { since: flags.since }); + // churn reads git, not the walk, so the global --scope/--include/--exclude + // are applied to its keys here — the same predicate the scan uses. + const scopeGlobs = flags.scope ? [`${flags.scope.replace(/\/+$/, "")}/**`] : []; + const globs = [...scopeGlobs, ...flags.include, ...flags.exclude.map((g) => `!${g}`)]; + const keep = compileGlobFilter(globs.length ? globs : undefined); const sorted: Record = {}; - for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!; - emit(JSON.stringify({ ok, churn: sorted }, null, 2) + "\n", flags.out); + for (const k of [...res.churn.keys()].sort()) if (!keep || keep(k)) sorted[k] = res.churn.get(k)!; + emit(JSON.stringify({ ok: res.ok, ...historyStatus(res), churn: sorted }, null, 2) + "\n", flags.out); } else if (cmd === "repomap") { const { scan, graph } = await readArtifacts(); emit(renderRepoMap(scan, graph, { budgetTokens: flags.budgetTokens }), flags.out); } else if (cmd === "hotspots") { const scan = await readScan(); - const { churn, ok } = gitChurn(flags.repo, { since: flags.since }); - emit(JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2) + "\n", flags.out); + const res = gitChurn(flags.repo, { since: flags.since }); + const hotspots = rankHotspots(scan, res.churn, flags.limit); + emit(JSON.stringify({ churnOk: res.ok, ...historyStatus(res), hotspots }, null, 2) + "\n", flags.out); } else if (cmd === "coupling") { - const { ok, couplings } = changeCoupling(flags.repo, { since: flags.since }); - emit(JSON.stringify({ ok, couplings }, null, 2) + "\n", flags.out); + // The graph restricts pairs to indexed files (no deleted paths; the scope + // flags apply) and says which pairs an edge already explains. + const { graph } = await readArtifacts(); + const res = changeCoupling(flags.repo, { + since: flags.since, + graph, + hidden: flags.hidden, + minTogether: flags.minTogether, + maxCommitFiles: flags.maxCommitFiles, + maxPairs: flags.limit, + }); + emit(JSON.stringify({ ok: res.ok, ...historyStatus(res), couplings: res.couplings }, null, 2) + "\n", flags.out); } else if (cmd === "deadcode") { emit(JSON.stringify(findDeadCode(await readScan()), null, 2) + "\n", flags.out); } else if (cmd === "literals") { @@ -1080,8 +1122,9 @@ export async function runCli(rawArgv: string[]): Promise { emit(JSON.stringify(symbolComplexity(scan, flags.positional), null, 2) + "\n", flags.out); } else if (cmd === "risk") { const scan = await readScan(); - const { churn, ok } = gitChurn(flags.repo, { since: flags.since }); - emit(JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2) + "\n", flags.out); + const res = gitChurn(flags.repo, { since: flags.since }); + const risks = riskHotspots(scan, res.churn, flags.limit); + emit(JSON.stringify({ churnOk: res.ok, ...historyStatus(res), risks }, null, 2) + "\n", flags.out); } else if (cmd === "delta") { const { graph, symbols } = await readArtifacts(); const res = deltaFor(flags.repo, graph, symbols, { diff --git a/src/mcp.ts b/src/mcp.ts index 724dfc6..789b168 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -19,7 +19,7 @@ import { callerIndexFor, hierarchyFor, symbolGraphFor } from "./derived.js"; import { implementationsOf } from "./relations.js"; import { neighborhood, type Direction } from "./symbolgraph.js"; import { detectWorkspaces } from "./workspaces.js"; -import { gitChurn } from "./git.js"; +import { gitChurn, historyStatus } from "./git.js"; import { grepRepo } from "./grep.js"; import { changeCoupling, rankHotspots } from "./coupling.js"; import { renderRepoMap } from "./repomap.js"; @@ -138,7 +138,7 @@ function errMessage(e: unknown): string { // the repo's grammars first; defaulting to "warm" keeps a newly added scan tool // correct without having to be listed here. const SCANLESS_TOOLS = new Set([ - "workspaces", "churn", "coupling", "grep", + "workspaces", "churn", "grep", "write_memory", "read_memory", "list_memories", "delete_memory", "embed_status", // scan_summary counts and classifies by path only — it never parses, so the @@ -229,10 +229,10 @@ async function callTool(name: string, args: Record, defaultRepo return JSON.stringify({ packages: info.packages, cycle: info.cycle ?? null, topoOrder: info.topoOrder }, null, 2); } if (name === "churn") { - const { churn, ok } = gitChurn(repo, { since: str(args.since) }); + const res = gitChurn(repo, { since: str(args.since) }); const sorted: Record = {}; - for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!; - return JSON.stringify({ ok, churn: sorted }, null, 2); + for (const k of [...res.churn.keys()].sort()) sorted[k] = res.churn.get(k)!; + return JSON.stringify({ ok: res.ok, ...historyStatus(res), churn: sorted }, null, 2); } if (name === "symbols_overview") { const file = str(args.file); @@ -335,8 +335,9 @@ async function callTool(name: string, args: Record, defaultRepo const scan = readScan(); if (args.risk === true) { // `since` was accepted by the CLI's `risk` but silently dropped here. - const { churn, ok } = gitChurn(repo, { since: str(args.since) }); - return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn, positiveNum(args.top)) }, null, 2); + const res = gitChurn(repo, { since: str(args.since) }); + const risks = riskHotspots(scan, res.churn, positiveNum(args.top)); + return JSON.stringify({ churnOk: res.ok, ...historyStatus(res), risks }, null, 2); } return JSON.stringify(symbolComplexity(scan, str(args.file), positiveNum(args.top)), null, 2); } @@ -365,12 +366,21 @@ async function callTool(name: string, args: Record, defaultRepo } if (name === "hotspots") { const scan = readScan(); - const { churn, ok } = gitChurn(repo, { since: str(args.since) }); - return JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2); + const res = gitChurn(repo, { since: str(args.since) }); + const hotspots = rankHotspots(scan, res.churn, positiveNum(args.limit)); + return JSON.stringify({ churnOk: res.ok, ...historyStatus(res), hotspots }, null, 2); } if (name === "coupling") { - const { ok, couplings } = changeCoupling(repo, { since: str(args.since) }); - return JSON.stringify({ ok, couplings }, null, 2); + const { graph } = readArtifacts(); + const res = changeCoupling(repo, { + since: str(args.since), + graph, + hidden: args.hidden === true, + minTogether: positiveNum(args.minTogether), + maxCommitFiles: positiveNum(args.maxCommitFiles), + maxPairs: positiveNum(args.limit), + }); + return JSON.stringify({ ok: res.ok, ...historyStatus(res), couplings: res.couplings }, null, 2); } if (name === "grep") { const pattern = str(args.pattern); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index af0d9f5..b64dc69 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -7,6 +7,10 @@ import { ANNOTATIONS_SINCE, PROTOCOL_VERSIONS, RICH_TOOLS_SINCE } from "./protocol.js"; const repoProp = { repo: { type: "string", description: "Absolute path to the repository root" } }; +const sinceProp = { + type: "string", + description: 'Only count commits after this ref (tag, branch, sha) or since this date ("2024-01-01", "6 months ago"); anything else is an error', +}; const conciseProp = { concise: { type: "boolean", description: "Return declaration locations (name/kind/file/line) without full symbol metadata. Keeps every result, reference tier and confidence label (default false)." } }; const scopeProps = { scope: { type: "string", description: "Restrict to one directory (repo-relative)" }, @@ -65,10 +69,11 @@ export const TOOLS = [ }, { name: "churn", - description: "Per-file git commit counts (whole history, or since a ref) — the churn half of hotspot analysis.", + description: + "Per-file git commit counts (whole history, or a since window) — the churn half of hotspot analysis. Paths are relative to repo; `shallow: true` means a shallow clone (counts are lower bounds), `error` says why `ok` is false.", inputSchema: { type: "object", - properties: { ...repoProp, since: { type: "string", description: "Only count commits after this ref" } }, + properties: { ...repoProp, since: sinceProp }, required: ["repo"], }, }, @@ -162,20 +167,31 @@ export const TOOLS = [ { name: "hotspots", description: - "Where does work concentrate? Files ranked by git churn × size (commits × log2 lines). High-scoring files are where changes and defects cluster.", + "Where does work concentrate? Files changed in the window, ranked by git churn × size (commits × log2 lines); test files carry `test: true`. High-scoring files are where changes and defects cluster.", inputSchema: { type: "object", - properties: { ...repoProp, since: { type: "string", description: "Only count commits after this ref" } }, + properties: { + ...repoProp, + since: sinceProp, + limit: { type: "number", minimum: 1, description: "Max files (default 20)" }, + }, required: ["repo"], }, }, { name: "coupling", description: - "Change coupling: pairs of files that repeatedly change in the same commits — hidden dependencies no import shows. strength 1.0 = every change to one touched the other.", + "Change coupling: pairs of indexed files that repeatedly change in the same commits. strength 1.0 = every change to one touched the other; ranked by `confidence` (the strength a pair's history supports); `linked: false` = no graph edge joins them — a hidden dependency.", inputSchema: { type: "object", - properties: { ...repoProp, since: { type: "string", description: "Only mine commits after this ref" } }, + properties: { + ...repoProp, + since: sinceProp, + hidden: { type: "boolean", description: "Only pairs no graph edge links (default false)" }, + minTogether: { type: "number", minimum: 1, description: "Commits a pair must share (default 3)" }, + maxCommitFiles: { type: "number", minimum: 1, description: "Skip commits touching more files, as mass refactors (default 30)" }, + limit: { type: "number", minimum: 1, description: "Max pairs (default 100)" }, + }, required: ["repo"], }, }, @@ -288,7 +304,7 @@ export const TOOLS = [ ...repoProp, file: { type: "string" }, risk: { type: "boolean", description: "Return complexity × git-churn risk ranking instead" }, - since: { type: "string", description: "Only count risk churn after this ref" }, + since: { ...sinceProp, description: "Only count risk churn after this ref (tag, branch, sha) or since this date" }, top: { type: "number", minimum: 1, description: "Cap ranked symbols" }, }, required: ["repo"], @@ -471,6 +487,9 @@ export const TOOLS = [ // engine adding a field must not turn a strict client's success into a failure. const strArr = { type: "array", items: { type: "string" } }; const anyObj = { type: "object" }; +// git-history answers (churn, hotspots, coupling): why `ok` is false, and +// whether a shallow clone truncated the history. Present only when they apply. +const historyProps = { error: { type: "string" }, shallow: { type: "boolean" }, note: { type: "string" } }; export const OUTPUT_SCHEMAS: Record> = { call_graph: { @@ -543,7 +562,7 @@ export const OUTPUT_SCHEMAS: Record> = { }, churn: { type: "object", - properties: { ok: { type: "boolean" }, churn: { type: "object", additionalProperties: { type: "integer" } } }, + properties: { ok: { type: "boolean" }, ...historyProps, churn: { type: "object", additionalProperties: { type: "integer" } } }, required: ["ok", "churn"], }, find_references: { @@ -596,12 +615,12 @@ export const OUTPUT_SCHEMAS: Record> = { }, hotspots: { type: "object", - properties: { churnOk: { type: "boolean" }, hotspots: { type: "array", items: anyObj } }, + properties: { churnOk: { type: "boolean" }, ...historyProps, hotspots: { type: "array", items: anyObj } }, required: ["churnOk", "hotspots"], }, coupling: { type: "object", - properties: { ok: { type: "boolean" }, couplings: { type: "array", items: anyObj } }, + properties: { ok: { type: "boolean" }, ...historyProps, couplings: { type: "array", items: anyObj } }, required: ["ok", "couplings"], }, duplicated_literals: { diff --git a/src/onboard.ts b/src/onboard.ts index 3609829..92374e6 100644 --- a/src/onboard.ts +++ b/src/onboard.ts @@ -40,6 +40,9 @@ export interface OnboardBrief { const README_NAMES = ["README.md", "README.markdown", "README.rst", "README.txt", "README"]; +// Fewest commits the "Where work concentrates" section ranks from. +const MIN_HOTSPOT_COMMITS = 10; + /** * The repository's own one-line self-description, when it has one. * @@ -114,13 +117,18 @@ export function onboardBrief(scan: RepoScan, graph: Graph, opts: OnboardOptions // Where work concentrates. Git-only, and silent when there is no history — // an empty section would read as "nothing is hot", which is not what an - // unmeasurable repository means. - const { churn, ok: churnOk } = gitChurn(scan.root); - if (churnOk && churn.size) { - const hotspots = rankHotspots(scan, churn, 8); + // unmeasurable repository means — or too little of it to rank: a handful of + // commits (a young repository, a CI clone of depth 2) says what changed + // lately, not where work concentrates. + const history = gitChurn(scan.root); + if (history.ok && history.commits >= MIN_HOTSPOT_COMMITS) { + const hotspots = rankHotspots(scan, history.churn, 8); if (hotspots.length) { - lines.push("## Where work concentrates", "", "Files ranked by commits × size — where changes and defects cluster.", ""); - for (const spot of hotspots) lines.push(`- \`${spot.rel}\` — ${spot.commits} commits, ${spot.lines} lines`); + const shallow = history.shallow ? ` Shallow clone: only ${history.commits} commits of history are visible.` : ""; + lines.push("## Where work concentrates", "", `Files ranked by commits × size — where changes and defects cluster.${shallow}`, ""); + for (const spot of hotspots) { + lines.push(`- \`${spot.rel}\` — ${spot.commits} commits, ${spot.lines} lines${spot.test ? " (test)" : ""}`); + } lines.push(""); } } diff --git a/tests/git-history.test.ts b/tests/git-history.test.ts index b415339..eb1b9ab 100644 --- a/tests/git-history.test.ts +++ b/tests/git-history.test.ts @@ -1,12 +1,20 @@ -// Git-history analytics (churn) and the diff plumbing behind `delta`, against -// real temporary repositories: --repo below the git toplevel, hostile user -// config, shallow clones, since windows and the history memo. -import { execFileSync } from "node:child_process"; +// Git-history analytics (churn, hotspots, coupling) and the diff plumbing +// behind `delta`, against real temporary repositories: --repo below the git +// toplevel, hostile user config, shallow clones, since windows, the history +// memo, and coupling's index filter and ranking. +import { execFileSync, spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { changedSince, diffFiles, diffHunks, gitChurn } from "../src/git.js"; +import { changeCoupling, rankHotspots } from "../src/coupling.js"; +import { onboardBrief } from "../src/onboard.js"; +import { buildIndexArtifacts } from "../src/pipeline.js"; +import { scanRepo } from "../src/scan.js"; + +const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); // Commits are made with signing off and a fixed identity so the fixture does // not depend on the machine's git config; the code under test still runs @@ -63,6 +71,21 @@ describe("history below the git toplevel (--repo is one package of a monorepo)", expect(top.commits).toBe(5); }); + it("keys coupling relative to --repo, and sizes the mass-refactor cut by the WHOLE commit", () => { + const root = monorepo(); + expect(changeCoupling(join(root, "pkg"), { minTogether: 3 }).couplings).toMatchObject([{ a: "a.ts", b: "b.ts", together: 4 }]); + // Four files in total, two of them in pkg/ — like the initial commit. Sized + // by what lands in pkg/ (2) both would pass a cap of 3; sized whole, only + // the three pair commits do. + commit(root, { "pkg/a.ts": bump(), "pkg/b.ts": bump(), "other/y.ts": bump(), "other/z.ts": bump() }); + expect(changeCoupling(join(root, "pkg"), { minTogether: 3, maxCommitFiles: 3 }).couplings).toMatchObject([ + { a: "a.ts", b: "b.ts", together: 3 }, + ]); + expect(changeCoupling(join(root, "pkg"), { minTogether: 3, maxCommitFiles: 4 }).couplings).toMatchObject([ + { a: "a.ts", b: "b.ts", together: 5 }, + ]); + }); + it("reports diffs, hunks and changed files relative to --repo, dropping changes outside it", () => { const root = monorepo(); writeFileSync(join(root, "pkg/a.ts"), "// 1\n// changed\n"); @@ -110,11 +133,13 @@ describe.skipIf(process.platform === "win32")("git output is immune to the user' for (const rel of [QUOTE, TAB, SPACE]) expect(hunks.get(rel)).toEqual([{ start: 1, end: 1 }]); }); - it("churn keys are the raw paths, root commit included", () => { + it("churn and coupling keys are the raw paths, root commit included", () => { const root = hostile(); const { churn, ok } = gitChurn(root); expect(ok).toBe(true); expect(sorted(churn)).toEqual({ [SPACE]: 2, [QUOTE]: 2, [TAB]: 2 }); + const pairs = changeCoupling(root, { minTogether: 2 }).couplings.map((c) => [c.a, c.b]); + expect(pairs).toContainEqual([QUOTE, TAB]); }); }); @@ -141,6 +166,7 @@ describe("shallow clones", () => { const one = gitChurn(clone(src, 1)); expect(one).toMatchObject({ ok: true, shallow: true, commits: 0 }); expect(one.churn.size).toBe(0); + expect(changeCoupling(clone(src, 1), { minTogether: 1 })).toMatchObject({ ok: true, shallow: true, couplings: [] }); // A complete clone is neither flagged nor truncated. const full = gitChurn(clone(src)); expect(full.shallow).toBeUndefined(); @@ -165,18 +191,21 @@ describe("since windows", () => { expect(sorted(gitChurn(root, { since: "HEAD~1" }).churn)).toEqual({ "new.ts": 1, "old.ts": 1 }); expect(sorted(gitChurn(root, { since: "2023-01-01" }).churn)).toEqual({ "new.ts": 2, "old.ts": 1 }); expect(sorted(gitChurn(root, { since: "100 years ago" }).churn)).toEqual({ "new.ts": 2, "old.ts": 3 }); + expect(changeCoupling(root, { since: "2024-06-15", minTogether: 1 }).couplings).toMatchObject([{ a: "new.ts", b: "old.ts" }]); }); it("refuses what is neither, instead of an empty window that reads as 'nothing changed'", () => { const { root } = dated(); for (const since of ["nosuchref", "HEAD~99", "v1.9.0"]) { expect(() => gitChurn(root, { since })).toThrow(/neither a commit .* nor a date/); + expect(() => changeCoupling(root, { since })).toThrow(/neither a commit .* nor a date/); } }); it("says why there is no history", () => { expect(gitChurn(mkdtempSync(join(tmpdir(), "ci-hist-nogit-")))).toMatchObject({ ok: false, error: expect.stringMatching(/not a git repository/) }); expect(gitChurn(newRepo("ci-hist-empty-"))).toMatchObject({ ok: false, error: expect.stringMatching(/no commits yet/) }); + expect(changeCoupling(newRepo("ci-hist-empty-"))).toMatchObject({ ok: false, error: expect.stringMatching(/no commits yet/) }); }); }); @@ -190,3 +219,123 @@ describe("history memo", () => { expect(gitChurn(root).churn.get("a.ts")).toBe(2); }); }); + +describe("change coupling over the index", () => { + // a↔b: 12 of their 13 commits each, and an import between them. c↔d: 3 of + // 3, no edge. gone.ts co-changes with c.ts and d.ts, then is deleted. + function coupled(): string { + const root = newRepo("ci-hist-coupling-"); + commit(root, { "a.ts": 'import { b } from "./b";\nexport const a = b;\n', "b.ts": "export const b = 1;\n" }); + for (let i = 0; i < 11; i++) commit(root, { "a.ts": `import { b } from "./b";\nexport const a = b + ${i};\n`, "b.ts": `export const b = ${i};\n` }); + commit(root, { "a.ts": 'import { b } from "./b";\nexport const a = b * 2;\n' }); + commit(root, { "b.ts": "export const b = 2;\n" }); + for (let i = 0; i < 3; i++) commit(root, { "c.ts": `export const c = ${i};\n`, "d.ts": `export const d = ${i};\n`, "gone.ts": bump() }); + git(root, ["rm", "-q", "gone.ts"]); + git(root, ["commit", "-qm", "rm"]); + return root; + } + + it("ranks by confidence, so 12/13 outranks a thin 3/3", () => { + const { couplings } = changeCoupling(coupled()); + expect(couplings.slice(0, 2).map((c) => [c.a, c.b, c.together, c.strength, c.confidence])).toEqual([ + ["a.ts", "b.ts", 12, 0.923, 0.667], + ["c.ts", "d.ts", 3, 1, 0.438], + ]); + }); + + it("with the graph: keeps indexed files only and marks which pairs an edge explains", () => { + const root = coupled(); + // Without the index the deleted file still couples. + expect(changeCoupling(root).couplings.some((c) => c.a === "gone.ts" || c.b === "gone.ts")).toBe(true); + const { graph } = buildIndexArtifacts(root); + const { couplings } = changeCoupling(root, { graph }); + expect(couplings.map((c) => [c.a, c.b, c.linked])).toEqual([ + ["a.ts", "b.ts", true], + ["c.ts", "d.ts", false], + ]); + expect(changeCoupling(root, { graph, hidden: true }).couplings.map((c) => [c.a, c.b])).toEqual([["c.ts", "d.ts"]]); + }); + + it("leaves out pairs the file names already declare (x.po/x.mo, x.ts/x.test.ts)", () => { + const root = newRepo("ci-hist-stem-"); + for (let i = 0; i < 3; i++) { + commit(root, { "l/x.po": bump(), "l/x.mo": bump(), "l/y.po": bump(), "src/.eslintrc.json": bump(), "src/.eslintrc.js": bump() }); + } + const pairs = changeCoupling(root, { maxPairs: 100 }).couplings.map((c) => `${c.a} ${c.b}`); + expect(pairs).not.toContain("l/x.mo l/x.po"); + expect(pairs).not.toContain("src/.eslintrc.js src/.eslintrc.json"); + expect(pairs).toContain("l/x.mo l/y.po"); + expect(pairs).toContain("l/x.po l/y.po"); + }); + + it("drops scope-excluded files through the graph", () => { + const root = coupled(); + const { graph } = buildIndexArtifacts(root, { exclude: ["d.ts"] }); + expect(changeCoupling(root, { graph }).couplings.map((c) => [c.a, c.b])).toEqual([["a.ts", "b.ts"]]); + }); +}); + +describe("hotspots", () => { + it("ranks only files that changed, and labels tests", () => { + const root = mkdtempSync(join(tmpdir(), "ci-hist-hot-")); + writeFileSync(join(root, "big.ts"), "export const x = 1;\n".repeat(200)); + writeFileSync(join(root, "hot.ts"), "export const y = 1;\n".repeat(10)); + writeFileSync(join(root, "hot.test.ts"), "export const z = 1;\n".repeat(10)); + const churn = new Map([ + ["hot.ts", 3], + ["hot.test.ts", 2], + ]); + const spots = rankHotspots(scanRepo(root), churn); + expect(spots.map((s) => [s.rel, s.commits, s.test])).toEqual([ + ["hot.ts", 3, undefined], + ["hot.test.ts", 2, true], + ]); + expect(rankHotspots(scanRepo(root), churn, 1)).toHaveLength(1); + }); +}); + +describe("onboard's 'Where work concentrates'", () => { + const brief = (root: string): string => { + const { scan, graph } = buildIndexArtifacts(root); + return onboardBrief(scan, graph, { remember: false }).brief; + }; + + it("ranks from ten commits up, never pads with unchanged files, and labels tests", () => { + const root = newRepo("ci-hist-onboard-"); + commit(root, { "still.ts": "export const s = 1;\n".repeat(50) }); + for (let i = 0; i < 8; i++) commit(root, { "hot.ts": bump(), "hot.test.ts": bump() }); + expect(brief(root)).not.toContain("Where work concentrates"); // 9 commits + commit(root, { "hot.ts": bump() }); + // Indexed, larger than anything else, never committed: it used to pad the + // list, ranked by size with 0 commits. + writeFileSync(join(root, "fresh.ts"), "export const f = 1;\n".repeat(300)); + const text = brief(root); + expect(text).toContain("## Where work concentrates"); + expect(text).toMatch(/- `hot\.ts` — 9 commits, \d+ lines\n/); + expect(text).toMatch(/- `hot\.test\.ts` — 8 commits, \d+ lines \(test\)\n/); + expect(text).toMatch(/- `still\.ts` — 1 commits, \d+ lines\n/); + expect(text).not.toMatch(/`fresh\.ts` — \d+ commits/); + }); +}); + +describe("CLI", () => { + const cli = (args: string[]) => spawnSync(process.execPath, [CLI, ...args], { encoding: "utf8" }); + + it("exits 2 on a --since that is neither a ref nor a date", () => { + const root = newRepo("ci-hist-cli-"); + commit(root, { "a.ts": bump() }); + const res = cli(["churn", "--repo", root, "--since", "nosuchref"]); + expect(res.status).toBe(2); + expect(res.stderr).toMatch(/since "nosuchref" is neither/); + }); + + it("applies --scope/--include/--exclude to churn keys and --limit to hotspots", () => { + const root = newRepo("ci-hist-cli-"); + commit(root, { "src/a.ts": bump(), "src/b.ts": bump(), "docs/x.ts": bump() }); + commit(root, { "src/a.ts": bump(), "docs/x.ts": bump() }); + const churn = JSON.parse(cli(["churn", "--repo", root, "--scope", "src", "--exclude", "src/b.ts"]).stdout); + expect(churn).toEqual({ ok: true, churn: { "src/a.ts": 2 } }); + const hot = JSON.parse(cli(["hotspots", "--repo", root, "--limit", "1", "--no-index-cache"]).stdout); + expect(hot.hotspots.map((h: { rel: string }) => h.rel)).toHaveLength(1); + }); +}); From 2188218ec71534fb254e61b27e42317a5f51d665 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:49:39 +0000 Subject: [PATCH 035/130] fix(cli): warn when a read command answers from an empty scan `index` and `scan` already warn on stderr when no file is kept. A read command such as `search --scope src --include '*.py'` answered from a 0-file scan with no warning, so the empty result looked like "no match". Read commands now print the same warning once, including the note that globs are rooted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/engine-cli.ts | 14 ++++++++++++-- tests/scope.test.ts | 9 +++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/engine-cli.ts b/src/engine-cli.ts index d8ed4b7..c0ab5f7 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -651,6 +651,16 @@ export async function runCli(rawArgv: string[]): Promise { loadArtifacts?: () => IndexArtifacts | undefined; } | undefined> | undefined; let preloaded: { scan: RepoScan; arts?: IndexArtifacts; loadArtifacts?: () => IndexArtifacts | undefined } | undefined; + // A read command answering from a scan that kept no file says so once, as + // `index` and `scan` do: an empty answer otherwise looks like "no match". + let warnedEmpty = false; + const noteEmpty = (scan: RepoScan): RepoScan => { + if (scan.files.length === 0 && !warnedEmpty) { + warnedEmpty = true; + warnEmptyScan(flags); + } + return scan; + }; const tryPreload = async (): Promise => { if (preloadPromise) return preloadPromise; if (preloadTried) return preloaded; @@ -662,7 +672,7 @@ export async function runCli(rawArgv: string[]): Promise { warmPresentGrammars, indexDir, ).then((p) => { - if (p) preloaded = { scan: p.scan, arts: p.arts, loadArtifacts: p.loadArtifacts }; + if (p) preloaded = { scan: noteEmpty(p.scan), arts: p.arts, loadArtifacts: p.loadArtifacts }; // The default location being empty is the normal first run; an index the // user NAMED being unusable is a mistake worth one line (a typo'd path // otherwise just looks like a slow command). @@ -683,7 +693,7 @@ export async function runCli(rawArgv: string[]): Promise { scanRepoParallel(flags.repo, { ...scanOptions(flags, precomputedWalk), workers: flags.workers, - }), + }).then(noteEmpty), )); }; let readArtifactsPromise: Promise | undefined; diff --git a/tests/scope.test.ts b/tests/scope.test.ts index ea8f7dc..e6412e2 100644 --- a/tests/scope.test.ts +++ b/tests/scope.test.ts @@ -213,6 +213,15 @@ describe("CLI warnings for path flags that match nothing", { timeout: 60_000 }, expect(slash.count).toBe(FILES.length - 1); }); + it("a read command answering from an empty scan says so", () => { + const root = fixture(); + const res = spawnSync(process.execPath, [CLI, "search", "flask", "--repo", root, "--no-ast", "--scope", "src", "--include", "*.py"], { + encoding: "utf8", + }); + expect(res.status).toBe(0); + expect(res.stderr).toMatch(/no file of .* was indexed.*'\*\*\/\*\.py' any depth/); + }); + it("an empty result under a rooted include glob names the any-depth spelling", () => { const root = fixture(); const { count, stderr } = run(root, "--scope", "src", "--include", "*.py"); From f3c31c22ecd18fde9c30b009a9649d92d5c44c34 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:50:07 +0000 Subject: [PATCH 036/130] fix(extract): index Ruby singleton-class, inline-visibility and factory-class methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three Ruby shapes the AST walk never visited: - `class << self; def create; end; end` — its methods are the class's own (the regex tier found them). The singleton body opens its own visibility section, as Ruby does: a bare `private` above it does not reach it, one inside it does. - `protected def weight`, `private_class_method def self.build`, `private attr_reader :x`, `public`/`module_function def …` — a definition wrapped in the visibility call that applies to it alone. The wrapped nodes are walked in the call's place with that visibility; the doc above the call is theirs. The regex tier reads the same form. - `Point = Struct.new(:x, :y) do … end` (and `Class.new`, `Module.new`, `Data.define`) builds a class; it is now declared as one, with its block's methods as members, instead of a constant with a lost body. Also: `klass.extend Mixin` / `NameError.prepend(Ext)` mix into their receiver, so they no longer state a relation about the enclosing method or module (the stdlib had "included implements ClassMethods"). On the Ruby 3.3 stdlib (996 files): +455 defs, +40 attrs, 29 constants become classes (+2 modules), 6 bogus relations dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 17 +++-- site/index.html | 8 +- site/quality.json | 8 +- src/ast/extract.ts | 81 +++++++++++--------- src/ast/specs.ts | 85 ++++++++++++++++++++- src/lang/ruby.ts | 7 ++ tests/extraction-shapes.test.ts | 91 +++++++++++++++++++++++ tests/fixtures/quality/ruby/expected.json | 6 +- tests/fixtures/quality/ruby/scheduler.rb | 20 +++++ tests/lang.test.ts | 6 ++ 10 files changed, 273 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 8ecd7ba..9cbb32a 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,9 @@ compares](#how-it-compares). aliases, record components, constructor `val` parameters and their TypeScript (`private readonly dep: Dep`) and PHP 8 (promoted) twins, C `#define` macros and the members of a `typedef struct`, Python declarations under - `if TYPE_CHECKING:` / `try:` / `with` blocks, Elixir clauses with a `when` - guard and `defguard`, and the members of a class + `if TYPE_CHECKING:` / `try:` / `with` blocks, Ruby `class << self` methods, + `private def x` definitions and the block of `Point = Struct.new(…) do`, + Elixir clauses with a `when` guard and `defguard`, and the members of a class bound by `module.exports =` or an anonymous `export default class` (a default export with no name of its own is named after the file stem). A doc comment is found across Rust attributes and TypeScript decorators; @@ -75,7 +76,7 @@ vocabulary: | **TypeScript compiler index** (`scip-typescript` 0.4.0) | an index built by the real TypeScript compiler — authoritative where every other check here is syntactic | **100%** of its 93 named declarations, against ctags' 94.6% on the same files | | **universal-ctags differential** (Universal Ctags 6.2.1) | an independent, mature indexer covering ~40 languages | reports **2,014** declarations ctags does not over 6 real repositories, and reproduces **61.7%–98.8%** of ctags' names — what is left bucketed by kind, per repo below | | **Official `tags.scm` queries** | the code-navigation patterns each grammar's own authors publish, and GitHub uses | **1** adjudicated difference, over the 14 of 17 languages that publish one | -| **Grammar vocabulary** | each tree-sitter grammar's own declared node types, read at runtime from the parser | 21 grammars audited, **211** declaration-ish node types still unhandled | +| **Grammar vocabulary** | each tree-sitter grammar's own declared node types, read at runtime from the parser | 21 grammars audited, **209** declaration-ish node types still unhandled | ### The one head-to-head @@ -165,12 +166,12 @@ terms live only in prose. | what is scored | score | measured on | |---|---|---| -| symbol precision / recall | **100% / 100%** | 332 labelled declarations in 23 files | -| kind accuracy | **100%** | the same 332 declarations | -| visibility accuracy | **100%** on 16 of 17 languages, 95.2% on Go | the same 332 declarations | -| doc comment attached | **100%** | the 181 declarations labelled with a doc | +| symbol precision / recall | **100% / 100%** | 336 labelled declarations in 23 files | +| kind accuracy | **100%** | the same 336 declarations | +| visibility accuracy | **100%** on 16 of 17 languages, 95.2% on Go | the same 336 declarations | +| doc comment attached | **100%** | the 185 declarations labelled with a doc | | complete signature | **100%** | the 32 declarations labelled with a signature | -| call edges / inheritance (F1) | **100% / 100%** | 51 labelled call sites, 24 relations | +| call edges / inheritance (F1) | **100% / 100%** | 54 labelled call sites, 24 relations | | search MRR / nDCG@10 / recall@5 | **93.8% / 86.0% / 84.4%** | 16 relevance-judged queries | `pnpm quality:report` reproduces every number; `tests/quality.test.ts` enforces diff --git a/site/index.html b/site/index.html index c134150..ca3a071 100644 --- a/site/index.html +++ b/site/index.html @@ -4469,10 +4469,10 @@

How it compares

"base": { "languages": 17, "files": 23, - "symbols": 332, - "calls": 51, + "symbols": 336, + "calls": 54, "relations": 24, - "docLabels": 181, + "docLabels": 185, "sigLabels": 32 }, "extraction": [ @@ -4727,7 +4727,7 @@

How it compares

"id": "grammar-vocabulary", "label": "Grammar vocabulary", "authority": "each tree-sitter grammar's own declared node types, read at runtime from the parser", - "value": "21 grammars, 211 declaration-ish types unhandled", + "value": "21 grammars, 209 declaration-ish types unhandled", "scope": "Names every construct a grammar declares that no extraction rule covers. The denominator is the grammar's, not ours — which is what makes it able to find what nobody thought to label." }, { diff --git a/site/quality.json b/site/quality.json index eca358a..74c11b2 100644 --- a/site/quality.json +++ b/site/quality.json @@ -4,10 +4,10 @@ "base": { "languages": 17, "files": 23, - "symbols": 332, - "calls": 51, + "symbols": 336, + "calls": 54, "relations": 24, - "docLabels": 181, + "docLabels": 185, "sigLabels": 32 }, "extraction": [ @@ -262,7 +262,7 @@ "id": "grammar-vocabulary", "label": "Grammar vocabulary", "authority": "each tree-sitter grammar's own declared node types, read at runtime from the parser", - "value": "21 grammars, 211 declaration-ish types unhandled", + "value": "21 grammars, 209 declaration-ish types unhandled", "scope": "Names every construct a grammar declares that no extraction rule covers. The denominator is the grammar's, not ours — which is what makes it able to find what nobody thought to label." }, { diff --git a/src/ast/extract.ts b/src/ast/extract.ts index 6d250ca..1f006ea 100644 --- a/src/ast/extract.ts +++ b/src/ast/extract.ts @@ -488,45 +488,55 @@ export function extractAst( } } const childCtx = sectionPublic === ctx.sectionPublic ? ctx : { ...ctx, sectionPublic }; - - // Enum members written without an initialiser are a bare identifier - // leaf, not a declaration node any table can key on. - if (bareKind && c.namedChildren.length === 0 && IDENT_LEAF.test(c.type)) { - emit({ - name: c.text, - kind: bareKind, - file: rel, - line: c.startPosition.row + 1, - endLine: endLineOf(c), - ...(childCtx.parent ? { parent: childCtx.parent } : {}), - exported: childCtx.forcePublic || childCtx.exported, - lang, - }); - continue; + const wrapped = spec.inlineVisibility?.(c); + if (wrapped) { + for (const inner of wrapped.nodes) walkMember(inner, bareKind, { ...childCtx, sectionPublic: wrapped.public }); + } else { + walkMember(c, bareKind, childCtx); } + } + }; - const extras = spec.extraMembers?.(c, { ownerKind: childCtx.ownerKind, inFunctionBody: childCtx.inFunctionBody, publicNames }); - for (const extra of extras ?? []) { - const at = extra.node ?? c; - const header = declHeader(at, content); - const doc = docCommentFor(at); - emit({ - name: extra.name, - kind: extra.kind, - file: rel, - line: at.startPosition.row + 1, - endLine: endLineOf(at), - ...(childCtx.parent ? { parent: childCtx.parent } : {}), - ...(childCtx.parentPath && childCtx.parentPath !== childCtx.parent ? { parentPath: childCtx.parentPath } : {}), - signature: header, - ...(doc ? { doc } : {}), - exported: visibilityOf(at, header, extra.name, childCtx), - lang, - }); - } + // One member of a container body: a bare enum member, the extras the spec + // reads off it, then the member itself. + const walkMember = (c: TSNode, bareKind: string | undefined, childCtx: WalkCtx): void => { + // Enum members written without an initialiser are a bare identifier + // leaf, not a declaration node any table can key on. + if (bareKind && c.namedChildren.length === 0 && IDENT_LEAF.test(c.type)) { + emit({ + name: c.text, + kind: bareKind, + file: rel, + line: c.startPosition.row + 1, + endLine: endLineOf(c), + ...(childCtx.parent ? { parent: childCtx.parent } : {}), + exported: childCtx.forcePublic || childCtx.exported, + lang, + }); + return; + } - walk(c, childCtx); + const extras = spec.extraMembers?.(c, { ownerKind: childCtx.ownerKind, inFunctionBody: childCtx.inFunctionBody, publicNames }); + for (const extra of extras ?? []) { + const at = extra.node ?? c; + const header = declHeader(at, content); + const doc = spec.docFrom?.(at) ?? docCommentFor(at); + emit({ + name: extra.name, + kind: extra.kind, + file: rel, + line: at.startPosition.row + 1, + endLine: endLineOf(at), + ...(childCtx.parent ? { parent: childCtx.parent } : {}), + ...(childCtx.parentPath && childCtx.parentPath !== childCtx.parent ? { parentPath: childCtx.parentPath } : {}), + signature: header, + ...(doc ? { doc } : {}), + exported: visibilityOf(at, header, extra.name, childCtx), + lang, + }); } + + walk(c, childCtx); }; // The context a declaration's body is walked in: its members hang off it, @@ -861,6 +871,7 @@ export function extractAst( inFunctionBody: ctx.inFunctionBody || entersFunction, funcDepth: ctx.funcDepth + (entersFunction ? 1 : 0), ...(qualifier ? { parent: qualifier, parentPath: qualifier, ownerKind: "type" } : {}), + ...(spec.sectionScopes?.has(type) ? { sectionPublic: true } : {}), }); } }; diff --git a/src/ast/specs.ts b/src/ast/specs.ts index 614d1d0..3859f81 100644 --- a/src/ast/specs.ts +++ b/src/ast/specs.ts @@ -14,6 +14,7 @@ import type { RawRelation } from "../types.js"; import type { TSNode } from "./node.js"; import { findFirst, nameOf, nextDeclarator, readTypeName } from "./node.js"; +import { docCommentFor } from "./doc.js"; /** Type names an inheritance clause lists, skipping type-argument noise. */ function heritageTargets(clause: TSNode | null | undefined): string[] { @@ -156,6 +157,21 @@ export interface LangSpec { */ sectionVisibility?: (node: TSNode) => boolean | undefined; + /** + * Containers that open a visibility section of their own, starting public + * whatever the enclosing body's section says: Ruby's `class << self`, whose + * methods a bare `private` above it does not reach. + */ + sectionScopes?: Set; + + /** + * A body child that WRAPS declarations and states their visibility inline — + * Ruby's `private def helper`, `protected attr_reader :x`, + * `private_class_method def self.build`. The walk visits the wrapped nodes in + * its place, as members of the same body, with that visibility for them alone. + */ + inlineVisibility?: (node: TSNode) => { nodes: TSNode[]; public: boolean } | undefined; + /** Force non-exported even inside an `export`ed declaration (TS `private`/`protected` members). */ privateMember?: (node: TSNode) => boolean; @@ -455,6 +471,33 @@ export function luaMember(target: TSNode | null | undefined): { name: string; ta return table && field && /^[\w.]+$/.test(table.text) ? { name: field.text, table: table.text } : undefined; } +// Ruby's visibility methods, by whether what they wrap ends up public. Called +// with a definition (`private def x`), they apply to it alone. +const RUBY_VISIBILITY: Record = { + public: true, + private: false, + protected: false, + module_function: true, + public_class_method: true, + private_class_method: false, +}; + +// `Point = Struct.new(:x, :y) do … end` defines a CLASS, whose block body holds +// its methods — `Class.new(Base)`, `Module.new` and `Data.define` likewise. +// Read as a plain constant, the block was never walked. +const RUBY_CLASS_FACTORY: Record = { + "Struct.new": "class", + "Class.new": "class", + "Data.define": "class", + "Module.new": "module", +}; +function rubyFactoryKind(node: TSNode): string | undefined { + if (node.childForFieldName("left")?.type !== "constant") return undefined; + const call = node.childForFieldName("right"); + if (call?.type !== "call") return undefined; + return RUBY_CLASS_FACTORY[`${call.childForFieldName("receiver")?.text}.${call.childForFieldName("method")?.text}`]; +} + // HCL/Terraform declares everything as a labelled block. Only these top-level // block types name something a reader looks up; `lifecycle`, `ingress` and the // rest are nested configuration, and treating them as symbols would bury the @@ -810,8 +853,37 @@ export const SPECS: Record = { ruby: { lang: "ruby", defs: { method: "def", singleton_method: "def", class: "class", module: "module" }, - containers: new Set(["class", "module", "body_statement", "program"]), + containers: new Set([ + "class", + "module", + "body_statement", + "program", + // `class << self` — its methods are the enclosing class's own. + "singleton_class", + // The `{ … }` body of a class factory's block (see rubyFactoryKind). + "block_body", + ]), + sectionScopes: new Set(["singleton_class"]), exported: always, + kindFrom: { assignment: rubyFactoryKind }, + nameFrom: { assignment: (node) => node.childForFieldName("left")?.text }, + bodyFrom: { assignment: (node) => node.childForFieldName("right")?.childForFieldName("block")?.childForFieldName("body") ?? undefined }, + inlineVisibility: (node) => { + if (node.type !== "call" || node.childForFieldName("receiver")) return undefined; + const pub = RUBY_VISIBILITY[node.childForFieldName("method")?.text ?? ""]; + if (pub === undefined) return undefined; + const nodes = (node.childForFieldName("arguments")?.namedChildren ?? []).filter( + (a) => a.type === "method" || a.type === "singleton_method" || a.type === "call", + ); + return nodes.length ? { nodes, public: pub } : undefined; + }, + // A wrapped definition's doc sits above the wrapping call. + docFrom: (node) => { + const call = node.parent?.type === "argument_list" ? node.parent.parent : null; + return call?.type === "call" && RUBY_VISIBILITY[call.childForFieldName("method")?.text ?? ""] !== undefined + ? docCommentFor(call) + : undefined; + }, // Ruby models every invocation — dotted, parenthesized, or bare command form // (`puts "x"`) — as a `call` node whose callee is the `method` field. calls: { call: "function" }, @@ -830,10 +902,14 @@ export const SPECS: Record = { return to ? [rel("extends", ctx.self, to, node)] : []; }, // `include Runnable` mixes a module in — Ruby's only `implements`. It is a - // method call, so nothing but the callee name identifies it. + // method call, so nothing but the callee name identifies it. With a + // receiver (`klass.extend Mixin`, inside a hook method) it mixes into + // something else, not into the enclosing declaration. call: (node, ctx) => { const method = node.childForFieldName("method"); + const receiver = node.childForFieldName("receiver"); if (!ctx.self || !method || !/^(include|prepend|extend)$/.test(method.text)) return []; + if (receiver && receiver.type !== "self") return []; const out: RawRelation[] = []; for (const a of node.childForFieldName("arguments")?.namedChildren ?? []) { const to = readTypeName(a); @@ -844,10 +920,11 @@ export const SPECS: Record = { }, extraMembers: (node, ctx) => { if (ctx.inFunctionBody) return []; - // `MAX_ATTEMPTS = 5` — a constant is an assignment to a `constant` node. + // `MAX_ATTEMPTS = 5` — a constant is an assignment to a `constant` node + // (unless it builds a class, which the walk declares as one). if (node.type === "assignment") { const left = node.childForFieldName("left"); - return left?.type === "constant" ? [{ name: left.text, kind: "const" }] : []; + return left?.type === "constant" && !rubyFactoryKind(node) ? [{ name: left.text, kind: "const" }] : []; } // `attr_reader :queue` declares real accessor methods; nothing else in the // file mentions `queue`, so without this the attribute does not exist. diff --git a/src/lang/ruby.ts b/src/lang/ruby.ts index e420c92..c38956b 100644 --- a/src/lang/ruby.ts +++ b/src/lang/ruby.ts @@ -4,6 +4,13 @@ import { scan, type Rule } from "./common.js"; // Ruby. `def` (instance/class methods), `class`, and `module` declarations. const RULES: Rule[] = [ { re: /^\s*def\s+(?:self\.)?(?[\w?!=]+)/, kind: "method", exported: true }, + // `private def helper` — a definition wrapped in the visibility call that + // applies to it alone. + { + re: /^\s*(?public|private|protected|module_function|(?:public|private)_class_method)\s+def\s+(?:self\.)?(?[\w?!=]+)/, + kind: "method", + exported: (m) => /^(public|module_function|public_class_method)$/.test(m.groups!.vis!), + }, { re: /^\s*class\s+(?[\w:]+)/, kind: "class", exported: true }, { re: /^\s*module\s+(?[\w:]+)/, kind: "module", exported: true }, ]; diff --git a/tests/extraction-shapes.test.ts b/tests/extraction-shapes.test.ts index f26f69d..6374c08 100644 --- a/tests/extraction-shapes.test.ts +++ b/tests/extraction-shapes.test.ts @@ -584,3 +584,94 @@ describe("a Python module that declares `__all__`", () => { expect(vis(syms("m.py", "from .x import Y\nOTHER = 1"))).toEqual(["const OTHER=1"]); }); }); + +describe("Ruby definitions outside a plain class body", () => { + const vis = (all: CodeSymbol[]) => all.map((s) => `${s.kind} ${ids([s])[0]}=${s.exported ? 1 : 0}`); + + it("indexes `class << self` methods as the class's, in a section of their own", () => { + const src = [ + "class W", + " private", + " class << self", + " def create; end", + " private", + " def build; end", + " end", + " def helper; end", + "end", + ].join("\n"); + expect(vis(syms("w.rb", src))).toEqual(["class W=1", "def W.create=1", "def W.build=0", "def W.helper=0"]); + }); + + it("gives a definition wrapped in a visibility call that visibility alone", () => { + const src = [ + "class W", + " # Weighs a job.", + " protected def weight; end", + " private def self.hidden; end", + " private_class_method def self.pcm; end", + " private attr_reader :secret", + " def open; end", + " private", + " public def shown; end", + " module_function def mf; end", + "end", + ].join("\n"); + const all = syms("w.rb", src); + expect(vis(all)).toEqual([ + "class W=1", + "def W.weight=0", + "def W.hidden=0", + "def W.pcm=0", + "attr W.secret=0", + "def W.open=1", + "def W.shown=1", + "def W.mf=1", + ]); + expect(find(all, "weight")?.doc).toBe("Weighs a job."); + }); + + it("walks the block of a class a factory builds", () => { + const src = [ + "Point = Struct.new(:x, :y) do", + " def dist; end", + "end", + "Pair = Struct.new(:a) { def sum; end }", + "Mixin = Module.new do", + " def helper; end", + "end", + "Value = Data.define(:v)", + "LIMIT = Limit.new(5)", + ].join("\n"); + expect(syms("p.rb", src).map((s) => `${s.kind} ${ids([s])[0]}`)).toEqual([ + "class Point", + "def Point.dist", + "class Pair", + "def Pair.sum", + "module Mixin", + "def Mixin.helper", + "class Value", + "const LIMIT", + ]); + }); +}); + +describe("a Ruby mixin call on another receiver", () => { + // `klass.extend Mixin` inside a hook mixes into `klass`, not into the method + // the call sits in; the Ruby stdlib produced "included implements + // ClassMethods" and "initialize implements TSort" this way. + it("states no relation about the enclosing declaration", () => { + const rels = (src: string) => (extractAst("m.rb", ".rb", src)?.relations ?? []).map((r) => `${r.kind} ${r.from} ${r.to}`); + const src = [ + "module Plugin", + " include Base", + " self.extend Helpers", + " def self.included(klass)", + " klass.extend ClassMethods", + " end", + "end", + "NameError.prepend(Plugin)", + ].join("\n"); + expect(rels(src)).toEqual(["implements Plugin Base", "implements Plugin Helpers"]); + }); +}); diff --git a/tests/fixtures/quality/ruby/expected.json b/tests/fixtures/quality/ruby/expected.json index 2592390..6a8a463 100644 --- a/tests/fixtures/quality/ruby/expected.json +++ b/tests/fixtures/quality/ruby/expected.json @@ -7,15 +7,19 @@ { "name": "MAX_ATTEMPTS", "parent": "Worker", "kind": "const", "doc": true }, { "name": "Runnable", "parent": "Worker", "kind": "module", "doc": true }, { "name": "start", "parent": "Runnable", "kind": "def", "doc": true }, + { "name": "JobSpec", "parent": "Worker", "kind": "class", "doc": true }, + { "name": "retry?", "parent": "JobSpec", "kind": "def", "doc": true }, { "name": "Scheduler", "parent": "Worker", "kind": "class", "doc": true }, { "name": "queue", "parent": "Scheduler", "kind": "attr" }, { "name": "initialize", "parent": "Scheduler", "kind": "def", "doc": true }, { "name": "start", "parent": "Scheduler", "kind": "def", "doc": true }, { "name": "dispatch", "parent": "Scheduler", "kind": "def", "doc": true }, { "name": "build", "parent": "Scheduler", "kind": "def" }, + { "name": "default", "parent": "Scheduler", "kind": "def", "doc": true }, + { "name": "weight", "parent": "Scheduler", "kind": "def", "exported": false, "doc": true }, { "name": "reset", "parent": "Scheduler", "kind": "def", "exported": false } ], - "calls": ["attr_reader", "clear", "dispatch", "each", "include", "new", "raise"], + "calls": ["attr_reader", "build", "clear", "dispatch", "each", "include", "new", "protected", "raise", "size"], "relations": [ { "kind": "extends", "from": "Scheduler", "to": "BaseWorker" }, { "kind": "implements", "from": "Scheduler", "to": "Runnable" } diff --git a/tests/fixtures/quality/ruby/scheduler.rb b/tests/fixtures/quality/ruby/scheduler.rb index 7a777c4..dfa9730 100644 --- a/tests/fixtures/quality/ruby/scheduler.rb +++ b/tests/fixtures/quality/ruby/scheduler.rb @@ -11,6 +11,14 @@ def start end end + # One queued unit of work. + JobSpec = Struct.new(:name, :attempts) do + # Whether the budget allows another attempt. + def retry? + attempts < MAX_ATTEMPTS + end + end + # Runs jobs with exponential backoff between retries. class Scheduler < BaseWorker include Runnable @@ -39,6 +47,18 @@ def self.build(queue) new(queue) end + class << self + # The scheduler shared by every caller that does not bring its own. + def default + @default ||= build("default") + end + end + + # Heavier jobs wait longer between attempts. + protected def weight(spec) + spec.size + end + private def reset diff --git a/tests/lang.test.ts b/tests/lang.test.ts index daaa313..23c37aa 100644 --- a/tests/lang.test.ts +++ b/tests/lang.test.ts @@ -148,6 +148,12 @@ load_plugin() { expect(names).toEqual(expect.arrayContaining(["Api", "Client", "fetch"])); }); + it("reads a Ruby definition's inline visibility (regex fallback)", () => { + const src = "class W\n protected def weight; end\n private_class_method def self.pcm; end\n public def shown; end\nend\n"; + const by = Object.fromEntries(extractSymbols("w.rb", ".rb", src).map((s) => [s.name, s.exported])); + expect(by).toEqual({ W: true, weight: false, pcm: false, shown: true }); + }); + it("extracts C functions and structs (regex fallback)", () => { const names = extractSymbols("a.c", ".c", "typedef struct Node Node;\nint compute(int x) {\n return x;\n}\n").map((s) => s.name); expect(names).toContain("compute"); From 6e963bb04c633ae5523411cf964b8810fedb54b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:50:09 +0000 Subject: [PATCH 037/130] fix(rewrite): restate grep/rg/git grep faithfully, or refuse A rewrite is only worth having if it means what the original meant. Several did not, and silently returned the wrong hits: - Globs. The base-name globs of grep --include and rg -g (which match at any depth) became rooted globs, and --scope was OR-ed with them. On gin, `--include=*.go` found 1 of 3 hits, and `!*_test.go` only dropped root-level tests. They now become `**/`, and the path becomes a separate --scope that is ANDed with them. - Arguments. rg -r (--replace) was read as --recursive. `./dir` and file paths matched nothing, while /etc and ../x searched the repo instead. BRE `+` became a quantifier. Backslashes and double quotes were mangled, and unquoted globs were taken literally. The tokenizer now follows POSIX quoting and refuses shell syntax only where the shell would see it. It also refuses an unquoted glob in a path, since the shell would expand it. Paths are normalised, and paths outside the tree are refused. - Dialects. BRE/ERE (grep, egrep, git grep) and Rust (rg) patterns are translated to the JS regex codeindex runs, including POSIX classes, \< \>, intervals and -F/-w/-i/-S. Syntax that cannot be translated exactly is refused. The result is validated with the engine's own compiler. - Flag injection. A pattern starting with `-` is emitted after `--`, so `grep -r -e --out src` no longer writes a file. - Universe. `--ignore-dir .codeindex` lifts the default vendor/build/out/tmp skips, which grep, rg and git grep do not make, so committed vendor/ and build/ matches are no longer dropped. - Coverage. `rg 'a|b'`, -t, -w, -F, -S, -l (a new `grep --files-with-matches` mode: one hit per file, with the cap applied to files) and `git grep` are now rewritten. Include/exclude orders where the later rule would win in grep/rg are refused. The rewrite tests now run each supported form through the real tool and through its rewrite on one fixture, and compare the lines found. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 34 +- src/engine-cli.ts | 11 +- src/grep.ts | 20 +- src/mcp.ts | 1 + src/mcp/tools.ts | 4 + src/rewrite.ts | 840 ++++++++++++++++++++++++++++++++++++------ tests/grep.test.ts | 12 + tests/rewrite.test.ts | 204 ++++++++-- 8 files changed, 979 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 7e37d0e..0d69c2b 100644 --- a/README.md +++ b/README.md @@ -888,15 +888,35 @@ its indexed equivalent, for agent harnesses that intercept shell commands ```sh $ codeindex rewrite 'grep -rn TODO src' -codeindex grep TODO --scope src +codeindex grep TODO --scope src --ignore-dir .codeindex +$ codeindex rewrite "rg -tpy -w 'def main'" +codeindex grep '\bdef main\b' --include '**/*.py' --include '**/*.pyi' --ignore-dir .codeindex ``` -It prints the replacement and exits `0`, or exits `1` with empty stdout when it -has no opinion — run the original. The parser is deliberately conservative: any -shell metacharacter (pipe, redirect, substitution, chaining), any unrecognized -flag, a non-recursive `grep`, or more than one search path all refuse the -rewrite. A refusal costs nothing; a wrong rewrite silently changes what the -agent asked for. +It prints the replacement and exits `0`. When it has no opinion, it exits `1` +with empty stdout, and the host should run the original command. It +understands recursive `grep`/`egrep`, `rg` and `git grep`: + +- **The pattern.** POSIX BRE and ERE and Rust regex syntax, plus `-F`, `-w` + and `-i`/`-S`, are restated as the JavaScript regex `codeindex grep` runs. + In a BRE, `x+y` stays a literal `+`. +- **The files.** A path becomes `--scope` (`./` stripped, a file allowed). An + `--include`/`-g` base-name glob becomes `**/`, and `-t` becomes the + globs of that ripgrep type. `--ignore-dir .codeindex` turns off the default + vendor/build/out/tmp skips, which none of these tools make. Gitignored files, + lockfiles and binaries are still left out, on purpose. +- **The flags.** `-l` becomes `--files-with-matches`, and a pattern that starts + with `-` goes behind `--`. + +The parser is deliberately conservative. The rewrite is refused when the line +contains shell syntax outside single quotes (pipe, redirect, substitution, +chaining, braces, an unquoted glob in a path), an unrecognized or +output-changing flag (`rg -r` is `--replace`), a non-recursive `grep`, a path +outside the tree, more than one path, include/exclude rules whose order +matters, or regex syntax that cannot be translated exactly. A refusal costs +nothing, while a wrong rewrite would silently change what the agent asked for. +The test suite runs each supported form through the real tool and through its +rewrite, and checks that both find the same lines. ## Versioning diff --git a/src/engine-cli.ts b/src/engine-cli.ts index 0d4c9d2..b99a5b1 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -142,8 +142,10 @@ Commands: rewrite Map an expensive tree-wide search onto its indexed equivalent: cli.mjs rewrite ''. Prints the replacement command and exits 0, or exits 1 when it has no opinion (run the original). - Deliberately conservative — any shell metacharacter or unknown - flag refuses the rewrite + Understands recursive grep/egrep, rg and git grep (BRE/ERE/Rust + patterns restated as JS; -F -w -i -S -l -t -g --include). + Deliberately conservative — shell syntax outside single quotes, + an unknown flag or an untranslatable pattern refuses the rewrite mcp Run as an MCP server over stdio (33 tools: scan_summary, graph, symbols, callers, workspaces, churn, symbols_overview, find_symbol, find_references, lsp_status, onboard, repo_map, @@ -214,6 +216,8 @@ Flags (accepted before OR after the subcommand: '--repo X scan' and --ignore-case \`grep\`: case-insensitive matching --max-hits \`grep\`: cap returned hits (default 200). A capped result says so on stderr, with the count of matching files + --files-with-matches \`grep\`: one hit per matching file (its first match), so + --max-hits caps files — \`grep -l\` with evidence --timeout-ms \`grep\`: wall-clock budget for the JavaScript regex engine (default 10000). ripgrep is linear-time; the JS fallback backtracks, so a pathological pattern is stopped at the @@ -245,6 +249,7 @@ interface CliFlags { ignoreCase?: boolean; maxHits?: number; timeoutMs?: number; // grep: JS regex engine wall-clock budget + filesWithMatches?: boolean; // grep: one hit (the first) per matching file budgetTokens?: number; config?: string; // rules config path limit?: number; // search result cap @@ -301,6 +306,7 @@ function parseFlags(args: string[]): CliFlags { else if (a === "--ignore-case") flags.ignoreCase = true; else if (a === "--max-hits") flags.maxHits = num(); else if (a === "--timeout-ms") flags.timeoutMs = num(); + else if (a === "--files-with-matches") flags.filesWithMatches = true; else if (a === "--budget-tokens") flags.budgetTokens = num(); else if (a === "--min-files") flags.minFiles = num(); else if (a === "--min-count") flags.minCount = num(); @@ -1140,6 +1146,7 @@ export async function runCli(rawArgv: string[]): Promise { scope: flags.scope, ignoreCase: flags.ignoreCase, maxHits: flags.maxHits, + filesWithMatches: flags.filesWithMatches, gitignore: flags.gitignore, ignoreDirs: flags.ignoreDirs.length ? flags.ignoreDirs : undefined, maxFileBytes: flags.maxBytes, diff --git a/src/grep.ts b/src/grep.ts index f81626a..7967192 100644 --- a/src/grep.ts +++ b/src/grep.ts @@ -46,6 +46,9 @@ export interface GrepOptions { scope?: string; maxHits?: number; // cap AFTER sorting (default 200) ignoreCase?: boolean; + // One hit per matching file — its first match — so maxHits caps FILES + // (`grep -l`, with the first match as evidence instead of a bare path). + filesWithMatches?: boolean; // The walk's universe knobs, honoured by both backends exactly as walk() // honours them (see WalkOptions): gitignore on by default, ignoreDirs // REPLACES the default set (`.git` stays skipped), maxFileBytes defaults to @@ -403,7 +406,7 @@ function rgBackend( // No file can contribute more than `remaining` hits to the answer; // stopping each file there bounds the output without changing it. "--max-count", - String(remaining), + String(opts.filesWithMatches ? 1 : remaining), "--regexp", rust, "--", @@ -460,6 +463,7 @@ interface ScanJob { flags: string; files: [rel: string, abs: string][]; // in path order want: number; // hits to collect; past it, a file is only checked for "matches at all" + firstOnly: boolean; // stop each file at its first match textMax: number; deadline: number; // epoch ms: a soft stop between files (the worker is also hard-stopped) } @@ -515,6 +519,7 @@ function scanFiles(job: ScanJob, io: ScanIo): void { if (got >= job.want) break; // cap reached: only "does it match" still counts got++; hits.push([n + 1, m.index + 1, io.clip(lines[n]!, m.index, job.textMax)]); + if (job.firstOnly) break; } if (matched) io.emit({ i, hits }); } @@ -604,7 +609,15 @@ function jsBackend( .filter((f) => !keep || keep(f.rel)) .map((f): [string, string] => [f.rel, f.abs]) .sort((a, b) => byStr(a[0], b[0])); - const job: ScanJob = { source: re.source, flags: re.flags, files, want: max + 1, textMax: MAX_TEXT, deadline }; + const job: ScanJob = { + source: re.source, + flags: re.flags, + files, + want: max + 1, + firstOnly: opts.filesWithMatches === true, + textMax: MAX_TEXT, + deadline, + }; const hits: SearchHit[] = []; let filesMatched = 0; const { stoppedAt } = runScan(job, (ev) => { @@ -648,8 +661,9 @@ export function grepRepoEx(root: string, pattern: string, opts: GrepOptions = {} out ??= jsBackend(root, re, opts, keep, max, deadline); notes.push(...out.notes); if (out.truncated) { + const shown = opts.filesWithMatches ? `${max} matching files by path` : `${max} hits by (file, line)`; notes.push( - `showing the first ${max} hits by (file, line); ${out.filesMatched} files match in all — raise maxHits (--max-hits) or narrow scope/globs for the rest`, + `showing the first ${shown}; ${out.filesMatched} files match in all — raise maxHits (--max-hits) or narrow scope/globs for the rest`, ); } return { hits: out.hits, truncated: out.truncated, filesMatched: out.filesMatched, timedOut: out.timedOut ?? false, notes }; diff --git a/src/mcp.ts b/src/mcp.ts index 39191c1..c430c56 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -383,6 +383,7 @@ async function callTool(name: string, args: Record, defaultRepo scope: str(args.scope), ignoreCase: args.ignoreCase === true, maxHits: positiveNum(args.maxHits), + filesWithMatches: args.filesWithMatches === true, timeoutMs: positiveNum(args.timeoutMs), }); // The bare array stays the default shape. `withMeta` opts into the diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 74d9cf9..47586f1 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -329,6 +329,10 @@ export const TOOLS = [ }, ignoreCase: { type: "boolean" }, maxHits: { type: "number", minimum: 1 }, + filesWithMatches: { + type: "boolean", + description: "One hit per matching file (its first match); maxHits then caps files", + }, withMeta: { type: "boolean", description: "Return { hits, truncated, filesMatched, notes? } instead of the bare hit array", diff --git a/src/rewrite.ts b/src/rewrite.ts index 7941aa7..6c7f591 100644 --- a/src/rewrite.ts +++ b/src/rewrite.ts @@ -9,64 +9,92 @@ // So this module is deliberately, aggressively conservative: // // * anything the parser cannot prove it understands → NO rewrite -// * any shell metacharacter at all → NO rewrite (we refuse to reason about -// pipelines, redirection, substitution or chaining) -// * any flag not on the explicit allowlist → NO rewrite +// * any shell construct outside single quotes that the shell would expand or +// interpret (pipes, redirection, substitution, chaining, globbing, braces, +// tildes) → NO rewrite +// * any flag not on the explicit allowlist of its binary → NO rewrite +// * a pattern whose dialect (POSIX BRE/ERE, Rust regex) cannot be restated +// exactly as the JavaScript regex `codeindex grep` runs → NO rewrite // // A refusal is cheap (the original command runs untouched); a bad rewrite is // not. Every branch here defaults to refusing. +// +// What a rewrite DOES change, on purpose: the output (JSON hits, capped, with +// the cap reported on stderr) and the universe's noise floor — gitignored +// files, lockfiles, binaries, minified bundles and files over 1 MiB are never +// searched. What it must NOT change: which committed text files are searched +// (hence `--ignore-dir .codeindex`, which lifts the default vendor/build/out/tmp +// skips none of grep, rg or git grep make) and what the pattern matches. +import { compilePattern } from "./grep.js"; +import { escapeRegExp } from "./util.js"; -// Shell constructs we will not reason about. Their presence anywhere in the -// line — quoted or not — refuses the rewrite. Over-refusing is the point: a -// pattern containing a literal `|` is rare, mis-rewriting a pipeline is fatal. -const SHELL_METACHARS = /[|&;<>`\n\r$(){}]/; - -// `grep` output is line-oriented text; `codeindex grep` returns JSON hits. That -// is the intended trade (bounded + structured), but it means we only rewrite -// invocations whose INTENT is "search the tree", never "search this one file" -// (cheap already) and never anything whose flags shape the output format. -const GREP_BINARIES = new Set(["grep", "egrep", "rg", "ripgrep"]); - -interface Parsed { - pattern?: string; - path?: string; - ignoreCase: boolean; - includes: string[]; - recursive: boolean; -} - -// Split on whitespace honouring single/double quotes. Returns undefined on an -// unterminated quote — an unparseable line is a refusal, not a guess. -export function tokenize(line: string): string[] | undefined { - const out: string[] = []; +// Split on whitespace the way a POSIX shell would, or return undefined the +// moment the line holds anything the shell would expand or interpret. Single +// quotes are opaque; inside double quotes only \$ \` \" \\ are escapes and a +// bare $ or ` is an expansion; outside quotes a backslash quotes the next +// character. Metacharacters are refused only where the shell sees them, so +// `rg 'foo|bar'` is a pattern, not a pipeline. An unquoted * ? [ is kept but +// marks its word `globbed`: the shell would expand it against the files in +// the cwd. +function lex(line: string): { value: string; globbed: boolean }[] | undefined { + const out: { value: string; globbed: boolean }[] = []; let cur = ""; - let quote: '"' | "'" | undefined; - let started = false; + let started = false; // a quoted empty string is still an argument + let globbed = false; for (let i = 0; i < line.length; i++) { - const c = line[i]; - if (quote) { - if (c === quote) quote = undefined; - else cur += c; - continue; - } - if (c === '"' || c === "'") { - quote = c; + const c = line[i]!; + if (c === "'") { + const end = line.indexOf("'", i + 1); + if (end === -1) return undefined; // unterminated quote + cur += line.slice(i + 1, end); started = true; - continue; - } - if (c === " " || c === "\t") { - if (started || cur) out.push(cur); + i = end; + } else if (c === '"') { + started = true; + for (i++; ; i++) { + const d = line[i]; + if (d === undefined) return undefined; // unterminated quote + if (d === '"') break; + if (d === "$" || d === "`") return undefined; // expansion + if (d === "\\") { + const n = line[i + 1]; + if (n === "$" || n === "`" || n === '"' || n === "\\") { + cur += n; + i++; + continue; + } + if (n === "\n") return undefined; + } + cur += d; + } + } else if (c === "\\") { + const n = line[i + 1]; + if (n === undefined || n === "\n" || n === "\r") return undefined; + cur += n; + started = true; + i++; + } else if (c === " " || c === "\t") { + if (started || cur) out.push({ value: cur, globbed }); cur = ""; started = false; - continue; + globbed = false; + } else if (/[|&;<>()$`\n\r{}]/.test(c)) { + return undefined; // control operators, expansions, brace expansion + } else if ((c === "#" || c === "~") && !started && !cur) { + return undefined; // a comment, a tilde expansion + } else { + if (c === "*" || c === "?" || c === "[") globbed = true; + cur += c; } - cur += c; } - if (quote) return undefined; // unterminated quote - if (started || cur) out.push(cur); + if (started || cur) out.push({ value: cur, globbed }); return out; } +export function tokenize(line: string): string[] | undefined { + return lex(line)?.map((t) => t.value); +} + // Wrap a token so the shell reproduces it verbatim. Single-quoting is total // (no escapes are interpreted inside), so the only case needing care is a // literal single quote, spliced via the standard '\'' idiom. @@ -75,95 +103,687 @@ export function shellQuote(s: string): string { return "'" + s.replace(/'/g, `'\\''`) + "'"; } -// Parse a grep/rg invocation into the fields we can faithfully re-express. -// Returns undefined the moment anything is unrecognized. -function parseSearch(bin: string, args: string[]): Parsed | undefined { - const p: Parsed = { ignoreCase: false, includes: [], recursive: bin !== "grep" && bin !== "egrep" }; - const positionals: string[] = []; +type Dialect = "bre" | "ere" | "rust"; +interface Parsed { + patterns: string[]; + dialect: Dialect; + fixed: boolean; + word: boolean; + ignoreCase: boolean; + smartCase: boolean; + // Already in the engine's rooted glob dialect, in the caller's order. + includes: string[]; + excludes: string[]; + excludeDirs: string[]; + // Set when an exclusion came before an inclusion: grep and rg then let the + // LATER rule win for a file matching both, where codeindex always excludes. + orderSensitive: boolean; + types: string[]; + paths: string[]; + recursive: boolean; + filesOnly: boolean; + noGitignore: boolean; +} + +// --------------------------------------------------------------------------- +// argv → flags +// --------------------------------------------------------------------------- + +interface Flag { + name: string; + value?: string; +} + +// Split argv into flags (short clusters expanded, `--x=v` split) and +// positionals. `takesValue` names the flags that consume an argument — a value +// attached to a short flag (`-tpy`, `-efoo`) ends its cluster. +function splitArgs( + args: string[], + takesValue: Set, +): { flags: Flag[]; positionals: string[]; afterDashDash: number } | undefined { + const flags: Flag[] = []; + const positionals: string[] = []; + let afterDashDash = -1; // index in positionals where `--` put us for (let i = 0; i < args.length; i++) { - const a = args[i]; - if (a === undefined) continue; + const a = args[i]!; if (a === "--") { - // Everything after `--` is positional by definition. + afterDashDash = positionals.length; positionals.push(...args.slice(i + 1)); break; } - if (!a.startsWith("-") || a === "-") { - positionals.push(a); - continue; + if (a.startsWith("--")) { + const eq = a.indexOf("="); + const name = eq === -1 ? a : a.slice(0, eq); + let value = eq === -1 ? undefined : a.slice(eq + 1); + if (takesValue.has(name) && value === undefined) { + value = args[++i]; + if (value === undefined) return undefined; + } + flags.push({ name, value }); + } else if (a.startsWith("-") && a !== "-") { + for (let j = 1; j < a.length; j++) { + const name = `-${a[j]}`; + if (!takesValue.has(name)) { + flags.push({ name }); + continue; + } + const value = j + 1 < a.length ? a.slice(j + 1) : args[++i]; + if (value === undefined) return undefined; + flags.push({ name, value }); + break; + } + } else positionals.push(a); + } + return { flags, positionals, afterDashDash }; +} + +function fresh(dialect: Dialect, recursive: boolean): Parsed { + return { + patterns: [], + dialect, + fixed: false, + word: false, + ignoreCase: false, + smartCase: false, + includes: [], + excludes: [], + excludeDirs: [], + orderSensitive: false, + types: [], + paths: [], + recursive, + filesOnly: false, + noGitignore: false, + }; +} + +function include(p: Parsed, glob: string | undefined): boolean { + if (glob === undefined) return false; + if (p.excludes.length) p.orderSensitive = true; + p.includes.push(glob); + return true; +} +function exclude(p: Parsed, glob: string | undefined): boolean { + if (glob === undefined) return false; + p.excludes.push(glob); + return true; +} +// A directory exclusion prunes whole trees and never competes with a file +// inclusion, so it does not make the rule order matter. +function excludeDir(p: Parsed, glob: string | undefined): boolean { + if (glob === undefined) return false; + p.excludeDirs.push(glob); + return true; +} + +// Glob characters the engine's dialect implements (`*`, `**`, `?`); classes, +// braces and escapes it does not. +const UNSUPPORTED_GLOB = /[[\]{}\\]/; + +// GNU grep --include/--exclude/--exclude-dir match a BASE name at any depth. +function baseNameGlob(g: string | undefined, dir = false): string | undefined { + if (!g || g.includes("/") || UNSUPPORTED_GLOB.test(g)) return undefined; + return dir ? `**/${g}/**` : `**/${g}`; +} + +// rg -g is gitignore-style: a slash-less glob matches a base name at any +// depth, one with a slash is anchored at the search root. +function rgGlob(g: string | undefined, rootIsRepo: boolean): string | undefined { + if (!g || UNSUPPORTED_GLOB.test(g) || g.endsWith("/")) return undefined; + if (!g.includes("/")) return `**/${g}`; + return rootIsRepo ? g.replace(/^\//, "") : undefined; +} + +// GNU grep / egrep. `-r` is required: a non-recursive `grep pattern file.ts` is +// already cheap and its semantics (one file, text output) are not what the +// indexed search provides. +function parseGrep(bin: string, args: string[]): Parsed | undefined { + const split = splitArgs(args, new Set(["-e", "--regexp", "--include", "--exclude", "--exclude-dir"])); + if (!split) return undefined; + const p = fresh(bin === "egrep" ? "ere" : "bre", false); + for (const { name, value } of split.flags) { + switch (name) { + case "-r": + case "-R": + case "--recursive": + case "--dereference-recursive": + p.recursive = true; + break; + case "-i": + case "-y": + case "--ignore-case": + p.ignoreCase = true; + break; + case "--no-ignore-case": + p.ignoreCase = false; + break; + case "-E": + case "--extended-regexp": + p.dialect = "ere"; + break; + case "-G": + case "--basic-regexp": + p.dialect = "bre"; + break; + case "-F": + case "--fixed-strings": + p.fixed = true; + break; + case "-w": + case "--word-regexp": + p.word = true; + break; + case "-e": + case "--regexp": + p.patterns.push(value!); + break; + case "-l": + case "--files-with-matches": + p.filesOnly = true; + break; + case "--include": + if (!include(p, baseNameGlob(value))) return undefined; + break; + case "--exclude": + if (!exclude(p, baseNameGlob(value))) return undefined; + break; + case "--exclude-dir": + if (!excludeDir(p, baseNameGlob(value, true))) return undefined; + break; + // Presentation only — every hit carries file + line, binaries are never + // searched, and there is no colour to turn off. + case "-n": + case "--line-number": + case "-H": + case "--with-filename": + case "-s": + case "--no-messages": + case "-I": + break; + case "--color": + case "--colour": + if (value !== undefined && !["never", "auto", "always"].includes(value)) return undefined; + break; + default: + return undefined; } - if (a === "-i" || a === "--ignore-case") { - p.ignoreCase = true; - } else if (a === "-r" || a === "-R" || a === "--recursive") { - p.recursive = true; - } else if (a === "-n" || a === "--line-number" || a === "-H" || a === "--with-filename" || a === "--no-heading") { - // Pure output-shaping flags whose effect codeindex already provides - // unconditionally (every hit carries file + line). Safe to drop. - } else if (a === "-e" || a === "--regexp") { - const v = args[++i]; - if (v === undefined || p.pattern !== undefined) return undefined; - p.pattern = v; - } else if (a.startsWith("--include=")) { - p.includes.push(a.slice("--include=".length)); - } else if (a === "--include" || a === "-g" || a === "--glob") { - const v = args[++i]; - if (v === undefined) return undefined; - p.includes.push(v); - } else if (a.length > 2 && /^-[a-zA-Z]+$/.test(a)) { - // A bundled short-flag cluster (-rn, -ri, -rni…). Expand and re-check; - // any member outside the allowlist refuses the whole line. - const expanded = a.slice(1).split("").map((c) => `-${c}`); - args.splice(i, 1, ...expanded); - i--; - } else { - return undefined; // unknown flag → refuse + } + if (!positionalsInto(p, split.positionals)) return undefined; + return p; +} + +// ripgrep. Recursive by default; `-r` is --replace here, NOT --recursive. +function parseRg(args: string[]): Parsed | undefined { + const split = splitArgs( + args, + new Set(["-e", "--regexp", "-g", "--glob", "-t", "--type", "-T", "--type-not", "-j", "--threads", "--sort"]), + ); + if (!split) return undefined; + const p = fresh("rust", true); + let unrestricted = 0; + const pending: { glob: string; negated: boolean }[] = []; + for (const { name, value } of split.flags) { + switch (name) { + case "-i": + case "--ignore-case": + p.ignoreCase = true; + p.smartCase = false; + break; + case "-s": + case "--case-sensitive": + p.ignoreCase = false; + p.smartCase = false; + break; + case "-S": + case "--smart-case": + p.smartCase = true; + p.ignoreCase = false; + break; + case "-F": + case "--fixed-strings": + p.fixed = true; + break; + case "-w": + case "--word-regexp": + p.word = true; + break; + case "-e": + case "--regexp": + p.patterns.push(value!); + break; + case "-l": + case "--files-with-matches": + p.filesOnly = true; + break; + case "-g": + case "--glob": + // Translated once the search path is known (a slashed glob is anchored + // at it). + pending.push({ glob: value!.replace(/^!/, ""), negated: value!.startsWith("!") }); + break; + case "-t": + case "--type": + case "-T": + case "--type-not": { + const globs = RG_TYPES[value!]; + if (!globs) return undefined; + if (name === "-t" || name === "--type") { + for (const g of globs) if (!include(p, `**/${g}`)) return undefined; + p.types.push(value!); + } else for (const g of globs) exclude(p, `**/${g}`); + break; + } + case "--no-ignore": + case "--no-ignore-vcs": + p.noGitignore = true; + break; + case "-u": + case "--unrestricted": + // -u lifts ignore files; -uu also hidden files (the engine searches + // those anyway); -uuu adds binaries, which the engine never searches. + if (++unrestricted > 2) return undefined; + p.noGitignore = true; + break; + case "--sort": + if (value !== "path") return undefined; // hits are always sorted by path + break; + case "-n": + case "--line-number": + case "-N": + case "--no-line-number": + case "-H": + case "--with-filename": + case "-I": + case "--no-filename": + case "--no-heading": + case "--heading": + case "--column": + case "-p": + case "--pretty": + case "--hidden": + case "-.": + case "--sort-files": + case "-j": + case "--threads": + break; + case "--color": + if (value !== undefined && !["never", "auto", "always", "ansi"].includes(value)) return undefined; + break; + default: + return undefined; // includes -r (--replace), -A/-B/-C, -c, -v, -o, -m… } } + if (!positionalsInto(p, split.positionals)) return undefined; + // rg lets an override glob beat the type filter: `-t py -g '*.md'` searches + // both. The engine ANDs them, so the mix refuses. + if (p.types.length && pending.some((g) => !g.negated)) return undefined; + for (const { glob, negated } of pending) { + const g = rgGlob(glob, p.paths.length === 0); + if (!(negated ? exclude(p, g) : include(p, g))) return undefined; + } + return p; +} - // With an -e pattern already bound, every positional is a path; otherwise the - // first positional is the pattern. Either way at most ONE path may remain — - // multi-path search has no single-scope equivalent, so it refuses. - if (p.pattern === undefined) { - const first = positionals.shift(); - if (first === undefined || first === "") return undefined; - p.pattern = first; +// `git grep [flags] [-- …]`. Searches the tree under the +// current directory (recursive by default). Anything before `--` after the +// pattern would be a revision: refused. +function parseGitGrep(args: string[]): Parsed | undefined { + const split = splitArgs(args, new Set(["-e"])); + if (!split) return undefined; + const p = fresh("bre", true); + for (const { name, value } of split.flags) { + switch (name) { + case "-i": + case "-y": + case "--ignore-case": + p.ignoreCase = true; + break; + case "-E": + case "--extended-regexp": + p.dialect = "ere"; + break; + case "-G": + case "--basic-regexp": + p.dialect = "bre"; + break; + case "-F": + case "--fixed-strings": + p.fixed = true; + break; + case "-w": + case "--word-regexp": + p.word = true; + break; + case "-e": + p.patterns.push(value!); + break; + case "-l": + case "--files-with-matches": + case "--name-only": + p.filesOnly = true; + break; + case "-n": + case "--line-number": + case "-I": + case "-r": + case "--recursive": + case "--no-color": + case "--full-name": + break; + default: + return undefined; + } } - if (positionals.length > 1) return undefined; - p.path = positionals[0]; + const { positionals, afterDashDash } = split; + const beforeDashDash = afterDashDash === -1 ? positionals.length : afterDashDash; + if (p.patterns.length === 0) { + if (beforeDashDash === 0) return undefined; + p.patterns.push(positionals[0]!); + if (beforeDashDash > 1) return undefined; // a revision + } else if (beforeDashDash > 0) return undefined; + // Pathspecs are OR-ed: one plain path (a scope), or `*`-globs without a + // slash (git's `*` crosses directories: `*.ts` ≡ `**/*.ts`), not a mix. + const specs = afterDashDash === -1 ? [] : positionals.slice(afterDashDash); + const plain = specs.filter((s) => !/[*?]/.test(s)); + const globbed = specs.filter((s) => /[*?]/.test(s)); + if (specs.some((s) => s.startsWith(":") || UNSUPPORTED_GLOB.test(s))) return undefined; // pathspec magic + if (plain.length > 1 || (plain.length && globbed.length)) return undefined; + for (const g of globbed) if (g.includes("/") || !include(p, `**/${g}`)) return undefined; + p.paths.push(...plain); return p; } +// With -e patterns bound, every positional is a path; otherwise the first is +// the pattern. +function positionalsInto(p: Parsed, positionals: string[]): boolean { + const rest = [...positionals]; + if (p.patterns.length === 0) { + const first = rest.shift(); + if (first === undefined) return false; + p.patterns.push(first); + } + p.paths = rest; + return true; +} + +// rg's built-in types (rg 14 `--type-list`), bracket classes expanded. +const RG_TYPES: Record = { + c: ["*.c", "*.h", "*.H", "*.c.in", "*.h.in", "*.H.in", "*.cats"], + cpp: [ + "*.C", "*.h", "*.H", "*.C.in", "*.h.in", "*.H.in", "*.cpp", "*.hpp", "*.cpp.in", "*.hpp.in", "*.cxx", + "*.hxx", "*.cxx.in", "*.hxx.in", "*.cc", "*.cc.in", "*.hh", "*.hh.in", "*.inl", + ], + cs: ["*.cs"], + csharp: ["*.cs"], + css: ["*.css", "*.scss"], + go: ["*.go"], + html: ["*.ejs", "*.htm", "*.html"], + java: ["*.java", "*.jsp", "*.jspx", "*.properties"], + js: ["*.cjs", "*.js", "*.jsx", "*.mjs", "*.vue"], + json: ["*.json", "*.sarif", "composer.lock"], + kotlin: ["*.kt", "*.kts"], + lua: ["*.lua"], + markdown: ["*.markdown", "*.md", "*.mdown", "*.mdwn", "*.mdx", "*.mkd", "*.mkdn"], + md: ["*.markdown", "*.md", "*.mdown", "*.mdwn", "*.mdx", "*.mkd", "*.mkdn"], + php: ["*.php", "*.php3", "*.php4", "*.php5", "*.php7", "*.php8", "*.pht", "*.phtml"], + py: ["*.py", "*.pyi"], + ruby: ["*.gemspec", "*.rb", "*.rbw", ".irbrc", "Gemfile", "Rakefile", "config.ru"], + rust: ["*.rs"], + scala: ["*.sbt", "*.scala"], + sql: ["*.psql", "*.sql"], + swift: ["*.swift"], + toml: ["*.toml", "Cargo.lock"], + ts: ["*.cts", "*.mts", "*.ts", "*.tsx"], + typescript: ["*.cts", "*.mts", "*.ts", "*.tsx"], + txt: ["*.txt"], + yaml: ["*.yaml", "*.yml"], +}; + +// --------------------------------------------------------------------------- +// Pattern dialects → JavaScript +// --------------------------------------------------------------------------- + +// POSIX named classes, as the C locale defines them (and Rust's ASCII classes). +const POSIX_CLASSES: Record = { + alpha: "A-Za-z", + digit: "0-9", + alnum: "A-Za-z0-9", + upper: "A-Z", + lower: "a-z", + space: " \\t\\n\\r\\f\\v", + blank: " \\t", + xdigit: "0-9A-Fa-f", + punct: "!-\\/:-@\\[-`{-~", + word: "A-Za-z0-9_", +}; + +// A bracket expression starting at p[i] === "[" → [JS class, index of its +// closing "]"], or undefined. POSIX brackets take a backslash literally; Rust +// ones treat it as an escape and allow nesting and set operators (refused). +function bracket(p: string, i: number, dialect: Dialect): [string, number] | undefined { + let j = i + 1; + let out = "["; + if (p[j] === "^") { + out += "^"; + j++; + } + for (let first = true; j < p.length; j++, first = false) { + const c = p[j]!; + if (c === "]" && !first) return [out + "]", j]; + if (c === "[" && p[j + 1] === ":") { + const m = /^\[:(\w+):\]/.exec(p.slice(j)); + const cls = m && POSIX_CLASSES[m[1]!]; + if (!cls || (m![1] === "word" && dialect !== "rust")) return undefined; + out += cls; + j += m![0].length - 1; + } else if (c === "[") { + if (dialect === "rust") return undefined; // a nested class + out += "\\["; + } else if (c === "\\") { + if (dialect !== "rust") { + out += "\\\\"; + continue; + } + const n = p[j + 1]; + if (n === undefined || /[A-Za-z0-9]/.test(n) && !"nrtfvsSwWdD".includes(n)) return undefined; + out += `\\${n}`; + j++; + } else if (dialect === "rust" && (c === "&" || c === "~" || c === "-") && p[j + 1] === c) { + return undefined; // set operators + } else if (c === "]") { + out += "\\]"; // a leading ] is a member + } else out += c; + } + return undefined; // unterminated +} + +// GNU BRE/ERE → JS. BRE's operators are the escaped forms (\( \) \| \+ \? \{ +// \}) and their bare spellings are literals; ERE is the other way round. Both +// have \< \> word edges, POSIX brackets and back-references. +function fromPosix(p: string, ere: boolean): string | undefined { + let out = ""; + // Where a `*` is a literal and a `^` an anchor: at the start of the pattern + // or of a group/alternative. + const atStart = (): boolean => out === "" || out.endsWith("(") || out.endsWith("|") || out === "^"; + for (let i = 0; i < p.length; i++) { + const c = p[i]!; + if (c === "\\") { + const n = p[++i]; + if (n === undefined) return undefined; + if (!ere && "()|+?".includes(n)) out += n; + else if (!ere && n === "{") { + const m = /^\\\{(\d*)(,?)(\d*)\\\}/.exec(p.slice(i - 1)); + if (!m || (m[1] === "" && m[2] === "")) return undefined; + out += `{${m[1] || "0"}${m[2]}${m[3]}}`; + i += m[0].length - 2; + } else if (n === "<" || n === ">") out += "\\b"; + else if ("wWsSbB".includes(n) || /[1-9]/.test(n)) out += `\\${n}`; + else if (/[.*[\]^$\\/+?(){}|]/.test(n)) out += `\\${n}`; + else return undefined; // \d, \n, \t… mean other things across grep versions + continue; + } + if (c === "[") { + const b = bracket(p, i, "bre"); + if (!b) return undefined; + out += b[0]; + i = b[1]; + } else if (c === "*") { + out += atStart() ? "\\*" : "*"; + } else if (c === "^") { + out += ere || atStart() ? "^" : "\\^"; + } else if (c === "$") { + const rest = p.slice(i + 1); + out += ere || rest === "" || rest.startsWith("\\)") || rest.startsWith("\\|") ? "$" : "\\$"; + } else if (!ere && "()|+?{}".includes(c)) { + out += `\\${c}`; + } else if (ere && c === "{") { + const m = /^\{(\d*)(,?)(\d*)\}/.exec(p.slice(i)); + if (m && (m[1] !== "" || m[2] !== "")) { + out += `{${m[1] || "0"}${m[2]}${m[3]}}`; + i += m[0].length - 1; + } else out += "\\{"; + } else if (ere && c === "}") { + out += "\\}"; + } else if (ere && (c === "+" || c === "?" || c === "*") && atStart()) { + return undefined; // a leading repetition: implementation-defined + } else out += c; + } + return out; +} + +// Rust regex (rg) → JS. The shared core passes through; Rust-only syntax is +// refused rather than approximated. \w \d \b are Unicode-aware in rg and ASCII +// in the engine — identical on ASCII text, which code overwhelmingly is. +function fromRust(p: string): string | undefined { + let out = ""; + for (let i = 0; i < p.length; i++) { + const c = p[i]!; + if (c === "\\") { + const n = p[++i]; + if (n === undefined) return undefined; + if ("wWdDsSbBnrtfv".includes(n)) out += `\\${n}`; + else if (n === "p" || n === "P") { + const m = /^\{[^}]+\}|^[A-Za-z]/.exec(p.slice(i + 1)); + if (!m) return undefined; + out += `\\${n}${m[0].startsWith("{") ? m[0] : `{${m[0]}}`}`; + i += m[0].length; + } else if (n === "x" || n === "u") { + const m = /^\{[0-9A-Fa-f]{1,6}\}|^[0-9A-Fa-f]{2}/.exec(p.slice(i + 1)); + if (!m || (n === "u" && !m[0].startsWith("{") && !/^[0-9A-Fa-f]{4}/.test(p.slice(i + 1)))) return undefined; + const hex = n === "u" && !m[0].startsWith("{") ? p.slice(i + 1, i + 5) : m[0].replace(/[{}]/g, ""); + out += `\\u{${hex}}`; + i += n === "u" && !m[0].startsWith("{") ? 4 : m[0].length; + } else if (/[A-Za-z0-9<>]/.test(n)) return undefined; // \A \z \Q \1 \< … + else out += `\\${n}`; + continue; + } + if (c === "[") { + const b = bracket(p, i, "rust"); + if (!b) return undefined; + out += b[0]; + i = b[1]; + } else if (c === "(" && p[i + 1] === "?") { + const m = /^\(\?(?::|P?<([A-Za-z_]\w*)>)/.exec(p.slice(i)); + if (!m) return undefined; // inline flags (?i), lookaround (an rg error) + out += m[1] ? `(?<${m[1]}>` : "(?:"; + i += m[0].length - 1; + } else if (c === "{") { + const m = /^\{\d+(?:,\d*)?\}/.exec(p.slice(i)); + if (!m) return undefined; // an rg syntax error + out += m[0]; + i += m[0].length - 1; + } else out += c; + } + return out; +} + +// rg's smart case: insensitive unless the pattern holds an uppercase LITERAL +// (escapes like \W or \p{Lu} are not literals). +function hasUpperLiteral(p: string): boolean { + const literal = p.replace(/\\[pP]\{[^}]*\}|\\[pP][A-Za-z]|\\./g, ""); + return literal !== literal.toLowerCase(); +} + +// The one JS pattern the parsed command means, or undefined. +function toPattern(p: Parsed): string | undefined { + const parts: string[] = []; + for (const raw of p.patterns) { + if (raw === "" || raw.includes("\n")) return undefined; + const js = p.fixed ? escapeRegExp(raw) : p.dialect === "rust" ? fromRust(raw) : fromPosix(raw, p.dialect === "ere"); + if (js === undefined) return undefined; + parts.push(js); + } + let pattern = parts.length === 1 ? parts[0]! : parts.map((s) => `(?:${s})`).join("|"); + if (p.word) { + // grep -w / rg -w: the match must not touch a word character on either + // side. For literals that start and end with one, \b says exactly that + // (and keeps the fast ripgrep path); otherwise spell the rule out. + const literals = p.patterns.every((s) => (p.fixed || !/[\\^$.*+?()[\]{}|]/.test(s)) && /^\w(?:.*\w)?$/.test(s)); + const body = parts.length === 1 && literals ? pattern : `(?:${pattern})`; + pattern = literals ? `\\b${body}\\b` : `(? seg === "..")) return undefined; + return s === "" || s === "." ? "" : s; +} + // Rewrite `cmd` to its codeindex equivalent, or return undefined to leave it // alone. `bin` is the codeindex executable name to emit (the host may have it // on PATH under a mount point of its choosing). export function rewriteCommand(cmd: string, bin = "codeindex"): string | undefined { - const line = cmd.trim(); - if (!line || SHELL_METACHARS.test(line)) return undefined; - - const tokens = tokenize(line); - if (!tokens || tokens.length < 2) return undefined; + const lexed = lex(cmd.trim()); + if (!lexed || lexed.length < 2) return undefined; + // An unquoted glob in a path or pattern is expanded by the shell against the + // cwd (`rg foo src/*.ts`, `-g *.ts`): what the command means depends on + // files we cannot see. In a flag word (`--include=*.ts`) it could only match + // a file whose name starts with `-`, so the word reaches grep verbatim. + if (lexed.some((t) => t.globbed && !t.value.startsWith("-"))) return undefined; + const tokens = lexed.map((t) => t.value); // Refuse env-prefixed or path-qualified invocations (`FOO=1 grep …`, // `/usr/bin/grep …`): resolving those faithfully is not worth the risk. const [head, ...args] = tokens; - if (head === undefined || !GREP_BINARIES.has(head)) return undefined; - - const p = parseSearch(head, args); - if (!p || p.pattern === undefined) return undefined; - const pattern = p.pattern; - // A non-recursive `grep pattern file.ts` is already cheap and its semantics - // (one file, text output) are not what the indexed search provides. - if (!p.recursive) return undefined; - - // A path that is not the tree root means "search this subtree"; express it as - // a scope rather than silently widening to the whole repo. - const path = p.path; - const out = [bin, "grep", shellQuote(pattern)]; - if (path && path !== "." && path !== "./") { - out.push("--scope", shellQuote(path.replace(/\/+$/, ""))); + let p: Parsed | undefined; + if (head === "grep" || head === "egrep") p = parseGrep(head, args); + else if (head === "rg") p = parseRg(args); + else if (head === "git" && args[0] === "grep") p = parseGitGrep(args.slice(1)); + if (!p || !p.recursive) return undefined; + + // Multi-path search has no single-scope equivalent. + if (p.paths.length > 1) return undefined; + const scope = p.paths.length ? toScope(p.paths[0]!) : ""; + if (scope === undefined) return undefined; + // grep and rg let the later of two conflicting include/exclude rules win; + // the engine lets exclusion win. Only an exclude-before-include order can + // tell the two apart. + if (p.orderSensitive) return undefined; + + const ignoreCase = p.smartCase ? !p.patterns.some(hasUpperLiteral) : p.ignoreCase; + const pattern = toPattern(p); + if (pattern === undefined) return undefined; + try { + compilePattern(pattern, ignoreCase); + } catch { + return undefined; // the engine would refuse what the original accepts } - if (p.ignoreCase) out.push("--ignore-case"); + + // `--` guards a pattern that starts with `-`: shell quoting does not stop + // `--out` from being parsed as a flag once it reaches argv. + const out = [bin, "grep", ...(pattern.startsWith("-") ? ["--"] : []), shellQuote(pattern)]; + if (scope) out.push("--scope", shellQuote(scope)); for (const g of p.includes) out.push("--include", shellQuote(g)); + for (const g of [...p.excludes, ...p.excludeDirs]) out.push("--exclude", shellQuote(g)); + if (ignoreCase) out.push("--ignore-case"); + if (p.filesOnly) out.push("--files-with-matches"); + if (p.noGitignore) out.push("--no-gitignore"); + out.push("--ignore-dir", ".codeindex"); return out.join(" "); } diff --git a/tests/grep.test.ts b/tests/grep.test.ts index 2b6adee..376d28f 100644 --- a/tests/grep.test.ts +++ b/tests/grep.test.ts @@ -138,6 +138,18 @@ describe("grep cap: truncation is reported, never silent", () => { expect(meta.hits).toHaveLength(1); }); + it("filesWithMatches returns each matching file's first hit and caps files", () => { + const r = both(root(), "hit", { filesWithMatches: true }); + expect(r.hits.map((h) => `${h.file}:${h.line}`)).toEqual(["a.txt:1", "b.txt:1"]); + expect(r).toMatchObject({ truncated: false, filesMatched: 2 }); + const capped = both(root(), "hit", { filesWithMatches: true, maxHits: 1 }); + expect(capped.hits.map((h) => h.file)).toEqual(["a.txt"]); + expect(capped.truncated).toBe(true); + expect(capped.notes.join("\n")).toMatch(/first 1 matching files by path; 2 files match/); + const out = cli(["grep", "hit", "--repo", root(), "--files-with-matches"]); + expect((JSON.parse(out.stdout) as { file: string }[]).map((h) => h.file)).toEqual(["a.txt", "b.txt"]); + }); + it("cuts a huge line to a window around the match and reports its column", () => { const line = `${"x".repeat(5000)}NEEDLE${"y".repeat(5000)}`; const r = both(repo({ "min.js": `${line}\n` }), "NEEDLE"); diff --git a/tests/rewrite.test.ts b/tests/rewrite.test.ts index 6503bde..ea236cb 100644 --- a/tests/rewrite.test.ts +++ b/tests/rewrite.test.ts @@ -1,10 +1,14 @@ -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { hoistLeadingFlags } from "../src/engine-cli.js"; import { rewriteCommand, shellQuote, tokenize } from "../src/rewrite.js"; const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); +const haveTools = ["rg", "git", "grep"].every((t) => spawnSync(t, ["--version"]).status === 0); describe("tokenize", () => { it("splits on whitespace", () => { @@ -49,48 +53,75 @@ describe("shellQuote", () => { }); describe("rewriteCommand — rewrites it understands", () => { - it("maps a recursive grep onto the indexed search", () => { - expect(rewriteCommand("grep -r foo .")).toBe("codeindex grep foo"); + it("maps a recursive grep onto the indexed search over the same committed files", () => { + // --ignore-dir .codeindex lifts the default vendor/build/out/tmp skips — + // grep, rg and git grep make none of them. + expect(rewriteCommand("grep -r foo .")).toBe("codeindex grep foo --ignore-dir .codeindex"); }); - it("treats a subdirectory argument as a scope, not a widened search", () => { - expect(rewriteCommand("grep -r foo src")).toBe("codeindex grep foo --scope src"); - expect(rewriteCommand("grep -r foo src/")).toBe("codeindex grep foo --scope src"); + it("treats a subdirectory or file argument as a scope, normalising ./ and a trailing /", () => { + expect(rewriteCommand("grep -r foo src")).toBe("codeindex grep foo --scope src --ignore-dir .codeindex"); + expect(rewriteCommand("grep -r foo ./src/")).toBe("codeindex grep foo --scope src --ignore-dir .codeindex"); + expect(rewriteCommand("grep -r foo gin.go")).toBe("codeindex grep foo --scope gin.go --ignore-dir .codeindex"); }); - it("expands bundled short flags", () => { - expect(rewriteCommand("grep -rn foo .")).toBe("codeindex grep foo"); - expect(rewriteCommand("grep -rni foo .")).toBe("codeindex grep foo --ignore-case"); + it("expands bundled short flags and carries --ignore-case across", () => { + expect(rewriteCommand("grep -rni foo .")).toBe("codeindex grep foo --ignore-case --ignore-dir .codeindex"); + expect(rewriteCommand("grep -r --ignore-case foo .")).toBe("codeindex grep foo --ignore-case --ignore-dir .codeindex"); }); - it("carries --ignore-case across", () => { - expect(rewriteCommand("grep -r --ignore-case foo .")).toBe("codeindex grep foo --ignore-case"); + it("turns base-name globs into any-depth globs, in every spelling", () => { + const want = "codeindex grep foo --include '**/*.ts' --ignore-dir .codeindex"; + expect(rewriteCommand("grep -r --include=*.ts foo .")).toBe(want); + expect(rewriteCommand("grep -r --include '*.ts' foo .")).toBe(want); + expect(rewriteCommand("rg -g '*.ts' foo")).toBe(want); + expect(rewriteCommand("rg -g '!*_test.go' foo")).toBe("codeindex grep foo --exclude '**/*_test.go' --ignore-dir .codeindex"); + // A scope stays a separate, ANDed predicate. + expect(rewriteCommand("grep -r --include=*.md foo binding")).toBe( + "codeindex grep foo --scope binding --include '**/*.md' --ignore-dir .codeindex", + ); }); - it("treats rg as recursive by default", () => { - expect(rewriteCommand("rg foo")).toBe("codeindex grep foo"); - expect(rewriteCommand("rg foo src")).toBe("codeindex grep foo --scope src"); + it("restates the pattern dialect as JavaScript", () => { + expect(rewriteCommand("grep -r 'x+y' .")).toBe("codeindex grep 'x\\+y' --ignore-dir .codeindex"); // BRE: + is literal + expect(rewriteCommand("grep -r 'a\\+b' .")).toBe("codeindex grep 'a+b' --ignore-dir .codeindex"); // GNU BRE \+ + expect(rewriteCommand("grep -rE 'a+b' .")).toBe("codeindex grep 'a+b' --ignore-dir .codeindex"); + expect(rewriteCommand("grep -r '[[:digit:]]x' .")).toBe("codeindex grep '[0-9]x' --ignore-dir .codeindex"); + expect(rewriteCommand("rg -F 'a.b('")).toBe("codeindex grep 'a\\.b\\(' --ignore-dir .codeindex"); + expect(rewriteCommand("rg -w id")).toBe("codeindex grep '\\bid\\b' --ignore-dir .codeindex"); }); - it("carries include globs across in both spellings", () => { - expect(rewriteCommand("grep -r --include=*.ts foo .")).toBe("codeindex grep foo --include '*.ts'"); - expect(rewriteCommand("rg -g *.ts foo")).toBe("codeindex grep foo --include '*.ts'"); + it("guards a pattern that starts with - behind --", () => { + expect(rewriteCommand("grep -r -e --out src")).toBe("codeindex grep -- --out --scope src --ignore-dir .codeindex"); + expect(rewriteCommand("grep -r -- --foo .")).toBe("codeindex grep -- --foo --ignore-dir .codeindex"); }); - it("honours -e for the pattern", () => { - expect(rewriteCommand("grep -r -e foo .")).toBe("codeindex grep foo"); + it("re-quotes a pattern with the shell's own backslash and quote rules", () => { + expect(rewriteCommand(`grep -r "two words" .`)).toBe("codeindex grep 'two words' --ignore-dir .codeindex"); + expect(rewriteCommand(`grep -r "say \\"hi\\"" .`)).toBe(`codeindex grep 'say "hi"' --ignore-dir .codeindex`); + expect(rewriteCommand("grep -r a\\.b .")).toBe("codeindex grep a.b --ignore-dir .codeindex"); // the shell eats the \\ }); - it("re-quotes a pattern containing spaces", () => { - expect(rewriteCommand(`grep -r "two words" .`)).toBe("codeindex grep 'two words'"); + it("reads shell metacharacters only where the shell does", () => { + expect(rewriteCommand(`rg 'foo|bar'`)).toBe("codeindex grep 'foo|bar' --ignore-dir .codeindex"); + expect(rewriteCommand(`rg "foo|bar"`)).toBe("codeindex grep 'foo|bar' --ignore-dir .codeindex"); + }); + + it("covers the common agent forms: types, smart case, -l, git grep", () => { + expect(rewriteCommand("rg -tpy 'def main'")).toBe("codeindex grep 'def main' --include '**/*.py' --include '**/*.pyi' --ignore-dir .codeindex"); + expect(rewriteCommand("rg -S foo")).toBe("codeindex grep foo --ignore-case --ignore-dir .codeindex"); + expect(rewriteCommand("rg -S Foo")).toBe("codeindex grep Foo --ignore-dir .codeindex"); + expect(rewriteCommand("rg -l foo")).toBe("codeindex grep foo --files-with-matches --ignore-dir .codeindex"); + expect(rewriteCommand("git grep -n foo -- '*.ts'")).toBe("codeindex grep foo --include '**/*.ts' --ignore-dir .codeindex"); }); it("drops purely presentational flags codeindex already satisfies", () => { - expect(rewriteCommand("grep -r -n -H foo .")).toBe("codeindex grep foo"); + expect(rewriteCommand("grep -r -n -H foo .")).toBe("codeindex grep foo --ignore-dir .codeindex"); + expect(rewriteCommand("rg --no-heading -n --color=never foo")).toBe("codeindex grep foo --ignore-dir .codeindex"); }); it("respects a caller-supplied binary name", () => { - expect(rewriteCommand("grep -r foo .", "/usr/local/bin/codeindex")).toBe("/usr/local/bin/codeindex grep foo"); + expect(rewriteCommand("grep -r foo .", "/usr/local/bin/codeindex")).toBe("/usr/local/bin/codeindex grep foo --ignore-dir .codeindex"); }); }); @@ -113,16 +144,41 @@ describe("rewriteCommand — refusals (a bad rewrite is worse than none)", () => ["command substitution", "grep -r $(cat pat) ."], ["a backtick", "grep -r `cat pat` ."], ["a variable", "grep -r $PATTERN ."], + ["a variable in double quotes", `grep -r "$PATTERN" .`], ["a brace group", "grep -r foo {a,b}"], + ["an unquoted glob the shell would expand", "rg foo src/*.ts"], + ["an unquoted glob value", "rg -g *.ts foo"], + ["a tilde", "grep -r foo ~/src"], ])("refuses %s", (_label, cmd) => { expect(rewriteCommand(cmd)).toBeUndefined(); }); it("refuses flags it cannot faithfully express", () => { expect(rewriteCommand("grep -r -A3 foo .")).toBeUndefined(); // context lines - expect(rewriteCommand("grep -r -l foo .")).toBeUndefined(); // files-with-matches expect(rewriteCommand("grep -r -v foo .")).toBeUndefined(); // inverted match expect(rewriteCommand("grep -rc foo .")).toBeUndefined(); // count only + expect(rewriteCommand("rg -r X ShouldBindJSON")).toBeUndefined(); // rg -r is --replace + expect(rewriteCommand("rg -C2 foo")).toBeUndefined(); + }); + + it("refuses a path outside the tree it would search", () => { + expect(rewriteCommand("grep -r foo /etc")).toBeUndefined(); + expect(rewriteCommand("grep -r foo ../other")).toBeUndefined(); + expect(rewriteCommand("grep -r foo 'src/*.ts'")).toBeUndefined(); + }); + + it("refuses rule orders where the later rule wins in grep/rg but exclusion wins here", () => { + expect(rewriteCommand("rg -g '!*.d.ts' -g '*.ts' x")).toBeUndefined(); + expect(rewriteCommand("grep -r --exclude=*_test.go --include=*.go x .")).toBeUndefined(); + expect(rewriteCommand("grep -r --include=*.go --exclude=*_test.go x .")).toBeDefined(); + }); + + it("refuses pattern syntax it cannot restate exactly", () => { + expect(rewriteCommand("rg '(?i)foo'")).toBeUndefined(); // inline flags + expect(rewriteCommand("rg '\\Afoo'")).toBeUndefined(); + expect(rewriteCommand("grep -r '[\\d]' .")).toBeDefined(); // POSIX: \\ and d, stated as such + expect(rewriteCommand("grep -r '\\d' .")).toBeUndefined(); // version-dependent in GNU grep + expect(rewriteCommand("rg '[a-z&&[^b]]'")).toBeUndefined(); }); it("refuses an env-prefixed or path-qualified invocation", () => { @@ -141,6 +197,104 @@ describe("rewriteCommand — refusals (a bad rewrite is worse than none)", () => }); }); +// The only real proof: run the original tool and the rewrite on one fixture +// and compare what they found. Paths under .git/ are dropped from grep's side +// (GNU grep -r searches VCS internals; the engine never does, on purpose). +describe.skipIf(!haveTools)("rewrite equivalence against the real tools", () => { + const root = mkdtempSync(join(tmpdir(), "ci-rewrite-eq-")); + const put = (rel: string, body: string): void => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body); + }; + put("gin.go", 'func Default() {}\nfunc New() {}\nsay "hi"\nx+y\nxxy\na+b\naab\na.b\naxb\ncall a.b(1)\n'); + put("binding/binding.go", "func Default(method string) {}\nfunc TestNot() {}\n"); + put("binding/binding_test.go", "func TestBind(t *T) {}\n"); + put("binding/doc.md", "Default docs\n"); + put("README.md", "Default readme\nfoo bar\n"); + put("sub/app.py", "def main():\n alpha = beta\n Foo = foo\n"); + put("sub/types.pyi", "def main() -> None: ...\n"); + put("src/x.ts", "const id = 1; // id\nlet idx = 2;\nfunc run(x)\n"); + put("vendor/lib/v.go", "func TestVendor() {}\nTARGET\n"); + put("build/b.sh", "TARGET\n"); + put("flags.txt", "use --out file\nuse -x here\nmy-x\n"); + put("words.txt", "Ax1 bx2\nfoo\nbar\nfoobar\nabbc\n"); + execFileSync("git", ["init", "-q", "."], { cwd: root }); + execFileSync("git", ["add", "-A"], { cwd: root }); + + // stdin from /dev/null: given a piped stdin and no path, rg searches stdin. + const sh = (line: string): string => + execFileSync("sh", ["-c", line], { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + // The original, with output-only flags the rewriter drops anyway, so every + // tool prints path:line:text. + function original(cmd: string, files: boolean): string[] { + const [bin, ...rest] = cmd.split(" "); + const extra = files ? "" : bin === "rg" ? " -n -H --no-heading" : bin === "git" ? "" : " -H -n"; + let out: string; + try { + out = sh(`${bin}${extra} ${rest.join(" ")}`); + } catch (e) { + out = (e as { stdout: string }).stdout; // exit 1: no match + } + return out + .split("\n") + .filter(Boolean) + .map((l) => l.replace(/^\.\//, "")) + .filter((l) => !l.startsWith(".git/")) + .map((l) => (files ? l : l.split(":").slice(0, 2).join(":"))) + .sort(); + } + function rewritten(cmd: string, files: boolean): string[] { + const line = rewriteCommand(cmd, `'${process.execPath}' '${CLI}'`); + expect(line, cmd).toBeDefined(); + const hits = JSON.parse(sh(line!)) as { file: string; line: number }[]; + return hits.map((h) => (files ? h.file : `${h.file}:${h.line}`)).sort(); + } + + it.each([ + `grep -rn --include=*.go "func Default" .`, + `rg -g '!*_test.go' 'func Test'`, + `grep -rn --include=*.md Default binding`, + `grep -rn 'func Default' ./binding`, + `grep -rn 'func Default' gin.go`, + `grep -rn 'x+y' .`, + `grep -r 'a\\+b' .`, + `grep -r "say \\"hi\\"" .`, + `grep -r a\\.b .`, + `grep -r '[[:alpha:]]x[[:digit:]]' .`, + `grep -r -e --out .`, + `rg "foo|bar"`, + `rg -tpy "def main"`, + `rg -w id`, + `rg -F 'a.b('`, + `grep -rnw foo .`, + `grep -rniE "foo|bar" .`, + `git grep -n foo`, + `git grep -n foo -- '*.py'`, + `rg 'func \\w+\\(' --type go`, + `rg -S Foo`, + `rg -S foo`, + `egrep -r 'ab{2,3}c' .`, + `grep -r '^foo\\|bar$' .`, + `grep -rw -- -x .`, + `rg -e alpha -e beta`, + `git grep -n -e alpha -e beta -- sub`, + `rg TARGET`, + `grep -r --exclude-dir=vendor TARGET .`, + ])("%s", (cmd) => { + const before = readdirSync(root).sort(); + const want = original(cmd, false); + expect(want.length).toBeGreaterThan(0); // the fixture exercises every case + expect(rewritten(cmd, false)).toEqual(want); + expect(readdirSync(root).sort()).toEqual(before); // nothing written (the --out case) + }); + + it.each([`rg -l foo`, `grep -rl foo .`, `git grep -l foo`])("%s", (cmd) => { + const want = original(cmd, true); + expect(want.length).toBeGreaterThan(1); + expect(rewritten(cmd, true)).toEqual(want); + }); +}); + describe("rewrite CLI contract", () => { // The host reads stdout only when the exit code says to. Exit 1 must stay // silent so a caller that ignores the code cannot run an empty command. @@ -157,7 +311,7 @@ describe("rewrite CLI contract", () => { it("prints the replacement and exits 0 when it has an opinion", () => { const { status, stdout } = run(["grep -r foo ."]); expect(status).toBe(0); - expect(stdout.trim()).toBe("codeindex grep foo"); + expect(stdout.trim()).toBe("codeindex grep foo --ignore-dir .codeindex"); }); it("exits 1 with empty stdout when it does not", () => { From 8ffefb523493d90e6c2dc6eae8f3eab2aaea07c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:51:45 +0000 Subject: [PATCH 038/130] fix(extract): index minified JavaScript without its symbols, and flag it Minified JS not named *.min.js went through full extraction: a 373KB bundle gave 1234 one- and two-letter symbols and 512 call sites in ~0.5s, polluting find-symbol, dead code and search. extract/minified.ts detects it from the content, deterministically: at least half of the file's code characters sit on lines of 300+ code characters with under 6 whitespace runs and at least 1 statement punctuation per 100. Measured on the masked text, so a long string, template or regex literal never counts, and calibrated on ~15k real JS files (every hit a real minified bundle; no compiled or hand-written file flagged). Such a file keeps its summary and imports, extracts nothing else, and is flagged `minified` on its FileRecord and graph node, so it is never silently emptied. Extraction drops to ~16ms. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 5 +- src/cache.ts | 2 +- src/extract/code.ts | 10 ++++ src/extract/minified.ts | 89 ++++++++++++++++++++++++++++ src/graph.ts | 1 + src/scan.ts | 1 + src/types.ts | 7 +++ tests/cache-validation.test.ts | 2 +- tests/minified.test.ts | 102 +++++++++++++++++++++++++++++++++ 9 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 src/extract/minified.ts create mode 100644 tests/minified.test.ts diff --git a/README.md b/README.md index bb625ef..b184c3e 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,10 @@ compares](#how-it-compares). `from pkg import name` also links `pkg/name.py` when `name` is a submodule (and nothing when it is a function or class); PHP group (`use A\{B, C}`) and comma `use` lists and `__DIR__`-anchored includes are followed, and a trait - `use` inside a class is not an import. + `use` inside a class is not an import. Minified JavaScript is recognised by + its content, not only by a `.min.js` name: it stays in the index with its + summary and imports, flagged `minified` on its `FileRecord` and graph node, + but its one-letter symbols and call sites are not extracted. - **Extract symbols** via tree-sitter (15 committed grammars, plus 6 more via `grammars pull`) or per-language regex rules (16 languages, always available). Each symbol carries its **complete signature** (parameters and return type, diff --git a/src/cache.ts b/src/cache.ts index 95c215b..33fdb68 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -59,7 +59,7 @@ function record(v: unknown, rel: string): v is FileRecord { strings(v.headings) && Array.isArray(v.symbols) && v.symbols.every((s) => symbol(s, rel)) && Array.isArray(v.refs) && v.refs.every(ref) && optionalArray(v.calls, call) && optionalArray(v.relations, relation) && optionalArray(v.literals, literal) && - (v.truncated === undefined || v.truncated === true); + (v.truncated === undefined || v.truncated === true) && (v.minified === undefined || v.minified === true); } function entry(v: unknown, rel: string): v is PersistedCacheEntry { diff --git a/src/extract/code.ts b/src/extract/code.ts index 81e4570..f6154dc 100644 --- a/src/extract/code.ts +++ b/src/extract/code.ts @@ -5,6 +5,7 @@ import { extractAst } from "../ast/extract.js"; import { extractReexports, extToLang, MAX_REEXPORTS } from "../lang/common.js"; import { extractImports, extractPackage } from "./imports.js"; import { sfcParts } from "./sfc.js"; +import { isMinified } from "./minified.js"; import { isBanner, isDirective, stripCommentMarkers } from "./doc-text.js"; import { subtokens } from "../util.js"; @@ -19,6 +20,9 @@ export interface CodeInfo { summary?: string; // A cap truncated `symbols` — propagated onto the FileRecord. truncated?: true; + // Minified JS: only the summary and imports were extracted (see + // extract/minified.ts) — propagated onto the FileRecord. + minified?: true; refs: RawRef[]; // import refs (raw specifiers, unresolved) pkg?: string; // Java: the file's own `package x.y.z;` — used to derive source roots idents?: string[]; // distinctive identifiers referenced (AST path) — feeds `use` edges @@ -308,6 +312,12 @@ function mergeCalls( // BOTH extraction tiers — AST and regex — so recall-oriented consumers can raise // it. Dedup/sort semantics are unchanged; absent, output is byte-identical. export function extractCode(rel: string, ext: string, content: string, opts: { maxCallsPerFile?: number } = {}): CodeInfo { + // A minified bundle keeps its place in the index, its summary and its imports + // (real edges, whoever wrote them) — and nothing else: its symbols and call + // sites are one-letter noise (see extract/minified.ts). The flag says so. + if (isMinified(ext, content)) { + return { symbols: [], minified: true, summary: topDocComment(content), refs: extractImports(ext, content) }; + } // A single-file component (.vue/.svelte/.astro) is extracted as its script: // the JS/TS tier runs over a copy with the markup blanked, lines unchanged // (see extract/sfc.ts). Its symbols keep the component's own language, which diff --git a/src/extract/minified.ts b/src/extract/minified.ts new file mode 100644 index 0000000..ca29598 --- /dev/null +++ b/src/extract/minified.ts @@ -0,0 +1,89 @@ +import { maskJs } from "./imports.js"; + +// Minified JavaScript that is not named `*.min.js` (the walk skips that one by +// name). A bundle like `dist.mjs` or a vendored `lib.js` otherwise goes through +// full extraction: hundreds of one- and two-letter symbols (`zf`, `O`, `lo`), +// thousands of call sites between them, and a parse that costs more than the +// rest of the repo's files together — all of it noise in find-symbol, dead code +// and search. extractCode keeps such a file in the index (path, summary, +// imports) but extracts nothing else, and flags the record `minified`. +// +// Only the JS extensions: minifiers emit `.js`/`.mjs`/`.cjs`, and a `.ts` or +// `.jsx` file is source by construction. +const MINIFIABLE = new Set([".js", ".mjs", ".cjs"]); + +// A line is minified code when it holds at least LONG_LINE code characters, +// with little whitespace between them and the punctuation of statements. +// Measured on the MASK (extract/imports.ts), so string, template, regex and +// comment text never counts: a long base64 or SVG literal, a long message or a +// long regex is data on an ordinary line, not minification. +// +// Calibrated on ~15k JS files (node_modules of this repo, the TypeScript repo's +// compiled baselines, this repo's own bundles). Minifiers wrap at 500 bytes +// (uglify's default, lodash.min.js) or never, so 300 code characters catches +// both while hand-written code, even unformatted, stays far below. Minified +// lines have 1-5 whitespace runs per 100 code characters; compiled and +// hand-written code with long lines (tsc's inlined helpers, JSX transforms) +// has 10 or more. A one-line numeric table (`[4,52,65,…]`) has neither +// statements nor calls, so it is data, not code: the `;{}(` count rejects it +// and the file's functions stay indexed. +const LONG_LINE = 300; +const MAX_GAPS_PER_100 = 6; +const MIN_PUNCT_PER_100 = 1; + +const SEMI = 59; +const LPAREN = 40; +const LBRACE = 123; +const RBRACE = 125; + +// True when at least half of a JS file's code characters sit on minified lines. +// A pure function of the bytes, so the flag is as deterministic as extraction. +export function isMinified(ext: string, content: string): boolean { + if (!MINIFIABLE.has(ext)) return false; + // Cheap exit for the common case: no physical line long enough to qualify. + let longest = 0; + for (let at = 0; at < content.length && longest < LONG_LINE; ) { + const nl = content.indexOf("\n", at); + const end = nl === -1 ? content.length : nl; + longest = Math.max(longest, end - at); + at = end + 1; + } + if (longest < LONG_LINE) return false; + + const masked = maskJs(content); + let total = 0; + let minified = 0; + let code = 0; // code characters on the current line + let gaps = 0; // whitespace runs between them — real whitespace only + let punct = 0; + let inGap = false; + let gapReal = true; // the current run is source whitespace, not a masked literal + const endLine = (): void => { + total += code; + if (code >= LONG_LINE && gaps * 100 <= code * MAX_GAPS_PER_100 && punct * 100 >= code * MIN_PUNCT_PER_100) { + minified += code; + } + code = gaps = punct = 0; + inGap = false; + }; + for (let i = 0; i < masked.length; i++) { + const c = masked.charCodeAt(i); + if (c === 10) endLine(); + else if (c > 32) { + if (inGap && gapReal && code > 0) gaps++; + inGap = false; + code++; + if (c === SEMI || c === LPAREN || c === LBRACE || c === RBRACE) punct++; + } else { + if (!inGap) { + inGap = true; + gapReal = true; + } + // A masked literal reads as spaces too; only whitespace the SOURCE has + // separates tokens (a string's body is one token, however long). + if (content.charCodeAt(i) > 32) gapReal = false; + } + } + endLine(); + return minified > 0 && minified * 2 >= total; +} diff --git a/src/graph.ts b/src/graph.ts index 4114684..d8add62 100644 --- a/src/graph.ts +++ b/src/graph.ts @@ -250,6 +250,7 @@ export function buildGraph( lines: f.lines, degIn: degIn.get(f.rel) ?? 0, degOut: degOut.get(f.rel) ?? 0, + ...(f.minified ? { minified: true as const } : {}), })) .sort((a, b) => byStr(a.rel, b.rel)); diff --git a/src/scan.ts b/src/scan.ts index a5e0dea..bd57124 100644 --- a/src/scan.ts +++ b/src/scan.ts @@ -132,6 +132,7 @@ export function buildCodeRecord( record.calls = code.calls; record.importedNames = code.importedNames; record.truncated = code.truncated; + record.minified = code.minified; record.relations = code.relations; record.terms = code.terms; record.literals = code.literals; diff --git a/src/types.ts b/src/types.ts index 5bae03b..3a8ff64 100644 --- a/src/types.ts +++ b/src/types.ts @@ -217,6 +217,11 @@ export interface FileRecord { // A per-file extraction cap truncated this record's symbols. Same doctrine as // the walk's `capped`: a bounded result says so instead of looking complete. truncated?: true; + // Minified JavaScript (detected from the content, whatever the file is + // named): the record keeps its summary and imports, but no symbols, calls or + // vocabulary — they would be one-letter noise. Set so an empty record is + // never mistaken for an empty file. + minified?: true; // Inheritance stated by declarations in this file (cap 256, deduped, sorted). // Resolved into `extends`/`implements` edges by the graph builder. relations?: RawRelation[]; @@ -251,6 +256,8 @@ export interface FileNode { pagerank?: number; // Present (true) only when the path classifies as a test file (tests-map.ts). testFile?: true; + // Present (true) only for minified JS, indexed without symbols (FileRecord.minified). + minified?: true; } export interface ModuleNode { diff --git a/tests/cache-validation.test.ts b/tests/cache-validation.test.ts index 6fa4337..84538bc 100644 --- a/tests/cache-validation.test.ts +++ b/tests/cache-validation.test.ts @@ -59,7 +59,7 @@ describe("persisted cache validation", () => { ["refs", [{ kind: "import", spec: ".util", soft: "yes" }]], ["calls", [{ name: "run", line: 0 }]], ["idents", [1]], ["terms", {}], ["importedNames", [null]], ["truncated", "yes"], ["relations", [{ kind: "extends", from: "A", to: null, line: 1 }]], - ["literals", [{ kind: "string", value: 10, line: 1 }]], + ["literals", [{ kind: "string", value: 10, line: 1 }]], ["minified", false], ])("rejects a malformed record field %s", (field, value) => { const current = cache(); Object.assign(current.files[REL]!.record, { [field]: value }); diff --git a/tests/minified.test.ts b/tests/minified.test.ts new file mode 100644 index 0000000..9ca1e8e --- /dev/null +++ b/tests/minified.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { isMinified } from "../src/extract/minified.js"; +import { extractCode } from "../src/extract/code.js"; +import { scanRepo } from "../src/scan.js"; +import { buildArtifactsFromScan } from "../src/pipeline.js"; + +// Minified JS not named *.min.js went through full extraction: a 373KB bundle +// gave 1234 one- and two-letter symbols (`zf`, `O`, `lo`) and 512 call sites in +// ~0.5s. It is now detected from its content, indexed with its summary and +// imports only, and flagged — never dropped. + +// Terser-style output: no whitespace but where a keyword needs it, statements +// run together, 30 functions per physical line. +function minifiedLine(seed: number): string { + const parts: string[] = []; + for (let i = 0; i < 30; i++) { + const n = `f${seed}_${i}`; + parts.push(`function ${n}(n,t){for(var r=-1,e=null==n?0:n.length;++r `export function g${i}(a, b) {\n return a + b * ${i};\n}`).join("\n"); + +describe("isMinified", () => { + it("flags terser-style output and 500-column wrapped output (uglify, lodash.min.js)", () => { + expect(isMinified(".js", MINIFIED)).toBe(true); + expect(isMinified(".mjs", MINIFIED)).toBe(true); + const wrapped = minifiedLine(3).match(/.{1,500}/g)!.join("\n"); + expect(isMinified(".cjs", wrapped)).toBe(true); + }); + + it("leaves ordinary code alone, whatever its long lines hold", () => { + expect(isMinified(".js", ORDINARY)).toBe(false); + // A long data literal is text, not code: a base64 blob, a long template. + expect(isMinified(".js", `const WASM = "${"QUJD".repeat(20000)}";\n${ORDINARY}`)).toBe(false); + expect(isMinified(".js", `const CSS = \`${"a{b:c}".repeat(5000)}\`;\n${ORDINARY}`)).toBe(false); + // A one-line numeric table (no statements, no calls) is data too. + const table = `const T = [${Array.from({ length: 2000 }, (_, i) => i).join(",")}];`; + expect(isMinified(".js", `${table}\n${ORDINARY}`)).toBe(false); + // Compiled, not minified: tsc's inlined helpers are long lines with spaces. + const helper = + 'var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } }); };'; + expect(isMinified(".js", `${helper}\n${helper}\nexports.run = run;`)).toBe(false); + }); + + it("only considers the extensions minifiers emit", () => { + expect(isMinified(".ts", MINIFIED)).toBe(false); + expect(isMinified(".jsx", MINIFIED)).toBe(false); + }); +}); + +describe("extracting a minified file", () => { + it("keeps the summary and the imports, drops the symbol and call noise, and says so", () => { + const info = extractCode("vendor.js", ".js", MINIFIED); + expect(info.minified).toBe(true); + expect(info.symbols).toEqual([]); + expect(info.calls).toBeUndefined(); + expect(info.terms).toBeUndefined(); + expect(info.refs.map((r) => r.spec)).toEqual(["./dep.js", "./other.js"]); + expect(info.summary).toBe("demo v1.0.0 | MIT License"); + // Ordinary code is untouched. + const ordinary = extractCode("lib.js", ".js", ORDINARY); + expect(ordinary.minified).toBeUndefined(); + expect(ordinary.symbols).toHaveLength(40); + }); + + it("flags the record and its graph node, and keeps the file's import edges", () => { + const root = mkdtempSync(join(tmpdir(), "ci-minified-")); + const files: Record = { + "public/bundle.js": MINIFIED, + "public/dep.js": "export const b = 1;\n", + "public/other.js": "module.exports = {};\n", + }; + for (const [rel, text] of Object.entries(files)) { + mkdirSync(dirname(join(root, rel)), { recursive: true }); + writeFileSync(join(root, rel), text); + } + const scan = scanRepo(root); + const record = scan.files.find((f) => f.rel === "public/bundle.js")!; + expect(record.minified).toBe(true); + expect(record.symbols).toEqual([]); + expect(scan.files.filter((f) => f.minified).map((f) => f.rel)).toEqual(["public/bundle.js"]); + const { graph } = buildArtifactsFromScan(scan); + expect(graph.files.find((f) => f.rel === "public/bundle.js")?.minified).toBe(true); + expect(graph.files.find((f) => f.rel === "public/dep.js")).not.toHaveProperty("minified"); + expect(graph.fileEdges.map((e) => `${e.from} -${e.kind}-> ${e.to}`)).toEqual([ + "public/bundle.js -import-> public/dep.js", + "public/bundle.js -import-> public/other.js", + ]); + }); +}); From 40a0f65cbb9902ad66e6494e6f0bdf813a707704 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:51:57 +0000 Subject: [PATCH 039/130] feat(grep): export grepRepoEx and keep a timed-out answer consistent Library consumers only had grepRepo's bare hit list, so they could not tell a capped or time-budget-cut answer from a complete one. grepRepoEx, which the CLI and MCP already use, is now part of the public API. It returns {hits, truncated, filesMatched, timedOut, notes}. On a timeout, the file in progress is read first, and only then are the results the worker already posted for earlier files drained. Before this, a file could finish between the last receive and the deadline check, and its hits were lost even though the note said the answer covered every file before the stop. Also pins .git/info/exclude parity between the two backends. The rg path used to pass --no-ignore-exclude while walk() reads that file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/engine.ts | 6 ++++-- src/grep.ts | 10 +++++++++- tests/engine.test.ts | 1 + tests/grep.test.ts | 8 ++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index f05f6f4..9015505 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -148,8 +148,10 @@ export { export type { DiffFile, DiffSpec, Hunk } from "./git.js"; // Repo text search (ripgrep when available, pure-JS fallback otherwise). -export { grepRepo } from "./grep.js"; -export type { SearchHit, GrepOptions } from "./grep.js"; +// grepRepoEx adds what the bare hit list cannot say: truncated, the matching +// file count, a time-budget stop, and the notes a caller should surface. +export { grepRepo, grepRepoEx } from "./grep.js"; +export type { SearchHit, GrepOptions, GrepResult } from "./grep.js"; // Keyless BM25 lexical search over symbols/paths/headings/summaries (issue #4). export { searchIndex, explainQuery, subtokens } from "./bm25.js"; diff --git a/src/grep.ts b/src/grep.ts index 7967192..d1fc349 100644 --- a/src/grep.ts +++ b/src/grep.ts @@ -585,7 +585,15 @@ function runScan(job: ScanJob, onEvent: (ev: ScanEvent) => void): { stoppedAt?: continue; } const left = job.deadline - Date.now(); - if (left <= 0) return { stoppedAt: Atomics.load(sig, 1) }; + if (left <= 0) { + // The file in progress bounds the answer; results the worker posted + // for earlier files since the last receive still belong to it. + const stoppedAt = Atomics.load(sig, 1); + for (let m; (m = receiveMessageOnPort(port) as { message: ScanEvent } | undefined); ) { + if ("i" in m.message && m.message.i < stoppedAt) onEvent(m.message); + } + return { stoppedAt }; + } // Short slices: a message posted between the load and the receive is // caught by the counter, and one still in flight by the next slice. Atomics.wait(sig, 0, seen, Math.min(left, 50)); diff --git a/tests/engine.test.ts b/tests/engine.test.ts index 4ae652c..82b5344 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -96,6 +96,7 @@ const CONTRACT = [ "gitChurn", "changedSince", "grepRepo", + "grepRepoEx", "searchIndex", "subtokens", "checkRules", diff --git a/tests/grep.test.ts b/tests/grep.test.ts index 376d28f..e12e36f 100644 --- a/tests/grep.test.ts +++ b/tests/grep.test.ts @@ -68,6 +68,14 @@ describe("grep universe: walk flags, scope and globs", () => { ]); }); + it("honours .git/info/exclude on both backends, as the walker does", () => { + const root = repo({ "kept.txt": "NEEDLE\n", "local.txt": "NEEDLE\n" }); + execFileSync("git", ["init", "-q", "."], { cwd: root }); + writeFileSync(join(root, ".git", "info", "exclude"), "local.txt\n"); + expect(files(both(root, "NEEDLE"))).toEqual(["kept.txt"]); + expect(files(both(root, "NEEDLE", { gitignore: false }))).toEqual(["kept.txt", "local.txt"]); + }); + it("never lets a positive glob resurrect ignored files (an rg whitelist glob overrides ignores)", () => { const root = repo({ ".gitignore": "gen.ts\n", From a99cb569f2d1c866db991757e6638544a95e268f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:53:36 +0000 Subject: [PATCH 040/130] fix(grep): pass rg only exclusions it reads the way the engine does Negated user globs went to ripgrep as a pre-filter, on the assumption that an exclusion can only narrow the search. That holds only while both glob dialects agree on what the glob matches. They disagree in two places: - rg reads `{a,b}` as alternation. The engine reads the braces literally. - rg prunes a DIRECTORY that matches `!x`. The engine's `x` names one path. In both cases rg dropped files the JS keep-predicate would have kept, and the two backends disagreed. Only `dir/**` tree exclusions without braces or classes are now passed, which are the cases that actually save rg a walk. The predicate still applies every glob afterwards. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/grep.ts | 12 ++++++++---- tests/grep.test.ts | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/grep.ts b/src/grep.ts index d1fc349..369fc31 100644 --- a/src/grep.ts +++ b/src/grep.ts @@ -333,12 +333,16 @@ function universeArgs(opts: GrepOptions): string[] { for (const l of LOCKFILES) args.push("--iglob", `!**/${l}`); for (const ext of BINARY_EXT) args.push("--iglob", `!**/*${ext}`); args.push("--glob", "!*.min.js", "--glob", "!*.min.css"); - // Negated user globs only NARROW, so they may prune rg's walk early; the JS - // keep-predicate re-applies every user glob afterwards regardless. Positive - // globs are never passed: an rg whitelist glob overrides .gitignore and the + // The JS keep-predicate applies every user glob afterwards, so rg only gets + // what can prune its walk without ever dropping a file that predicate keeps: + // a tree exclusion (`!dir/**`) with no braces or classes, where the two glob + // dialects agree. Anything else could over-exclude — rg reads `{a,b}` as + // alternation and prunes a DIRECTORY matching `!x` — so it is not passed. + // Positive globs never are: an rg whitelist glob overrides .gitignore and the // junk-dir exclusions above (`-g 'src/**'` searched src/node_modules). for (const g of opts.globs ?? []) { - if (g.startsWith("!")) args.push("--glob", `!/${g.slice(1).replace(/^\//, "")}`); + const body = g.slice(1).replace(/^\//, ""); + if (g.startsWith("!") && body.endsWith("/**") && !/[{}[\]\\!]/.test(body)) args.push("--glob", `!/${body}`); } if (opts.ignoreCase) args.push("--ignore-case"); return args; diff --git a/tests/grep.test.ts b/tests/grep.test.ts index e12e36f..4ac74b4 100644 --- a/tests/grep.test.ts +++ b/tests/grep.test.ts @@ -87,6 +87,13 @@ describe("grep universe: walk flags, scope and globs", () => { expect(files(both(root, "NEEDLE", { globs: ["**/*.ts"] }))).toEqual(["src/a.ts"]); }); + it("hands rg only the exclusions both glob dialects read alike", () => { + const root = repo({ "sub/a.txt": "NEEDLE\n", "a.txt": "NEEDLE\n", "gen/g.txt": "NEEDLE\n" }); + // `!sub` names a path, not a tree; `{a,b}` is not alternation here. + expect(files(both(root, "NEEDLE", { globs: ["!sub", "!{a,b}.txt"] }))).toEqual(["a.txt", "gen/g.txt", "sub/a.txt"]); + expect(files(both(root, "NEEDLE", { globs: ["!gen/**"] }))).toEqual(["a.txt", "sub/a.txt"]); + }); + it("ANDs scope with globs, and a scope may be a file or a ./ or absolute spelling", () => { const root = repo({ "binding/a.go": "Default\n", From e825db7292608ba36e82a2959139900030bf7edd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:54:51 +0000 Subject: [PATCH 041/130] fix(rewrite): smart case on fixed strings, refuse an empty path For a -F pattern, rg's smart case looks at the literal text. The escape stripping meant for regexes dropped the uppercase letter from `-F '\Q'`, and the search came out case-insensitive. An empty path argument (`grep -r foo ''`) is an error for grep, not the repo root. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/rewrite.ts | 5 +++-- tests/rewrite.test.ts | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/rewrite.ts b/src/rewrite.ts index 6c7f591..0705ea3 100644 --- a/src/rewrite.ts +++ b/src/rewrite.ts @@ -729,7 +729,7 @@ function toPattern(p: Parsed): string | undefined { // Normalise a search path to the engine's repo-relative scope, or undefined // when it points outside the tree the rewrite searches (`..`, absolute, `~`). function toScope(path: string): string | undefined { - if (path.startsWith("/") || path.startsWith("~") || /[*?[\]{}\\]/.test(path)) return undefined; + if (path === "" || path.startsWith("/") || path.startsWith("~") || /[*?[\]{}\\]/.test(path)) return undefined; const s = path.replace(/^(?:\.\/)+/, "").replace(/\/+$/, ""); if (s.split("/").some((seg) => seg === "..")) return undefined; return s === "" || s === "." ? "" : s; @@ -766,7 +766,8 @@ export function rewriteCommand(cmd: string, bin = "codeindex"): string | undefin // tell the two apart. if (p.orderSensitive) return undefined; - const ignoreCase = p.smartCase ? !p.patterns.some(hasUpperLiteral) : p.ignoreCase; + const upper = (s: string): boolean => (p.fixed ? s !== s.toLowerCase() : hasUpperLiteral(s)); + const ignoreCase = p.smartCase ? !p.patterns.some(upper) : p.ignoreCase; const pattern = toPattern(p); if (pattern === undefined) return undefined; try { diff --git a/tests/rewrite.test.ts b/tests/rewrite.test.ts index ea236cb..9d7e000 100644 --- a/tests/rewrite.test.ts +++ b/tests/rewrite.test.ts @@ -111,6 +111,7 @@ describe("rewriteCommand — rewrites it understands", () => { expect(rewriteCommand("rg -tpy 'def main'")).toBe("codeindex grep 'def main' --include '**/*.py' --include '**/*.pyi' --ignore-dir .codeindex"); expect(rewriteCommand("rg -S foo")).toBe("codeindex grep foo --ignore-case --ignore-dir .codeindex"); expect(rewriteCommand("rg -S Foo")).toBe("codeindex grep Foo --ignore-dir .codeindex"); + expect(rewriteCommand("rg -S -F '\\Q'")).toBe("codeindex grep '\\\\Q' --ignore-dir .codeindex"); // a literal Q is uppercase expect(rewriteCommand("rg -l foo")).toBe("codeindex grep foo --files-with-matches --ignore-dir .codeindex"); expect(rewriteCommand("git grep -n foo -- '*.ts'")).toBe("codeindex grep foo --include '**/*.ts' --ignore-dir .codeindex"); }); @@ -165,6 +166,7 @@ describe("rewriteCommand — refusals (a bad rewrite is worse than none)", () => expect(rewriteCommand("grep -r foo /etc")).toBeUndefined(); expect(rewriteCommand("grep -r foo ../other")).toBeUndefined(); expect(rewriteCommand("grep -r foo 'src/*.ts'")).toBeUndefined(); + expect(rewriteCommand("grep -r foo ''")).toBeUndefined(); }); it("refuses rule orders where the later rule wins in grep/rg but exclusion wins here", () => { From 9b794c0043949c1b8e805471872743ebd1894bbe Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:55:03 +0000 Subject: [PATCH 042/130] fix(extract): declare every name of a multi-name declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `var Single, Double = 1, 2`, a Go struct's `a, b int`, Java's `private List items = …, others;`, C#'s `_items = new(), _other`, PHP's `public int $count = 0, $other = 1;` and `const A = 1, B = 2;`, C/C++'s `int x, *y, z[3];` and Python's `a, b = 1, 2` / `a = b = 0` indexed only their first name: every single-name reader stopped there. A spec can now list every name one node binds (`namesFrom`); each becomes a symbol sharing the node's line, signature and doc, with its own visibility (Go's capitalisation, the modifier prefix before that name). A node binding one name is read exactly as before. Python's assignment reader binds the targets of a tuple/list pattern (a starred one included) and of a chained assignment, never an attribute or subscript target. On the CPython stdlib: +145 module constants and class fields (calendar's `MONDAY, …, SUNDAY = range(7)`). The labelled fixtures gain the shape in all seven languages; Go's visibility accuracy moves 0.9524 -> 0.9583 only because its denominator grew. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 41 +++++------ site/index.html | 6 +- site/quality.json | 6 +- src/ast/extract.ts | 30 ++++---- src/ast/node.ts | 3 + src/ast/specs.ts | 65 +++++++++++++++++- tests/extraction-shapes.test.ts | 76 +++++++++++++++++++++ tests/fixtures/quality/c/expected.json | 1 + tests/fixtures/quality/c/scheduler.h | 2 +- tests/fixtures/quality/cpp/expected.json | 1 + tests/fixtures/quality/cpp/service.hpp | 2 +- tests/fixtures/quality/csharp/Scheduler.cs | 2 +- tests/fixtures/quality/csharp/expected.json | 6 ++ tests/fixtures/quality/go/expected.json | 18 +++++ tests/fixtures/quality/go/service.go | 4 +- tests/fixtures/quality/java/Scheduler.java | 2 +- tests/fixtures/quality/java/expected.json | 6 ++ tests/fixtures/quality/php/Scheduler.php | 2 +- tests/fixtures/quality/php/expected.json | 7 ++ tests/fixtures/quality/python/expected.json | 2 + tests/fixtures/quality/python/service.py | 1 + tests/quality/baseline.json | 2 +- 22 files changed, 237 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 9cbb32a..b43edc5 100644 --- a/README.md +++ b/README.md @@ -36,21 +36,24 @@ compares](#how-it-compares). interface members, class fields, enum members, every `declare`/`.d.ts` declaration, Rust trait method signatures, Go interface method sets and type aliases, record components, constructor `val` parameters and their - TypeScript (`private readonly dep: Dep`) and PHP 8 (promoted) twins, C `#define` - macros and the members of a `typedef struct`, Python declarations under - `if TYPE_CHECKING:` / `try:` / `with` blocks, Ruby `class << self` methods, - `private def x` definitions and the block of `Point = Struct.new(…) do`, - Elixir clauses with a `when` guard and `defguard`, and the members of a class - bound by `module.exports =` or an anonymous `export default class` (a - default export with no name of its own is named after the file stem). A doc - comment is found across Rust attributes and TypeScript decorators; - visibility is read from a declaration's modifiers, never from its parameter - names or default values, and a Python module's `__all__` (when written as a - literal) decides which of its top-level names are public — the names it - imports and lists become `reexport` symbols; an out-of-line C++ definition (`void - Widget::draw()`) belongs to its class, and a Lua `function M.go()` to its - table; and a `.h` header is parsed as C++ when its content is (a namespace, - class or template), as C otherwise. + TypeScript (`private readonly dep: Dep`) and PHP 8 (promoted) twins, every + name of a multi-name declaration (`var a, b int`, `int x, y;`, + `a, b = 1, 2`), C `#define` macros and the members of a `typedef struct`, + Python declarations under `if TYPE_CHECKING:` / `try:` / `with` blocks, Ruby + `class << self` methods, `private def x` definitions and the block of + `Point = Struct.new(…) do`, Elixir clauses with a `when` guard and + `defguard`, and the members of a class bound by `module.exports =` or an + anonymous `export default class` (a default export with no name of its own + is named after the file stem). A doc comment is found across Rust + attributes and TypeScript decorators. Visibility is read from a + declaration's modifiers, never from its parameter names or default values; + an `export { … }` list marks only the bindings of its own scope; and a + Python module's `__all__` (when written as literals) decides which of its + top-level names are public, the names it imports and lists becoming + `reexport` symbols. An out-of-line C++ definition (`void Widget::draw()`) + belongs to its class, and a Lua `function M.go()` to its table; a `.h` + header is parsed as C++ when its content is (a namespace, class or + template), as C otherwise. - **Resolve imports** across languages: tsconfig paths, package `exports`, go.mod, Cargo, Java packages, PSR-4, C# namespaces. - **Build a typed link-graph**: `import` / `call` / `extends` / `implements` / @@ -166,10 +169,10 @@ terms live only in prose. | what is scored | score | measured on | |---|---|---| -| symbol precision / recall | **100% / 100%** | 336 labelled declarations in 23 files | -| kind accuracy | **100%** | the same 336 declarations | -| visibility accuracy | **100%** on 16 of 17 languages, 95.2% on Go | the same 336 declarations | -| doc comment attached | **100%** | the 185 declarations labelled with a doc | +| symbol precision / recall | **100% / 100%** | 346 labelled declarations in 23 files | +| kind accuracy | **100%** | the same 346 declarations | +| visibility accuracy | **100%** on 16 of 17 languages, 95.8% on Go | the same 346 declarations | +| doc comment attached | **100%** | the 188 declarations labelled with a doc | | complete signature | **100%** | the 32 declarations labelled with a signature | | call edges / inheritance (F1) | **100% / 100%** | 54 labelled call sites, 24 relations | | search MRR / nDCG@10 / recall@5 | **93.8% / 86.0% / 84.4%** | 16 relevance-judged queries | diff --git a/site/index.html b/site/index.html index ca3a071..0874a09 100644 --- a/site/index.html +++ b/site/index.html @@ -4469,10 +4469,10 @@

How it compares

"base": { "languages": 17, "files": 23, - "symbols": 336, + "symbols": 346, "calls": 54, "relations": 24, - "docLabels": 185, + "docLabels": 188, "sigLabels": 32 }, "extraction": [ @@ -4553,7 +4553,7 @@

How it compares

"symbolsRecall": 1, "symbolsF1": 1, "kindAccuracy": 1, - "exportedAccuracy": 0.9524, + "exportedAccuracy": 0.9583, "docCoverage": 1, "sigCoverage": 1, "callsF1": 1, diff --git a/site/quality.json b/site/quality.json index 74c11b2..5dcd783 100644 --- a/site/quality.json +++ b/site/quality.json @@ -4,10 +4,10 @@ "base": { "languages": 17, "files": 23, - "symbols": 336, + "symbols": 346, "calls": 54, "relations": 24, - "docLabels": 185, + "docLabels": 188, "sigLabels": 32 }, "extraction": [ @@ -88,7 +88,7 @@ "symbolsRecall": 1, "symbolsF1": 1, "kindAccuracy": 1, - "exportedAccuracy": 0.9524, + "exportedAccuracy": 0.9583, "docCoverage": 1, "sigCoverage": 1, "callsF1": 1, diff --git a/src/ast/extract.ts b/src/ast/extract.ts index 1f006ea..cb9e443 100644 --- a/src/ast/extract.ts +++ b/src/ast/extract.ts @@ -833,19 +833,23 @@ export function extractAst( const doc = docOf(node); const parent = qualifier ?? ctx.parent; const parentPath = qualifier ?? ctx.parentPath; - emit({ - name, - kind, - file: rel, - line: node.startPosition.row + 1, - endLine: endLineOf(node), - ...(parent ? { parent } : {}), - ...(parentPath && parentPath !== parent ? { parentPath } : {}), - signature: header, - ...(doc ? { doc } : {}), - exported: visibilityOf(node, header, name, { ...ctx, exported: nowExported }), - lang, - }); + // `var a, b int` / `int x, y;` declare every name they list. + const several = spec.namesFrom?.[type]?.(node); + for (const each of several && several.length > 1 ? several : [name]) { + emit({ + name: each, + kind, + file: rel, + line: node.startPosition.row + 1, + endLine: endLineOf(node), + ...(parent ? { parent } : {}), + ...(parentPath && parentPath !== parent ? { parentPath } : {}), + signature: header, + ...(doc ? { doc } : {}), + exported: visibilityOf(node, header, each, { ...ctx, exported: nowExported }), + lang, + }); + } collectRelations(node, name); walkBody(node, bodyCtx(name, kind, parentPath, ctx, nowExported)); return; diff --git a/src/ast/node.ts b/src/ast/node.ts index 528130f..bacf8ca 100644 --- a/src/ast/node.ts +++ b/src/ast/node.ts @@ -14,6 +14,9 @@ export interface TSNode { namedChildCount: number; namedChild(i: number): TSNode | null; childForFieldName(name: string): TSNode | null; + // Every child under a field a grammar repeats — the several `declarator`s of + // `int x, y;`, the several `name`s of Go's `var a, b int`. + childrenForFieldName(name: string): TSNode[]; children: TSNode[]; // ONE marshal + ONE wasm call for the whole child list, memoized on the node — // versus `namedChildCount` plus a `namedChild(i)` round-trip per index. Every diff --git a/src/ast/specs.ts b/src/ast/specs.ts index 3859f81..08b4653 100644 --- a/src/ast/specs.ts +++ b/src/ast/specs.ts @@ -106,6 +106,15 @@ export interface LangSpec { */ nameFrom?: Record string | undefined>; + /** + * Every name a declaration node binds, for a node that can bind SEVERAL — + * Go's `var a, b = 1, 2`, Java's `int x, y;`, PHP's `public $a, $b;`, C's + * `int x, *y;`. The single-name readers stop at the first, so the rest of the + * list was never indexed. Each name becomes a symbol sharing the node's line, + * signature and doc; a node binding one name is read as before. + */ + namesFrom?: Record string[]>; + /** * Per-node-type kind chooser, for one node type that is two declarations. * C++ spells a method declaration and a data member both `field_declaration`, @@ -364,8 +373,28 @@ function pythonAll(root: TSNode): Set | undefined { const hasFunctionDeclarator = (node: TSNode): boolean => findFirst(node, (n) => n.type === "function_declarator" || n.type === "operator_cast") !== undefined; +// The names a repeated field holds, in source order: Go's `name`s, the +// `variable_declarator`s of Java and C#. A grammar can file the separating +// commas under the field too (Go's const_spec does), so only identifiers count. +const fieldNames = (field: string) => (node: TSNode): string[] => + node.childrenForFieldName(field).flatMap((n) => (/identifier$/.test(n.type) ? n.text : [])); +const csharpDeclaratorNames = (node: TSNode): string[] => + (childOfType(node, "variable_declaration")?.namedChildren ?? []).flatMap((d) => + d.type === "variable_declarator" ? (d.childForFieldName("name")?.text ?? []) : [], + ); +const declaratorNames = (node: TSNode): string[] => + node.childrenForFieldName("declarator").flatMap((d) => d.childForFieldName("name")?.text ?? []); + // --- C and C++ share their preprocessor and their `typedef struct` idiom ----- +// `int x, *y, z[3];` — one name per `declarator`, each at the end of its own +// chain (pointer, array, init, function declarators wrap it). A bare +// identifier declarator IS the name. +const cDeclaratorNames = (node: TSNode): string[] => + node + .childrenForFieldName("declarator") + .flatMap((d) => (d.namedChildren.length > 0 ? (nameOf(d) ?? []) : /identifier$/.test(d.type) ? d.text : [])); + const TAGGED_SPECIFIER: Record = { struct_specifier: "struct", union_specifier: "union", @@ -780,10 +809,22 @@ export const SPECS: Record = { if (node.type !== "expression_statement") return []; const assign = node.namedChildren[0]; if (!assign || assign.type !== "assignment") return []; - const left = assign.childForFieldName("left"); - if (!left || left.type !== "identifier") return []; if (underMainGuard(node)) return []; - return [{ name: left.text, kind: ctx.ownerKind === "class" ? "field" : "const" }]; + // `a, b = 1, 2` and `a = b = 0` bind every name they list; an attribute + // or subscript target (`self.x, y = …`) binds nothing here. + const names: string[] = []; + for (let a: TSNode | null = assign; a?.type === "assignment"; a = a.childForFieldName("right")) { + const left = a.childForFieldName("left"); + if (left?.type === "identifier") names.push(left.text); + else if (left && /^(pattern_list|tuple_pattern|list_pattern)$/.test(left.type)) { + for (const t of left.namedChildren) { + const bound = t.type === "list_splat_pattern" ? t.namedChildren[0] : t; + if (bound?.type === "identifier") names.push(bound.text); + } + } + } + const kind = ctx.ownerKind === "class" ? "field" : "const"; + return names.map((name) => ({ name, kind })); }, }, go: { @@ -840,6 +881,7 @@ export const SPECS: Record = { // undefined skips it, and the relation below records the embedding instead. field_declaration: (node) => node.childForFieldName("name")?.text, }, + namesFrom: { var_spec: fieldNames("name"), const_spec: fieldNames("name"), field_declaration: fieldNames("name") }, relationsFrom: { // Embedding IS Go's inheritance: `type Audited struct { Scheduler }` // promotes every Scheduler method onto Audited. @@ -991,6 +1033,7 @@ export const SPECS: Record = { // Same declarator shape as a field. constant_declaration: (node) => findFirst(node, (n) => n.type === "variable_declarator")?.childForFieldName("name")?.text, }, + namesFrom: { field_declaration: declaratorNames, constant_declaration: declaratorNames }, relationsFrom: { class_declaration: (node, ctx) => { if (!ctx.self) return []; @@ -1141,6 +1184,12 @@ export const SPECS: Record = { // reader cannot see, so every indexer was dropped. indexer_declaration: () => "this[]", }, + namesFrom: { + // One level deeper than Java's: the declarators hang off a + // variable_declaration. + field_declaration: (node) => csharpDeclaratorNames(node), + event_field_declaration: (node) => csharpDeclaratorNames(node), + }, relationsFrom: { class_declaration: (node, ctx) => (ctx.self ? firstIsBase(childOfType(node, "base_list"), ctx.self, node) : []), struct_declaration: (node, ctx) => (ctx.self ? firstIsBase(childOfType(node, "base_list"), ctx.self, node) : []), @@ -1179,6 +1228,14 @@ export const SPECS: Record = { property_declaration: (node) => findFirst(node, (n) => n.type === "variable_name")?.text.replace(/^\$/, ""), const_declaration: (node) => findFirst(node, (n) => n.type === "const_element")?.namedChildren[0]?.text, }, + namesFrom: { + property_declaration: (node) => + node.namedChildren.flatMap((e) => + e.type === "property_element" ? (e.childForFieldName("name")?.text.replace(/^\$/, "") ?? []) : [], + ), + const_declaration: (node) => + node.namedChildren.flatMap((e) => (e.type === "const_element" ? (e.namedChildren[0]?.text ?? []) : [])), + }, relationsFrom: { class_declaration: (node, ctx) => { if (!ctx.self) return []; @@ -1237,6 +1294,7 @@ export const SPECS: Record = { nameFrom: { preproc_def: (node) => (isIncludeGuard(node) ? undefined : node.childForFieldName("name")?.text), }, + namesFrom: { field_declaration: cDeclaratorNames, declaration: cDeclaratorNames }, bodyFrom: { type_definition: typedefBody }, extraMembers: typedefTag, }, @@ -1294,6 +1352,7 @@ export const SPECS: Record = { friend_declaration: (node) => nameOf(node) ?? (node.namedChildren[0] ? nameOf(node.namedChildren[0]) : undefined), preproc_def: (node) => (isIncludeGuard(node) ? undefined : node.childForFieldName("name")?.text), }, + namesFrom: { field_declaration: cDeclaratorNames, declaration: cDeclaratorNames }, parentFrom: { function_definition: cppMemberScope, declaration: cppMemberScope, diff --git a/tests/extraction-shapes.test.ts b/tests/extraction-shapes.test.ts index 6374c08..b38173d 100644 --- a/tests/extraction-shapes.test.ts +++ b/tests/extraction-shapes.test.ts @@ -675,3 +675,79 @@ describe("a Ruby mixin call on another receiver", () => { expect(rels(src)).toEqual(["implements Plugin Base", "implements Plugin Helpers"]); }); }); + +describe("a declaration that binds several names", () => { + // The single-name readers stopped at the first, so `var a, b` or `int x, y;` + // indexed `a` and `x` and silently dropped the rest. + const vis = (rel: string, src: string) => syms(rel, src).map((s) => `${s.kind} ${ids([s])[0]}=${s.exported ? 1 : 0}`); + + it("declares every name it lists, each with its own visibility", () => { + expect(vis("s.go", "package p\nvar Single, double = 1, 2\nconst A, b = 1, 2\ntype Box struct {\n\ta, B int\n\tEmbedded\n}")).toEqual([ + "package p=0", + "var Single=1", + "var double=0", + "const A=1", + "const b=0", + "type Box=1", + "field Box.a=0", + "field Box.B=1", + ]); + expect(vis("S.java", "class S {\n private final List items = List.of(), others;\n public int a = 1, b;\n}")).toEqual([ + "class S=0", + "field S.items=0", + "field S.others=0", + "field S.a=1", + "field S.b=1", + ]); + expect(vis("S.cs", "class S {\n private List _items = new(), _other;\n public event EventHandler A, B;\n}")).toEqual([ + "class S=0", + "field S._items=0", + "field S._other=0", + "event S.A=1", + "event S.B=1", + ]); + expect(vis("S.php", " { + expect(vis("p.c", "struct P { int x, *y, z[3]; };\nint ga, *gb;")).toEqual([ + "struct P=1", + "field P.x=1", + "field P.y=1", + "field P.z=1", + "const ga=1", + "const gb=1", + ]); + expect(vis("p.cpp", "class W {\n public:\n std::string s, t;\n int a, &b = a;\n};")).toEqual([ + "class W=1", + "field W.s=1", + "field W.t=1", + "field W.a=1", + "field W.b=1", + ]); + }); + + it("binds every Python target of a tuple or chained assignment", () => { + const src = "a, b = 1, 2\n(c, d) = 3, 4\ng, *rest = [1, 2]\nx, obj.attr = 1, 2\nh = _i = 7\nclass K:\n lo, hi = 0, 9"; + expect(vis("m.py", src)).toEqual([ + "const a=1", + "const b=1", + "const c=1", + "const d=1", + "const g=1", + "const rest=1", + "const x=1", + "const h=1", + "const _i=0", + "class K=1", + "field K.lo=1", + "field K.hi=1", + ]); + }); +}); diff --git a/tests/fixtures/quality/c/expected.json b/tests/fixtures/quality/c/expected.json index da251a4..896e677 100644 --- a/tests/fixtures/quality/c/expected.json +++ b/tests/fixtures/quality/c/expected.json @@ -25,6 +25,7 @@ { "name": "acme_stats_t", "kind": "type", "doc": true }, { "name": "done", "parent": "acme_stats_t", "kind": "field", "doc": true }, { "name": "failed", "parent": "acme_stats_t", "kind": "field" }, + { "name": "retried", "parent": "acme_stats_t", "kind": "field" }, { "name": "on_done", "parent": "acme_stats_t", "kind": "method", "doc": true }, { "name": "acme_status_t", "kind": "type", "doc": true }, { "name": "ACME_OK", "parent": "acme_status_t", "kind": "enum-member" }, diff --git a/tests/fixtures/quality/c/scheduler.h b/tests/fixtures/quality/c/scheduler.h index 1e4de17..e153272 100644 --- a/tests/fixtures/quality/c/scheduler.h +++ b/tests/fixtures/quality/c/scheduler.h @@ -39,7 +39,7 @@ typedef struct acme_scheduler acme_scheduler_t; typedef struct acme_stats { /** Jobs that ran to completion. */ unsigned done; - unsigned failed; + unsigned failed, retried; /** Called after every finished job. */ void (*on_done)(const struct acme_job *job); } acme_stats_t; diff --git a/tests/fixtures/quality/cpp/expected.json b/tests/fixtures/quality/cpp/expected.json index b7cc6cf..843b155 100644 --- a/tests/fixtures/quality/cpp/expected.json +++ b/tests/fixtures/quality/cpp/expected.json @@ -20,6 +20,7 @@ { "name": "JobSpec", "parent": "worker", "kind": "struct", "doc": true }, { "name": "name", "parent": "JobSpec", "kind": "field", "doc": true }, { "name": "attempts", "parent": "JobSpec", "kind": "field" }, + { "name": "priority", "parent": "JobSpec", "kind": "field" }, { "name": "Runnable", "parent": "worker", "kind": "class", "doc": true }, { "name": "start", "parent": "Runnable", "kind": "method" }, { "name": "depth", "parent": "Runnable", "kind": "method" }, diff --git a/tests/fixtures/quality/cpp/service.hpp b/tests/fixtures/quality/cpp/service.hpp index e5d185c..abe3492 100644 --- a/tests/fixtures/quality/cpp/service.hpp +++ b/tests/fixtures/quality/cpp/service.hpp @@ -27,7 +27,7 @@ enum class Outcome { struct JobSpec { /// Identifies the job. std::string name; - int attempts; + int attempts, priority; }; /// Anything the scheduler can drive. diff --git a/tests/fixtures/quality/csharp/Scheduler.cs b/tests/fixtures/quality/csharp/Scheduler.cs index dd6ce40..b5fcfdc 100644 --- a/tests/fixtures/quality/csharp/Scheduler.cs +++ b/tests/fixtures/quality/csharp/Scheduler.cs @@ -31,7 +31,7 @@ public class Scheduler : BaseWorker, IRunnable /// Bounds how often a job is retried. public const int MaxAttempts = 5; - private readonly List pending = new(); + private readonly List pending = new(), failed = new(); /// Raised after every attempt. public event AttemptHandler? Attempted; diff --git a/tests/fixtures/quality/csharp/expected.json b/tests/fixtures/quality/csharp/expected.json index 38699f0..953d665 100644 --- a/tests/fixtures/quality/csharp/expected.json +++ b/tests/fixtures/quality/csharp/expected.json @@ -71,6 +71,12 @@ "kind": "field", "exported": false }, + { + "name": "failed", + "parent": "Scheduler", + "kind": "field", + "exported": false + }, { "name": "Attempted", "parent": "Scheduler", diff --git a/tests/fixtures/quality/go/expected.json b/tests/fixtures/quality/go/expected.json index 8b0daa1..f4e0a6f 100644 --- a/tests/fixtures/quality/go/expected.json +++ b/tests/fixtures/quality/go/expected.json @@ -17,6 +17,11 @@ "kind": "var", "exported": false }, + { + "name": "deadQueue", + "kind": "var", + "exported": false + }, { "name": "Runnable", "kind": "type", @@ -49,6 +54,19 @@ "parent": "JobSpec", "kind": "field" }, + { + "name": "Priority", + "parent": "JobSpec", + "kind": "field", + "doc": true + }, + { + "name": "weight", + "parent": "JobSpec", + "kind": "field", + "exported": false, + "doc": true + }, { "name": "Scheduler", "kind": "type", diff --git a/tests/fixtures/quality/go/service.go b/tests/fixtures/quality/go/service.go index 1f4b02c..b2ad2b3 100644 --- a/tests/fixtures/quality/go/service.go +++ b/tests/fixtures/quality/go/service.go @@ -6,7 +6,7 @@ import "fmt" // MaxAttempts bounds how often a job is retried. const MaxAttempts = 5 -var defaultQueue = "jobs" +var defaultQueue, deadQueue = "jobs", "dead" // Runnable is anything the scheduler can drive. type Runnable interface { @@ -20,6 +20,8 @@ type JobSpec struct { // Name identifies the job. Name string Attempts int + // Priority and weight order the queue. + Priority, weight int } // Scheduler runs jobs with exponential backoff between retries. diff --git a/tests/fixtures/quality/java/Scheduler.java b/tests/fixtures/quality/java/Scheduler.java index 3ed91e4..f763ad6 100644 --- a/tests/fixtures/quality/java/Scheduler.java +++ b/tests/fixtures/quality/java/Scheduler.java @@ -41,7 +41,7 @@ public class Scheduler extends BaseWorker implements Runnable { /** Bounds how often a job is retried. */ public static final int MAX_ATTEMPTS = 5; - private final List pending = new ArrayList<>(); + private final List pending = new ArrayList<>(), failed = new ArrayList<>(); /** Drain the pending queue. */ @Override diff --git a/tests/fixtures/quality/java/expected.json b/tests/fixtures/quality/java/expected.json index c1cf791..56c04c6 100644 --- a/tests/fixtures/quality/java/expected.json +++ b/tests/fixtures/quality/java/expected.json @@ -99,6 +99,12 @@ "kind": "field", "exported": false }, + { + "name": "failed", + "parent": "Scheduler", + "kind": "field", + "exported": false + }, { "name": "start", "parent": "Scheduler", diff --git a/tests/fixtures/quality/php/Scheduler.php b/tests/fixtures/quality/php/Scheduler.php index 4b38996..f3694fe 100644 --- a/tests/fixtures/quality/php/Scheduler.php +++ b/tests/fixtures/quality/php/Scheduler.php @@ -25,7 +25,7 @@ class Scheduler extends BaseWorker implements Runnable public const MAX_ATTEMPTS = 5; /** Jobs waiting for a slot. */ - private array $pending = []; + private array $pending = [], $failed = []; public function __construct( private readonly Clock $clock, diff --git a/tests/fixtures/quality/php/expected.json b/tests/fixtures/quality/php/expected.json index c0f4d2d..3dd393d 100644 --- a/tests/fixtures/quality/php/expected.json +++ b/tests/fixtures/quality/php/expected.json @@ -56,6 +56,13 @@ "exported": false, "doc": true }, + { + "name": "failed", + "parent": "Scheduler", + "kind": "property", + "exported": false, + "doc": true + }, { "name": "clock", "parent": "Scheduler", diff --git a/tests/fixtures/quality/python/expected.json b/tests/fixtures/quality/python/expected.json index e353a84..f208899 100644 --- a/tests/fixtures/quality/python/expected.json +++ b/tests/fixtures/quality/python/expected.json @@ -4,6 +4,8 @@ "service.py": { "symbols": [ { "name": "MAX_ATTEMPTS", "kind": "const" }, + { "name": "BASE_DELAY", "kind": "const" }, + { "name": "MAX_DELAY", "kind": "const" }, { "name": "_DEFAULT_QUEUE", "kind": "const", "exported": false }, { "name": "_fast_loads", "kind": "const", "exported": false }, { "name": "parse_payload", "kind": "function", "doc": true }, diff --git a/tests/fixtures/quality/python/service.py b/tests/fixtures/quality/python/service.py index 681e49b..6614f3d 100644 --- a/tests/fixtures/quality/python/service.py +++ b/tests/fixtures/quality/python/service.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Optional MAX_ATTEMPTS = 5 +BASE_DELAY, MAX_DELAY = 1.0, 60.0 _DEFAULT_QUEUE = "jobs" try: diff --git a/tests/quality/baseline.json b/tests/quality/baseline.json index 7effaca..2fdc22e 100644 --- a/tests/quality/baseline.json +++ b/tests/quality/baseline.json @@ -60,7 +60,7 @@ "symbolsRecall": 1, "symbolsF1": 1, "kindAccuracy": 1, - "exportedAccuracy": 0.9524, + "exportedAccuracy": 0.9583, "docCoverage": 1, "sigCoverage": 1, "callsF1": 1, From 6d0b2969754ebc0b6ff50234f7ae3244ce3ac735 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:57:14 +0000 Subject: [PATCH 043/130] fix(delta): trace importers of deleted and renamed files Deleting a file that twelve modules import scored nothing: the worktree's graph no longer holds the file, so no edge points at it, and the dangling import filter only looked at files the diff changed. delta now puts the removed paths back into a resolve context and re-resolves the graph's dangling specs; those landing on a removed path are reported under `broken` (with renamedTo for a move), charged a new brokenImport weight (40) to the module the file left, and their importers count as its direct dependents. The scan is passed through DeltaOptions.scan. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 19 +++ src/delta.ts | 226 ++++++++++++++++++++++++++++++----- src/engine-cli.ts | 8 +- src/engine.ts | 4 +- tests/delta.test.ts | 163 +++++++++++++++++++++++++ tests/traverse-delta.test.ts | 1 + 6 files changed, 385 insertions(+), 36 deletions(-) create mode 100644 tests/delta.test.ts diff --git a/README.md b/README.md index f8a2db0..2fe1270 100644 --- a/README.md +++ b/README.md @@ -472,6 +472,25 @@ codeindex churn --repo packages/api # one package of a monorepo all four commands and reused while HEAD stays the same, so an MCP session asking for `onboard`, `hotspots` and `risk` reads the history once. +## Reviewing a diff + +`codeindex delta` maps the git diff onto the graph: changed files, the symbols +enclosing each hunk, the blast radius, and a risk score per module in which +every point comes with the reason that fired it. + +```sh +codeindex delta --repo . # the branch vs its merge-base with the default branch +codeindex delta --repo . --staged --json # the staged changeset, as JSON +``` + +- **A removed file that is still imported is the highest-weighted signal** + (`brokenImport`, 40). The worktree's graph no longer holds a deleted or + renamed file, so delta puts the removed paths back and re-resolves the + graph's dangling imports: the ones that land on a removed path are listed + under `broken` with their importer (and `renamedTo` for a move), the module + the file was removed from is scored even when nothing else in it changed, + and the importers count as its direct dependents. + ## Docker `ghcr.io/maxgfr/codeindex` ships the same zero-dependency bundle (`engine.mjs` diff --git a/src/delta.ts b/src/delta.ts index 54c8771..557e7e8 100644 --- a/src/delta.ts +++ b/src/delta.ts @@ -5,18 +5,27 @@ // CORRECT is not this engine's job. Reasons matter more than the number: every // point of the score is explained by one reason string carrying its numbers, so // a reviewer can disagree with a signal instead of with an opaque total. +import { posix } from "node:path"; import type { Graph, ModuleNode, SymbolIndex } from "./types.js"; +import type { RepoScan } from "./scan.js"; import type { DiffFile, DiffSpec, Hunk } from "./git.js"; import { isGitWorktree, resolveBaseRef, diffFiles, diffHunks, untrackedFiles } from "./git.js"; +import { buildResolveContext, resolveDocLink, resolveImport } from "./resolve.js"; import { byStr } from "./sort.js"; import { IGNORE_DIRS } from "./walk.js"; -import { have, sh } from "./util.js"; -import { impactOf } from "./traverse.js"; +import { have, sh, slugify } from "./util.js"; +import { sha1 } from "./hash.js"; +import { impactOf, reverseClosure } from "./traverse.js"; export interface DeltaOptions { base?: string; staged?: boolean; depth?: number; // blast-radius hops, default 2 + // The scan the graph was built from. With it, delta re-resolves the graph's + // dangling imports against the files the diff removed, which is the only + // way to see who still imports a deleted or renamed file: the graph of the + // worktree no longer holds that file, so no edge points at it. + scan?: RepoScan; } export interface ChangedSymbol { @@ -54,13 +63,25 @@ export interface DeltaModule { open: string[]; // best changed files to open first } +// An import (or doc link) of a file the diff removed: it resolves to the +// removed path once that path is put back. `renamedTo` when the file moved. +export interface BrokenImport { + from: string; + spec: string; + kind: "import" | "doc-link"; + target: string; + renamedTo?: string; +} + export interface DeltaResult { base: { ref: string; mergeBase: string; staged: boolean }; indexCommit?: string; depth: number; changes: DeltaChange[]; modules: DeltaModule[]; + // Dangling imports leaving changed files, minus the ones `broken` explains. dangling: { from: string; spec: string; reason: string }[]; + broken: BrokenImport[]; deleted: string[]; unindexed: string[]; notes: string[]; @@ -79,6 +100,10 @@ export const RISK_WEIGHTS = { testGap: 20, // a testable module with no covering test surprise: 10, // the module sits on a surprising cross-community edge dangling: 15, // a changed file carries a dangling import + // A file the diff deleted or renamed is still imported. Weighs more than + // `dangling`: that one may predate the diff or be a resolver blind spot, + // while this breakage is the diff's own, and certain. + brokenImport: 40, } as const; const HIGH_MIN = 60; @@ -141,18 +166,68 @@ function percentile(values: number[], mine: number): number { return smaller / (values.length - 1); } +// Who still imports a file the diff removed (deleted, or the old side of a +// rename). The worktree's graph cannot say directly — the file is gone, so the +// importer's edge is now dangling with only its spec left. Putting the removed +// paths back into a resolve context and re-resolving just the dangling specs +// answers it for every language the resolver knows, with no language logic +// here. Everything else about the context is the live scan's. +export function brokenImports( + scan: RepoScan, + graph: Graph, + removed: { path: string; renamedTo?: string }[], +): BrokenImport[] { + const dangling = graph.fileEdges.filter((e) => e.dangling && (e.kind === "import" || e.kind === "doc-link")); + if (!dangling.length || !removed.length) return []; + const ctx = buildResolveContext(scan); + const renamedTo = new Map(); + for (const r of removed) { + // Re-created at the same path: nothing is missing there. + if (ctx.fileSet.has(r.path)) continue; + renamedTo.set(r.path, r.renamedTo); + ctx.fileSet.add(r.path); + const dir = r.path.includes("/") ? posix.dirname(r.path) : ""; + const list = ctx.filesByDir.get(dir); + if (list) list.push(r.path); + else ctx.filesByDir.set(dir, [r.path]); + for (let d = dir; d && !ctx.dirSet.has(d); d = d.includes("/") ? posix.dirname(d) : "") ctx.dirSet.add(d); + } + if (!renamedTo.size) return []; + const extOf = new Map(scan.files.map((f) => [f.rel, f.ext])); + const out: BrokenImport[] = []; + for (const e of dangling) { + const ext = extOf.get(e.from); + if (ext === undefined) continue; + const kind = e.kind === "doc-link" ? "doc-link" : "import"; + const r = kind === "doc-link" ? resolveDocLink(e.from, e.to, ctx) : resolveImport(e.from, ext, e.to, ctx); + if (r.kind !== "resolved" || !renamedTo.has(r.target)) continue; + const to = renamedTo.get(r.target); + out.push({ from: e.from, spec: e.to, kind, target: r.target, ...(to !== undefined ? { renamedTo: to } : {}) }); + } + return out.sort((a, b) => byStr(a.target, b.target) || byStr(a.from, b.from) || byStr(a.spec, b.spec)); +} + // The pure core: graph + symbols + parsed diff → the full result. No git, no -// filesystem — unit-testable with synthetic inputs. +// filesystem — unit-testable with synthetic inputs. `broken` comes from +// brokenImports (deltaFor computes it when it has the scan). export function computeDelta( graph: Graph, symbols: SymbolIndex | undefined, - diff: { files: DiffFile[]; hunks: Map; base: DeltaResult["base"]; notes?: string[] }, + diff: { + files: DiffFile[]; + hunks: Map; + base: DeltaResult["base"]; + notes?: string[]; + broken?: BrokenImport[]; + }, depth: number = DEFAULT_DELTA_DEPTH, ): DeltaResult { const notes = [...(diff.notes ?? [])]; if (!symbols) notes.push("symbol index missing — symbol-level attribution disabled"); + const broken = diff.broken ?? []; const fileByRel = new Map(graph.files.map((f) => [f.rel, f])); + const moduleBySlug = new Map(graph.modules.map((m) => [m.slug, m])); const defsByFile = new Map(); if (symbols) { @@ -202,10 +277,18 @@ export function computeDelta( } // Dangling imports leaving any changed indexed file — broken references the - // diff either introduced or now sits on top of. + // diff either introduced or now sits on top of. One that points at a file + // the diff removed is listed under `broken` instead, with its cause. const changedRels = new Set(changes.filter((c) => c.status !== "deleted").map((c) => c.path)); + const explained = new Set(broken.map((b) => `${b.from}\0${b.spec}`)); const dangling = graph.fileEdges - .filter((e) => e.dangling && (e.kind === "import" || e.kind === "doc-link") && changedRels.has(e.from)) + .filter( + (e) => + e.dangling && + (e.kind === "import" || e.kind === "doc-link") && + changedRels.has(e.from) && + !explained.has(`${e.from}\0${e.to}`), + ) .map((e) => ({ from: e.from, spec: e.to, reason: e.reason ?? "unknown" })) .sort((a, b) => byStr(a.from, b.from) || byStr(a.spec, b.spec)); @@ -236,6 +319,28 @@ export function computeDelta( arr.push(c); } + // A removed file's breakage is charged to the module it was removed FROM — + // the change a reviewer is looking at — even when no indexed file there + // changed, and even when the whole directory is gone (the module then exists + // only in this panel, under the slug the directory would have had). + const moduleOfDir = new Map(graph.modules.map((m) => [m.path, m])); + const slugOfRemoved = (rel: string): string => { + const dir = rel.includes("/") ? posix.dirname(rel) : "(root)"; + const m = moduleOfDir.get(dir); + if (m) return m.slug; + const base = dir === "(root)" ? "root" : slugify(dir); + return base && !moduleBySlug.has(base) ? base : `${base || "module"}-${sha1(dir).slice(0, 8)}`; + }; + const brokenByModule = new Map(); + const removedPathOf = new Map(); // synthetic slug → removed dir + for (const b of broken) { + const slug = slugOfRemoved(b.target); + if (!moduleBySlug.has(slug)) removedPathOf.set(slug, b.target.includes("/") ? posix.dirname(b.target) : "(root)"); + let arr = brokenByModule.get(slug); + if (!arr) brokenByModule.set(slug, (arr = [])); + arr.push(b); + } + const nonTestCode = new Set(); for (const f of graph.files) { if (f.fileKind === "code" && !f.testFile) nonTestCode.add(f.module); @@ -246,13 +351,29 @@ export function computeDelta( const metricName = pagerankKnown ? "pagerank" : "degree"; const modules: DeltaModule[] = []; - for (const slug of [...byModule.keys()].sort(byStr)) { - const m = graph.modules.find((x) => x.slug === slug); - if (!m) continue; - const moduleChanges = byModule.get(slug)!; + for (const slug of [...new Set([...byModule.keys(), ...brokenByModule.keys()])].sort(byStr)) { + const m = moduleBySlug.get(slug); + const gonePath = removedPathOf.get(slug); + if (!m && gonePath === undefined) continue; + const moduleChanges = byModule.get(slug) ?? []; + const moduleBroken = brokenByModule.get(slug) ?? []; const reasons: string[] = []; let score = 0; + // 0. A removed file that is still imported: the build breaks. + const brokenImportsOnly = moduleBroken.filter((b) => b.kind === "import"); + if (brokenImportsOnly.length) { + score += RISK_WEIGHTS.brokenImport; + const targets = [...new Set(brokenImportsOnly.map((b) => b.target))].sort(byStr); + const first = targets[0]!; + const importers = [...new Set(brokenImportsOnly.filter((b) => b.target === first).map((b) => b.from))].sort(byStr); + const firstBroken = brokenImportsOnly.find((b) => b.target === first)!; + const what = firstBroken.renamedTo !== undefined ? `renamed ${first} (now ${firstBroken.renamedTo})` : `removed ${first}`; + const shown = importers.slice(0, 3).join(", ") + (importers.length > 3 ? ", …" : ""); + const more = targets.length > 1 ? ` (+${targets.length - 1} more removed file${targets.length > 2 ? "s" : ""})` : ""; + reasons.push(`${what} is still imported by ${importers.length} file${importers.length === 1 ? "" : "s"} (${shown})${more}`); + } + // 1. Exported API changed. const exportedNames = [...new Set(moduleChanges.flatMap((c) => c.symbols.filter((s) => s.exported).map((s) => s.name)))].sort(byStr); if (exportedNames.length) { @@ -262,7 +383,7 @@ export function computeDelta( } // 2. Structural importance of the touched module. - const pct = percentile(metricValues, metricOf(m)); + const pct = m ? percentile(metricValues, metricOf(m)) : 0; if (pct >= 0.9) { score += RISK_WEIGHTS.hubHigh; reasons.push(`${metricName} p${Math.round(pct * 100)} hub`); @@ -271,18 +392,31 @@ export function computeDelta( reasons.push(`${metricName} p${Math.round(pct * 100)} hub`); } - // 3. Blast radius — union of the reverse closure of each changed file. + // 3. Blast radius — union of the reverse closure of each changed file. A + // removed file has no node left to walk from, so its importers stand in as + // its direct dependents and the walk continues from them. const depthByRel = new Map(); const impModules = new Set(); + const reach = (rel: string, d: number): void => { + const prev = depthByRel.get(rel); + if (prev === undefined || d < prev) depthByRel.set(rel, d); + }; for (const c of moduleChanges) { const imp = impactOf(graph, c.path, depth); if (!imp) continue; - for (const f of imp.files) { - const prev = depthByRel.get(f.rel); - if (prev === undefined || f.depth < prev) depthByRel.set(f.rel, f.depth); - } + for (const f of imp.files) reach(f.rel, f.depth); for (const im of imp.modules) if (im !== slug) impModules.add(im); } + const importers = [...new Set(moduleBroken.map((b) => b.from))].sort(byStr); + if (importers.length && depth >= 1) { + const hops = new Map(importers.map((rel) => [rel, 1])); + for (const [rel, d] of reverseClosure(graph.fileEdges, importers, depth - 1)) hops.set(rel, d + 1); + for (const [rel, d] of hops) { + reach(rel, d); + const im = fileByRel.get(rel)?.module ?? "root"; + if (im !== slug) impModules.add(im); + } + } const transitiveFiles = depthByRel.size; const directFiles = [...depthByRel.values()].filter((d) => d === 1).length; const impact = { directFiles, transitiveFiles, modules: [...impModules].sort(byStr) }; @@ -295,8 +429,8 @@ export function computeDelta( } // 4. Test gap — only for modules that should have tests at all. - const testable = m.tier <= 1 && m.symbols > 0 && nonTestCode.has(slug); - const coveredBy = m.testedBy ?? []; + const testable = m !== undefined && m.tier <= 1 && m.symbols > 0 && nonTestCode.has(slug); + const coveredBy = m?.testedBy ?? []; const tests: DeltaModule["tests"] = testable ? coveredBy.length ? { status: "covered", files: coveredBy } @@ -326,22 +460,27 @@ export function computeDelta( } score = Math.min(100, score); - const changedFiles = moduleChanges.map((c) => c.path).sort(byStr); + const removedHere = [...new Set(moduleBroken.map((b) => b.target))].filter((t) => deleted.includes(t)); + const changedFiles = [...moduleChanges.map((c) => c.path), ...removedHere].sort(byStr); const allSyms = moduleChanges.flatMap((c) => c.symbols); - const open = moduleChanges - .slice() - .sort( - (a, b) => - b.symbols.filter((s) => s.exported).length - a.symbols.filter((s) => s.exported).length || - b.symbols.length - a.symbols.length || - byStr(a.path, b.path), - ) - .slice(0, OPEN_CAP) - .map((c) => c.path); + // The changed files carrying the most exported surface first; after a + // removal, the importers that must now be fixed. + const open = [ + ...moduleChanges + .slice() + .sort( + (a, b) => + b.symbols.filter((s) => s.exported).length - a.symbols.filter((s) => s.exported).length || + b.symbols.length - a.symbols.length || + byStr(a.path, b.path), + ) + .map((c) => c.path), + ...importers.filter((rel) => !changedRels.has(rel)), + ].slice(0, OPEN_CAP); modules.push({ slug, - path: m.path, + path: m?.path ?? gonePath!, score, bucket: score >= HIGH_MIN ? "HIGH" : score >= MEDIUM_MIN ? "MEDIUM" : "LOW", reasons, @@ -364,6 +503,7 @@ export function computeDelta( changes, modules, dangling, + broken, deleted: deleted.sort(byStr), unindexed: unindexed.sort(byStr), notes, @@ -406,7 +546,23 @@ export function deltaFor( } } - return computeDelta(graph, symbols, { files, hunks: diffHunks(repo, spec), base, notes }, opts.depth ?? DEFAULT_DELTA_DEPTH); + const removed = files.flatMap((f) => + f.status === "deleted" + ? [{ path: f.path }] + : f.status === "renamed" && f.oldPath !== undefined + ? [{ path: f.oldPath, renamedTo: f.path }] + : [], + ); + let broken: BrokenImport[] = []; + if (removed.length && opts.scan) broken = brokenImports(opts.scan, graph, removed); + else if (removed.length) notes.push("no scan supplied — importers of removed files were not traced"); + + return computeDelta( + graph, + symbols, + { files, hunks: diffHunks(repo, spec), base, notes, broken }, + opts.depth ?? DEFAULT_DELTA_DEPTH, + ); } // The human panel. Stdout-only by design: delta output is ephemeral per-worktree @@ -432,6 +588,14 @@ export function formatDeltaPanel(res: DeltaResult): string { if (res.dangling.length) { lines.push(` dangling: ${res.dangling.map((d) => `${d.spec} (from ${d.from})`).join(" · ")}`); } + // One line per removed file, naming everything that still imports it. + const brokenTargets = [...new Set(res.broken.map((b) => b.target))]; + for (const target of brokenTargets) { + const hits = res.broken.filter((b) => b.target === target); + const moved = hits[0]!.renamedTo !== undefined ? ` (renamed to ${hits[0]!.renamedTo})` : ""; + const from = [...new Set(hits.map((b) => b.from))]; + lines.push(` broken: ${target}${moved} still imported by ${from.join(", ")}`); + } if (res.deleted.length) lines.push(` deleted: ${res.deleted.join(", ")}`); if (res.unindexed.length) lines.push(` unindexed: ${res.unindexed.join(", ")}`); return lines.join("\n") + "\n"; diff --git a/src/engine-cli.ts b/src/engine-cli.ts index 2e0e9fb..db00687 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -132,8 +132,9 @@ Commands: positional for one file; omit for the repo-wide top risk Complexity × git-churn ranking (JSON; --since to bound, --limit) delta Review panel for the git diff: changed files -> enclosing symbols -> - blast radius -> risk score with explained reasons - (--base | --staged, --depth , --json) + blast radius -> risk score with explained reasons; a deleted or + renamed file that is still imported is listed under \`broken\` + with its importers (--base | --staged, --depth , --json) impact Reverse dependency closure of a file or module: everything that transitively imports/uses/calls it (--depth ; JSON) neighbors Graph neighbours of a file or module, both directions @@ -1126,11 +1127,12 @@ export async function runCli(rawArgv: string[]): Promise { const risks = riskHotspots(scan, res.churn, flags.limit); emit(JSON.stringify({ churnOk: res.ok, ...historyStatus(res), risks }, null, 2) + "\n", flags.out); } else if (cmd === "delta") { - const { graph, symbols } = await readArtifacts(); + const { scan, graph, symbols } = await readArtifacts(); const res = deltaFor(flags.repo, graph, symbols, { base: flags.base, staged: flags.staged, depth: flags.depth, + scan, }); if ("error" in res) throw new Error(res.error); emit(flags.json ? JSON.stringify(res, null, 2) + "\n" : formatDeltaPanel(res), flags.out); diff --git a/src/engine.ts b/src/engine.ts index f05f6f4..33d6fa0 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -252,8 +252,8 @@ export type { ImpactResult, ImpactedFile, NeighborResult, NeighborLink } from ". // Diff review: git diff -> enclosing symbols -> blast radius -> risk-scored, // reasons-first panel. `computeDelta` is the pure core (no git, no fs); // `deltaFor` adds the git plumbing against a graph the caller supplies. -export { computeDelta, deltaFor, formatDeltaPanel, symbolsInHunks, RISK_WEIGHTS, DEFAULT_DELTA_DEPTH } from "./delta.js"; -export type { DeltaOptions, DeltaResult, DeltaError, DeltaModule, DeltaChange, ChangedSymbol } from "./delta.js"; +export { brokenImports, computeDelta, deltaFor, formatDeltaPanel, symbolsInHunks, RISK_WEIGHTS, DEFAULT_DELTA_DEPTH } from "./delta.js"; +export type { BrokenImport, DeltaOptions, DeltaResult, DeltaError, DeltaModule, DeltaChange, ChangedSymbol } from "./delta.js"; export type { RepoMapOptions } from "./repomap.js"; // MCP server over stdio (also reachable as `engine.mjs mcp`). diff --git a/tests/delta.test.ts b/tests/delta.test.ts new file mode 100644 index 0000000..a6fd83a --- /dev/null +++ b/tests/delta.test.ts @@ -0,0 +1,163 @@ +// `delta` against real temporary git repositories: what a removal breaks, what +// the diff side must ignore, and the CI gate. +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { brokenImports, computeDelta, deltaFor, formatDeltaPanel, RISK_WEIGHTS } from "../src/delta.js"; +import type { DeltaResult } from "../src/delta.js"; +import { buildIndexArtifacts } from "../src/pipeline.js"; + +function git(dir: string, args: string[]): string { + return execFileSync("git", ["-C", dir, "-c", "commit.gpgsign=false", ...args], { + encoding: "utf8", + env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" }, + }); +} + +function write(root: string, files: Record): void { + for (const [rel, content] of Object.entries(files)) { + mkdirSync(dirname(join(root, rel)), { recursive: true }); + writeFileSync(join(root, rel), content); + } +} + +// A committed repo on `main`, so delta has a base to diff against. +function repo(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "ci-delta-")); + git(root, ["init", "-q", "-b", "main"]); + write(root, files); + git(root, ["add", "-A"]); + git(root, ["commit", "-qm", "init"]); + return root; +} + +function delta(root: string, opts: { staged?: boolean } = {}): DeltaResult { + const { scan, graph, symbols } = buildIndexArtifacts(root); + const res = deltaFor(root, graph, symbols, { ...opts, scan }); + if ("error" in res) throw new Error(res.error); + return res; +} + +// lib/hub.ts is imported by five files in three directories. +const HUB_REPO = { + "lib/hub.ts": "export function hub(): number {\n return 1;\n}\n", + "lib/other.ts": "export const other = 2;\n", + "app/a.ts": 'import { hub } from "../lib/hub";\nexport const a = hub();\n', + "app/b.ts": 'import { hub } from "../lib/hub";\nexport const b = hub();\n', + "app/c.ts": 'import { hub } from "../lib/hub";\nexport const c = hub();\n', + "web/d.ts": 'import { hub } from "../lib/hub";\nexport const d = hub();\n', + "web/e.ts": 'import { a } from "../app/a";\nexport const e = a;\n', + "cli/f.ts": 'import { hub } from "../lib/hub";\nexport const f = hub();\n', +}; + +describe("delta: importers of a removed file", () => { + it("scores the module a still-imported file was deleted from, naming its importers", () => { + const root = repo(HUB_REPO); + try { + rmSync(join(root, "lib/hub.ts")); + const res = delta(root); + expect(res.deleted).toEqual(["lib/hub.ts"]); + expect(res.broken.map((b) => b.from)).toEqual(["app/a.ts", "app/b.ts", "app/c.ts", "cli/f.ts", "web/d.ts"]); + expect(res.broken.every((b) => b.target === "lib/hub.ts" && b.kind === "import" && b.renamedTo === undefined)).toBe(true); + const lib = res.modules.find((m) => m.slug === "lib")!; + expect(lib.reasons[0]).toBe("removed lib/hub.ts is still imported by 5 files (app/a.ts, app/b.ts, app/c.ts, …)"); + expect(lib.score).toBeGreaterThanOrEqual(RISK_WEIGHTS.brokenImport); + expect(lib.changedFiles).toEqual(["lib/hub.ts"]); + // The importers are the blast radius, and the ones to open. + expect(lib.impact.directFiles).toBe(5); + expect(lib.impact.transitiveFiles).toBe(6); // + web/e.ts through app/a.ts + expect(lib.impact.modules).toEqual(["app", "cli", "web"]); + expect(lib.open).toEqual(["app/a.ts", "app/b.ts", "app/c.ts"]); + // Explained by the removal, so not repeated as an unexplained dangling import. + expect(res.dangling).toEqual([]); + expect(formatDeltaPanel(res)).toContain("broken: lib/hub.ts still imported by app/a.ts, app/b.ts, app/c.ts, cli/f.ts, web/d.ts"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("says where a renamed file went, even when an importer changed too", () => { + const root = repo(HUB_REPO); + try { + git(root, ["mv", "lib/hub.ts", "lib/core.ts"]); + write(root, { "app/a.ts": 'import { hub } from "../lib/hub";\nexport const a = hub() + 1;\n' }); + const res = delta(root); + expect(res.broken.filter((b) => b.from === "app/a.ts")).toEqual([ + { from: "app/a.ts", spec: "../lib/hub", kind: "import", target: "lib/hub.ts", renamedTo: "lib/core.ts" }, + ]); + expect(res.dangling).toEqual([]); + expect(res.modules.find((m) => m.slug === "lib")!.reasons[0]).toMatch(/^renamed lib\/hub\.ts \(now lib\/core\.ts\) is still imported by 5 files/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("scores a directory that is gone entirely under the slug it would have had", () => { + const root = repo(HUB_REPO); + try { + rmSync(join(root, "lib"), { recursive: true }); + const res = delta(root); + const lib = res.modules.find((m) => m.slug === "lib")!; + expect(lib.path).toBe("lib"); + expect(lib.tests.status).toBe("n/a"); + expect(lib.reasons[0]).toMatch(/^removed lib\/hub\.ts is still imported by 5 files/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("finds nothing broken when nobody imported the removed file", () => { + const root = repo(HUB_REPO); + try { + rmSync(join(root, "lib/other.ts")); + const res = delta(root); + expect(res.broken).toEqual([]); + expect(res.modules).toEqual([]); + expect(res.deleted).toEqual(["lib/other.ts"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("re-resolves against the live scan: a removed path re-created in place breaks nothing", () => { + const root = repo(HUB_REPO); + try { + const { scan, graph } = buildIndexArtifacts(root); + expect(brokenImports(scan, graph, [{ path: "lib/hub.ts" }])).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("notes when no scan was supplied instead of silently skipping the trace", () => { + const root = repo(HUB_REPO); + try { + rmSync(join(root, "lib/hub.ts")); + const { graph, symbols } = buildIndexArtifacts(root); + const res = deltaFor(root, graph, symbols) as DeltaResult; + expect(res.broken).toEqual([]); + expect(res.notes).toContain("no scan supplied — importers of removed files were not traced"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("keeps computeDelta pure: broken imports arrive as input", () => { + const root = repo(HUB_REPO); + try { + rmSync(join(root, "lib/hub.ts")); + const { graph, symbols } = buildIndexArtifacts(root); + const base = { ref: "main", mergeBase: "0000000", staged: false }; + const files = [{ path: "lib/hub.ts", status: "deleted" as const }]; + expect(computeDelta(graph, symbols, { files, hunks: new Map(), base }).modules).toEqual([]); + const broken = [{ from: "cli/f.ts", spec: "../lib/hub", kind: "import" as const, target: "lib/hub.ts" }]; + const res = computeDelta(graph, symbols, { files, hunks: new Map(), base, broken }); + expect(res.modules.map((m) => m.slug)).toEqual(["lib"]); + expect(res.modules[0]!.impact).toEqual({ directFiles: 1, transitiveFiles: 1, modules: ["cli"] }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/traverse-delta.test.ts b/tests/traverse-delta.test.ts index d72d457..940ef84 100644 --- a/tests/traverse-delta.test.ts +++ b/tests/traverse-delta.test.ts @@ -218,6 +218,7 @@ describe("computeDelta", () => { testGap: 20, surprise: 10, dangling: 15, + brokenImport: 40, }); }); From 268151ed511aacc6884c47bd5cb8814048a41bbc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:14:35 +0000 Subject: [PATCH 044/130] fix(resolution): count the new language resolvers as supported The resolution report labels an importer "unsupported" when resolveImport has no branch for its extension. Kotlin, Scala, Dart, Lua, shell and Elixir gained resolvers on a parallel branch, so the registry that mirrors the dispatch now lists them too; their out-of-repo imports read as external instead of "never become edges". The report tests move the no-resolver case to Swift, which still has neither extractor nor resolver. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/resolve.ts | 12 ++++++++++-- tests/resolution.test.ts | 14 +++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/resolve.ts b/src/resolve.ts index d447a6f..86c27d3 100644 --- a/src/resolve.ts +++ b/src/resolve.ts @@ -1497,9 +1497,17 @@ function resolveShell(fromRel: string, spec: string, ctx: ResolveContext): Resol // `external`, so its imports can never become edges — the resolution report // calls those `unsupported` rather than letting them pass as third-party. // Keep in step with the dispatch below. -const RESOLVER_EXTS = new Set([".go", ".rs", ".java", ".rb", ".rake", ".php", ".cs"]); +const RESOLVER_EXTS = new Set([".go", ".rs", ".java", ".rb", ".rake", ".php", ".cs", ".dart", ".lua", ".ex", ".exs"]); export function hasImportResolver(ext: string): boolean { - return JS_TS.has(ext) || SFC_HTML.has(ext) || PY.has(ext) || C_CPP.has(ext) || RESOLVER_EXTS.has(ext); + return ( + JS_TS.has(ext) || + SFC_HTML.has(ext) || + PY.has(ext) || + C_CPP.has(ext) || + JVM_OTHER.has(ext) || + SHELL.has(ext) || + RESOLVER_EXTS.has(ext) + ); } // Resolve an import specifier for a file of the given extension. diff --git a/tests/resolution.test.ts b/tests/resolution.test.ts index 65bd15d..e2c27cc 100644 --- a/tests/resolution.test.ts +++ b/tests/resolution.test.ts @@ -40,6 +40,7 @@ const REPO_FILES = { "src/b.ts": 'import { gone } from "./gone";\nexport const b = gone;\n', "src/c.ts": 'import { gone } from "./gone";\nexport const c = 1;\n', "src/Main.kt": "package a\nimport b.Util\nfun main() { Util.x() }\n", + "src/App.swift": "import Foundation\nfunc main() {}\n", "docs/guide.md": "# Guide\n\nSee [a](../src/a.ts), [missing](./nope.md), [the sources](../src/) and [site](https://example.com/x).\n", "config.json": '{ "k": 1 }\n', }; @@ -76,7 +77,10 @@ describe("resolutionReport", () => { }); it("says when a code language yields no import edges, and skips config-only languages", () => { - expect(row("kotlin")).toMatchObject({ files: 1, refs: 0, note: "no imports extracted from 1 file — this language gets no import edges" }); + expect(row("swift")).toMatchObject({ files: 1, refs: 0, note: "no imports extracted from 1 file — this language gets no import edges" }); + // Kotlin has an extractor and a resolver: an import outside the repo is + // simply external, never "unsupported". + expect(row("kotlin")).toMatchObject({ files: 1, refs: 1, external: 1, unsupported: 0 }); expect(report.languages.map((l) => l.lang)).not.toContain("json"); }); @@ -92,10 +96,10 @@ describe("resolutionReport", () => { it("labels imports from a language with no resolver as unsupported", () => { const scan = scanRepo(root); - scan.files.find((f) => f.rel === "src/Main.kt")!.refs.push({ kind: "import", spec: "b.Util" }); - const kt = resolutionReport(scan, { lang: "kotlin" }).languages; - expect(kt).toHaveLength(1); - expect(kt[0]).toMatchObject({ refs: 1, unsupported: 1, external: 0, note: "no import resolver for this language — its 1 import never become edges" }); + scan.files.find((f) => f.rel === "src/App.swift")!.refs.push({ kind: "import", spec: "Foundation" }); + const sw = resolutionReport(scan, { lang: "swift" }).languages; + expect(sw).toHaveLength(1); + expect(sw[0]).toMatchObject({ refs: 1, unsupported: 1, external: 0, note: "no import resolver for this language — its 1 import never become edges" }); }); it("filters to one language, caps the top lists, and rejects an unknown language", () => { From 7cd57600ddd242d7b9d575ce9cf2a3bb95d4ab47 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:17:57 +0000 Subject: [PATCH 045/130] perf(delta): read the diff before the index, and leave the index out of it On a clean worktree delta walked and loaded the whole index only to report the three files `index --out .codeindex` had just written, as unindexed changes (15.6 s on a 66k-file repo). The git side is now its own step (readDeltaDiff): paths under the --index directory are dropped, and so are untracked paths in directories the walker never indexes. When nothing is left, the CLI answers without walking or loading anything (0.36 s). computeDelta collects defs for the changed files only instead of materialising every def in the repo (600 ms -> 180 ms on the same index). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 9 ++++ src/delta.ts | 104 ++++++++++++++++++++++++++++++++------------ src/engine-cli.ts | 37 +++++++++------- src/engine.ts | 19 ++++++-- tests/delta.test.ts | 72 +++++++++++++++++++++++++++++- 5 files changed, 193 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 2fe1270..a93bbc5 100644 --- a/README.md +++ b/README.md @@ -490,6 +490,15 @@ codeindex delta --repo . --staged --json # the staged changeset, as JSON under `broken` with their importer (and `renamedTo` for a move), the module the file was removed from is scored even when nothing else in it changed, and the importers count as its direct dependents. +- **The engine's own output is not part of a review.** Paths under the index + directory (`--index`, default `.codeindex`) are dropped from the diff, and so + are untracked files in directories the walker never indexes (`node_modules/`, + `dist/`, …). A tracked change in such a directory stays listed as + `unindexed`. +- **The diff is read before the index.** A clean worktree answers + `no changes` without loading or walking anything (0.4 s instead of 15 s on a + 66k-file repository), and symbol attribution reads only the changed files' + definitions. ## Docker diff --git a/src/delta.ts b/src/delta.ts index 557e7e8..0b645bc 100644 --- a/src/delta.ts +++ b/src/delta.ts @@ -5,7 +5,7 @@ // CORRECT is not this engine's job. Reasons matter more than the number: every // point of the score is explained by one reason string carrying its numbers, so // a reviewer can disagree with a signal instead of with an opaque total. -import { posix } from "node:path"; +import { isAbsolute, posix, relative, resolve, sep } from "node:path"; import type { Graph, ModuleNode, SymbolIndex } from "./types.js"; import type { RepoScan } from "./scan.js"; import type { DiffFile, DiffSpec, Hunk } from "./git.js"; @@ -26,6 +26,9 @@ export interface DeltaOptions { // way to see who still imports a deleted or renamed file: the graph of the // worktree no longer holds that file, so no edge points at it. scan?: RepoScan; + // The persisted index directory, relative to the repo (default .codeindex): + // its files are the engine's output, never part of a review. + indexDir?: string; } export interface ChangedSymbol { @@ -209,7 +212,7 @@ export function brokenImports( // The pure core: graph + symbols + parsed diff → the full result. No git, no // filesystem — unit-testable with synthetic inputs. `broken` comes from -// brokenImports (deltaFor computes it when it has the scan). +// brokenImports (deltaOfDiff computes it when it has the scan). export function computeDelta( graph: Graph, symbols: SymbolIndex | undefined, @@ -229,14 +232,14 @@ export function computeDelta( const fileByRel = new Map(graph.files.map((f) => [f.rel, f])); const moduleBySlug = new Map(graph.modules.map((m) => [m.slug, m])); + // Defs of the changed indexed files only: symbols.defs spans the whole repo + // (hundreds of thousands of entries on a large one), and a review touches a + // handful of files. const defsByFile = new Map(); - if (symbols) { - for (const [name, entries] of Object.entries(symbols.defs)) { - for (const d of entries) { - let arr = defsByFile.get(d.file); - if (!arr) defsByFile.set(d.file, (arr = [])); - arr.push({ name, ...d }); - } + for (const df of diff.files) if (df.status !== "deleted" && fileByRel.has(df.path)) defsByFile.set(df.path, []); + if (symbols && defsByFile.size) { + for (const name of Object.keys(symbols.defs)) { + for (const d of symbols.defs[name]!) defsByFile.get(d.file)?.push({ name, ...d }); } for (const arr of defsByFile.values()) arr.sort((a, b) => a.line - b.line || byStr(a.name, b.name)); } @@ -510,17 +513,35 @@ export function computeDelta( }; } -// Git plumbing → computeDelta, against a graph the caller already built. The -// caller owns index freshness: a consumer serving a PERSISTED graph must gate -// on its own staleness oracle first, because symbol line-mapping is only -// correct against an index built from the same bytes, and a confidently wrong -// attribution is worse than "rebuild first". -export function deltaFor( - repo: string, - graph: Graph, - symbols: SymbolIndex | undefined, - opts: DeltaOptions = {}, -): DeltaResult | DeltaError { +// The git side of a review, on its own: the base, the changed files and their +// hunks. It needs no index, so a caller can run it first and skip loading the +// artifacts when there is nothing to review. +export interface DeltaDiff { + base: DeltaResult["base"]; + files: DiffFile[]; + hunks: Map; + notes: string[]; +} + +// The engine's own output directory, and anything else the walker would never +// index, is not part of a review: `index --out .codeindex` (the documented +// default) otherwise showed up as three unindexed files in every delta, and an +// untracked node_modules/ as thousands. Tracked paths are kept outside the +// index directory: a committed change is the diff's own even where the walker +// does not look, and the panel lists it as unindexed. Untracked paths are +// judged by the walker's default ignore set. +function reviewFilter(repo: string, indexDir: string | undefined): (path: string, tracked: boolean) => boolean { + const idx = relative(resolve(repo), resolve(repo, indexDir ?? ".codeindex")).split(sep).join("/"); + const inIndex = idx && !idx.startsWith("..") && !isAbsolute(idx) ? (p: string) => p === idx || p.startsWith(`${idx}/`) : () => false; + return (path, tracked) => { + if (inIndex(path)) return false; + if (tracked) return true; + const dirs = path.split("/").slice(0, -1); + return !dirs.some((d) => IGNORE_DIRS.has(d) || d.startsWith(".codeindex-edit-")); + }; +} + +export function readDeltaDiff(repo: string, opts: DeltaOptions = {}): DeltaDiff | DeltaError { if (!have("git")) return { error: "git is required for delta and was not found on PATH" }; if (!isGitWorktree(repo)) return { error: `delta needs a git worktree — ${repo} is not inside one` }; @@ -537,16 +558,36 @@ export function deltaFor( base = { ref: r.ref, mergeBase: r.mergeBase, staged: false }; } + const keep = reviewFilter(repo, opts.indexDir); const spec: DiffSpec = opts.staged ? { staged: true } : { mergeBase: base.mergeBase }; - const files = diffFiles(repo, spec); + const files = diffFiles(repo, spec).filter((f) => keep(f.path, true)); if (!opts.staged) { const known = new Set(files.map((f) => f.path)); for (const u of untrackedFiles(repo)) { - if (!known.has(u)) files.push({ path: u, status: "added" }); + if (!known.has(u) && keep(u, false)) files.push({ path: u, status: "added" }); } } + return { base, files, hunks: files.length ? diffHunks(repo, spec) : new Map(), notes }; +} - const removed = files.flatMap((f) => +// A review with nothing in it, answered without an index. +export function emptyDelta(diff: DeltaDiff, depth: number = DEFAULT_DELTA_DEPTH): DeltaResult { + return { base: diff.base, depth, changes: [], modules: [], dangling: [], broken: [], deleted: [], unindexed: [], notes: diff.notes }; +} + +// A read diff → computeDelta, against a graph the caller already built. The +// caller owns index freshness: a consumer serving a PERSISTED graph must gate +// on its own staleness oracle first, because symbol line-mapping is only +// correct against an index built from the same bytes, and a confidently wrong +// attribution is worse than "rebuild first". +export function deltaOfDiff( + diff: DeltaDiff, + graph: Graph, + symbols: SymbolIndex | undefined, + opts: DeltaOptions = {}, +): DeltaResult { + const notes = [...diff.notes]; + const removed = diff.files.flatMap((f) => f.status === "deleted" ? [{ path: f.path }] : f.status === "renamed" && f.oldPath !== undefined @@ -556,13 +597,18 @@ export function deltaFor( let broken: BrokenImport[] = []; if (removed.length && opts.scan) broken = brokenImports(opts.scan, graph, removed); else if (removed.length) notes.push("no scan supplied — importers of removed files were not traced"); + return computeDelta(graph, symbols, { ...diff, notes, broken }, opts.depth ?? DEFAULT_DELTA_DEPTH); +} - return computeDelta( - graph, - symbols, - { files, hunks: diffHunks(repo, spec), base, notes, broken }, - opts.depth ?? DEFAULT_DELTA_DEPTH, - ); +// Git plumbing → computeDelta in one call (see deltaOfDiff on freshness). +export function deltaFor( + repo: string, + graph: Graph, + symbols: SymbolIndex | undefined, + opts: DeltaOptions = {}, +): DeltaResult | DeltaError { + const diff = readDeltaDiff(repo, opts); + return "error" in diff ? diff : deltaOfDiff(diff, graph, symbols, opts); } // The human panel. Stdout-only by design: delta output is ephemeral per-worktree diff --git a/src/engine-cli.ts b/src/engine-cli.ts index db00687..21c196b 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -36,7 +36,7 @@ import { findLiteralDuplications } from "./literals.js"; import { symbolComplexity, riskHotspots } from "./complexity.js"; import { renderMermaid } from "./viz.js"; import { impactOf, neighborsOf } from "./traverse.js"; -import { deltaFor, formatDeltaPanel } from "./delta.js"; +import { deltaOfDiff, emptyDelta, formatDeltaPanel, readDeltaDiff } from "./delta.js"; import { explainQuery, searchIndex } from "./bm25.js"; import { checkRules, parseRules } from "./rules.js"; import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, parseEmbedModel, resolveEmbedPullUrl, fetchEmbedModel } from "./embed/model.js"; @@ -134,7 +134,8 @@ Commands: delta Review panel for the git diff: changed files -> enclosing symbols -> blast radius -> risk score with explained reasons; a deleted or renamed file that is still imported is listed under \`broken\` - with its importers (--base | --staged, --depth , --json) + with its importers (--base | --staged, --depth , --json). + Paths under the --index directory are not part of the review impact Reverse dependency closure of a file or module: everything that transitively imports/uses/calls it (--depth ; JSON) neighbors Graph neighbours of a file or module, both directions @@ -549,17 +550,19 @@ export async function runCli(rawArgv: string[]): Promise { // already returned above. The walk is done ONCE here to derive the present // extensions, then handed to the scan via precomputedWalk so the tree is // traversed a single time. --no-ast keeps the regex tier: no walk, no warm — - // scanRepo walks itself, exactly as before. + // scanRepo walks itself, exactly as before. `delta` walks only once it knows + // the diff is not empty (see there): the walk alone is seconds on a large + // repo, and a clean worktree needs no index at all. const scans = !SCANLESS_COMMANDS.has(cmd) && !(cmd === "embed" && flags.positional !== "build"); - let precomputedWalk: WalkResult | undefined; - if (scans && !flags.noAst) { - precomputedWalk = walk(flags.repo, { + const walkRepo = (): WalkResult => + walk(flags.repo, { maxFileBytes: flags.maxBytes, maxFiles: flags.maxFiles, gitignore: flags.gitignore, ignoreDirs: flags.ignoreDirs.length ? flags.ignoreDirs : undefined, }); - } + let precomputedWalk: WalkResult | undefined; + if (scans && !flags.noAst && cmd !== "delta") precomputedWalk = walkRepo(); let grammarsWarmed = false; const warmPresentGrammars = async (): Promise => { if (grammarsWarmed || flags.noAst || !precomputedWalk) return; @@ -1127,14 +1130,18 @@ export async function runCli(rawArgv: string[]): Promise { const risks = riskHotspots(scan, res.churn, flags.limit); emit(JSON.stringify({ churnOk: res.ok, ...historyStatus(res), risks }, null, 2) + "\n", flags.out); } else if (cmd === "delta") { - const { scan, graph, symbols } = await readArtifacts(); - const res = deltaFor(flags.repo, graph, symbols, { - base: flags.base, - staged: flags.staged, - depth: flags.depth, - scan, - }); - if ("error" in res) throw new Error(res.error); + // The git side first: it needs no index, and on a clean worktree it is the + // whole answer. Loading the artifacts to report "no changes" cost 8 s on a + // 66k-file repo. + const opts = { base: flags.base, staged: flags.staged, depth: flags.depth, indexDir }; + const diff = readDeltaDiff(flags.repo, opts); + if ("error" in diff) throw new Error(diff.error); + let res = emptyDelta(diff, flags.depth); + if (diff.files.length) { + if (!flags.noAst) precomputedWalk = walkRepo(); + const { scan, graph, symbols } = await readArtifacts(); + res = deltaOfDiff(diff, graph, symbols, { ...opts, scan }); + } emit(flags.json ? JSON.stringify(res, null, 2) + "\n" : formatDeltaPanel(res), flags.out); } else if (cmd === "impact") { if (!flags.positional) throw new Error("impact needs a target: cli.mjs impact --repo "); diff --git a/src/engine.ts b/src/engine.ts index 33d6fa0..aeee414 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -251,9 +251,22 @@ export type { ImpactResult, ImpactedFile, NeighborResult, NeighborLink } from ". // Diff review: git diff -> enclosing symbols -> blast radius -> risk-scored, // reasons-first panel. `computeDelta` is the pure core (no git, no fs); -// `deltaFor` adds the git plumbing against a graph the caller supplies. -export { brokenImports, computeDelta, deltaFor, formatDeltaPanel, symbolsInHunks, RISK_WEIGHTS, DEFAULT_DELTA_DEPTH } from "./delta.js"; -export type { BrokenImport, DeltaOptions, DeltaResult, DeltaError, DeltaModule, DeltaChange, ChangedSymbol } from "./delta.js"; +// `deltaFor` adds the git plumbing against a graph the caller supplies; +// `readDeltaDiff` + `deltaOfDiff` split it so a caller can skip loading the +// index when the diff is empty (`emptyDelta`). +export { + brokenImports, + computeDelta, + deltaFor, + deltaOfDiff, + emptyDelta, + formatDeltaPanel, + readDeltaDiff, + symbolsInHunks, + RISK_WEIGHTS, + DEFAULT_DELTA_DEPTH, +} from "./delta.js"; +export type { BrokenImport, DeltaDiff, DeltaOptions, DeltaResult, DeltaError, DeltaModule, DeltaChange, ChangedSymbol } from "./delta.js"; export type { RepoMapOptions } from "./repomap.js"; // MCP server over stdio (also reachable as `engine.mjs mcp`). diff --git a/tests/delta.test.ts b/tests/delta.test.ts index a6fd83a..56dfef2 100644 --- a/tests/delta.test.ts +++ b/tests/delta.test.ts @@ -5,7 +5,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { describe, expect, it } from "vitest"; -import { brokenImports, computeDelta, deltaFor, formatDeltaPanel, RISK_WEIGHTS } from "../src/delta.js"; +import { brokenImports, computeDelta, deltaFor, deltaOfDiff, emptyDelta, formatDeltaPanel, readDeltaDiff, RISK_WEIGHTS } from "../src/delta.js"; import type { DeltaResult } from "../src/delta.js"; import { buildIndexArtifacts } from "../src/pipeline.js"; @@ -161,3 +161,73 @@ describe("delta: importers of a removed file", () => { } }); }); + +describe("delta: what the diff side ignores", () => { + it("drops the engine's own index and untracked paths the walker never indexes", () => { + const root = repo(HUB_REPO); + try { + write(root, { + ".codeindex/graph.json": "{}\n", + ".codeindex/memories/onboarding.md": "# brief\n", + "node_modules/pkg/index.js": "module.exports = 1;\n", + ".codeindex-edit-x1/hub.ts": "export const x = 1;\n", + }); + const diff = readDeltaDiff(root); + if ("error" in diff) throw new Error(diff.error); + expect(diff.files).toEqual([]); + expect(formatDeltaPanel(emptyDelta(diff))).toMatch(/^codeindex: no changes vs main/); + // A real untracked source file is still part of the review. + write(root, { "app/new.ts": "export const n = 1;\n" }); + const res = delta(root); + expect(res.changes.map((c) => c.path)).toEqual(["app/new.ts"]); + expect(res.unindexed).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("drops a custom --index directory, tracked or not, and keeps other tracked paths", () => { + const root = repo({ ...HUB_REPO, "vendor/dep.js": "module.exports = 1;\n", "idx/old.json": "{}\n" }); + try { + write(root, { + "idx/graph.json": "{}\n", + "idx/old.json": "{\"x\":1}\n", + "vendor/dep.js": "module.exports = 2;\n", + }); + const withDefault = readDeltaDiff(root); + if ("error" in withDefault) throw new Error(withDefault.error); + expect(withDefault.files.map((f) => f.path).sort()).toEqual(["idx/graph.json", "idx/old.json", "vendor/dep.js"]); + const diff = readDeltaDiff(root, { indexDir: "idx" }); + if ("error" in diff) throw new Error(diff.error); + // A tracked change under an ignored directory is still the diff's: the + // panel lists it as unindexed rather than hiding it. + expect(diff.files.map((f) => f.path)).toEqual(["vendor/dep.js"]); + expect(deltaOfDiff(diff, buildIndexArtifacts(root).graph, undefined).unindexed).toEqual(["vendor/dep.js"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("delta: symbol attribution reads only the changed files' defs", () => { + it("attributes hunks exactly as before when the repo holds many other defs", () => { + const root = repo(HUB_REPO); + try { + write(root, { "app/b.ts": 'import { hub } from "../lib/hub";\nexport const b = hub() * 2;\n' }); + const res = delta(root); + expect(res.changes).toEqual([ + { + path: "app/b.ts", + status: "modified", + linesAdded: 1, + linesDeleted: 1, + module: "app", + hunks: [{ start: 2, end: 2 }], + symbols: [{ name: "b", kind: "const", exported: true, line: 2, endLine: 2 }], + }, + ]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 17f41cdb7f113ab57e71e24e492de3b7169b202c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:19:36 +0000 Subject: [PATCH 046/130] fix(mcp): make every result honour the official SDK's schema contract The symbols and callers outputSchemas were bare `{ oneOf: [...] }` roots. The MCP TypeScript SDK types Tool.outputSchema as `type: "object"` and rejects the whole tools/list otherwise, so every SDK-based host negotiating 2025-06-18+ saw zero tools. Both roots now carry `type: "object"`. Once listing works the SDK also validates each non-error result: the size-guard notice (no structuredContent) and call_graph's not-found text (fails its schema) both became thrown client errors. The notice and the call_graph / type_hierarchy / implementations lookup misses are now sent with isError, their text unchanged. Tests iterate every advertised schema for the root type and the result contract, so tools added later are covered. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 14 ++++-- src/engine-cli.ts | 5 +- src/mcp.ts | 26 +++++++--- src/mcp/protocol.ts | 7 ++- src/mcp/tools.ts | 9 ++++ tests/mcp-output.test.ts | 102 ++++++++++++++++++++++++++++++++++++--- 6 files changed, 144 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 17af6f6..2a6a7d8 100644 --- a/README.md +++ b/README.md @@ -835,14 +835,22 @@ which is what lets a host auto-approve reads and confirm only writes. From the result instead of re-parsing a string. The remaining tools return arrays, argument-dependent shapes or plain text, which cannot yield a conforming structured result without diverging from the text block — they are left -unschema'd rather than described inaccurately. +unschema'd rather than described inaccurately. Every schema is rooted at +`type: "object"`, as the official TypeScript SDK requires to list tools at all. +A lookup miss (`call_graph`, `type_hierarchy` or `implementations` naming +nothing in the repo) keeps its `{ "error": ... }` text but is flagged +`isError`, so a client validating against the schema reads it as the tool +error it is. Responses are capped (`--max-response-bytes`, default 1 MB). Under the cap nothing changes. Over it — where a whole-repo `graph` on a large monorepo runs to millions of tokens and no client can accept it — the response is replaced by a short notice naming the size, the artifact already on disk, and the narrower -tool that answers the question. Most tools also take a `limit`/`maxResults`/ -`top`/`maxEdges` argument to stay well under it. +tool that answers the question. The notice is sent as a tool error +(`isError: true`): the model reads it and narrows the call, and a client that +validates `structuredContent` is not handed a result that cannot conform. Most +tools also take a `limit`/`maxResults`/`top`/`maxEdges` argument to stay well +under it. `engine.mjs` is a pure side-effect-free library (safe for consumers to inline into their own CLIs); `cli.mjs` is the thin standalone CLI/MCP wrapper. diff --git a/src/engine-cli.ts b/src/engine-cli.ts index a03a3b3..069c49f 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -149,8 +149,9 @@ Commands: explicit per-call repo still wins); --server-name overrides the announced serverInfo; --max-response-bytes caps a single tool response (default 1e6; a response under the cap is - byte-identical, one over it is replaced by an actionable notice - instead of an unusable blob); --tools + byte-identical, one over it is replaced by an actionable notice, + sent as a tool error, instead of an unusable blob); + --tools advertises a named subset (all | orient | find | impact | edit | risk, default all) — every advertised tool's schema costs an agent context on EVERY turn, and a tool left out is still answerable diff --git a/src/mcp.ts b/src/mcp.ts index 724dfc6..041a743 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -133,6 +133,14 @@ function errMessage(e: unknown): string { return e instanceof Error ? e.message : String(e); } +// "There is no such symbol/type" from a lookup tool. Its text stays the +// `{ "error": ... }` JSON it always was, but it travels as a tool execution +// error (isError): it is not a result, and a declared outputSchema describes +// results. An SDK client validates structuredContent on every NON-error +// response, so `call_graph`'s notice used to surface as a -32602 protocol +// failure instead of the sentence the model needed to read. +class NotFound extends Error {} + // Tools that never scan the file tree (git/grep/memory/embed-status only) — they // must not trigger a grammar warm. Every other tool is scan-needing and warms // the repo's grammars first; defaulting to "warm" keeps a newly added scan tool @@ -488,14 +496,14 @@ async function callTool(name: string, args: Record, defaultRepo return JSON.stringify(obj, null, 2); } const entry = hierarchy.get(wanted); - if (!entry) return JSON.stringify({ error: `no type named ${wanted}` }, null, 2); + if (!entry) throw new NotFound(`no type named ${wanted}`); return JSON.stringify(entry, null, 2); } if (name === "implementations") { const wanted = str(args.name); if (!wanted) throw new Error("`name` is required"); const hierarchy = hierarchyFor(readScan()); - if (!hierarchy.has(wanted)) return JSON.stringify({ error: `no type named ${wanted}` }, null, 2); + if (!hierarchy.has(wanted)) throw new NotFound(`no type named ${wanted}`); return JSON.stringify({ name: wanted, implementations: implementationsOf(hierarchy, wanted) }, null, 2); } if (name === "call_graph") { @@ -507,7 +515,7 @@ async function callTool(name: string, args: Record, defaultRepo ...(positiveNum(args.depth) !== undefined ? { depth: positiveNum(args.depth)! } : {}), direction: dir, }); - if (!result.root.length) return JSON.stringify({ error: `no symbol named ${symbol}` }, null, 2); + if (!result.root.length) throw new NotFound(`no symbol named ${symbol}`); return JSON.stringify(result, null, 2); } if (name === "check_rules") { @@ -704,12 +712,18 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise { result: { content: link ? [{ type: "text", text }, link] : [{ type: "text", text }], ...(structured ? { structuredContent: structured } : {}), + // The withheld-payload notice is a tool execution error: the call + // did not deliver what was asked, and the notice is exactly the + // actionable text such an error exists to carry. It is also the + // only honest option for a tool with an outputSchema — the notice + // cannot conform to it, and SDK clients reject a non-error result + // that lacks conforming structuredContent. + ...(capped ? { isError: true } : {}), }, }); } catch (e) { - return respond({ - result: { content: [{ type: "text", text: e instanceof Error ? e.message : String(e) }], isError: true }, - }); + const text = e instanceof NotFound ? JSON.stringify({ error: e.message }, null, 2) : errMessage(e); + return respond({ result: { content: [{ type: "text", text }], isError: true } }); } } else { return respond({ error: { code: -32601, message: `method not found: ${req.method}` } }); diff --git a/src/mcp/protocol.ts b/src/mcp/protocol.ts index e190943..0835892 100644 --- a/src/mcp/protocol.ts +++ b/src/mcp/protocol.ts @@ -88,7 +88,8 @@ export function validateArgs( // outputSchema to be honoured by every structured result: // * the tool declares an outputSchema (see OUTPUT_SCHEMAS), // * the response was NOT replaced by the size guard — the truncation notice -// is a different shape and would not conform, +// is a different shape and would not conform (it is sent with isError, +// which is what exempts it from the schema), // * the text parses to a JSON object (never an array: structuredContent is // specified as an object). // The text block is left exactly as it was, so this is purely additive and @@ -123,7 +124,9 @@ export function negotiateProtocol(requested: unknown): string { // consumed by any client anyway, so replacing it with something actionable // cannot regress a working call — it converts a hard failure into a usable // answer that says how big the payload is, where the artifact already sits on -// disk, and which narrower tool answers the question. +// disk, and which narrower tool answers the question. The server sends that +// notice as a tool execution error (isError): the model reads it and retries +// narrower, and a client validating against the tool's outputSchema skips it. export const DEFAULT_MAX_RESPONSE_BYTES = 1_000_000; // What to steer a caller toward when their whole-repo request is too large. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index af0d9f5..a3808a6 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -469,6 +469,13 @@ export const TOOLS = [ // // Shapes are deliberately open (no `additionalProperties: false`): a later // engine adding a field must not turn a strict client's success into a failure. +// +// Every root is `type: "object"`, including the ones whose alternatives live in +// a `oneOf`. The spec types Tool.outputSchema as an object schema, and the +// official TypeScript SDK enforces it when it parses tools/list: a bare +// `{ oneOf: [...] }` root made its client reject the WHOLE list, so every +// SDK-based host saw zero tools. tests/mcp-output.test.ts pins this for every +// declared schema, including ones added later. const strArr = { type: "array", items: { type: "string" } }; const anyObj = { type: "object" }; @@ -511,6 +518,7 @@ export const OUTPUT_SCHEMAS: Record> = { }, // Two shapes, both objects: the whole index, or one symbol's entry. symbols: { + type: "object", oneOf: [ { type: "object", @@ -526,6 +534,7 @@ export const OUTPUT_SCHEMAS: Record> = { }, // The whole index (symbol name -> entry), one entry, or the not-found notice. callers: { + type: "object", oneOf: [ { type: "object", additionalProperties: anyObj }, { type: "object", properties: { def: anyObj, callers: { type: "array", items: anyObj }, lsp: anyObj }, required: ["def", "callers"] }, diff --git a/tests/mcp-output.test.ts b/tests/mcp-output.test.ts index 6f52de1..38cbb0f 100644 --- a/tests/mcp-output.test.ts +++ b/tests/mcp-output.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from "vitest"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { OUTPUT_SCHEMAS, structuredContentFor, toolsFor } from "../src/mcp.js"; +import { OUTPUT_SCHEMAS, PROTOCOL_VERSIONS, structuredContentFor, toolsFor } from "../src/mcp.js"; const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); const REPO = fileURLToPath(new URL("./fixtures/mini-repo", import.meta.url)); @@ -61,8 +61,12 @@ function validate(schema: Record, value: unknown, path = "$"): } // One server session, driven over the real stdio transport. -async function session(calls: { name: string; arguments: Record }[], version: string) { - const proc = spawn("node", [CLI, "mcp", "--repo", REPO], { stdio: ["pipe", "pipe", "ignore"] }); +async function session( + calls: { name: string; arguments: Record }[], + version: string, + flags: string[] = [], +) { + const proc = spawn("node", [CLI, "mcp", "--repo", REPO, ...flags], { stdio: ["pipe", "pipe", "ignore"] }); const pending = new Map) => void>(); let buf = ""; proc.stdout.on("data", (d: Buffer) => { @@ -92,11 +96,13 @@ async function session(calls: { name: string; arguments: Record await call("initialize", { protocolVersion: version, capabilities: {}, clientInfo: { name: "t", version: "1" } }); const tools = ((await call("tools/list", {})) as { result: { tools: Record[] } }).result.tools; const results: Record> = {}; + const ordered: Record[] = []; for (const c of calls) { - results[c.name] = ((await call("tools/call", { name: c.name, arguments: c.arguments })) as { result: Record }) - .result; + const result = ((await call("tools/call", { name: c.name, arguments: c.arguments })) as { result: Record }).result; + results[c.name] = result; + ordered.push(result); } - return { tools, results }; + return { tools, results, ordered }; } finally { proc.kill(); } @@ -188,6 +194,90 @@ describe("outputSchema / structuredContent", () => { }, 60_000); }); +// What the official TypeScript SDK client enforces on a tools/call result once +// the tool advertised an outputSchema (client/index.js callTool): a non-error +// result MUST carry structuredContent, and it MUST validate. Error results are +// exempt. Returns the violation, or undefined. +function sdkContractViolation(name: string, result: Record): string | undefined { + const schema = OUTPUT_SCHEMAS[name]; + if (!schema || result.isError === true) return undefined; + if (result.structuredContent === undefined) return `${name}: has an output schema but did not return structured content`; + return validate(schema, result.structuredContent); +} + +describe("SDK conformance of the advertised tool list", () => { + // The SDK's ToolSchema types inputSchema and outputSchema as + // `{ type: "object", properties?, required? }`. A root without `type: + // "object"` (the symbols/callers `oneOf` schemas once) makes its client + // reject the ENTIRE tools/list, so an SDK-based host sees no tools at all. + // Iterates the live list, so a tool added later is covered automatically. + it("roots every inputSchema and outputSchema at type: object, for every version and pin", () => { + for (const version of PROTOCOL_VERSIONS) { + for (const pin of [undefined, "/pinned/repo"]) { + const tools = toolsFor(pin, version) as { + name: string; + inputSchema: Record; + outputSchema?: Record; + }[]; + for (const tool of tools) { + for (const [kind, schema] of [["inputSchema", tool.inputSchema], ["outputSchema", tool.outputSchema]] as const) { + if (schema === undefined) continue; + const where = `${version} ${pin ?? "unpinned"} ${tool.name}.${kind}`; + expect(schema.type, where).toBe("object"); + const props = (schema.properties ?? {}) as Record; + expect(typeof props, where).toBe("object"); + // A required name with no property is a typo nobody can satisfy. + for (const key of (schema.required as string[] | undefined) ?? []) { + expect(Object.keys(props), `${where}: required \`${key}\``).toContain(key); + } + } + } + } + } + }); + + it("sends a capped response as a tool error without structuredContent, for every schema-declaring tool", async () => { + // A cap this small withholds every payload, so each call exercises the + // notice path. Before, the notice was a non-error result with no + // structuredContent — which an SDK client turns into a thrown -32600. + const names = Object.keys(CASES).filter((n) => !EDIT_TOOLS.includes(n) && n !== "write_memory" && n !== "delete_memory"); + const { ordered } = await session( + names.map((name) => ({ name, arguments: CASES[name]! })), + "2025-11-25", + ["--max-response-bytes", "40"], + ); + names.forEach((name, i) => { + const res = ordered[i] as Record; + expect(res.isError, name).toBe(true); + expect(res.structuredContent, name).toBeUndefined(); + const notice = JSON.parse((res.content as { text: string }[])[0]!.text) as { truncated: boolean; tool: string }; + expect(notice).toMatchObject({ truncated: true, tool: name }); + }); + }, 120_000); + + it("sends lookup misses as tool errors, keeping their { error } text", async () => { + const misses: [string, Record, string][] = [ + ["call_graph", { symbol: "NoSuchSymbolAnywhere" }, "no symbol named NoSuchSymbolAnywhere"], + ["type_hierarchy", { name: "NoSuchType" }, "no type named NoSuchType"], + ["implementations", { name: "NoSuchType" }, "no type named NoSuchType"], + ]; + const { ordered } = await session(misses.map(([name, args]) => ({ name, arguments: args })), "2025-11-25"); + misses.forEach(([name, , message], i) => { + const res = ordered[i] as Record; + expect(res.isError, name).toBe(true); + expect(res.structuredContent, name).toBeUndefined(); + expect(JSON.parse((res.content as { text: string }[])[0]!.text), name).toEqual({ error: message }); + expect(sdkContractViolation(name, res)).toBeUndefined(); + }); + }, 60_000); + + it("meets the SDK result contract on the ordinary answers too", async () => { + const names = Object.keys(CASES).filter((n) => !EDIT_TOOLS.includes(n)); + const { ordered } = await session(names.map((name) => ({ name, arguments: CASES[name]! })), "2025-06-18"); + names.forEach((name, i) => expect(sdkContractViolation(name, ordered[i] as Record)).toBeUndefined()); + }, 120_000); +}); + describe("structuredContentFor", () => { it("returns the parsed object for a schema-declaring tool", () => { expect(structuredContentFor('{"a":1}', false, true)).toEqual({ a: 1 }); From dfcced9e55b1b6f75d5035dfb49808b55dd8adad Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:21:29 +0000 Subject: [PATCH 047/130] feat(delta): serve the review over MCP and gate CI with --fail-on An agent that had just edited files over MCP had to shell out to the CLI to learn what its change broke. The new `delta` tool returns the same result (the diff read first, the session's freshness-proven graph after), with `concise` dropping hunks, `limit` keeping the riskiest modules and saying it truncated, and `format: "text"` returning the panel; it sits in the impact and risk profiles. The CLI gains `--fail-on HIGH|MEDIUM|LOW`, which exits 1 like `rules` does once a module reaches the bucket. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 17 ++++++++--- src/engine-cli.ts | 21 ++++++++++--- src/mcp.ts | 24 ++++++++++++++- src/mcp/concise.ts | 16 ++++++++++ src/mcp/protocol.ts | 1 + src/mcp/tools.ts | 27 +++++++++++++--- tests/delta.test.ts | 66 +++++++++++++++++++++++++++++++++++++++- tests/mcp-output.test.ts | 2 +- tests/mcp.test.ts | 2 +- 9 files changed, 158 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a93bbc5..6877cbc 100644 --- a/README.md +++ b/README.md @@ -481,8 +481,15 @@ every point comes with the reason that fired it. ```sh codeindex delta --repo . # the branch vs its merge-base with the default branch codeindex delta --repo . --staged --json # the staged changeset, as JSON +codeindex delta --repo . --fail-on HIGH # CI gate: exit 1 when a module scores HIGH ``` +The MCP `delta` tool answers the same question for an agent that has just +edited files: `{base?, staged?, depth?}` return the JSON result, `concise` +drops the hunks and reduces each enclosing symbol to `name/kind/line`, `limit` +keeps the highest-scoring modules (and says it truncated), and +`format: "text"` returns the panel. It is in the `impact` and `risk` profiles. + - **A removed file that is still imported is the highest-weighted signal** (`brokenImport`, 40). The worktree's graph no longer holds a deleted or renamed file, so delta puts the removed paths back and re-resolves the @@ -814,13 +821,13 @@ Register it in Claude Code with: claude mcp add codeindex -- codeindex mcp ``` -**33 tools**, grouped by what they answer: +**34 tools**, grouped by what they answer: | group | tools | |---|---| | orient | `scan_summary`, `onboard` *(write)*, `repo_map`, `graph`, `mermaid`, `workspaces` | | find | `search`, `explain_search`, `grep`, `find_symbol`, `symbols`, `symbols_overview` | -| impact | `find_references`, `callers`, `call_graph`, `dead_code` | +| impact | `find_references`, `callers`, `call_graph`, `dead_code`, `delta` | | types | `type_hierarchy`, `implementations` | | risk | `hotspots`, `churn`, `coupling`, `complexity`, `check_rules`, `duplicated_literals` | | edit *(write)* | `replace_symbol_body`, `insert_after_symbol`, `insert_before_symbol` | @@ -835,7 +842,7 @@ of rebuilding. ### Smaller read responses MCP `find_symbol`, `find_references`, `callers`, `symbols_overview` and `symbols` -accept `concise: true`. Declarations are reduced to `name/kind/file/line` while +accept `concise: true` (and `delta`, where it drops each change's hunks). Declarations are reduced to `name/kind/file/line` while result membership, order, reference groups, call-site locations, confidence labels and LSP metadata stay intact. Defaults retain their full existing shape. `symbols` keeps its name-keyed groups and references for full-index requests. @@ -902,7 +909,7 @@ introduced are only sent to clients that asked for it, so an older client sees exactly what it saw before. From `2025-03-26` every tool carries behaviour annotations — `readOnlyHint` on -the 27 read tools, `destructiveHint`/`idempotentHint` on the six that write — +the 28 read tools, `destructiveHint`/`idempotentHint` on the six that write — which is what lets a host auto-approve reads and confirm only writes. From `2025-06-18`, the 20 tools whose result is always a JSON object also declare an `outputSchema` and return `structuredContent`, so a client can validate and type @@ -980,7 +987,7 @@ dates in one table, said out loud rather than implied._ | language coverage | 16 regex extractors, 21 tree-sitter grammars | **~40**, generic parser rules | any language with an LSP server | 36 via tree-sitter | **ctags / Serena** | | type-aware references | opt-in LSP tier, annotating the static answer | none | **native** | none | **Serena** | | install footprint | **23.5 MB, zero runtime deps** | single binary | 114.3 MB venv + language servers | 140.1 MB Python venv | **ctags** | -| MCP server | **33 tools**, subsettable by profile | none | yes, LSP-backed | yes | **codeindex** | +| MCP server | **34 tools**, subsettable by profile | none | yes, LSP-backed | yes | **codeindex** | | onboarding brief | `onboard`, one call, persisted as a memory | none | `onboarding` | none | tie | | says when a query matched nothing | **verdict on every search** (`match`/`weak`/`none`) | no | not measured | not measured | — | diff --git a/src/engine-cli.ts b/src/engine-cli.ts index 21c196b..7994b82 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -134,8 +134,9 @@ Commands: delta Review panel for the git diff: changed files -> enclosing symbols -> blast radius -> risk score with explained reasons; a deleted or renamed file that is still imported is listed under \`broken\` - with its importers (--base | --staged, --depth , --json). - Paths under the --index directory are not part of the review + with its importers (--base | --staged, --depth , --json, + --fail-on HIGH|MEDIUM|LOW to exit 1 as a CI gate). Paths under + the --index directory are not part of the review impact Reverse dependency closure of a file or module: everything that transitively imports/uses/calls it (--depth ; JSON) neighbors Graph neighbours of a file or module, both directions @@ -147,12 +148,12 @@ Commands: and exits 0, or exits 1 when it has no opinion (run the original). Deliberately conservative — any shell metacharacter or unknown flag refuses the rewrite - mcp Run as an MCP server over stdio (33 tools: scan_summary, graph, + mcp Run as an MCP server over stdio (34 tools: scan_summary, graph, symbols, callers, workspaces, churn, symbols_overview, find_symbol, find_references, lsp_status, onboard, repo_map, hotspots, coupling, dead_code, complexity, mermaid, grep, search, - explain_search, embed_status, check_rules, the memory quartet and - the three symbolic-edit writes). Flags: --repo pins ONE + explain_search, embed_status, check_rules, delta, the memory + quartet and the three symbolic-edit writes). Flags: --repo pins ONE repository so the per-tool repo argument becomes optional (an explicit per-call repo still wins); --server-name overrides the announced serverInfo; --max-response-bytes caps a single @@ -277,6 +278,7 @@ interface CliFlags { direction?: "out" | "in" | "both"; // callgraph: which way to walk rank?: "graph" | "lexical"; // search: structural prior (default lexical) json?: boolean; // delta: emit JSON instead of the human panel + failOn?: "HIGH" | "MEDIUM" | "LOW"; // delta: exit 1 when a module reaches this bucket positional?: string; // e.g. the grep pattern or search query } @@ -353,6 +355,11 @@ function parseFlags(args: string[]): CliFlags { flags.direction = v; } else if (a === "--json") flags.json = true; + else if (a === "--fail-on") { + const v = next().toUpperCase(); + if (v !== "HIGH" && v !== "MEDIUM" && v !== "LOW") throw new Error(`--fail-on expects HIGH, MEDIUM or LOW, got "${v}"`); + flags.failOn = v; + } else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a; else throw new Error(`unknown flag: ${a}`); } @@ -388,6 +395,8 @@ function scanOptions(flags: CliFlags, precomputedWalk?: WalkResult): BuildIndexO // excluded by the positional check at the warm site. `grammars` (status/pull) // resolves/downloads the wasms itself and must not warm them. // version/help/mcp return before we get there. +const bucketRank = (b: "HIGH" | "MEDIUM" | "LOW"): number => (b === "HIGH" ? 2 : b === "MEDIUM" ? 1 : 0); + const SCANLESS_COMMANDS = new Set(["grep", "churn", "workspaces", "grammars"]); // Flags for `codeindex mcp`. Kept separate from parseFlags on purpose (see the @@ -1143,6 +1152,8 @@ export async function runCli(rawArgv: string[]): Promise { res = deltaOfDiff(diff, graph, symbols, { ...opts, scan }); } emit(flags.json ? JSON.stringify(res, null, 2) + "\n" : formatDeltaPanel(res), flags.out); + // The CI gate, like `rules`: the output is written either way. + if (flags.failOn && res.modules.some((m) => bucketRank(m.bucket) >= bucketRank(flags.failOn!))) process.exitCode = 1; } else if (cmd === "impact") { if (!flags.positional) throw new Error("impact needs a target: cli.mjs impact --repo "); const { graph } = await readArtifacts(); diff --git a/src/mcp.ts b/src/mcp.ts index 789b168..594f9b1 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -29,12 +29,13 @@ import { symbolComplexity, riskHotspots } from "./complexity.js"; import { renderMermaid } from "./viz.js"; import { symbolsOverview, findSymbol, findReferences } from "./query.js"; import { lspStatus, referencesWithLsp, callersWithLsp } from "./lsp/index.js"; -import { conciseCaller, conciseReferences, conciseSymbolIndex, symbolLocation } from "./mcp/concise.js"; +import { conciseCaller, conciseDelta, conciseReferences, conciseSymbolIndex, symbolLocation } from "./mcp/concise.js"; import { onboardBrief } from "./onboard.js"; import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit.js"; import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js"; import { explainQuery, searchIndex, type RankMode } from "./bm25.js"; import { checkRules, parseRules } from "./rules.js"; +import { deltaOfDiff, emptyDelta, formatDeltaPanel, readDeltaDiff } from "./delta.js"; import { EMBED_VERSION, resolveEmbedModelDir } from "./embed/model.js"; import { buildEmbeddingIndex } from "./embed/index.js"; import { searchSemantic } from "./embed/search.js"; @@ -539,6 +540,27 @@ async function callTool(name: string, args: Record, defaultRepo const { graph } = readArtifacts(); return JSON.stringify(checkRules(graph, rules), null, 2); } + if (name === "delta") { + // The CLI's review panel, for an agent that just edited files over this + // server. The session scan is re-proven fresh on every call, so the graph + // is the worktree's as it sits now. The diff is read first: when it is + // empty the artifacts are not needed. + const diff = readDeltaDiff(repo, { base: str(args.base), staged: args.staged === true }); + if ("error" in diff) throw new Error(diff.error); + const depth = positiveNum(args.depth); + let res = emptyDelta(diff, depth); + if (diff.files.length) { + const { scan, graph, symbols } = readArtifacts(); + res = deltaOfDiff(diff, graph, symbols, { depth, scan }); + } + if (str(args.format) === "text") return formatDeltaPanel(res); + const out = args.concise === true ? conciseDelta(res) : res; + const limit = positiveNum(args.limit); + // Modules are ranked highest score first, so a cap keeps the riskiest; it + // says so, same doctrine as dead_code. + if (limit === undefined || out.modules.length <= limit) return JSON.stringify(out, null, 2); + return JSON.stringify({ ...out, modules: out.modules.slice(0, limit), totalModules: out.modules.length, truncated: true }, null, 2); + } throw new Error(`unknown tool: ${name}`); } diff --git a/src/mcp/concise.ts b/src/mcp/concise.ts index fe8add3..f0a8948 100644 --- a/src/mcp/concise.ts +++ b/src/mcp/concise.ts @@ -3,6 +3,7 @@ import type { CodeSymbol, SymbolIndex } from "../types.js"; import type { CallerEntry } from "../callers.js"; import type { SymbolReferences } from "../query.js"; +import type { DeltaChange, DeltaResult } from "../delta.js"; export type SymbolLocation = Pick; @@ -24,3 +25,18 @@ export function conciseSymbolIndex(index: SymbolIndex) { defs: Object.fromEntries(Object.entries(index.defs).map(([name, defs]) => [name, defs.map((s) => symbolLocation(s, name))])), }; } + +// A review's changes without their hunks and line counts, each enclosing symbol +// reduced to where it is. Modules, reasons and broken imports are untouched. +export function conciseDelta(res: T) { + return { + ...res, + changes: res.changes.map((c: DeltaChange) => ({ + path: c.path, + status: c.status, + ...(c.oldPath !== undefined ? { oldPath: c.oldPath } : {}), + ...(c.module !== undefined ? { module: c.module } : {}), + symbols: c.symbols.map((s) => ({ name: s.name, kind: s.kind, line: s.line })), + })), + }; +} diff --git a/src/mcp/protocol.ts b/src/mcp/protocol.ts index e190943..9995c77 100644 --- a/src/mcp/protocol.ts +++ b/src/mcp/protocol.ts @@ -134,6 +134,7 @@ const NARROWER: Record = { dead_code: "pass `scope` to a subdirectory", find_references: "the symbol is referenced very widely — narrow with `scope` on a graph query", check_rules: "narrow the rule set, or pass `scope` to a subdirectory", + delta: 'pass `concise: true` (no hunks), a `limit` on modules, or `format: "text"` for the panel', }; // The persisted artifact backing a tool, when a `codeindex index` already wrote diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index b64dc69..d5f9188 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -1,4 +1,4 @@ -// The MCP tool catalogue: the 29 tool definitions, their display metadata, and +// The MCP tool catalogue: the tool definitions, their display metadata, and // the per-protocol-version view of the list a client actually receives. // // Split out of mcp.ts because it is pure data plus one projection function — @@ -458,6 +458,24 @@ export const TOOLS = [ required: ["repo"], }, }, + { + name: "delta", + description: + "What does my change break? Maps the git diff (the branch against its merge-base with the default branch, uncommitted and untracked work included; or the staged changeset) onto the graph: each changed file with the symbols enclosing its hunks, and per module a 0-100 risk score (HIGH/MEDIUM/LOW) in which every point comes with its reason — exported API changed, hub, blast radius, test gap, a deleted or renamed file that is still imported (`broken`, with its importers), dangling imports. `open` names the files to read first. Call it after editing.", + inputSchema: { + type: "object", + properties: { + ...repoProp, + ...conciseProp, + base: { type: "string", description: "Branch or ref to review against (default: origin/HEAD, origin/main, origin/master, main, master; else HEAD)" }, + staged: { type: "boolean", description: "Review the staged changeset against HEAD instead (default false)" }, + depth: { type: "number", minimum: 1, description: "Blast-radius hops (default 2)" }, + limit: { type: "number", minimum: 1, description: "Max modules, highest score first (default: all)" }, + format: { type: "string", enum: ["json", "text"], description: '"text" returns the compact human panel instead of JSON (default "json")' }, + }, + required: ["repo"], + }, + }, ] as const; @@ -477,7 +495,7 @@ export const TOOLS = [ // option that breaks neither. // * argument-dependent shapes — dead_code (array, object with `limit`), // complexity (array, object with `risk`), search (array, object with -// `semantic` or `explain`). A schema that cannot describe every response is +// `semantic` or `explain`), delta (object, text with `format: "text"`). A schema that cannot describe every response is // worse than none: it would make a conforming client reject valid output. // `explain_search` exists precisely because of this rule — it is the same // answer with ONE shape, so it can carry a schema where `search` cannot. @@ -722,6 +740,7 @@ export const TOOL_META: Record = { implementations: { title: "Implementations" }, call_graph: { title: "Call graph neighborhood" }, check_rules: { title: "Check architecture rules" }, + delta: { title: "Review the diff" }, }; export function annotationsFor(name: string): Record | undefined { @@ -759,11 +778,11 @@ export const TOOL_PROFILES: Record = { // Locate a thing. find: ["search", "explain_search", "grep", "find_symbol", "symbols", "symbols_overview"], // Decide whether changing it is safe. - impact: ["find_references", "callers", "call_graph", "dead_code", "type_hierarchy", "implementations", "lsp_status"], + impact: ["find_references", "callers", "call_graph", "dead_code", "type_hierarchy", "implementations", "lsp_status", "delta"], // Change it. edit: ["find_symbol", "symbols_overview", "replace_symbol_body", "insert_after_symbol", "insert_before_symbol"], // Where the work and the risk concentrate. - risk: ["hotspots", "churn", "coupling", "complexity", "check_rules", "duplicated_literals", "dead_code"], + risk: ["hotspots", "churn", "coupling", "complexity", "check_rules", "duplicated_literals", "dead_code", "delta"], }; export function profileNames(): string[] { diff --git a/tests/delta.test.ts b/tests/delta.test.ts index 56dfef2..501f742 100644 --- a/tests/delta.test.ts +++ b/tests/delta.test.ts @@ -1,14 +1,18 @@ // `delta` against real temporary git repositories: what a removal breaks, what // the diff side must ignore, and the CI gate. -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { brokenImports, computeDelta, deltaFor, deltaOfDiff, emptyDelta, formatDeltaPanel, readDeltaDiff, RISK_WEIGHTS } from "../src/delta.js"; import type { DeltaResult } from "../src/delta.js"; import { buildIndexArtifacts } from "../src/pipeline.js"; +const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); +const clientModule = new URL("../scripts/bench/mcp-client.mjs", import.meta.url).href; + function git(dir: string, args: string[]): string { return execFileSync("git", ["-C", dir, "-c", "commit.gpgsign=false", ...args], { encoding: "utf8", @@ -231,3 +235,63 @@ describe("delta: symbol attribution reads only the changed files' defs", () => { } }); }); + +describe("delta: the CI gate", () => { + it("--fail-on exits 1 once a module reaches the bucket, and writes the panel either way", () => { + const root = repo(HUB_REPO); + try { + rmSync(join(root, "lib/hub.ts")); // lib scores >= brokenImport (40): MEDIUM at least + const run = (...args: string[]) => spawnSync(process.execPath, [CLI, "delta", "--repo", root, ...args], { encoding: "utf8" }); + const plain = run(); + expect(plain.status).toBe(0); + const lib = (JSON.parse(run("--json").stdout) as DeltaResult).modules.find((m) => m.slug === "lib")!; + expect(lib.bucket).not.toBe("LOW"); + const gated = run("--fail-on", "medium"); + expect(gated.status).toBe(1); + expect(gated.stdout).toBe(plain.stdout); + expect(run("--fail-on", "HIGH").status).toBe(lib.bucket === "HIGH" ? 1 : 0); + expect(run("--fail-on", "urgent").stderr).toMatch(/--fail-on expects HIGH, MEDIUM or LOW/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("delta over MCP", () => { + it("returns the review as JSON, concise, capped or as the panel", async () => { + const root = repo(HUB_REPO); + const { startMcpClient } = await import(/* @vite-ignore */ clientModule); + const client = startMcpClient(process.execPath, [CLI, "mcp", "--repo", root, "--tools", "risk"], { timeoutMs: 30_000 }); + try { + expect((await client.handshake()).ok).toBe(true); + const listed = await client.request("tools/list", {}); + expect(listed.result.tools.map((t: { name: string }) => t.name)).toContain("delta"); + const call = async (args: Record): Promise => { + const r = await client.request("tools/call", { name: "delta", arguments: args }); + expect(r.result.isError, JSON.stringify(r.result)).not.toBe(true); + return r.result.content[0].text as string; + }; + expect(JSON.parse(await call({}))).toMatchObject({ changes: [], modules: [] }); + + rmSync(join(root, "lib/hub.ts")); + write(root, { "app/b.ts": 'import { hub } from "../lib/hub";\nexport const b = hub() * 2;\n' }); + const full = JSON.parse(await call({})) as DeltaResult; + expect(full).toEqual(delta(root)); + expect(full.modules.map((m) => m.slug)).toEqual(["lib", "app"]); + + const concise = JSON.parse(await call({ concise: true })); + expect(concise).toEqual({ + ...full, + changes: [ + { path: "app/b.ts", status: "modified", module: "app", symbols: [{ name: "b", kind: "const", line: 2 }] }, + { path: "lib/hub.ts", status: "deleted", symbols: [] }, + ], + }); + expect(JSON.parse(await call({ limit: 1 }))).toEqual({ ...full, modules: full.modules.slice(0, 1), totalModules: 2, truncated: true }); + expect(await call({ format: "text" })).toBe(formatDeltaPanel(full)); + } finally { + await client.close(); + rmSync(root, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/tests/mcp-output.test.ts b/tests/mcp-output.test.ts index 6f52de1..a732fbc 100644 --- a/tests/mcp-output.test.ts +++ b/tests/mcp-output.test.ts @@ -140,7 +140,7 @@ describe("outputSchema / structuredContent", () => { // The tools deliberately left out: array responses, argument-dependent // shapes, and plain text. Pinned so a future "just add a schema" does not // silently start emitting a structuredContent that cannot conform. - for (const name of ["symbols_overview", "find_symbol", "grep", "check_rules", "list_memories", "dead_code", "complexity", "search", "repo_map", "mermaid", "read_memory"]) { + for (const name of ["symbols_overview", "find_symbol", "grep", "check_rules", "list_memories", "dead_code", "complexity", "search", "repo_map", "mermaid", "read_memory", "delta"]) { expect(OUTPUT_SCHEMAS[name], name).toBeUndefined(); } }, 60_000); diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index 0e4e90a..8616ce3 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -362,7 +362,7 @@ describe("MCP server", () => { expect(res.get(1)!.result!.serverInfo!.name).toBe("codeindex"); const toolNames = res.get(2)!.result!.tools!.map((t) => t.name); - expect(toolNames).toEqual(["scan_summary", "graph", "symbols", "callers", "workspaces", "churn", "symbols_overview", "find_symbol", "find_references", "lsp_status", "onboard", "repo_map", "hotspots", "coupling", "replace_symbol_body", "insert_after_symbol", "insert_before_symbol", "write_memory", "read_memory", "list_memories", "delete_memory", "dead_code", "duplicated_literals", "complexity", "mermaid", "grep", "search", "explain_search", "embed_status", "type_hierarchy", "implementations", "call_graph", "check_rules"]); + expect(toolNames).toEqual(["scan_summary", "graph", "symbols", "callers", "workspaces", "churn", "symbols_overview", "find_symbol", "find_references", "lsp_status", "onboard", "repo_map", "hotspots", "coupling", "replace_symbol_body", "insert_after_symbol", "insert_before_symbol", "write_memory", "read_memory", "list_memories", "delete_memory", "dead_code", "duplicated_literals", "complexity", "mermaid", "grep", "search", "explain_search", "embed_status", "type_hierarchy", "implementations", "call_graph", "check_rules", "delta"]); const summary = JSON.parse(res.get(3)!.result!.content![0]!.text) as { fileCount: number }; expect(summary.fileCount).toBeGreaterThan(0); From 13fd914c501773d58a4333099d1c1dd26b6075ab Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:21:57 +0000 Subject: [PATCH 048/130] fix(mcp): validate every tools/call before the repo is walked Validation looked the declaration up in the profile-filtered advertised list, so under `--tools find` a call_graph with `depth: "abc"` or a dead_code with `limit: "x"` silently ran with defaults. Calls are now checked against the complete tool list (shaped by the pin only). Unknown tools, missing required arguments and bad types were reported only after callTool had walked and scanned the repo: 13.5 s for a missing `namePath` on the first call against microsoft/TypeScript. validateArgs now checks the declared `required` list, and all request-only checks run before callTool (about 1 ms there). A tools/call whose `name` is not a string or whose `arguments` is not an object is a -32602 protocol error, as the SDK server answers it, instead of running the tool with none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/mcp.ts | 38 +++++++++++++++++++++---- src/mcp/protocol.ts | 17 +++++++++--- tests/mcp.test.ts | 68 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/src/mcp.ts b/src/mcp.ts index 041a743..ab60744 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -578,6 +578,18 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise { // Rebuilt when negotiation lands: the pin cannot change mid-session, but the // fields we are allowed to advertise depend on the version. let tools = toolsFor(opts.defaultRepo, protocolVersion, opts.profile); + // What a call is validated against: EVERY tool, whatever the profile. A + // profile trims what is advertised, not what is answerable, so a tool called + // by name from outside it must be checked like any other — looking it up in + // the advertised list found nothing and silently skipped validation. The + // pin is what shapes `required` (it drops `repo`); the protocol version + // never touches an inputSchema, so one map serves the whole session. + const callable = new Map( + (toolsFor(opts.defaultRepo) as { name: string; inputSchema: Parameters[0] }[]).map((t) => [ + t.name, + t.inputSchema, + ]), + ); let watcher: FSWatcher | undefined; if (opts.watch && opts.defaultRepo) { try { @@ -681,13 +693,27 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise { return respond({ result: { tools } }); } else if (req.method === "tools/call") { const params = req.params ?? {}; - const name = str(params.name) ?? ""; - const args = (params.arguments ?? {}) as Record; + // A call whose params do not have the CallToolRequest shape is a + // malformed request — a protocol error, as the SDK server answers it. + // `arguments: "xyz"` used to run the tool with no arguments at all. + const rawArgs = params.arguments; + if (typeof params.name !== "string") { + return respond({ error: { code: -32602, message: "invalid params: tools/call requires a string `name`" } }); + } + if (rawArgs !== undefined && rawArgs !== null && (typeof rawArgs !== "object" || Array.isArray(rawArgs))) { + return respond({ error: { code: -32602, message: "invalid params: tools/call `arguments` must be an object" } }); + } + const name = params.name; + const args = (rawArgs ?? {}) as Record; try { - const decl = (tools as { name: string; inputSchema: { properties?: Record } }[]).find( - (t) => t.name === name, - ); - const invalid = decl ? validateArgs(decl.inputSchema, args) : undefined; + // Everything checkable from the request alone is checked BEFORE + // callTool, which walks and scans the repo first: an unknown tool or + // a missing argument used to cost a full walk to report. An unknown + // tool stays a tool error rather than -32602 — what the reference + // SDK server puts on the wire, and what clients already handle. + const schema = callable.get(name); + if (!schema) throw new Error(`unknown tool: ${name}`); + const invalid = validateArgs(schema, args); if (invalid) throw new Error(invalid); const raw = await callTool(name, args, opts.defaultRepo); const repo = str(args.repo) ?? opts.defaultRepo ?? ""; diff --git a/src/mcp/protocol.ts b/src/mcp/protocol.ts index 0835892..82b7cb3 100644 --- a/src/mcp/protocol.ts +++ b/src/mcp/protocol.ts @@ -37,11 +37,14 @@ export const RICH_TOOLS_SINCE = "2025-06-18"; // Tool.title, resource_link conte // JSON Schema implementation. The spec (2025-11-25) is explicit that input // validation failures belong in a Tool Execution Error, not a protocol error, // precisely so the model can read the message and retry. -// Required-ness stays with callTool, which raises tool-specific messages -// ("`rules` (or `configPath`) is required"); duplicating it here would only let -// the two drift. +// +// The declared `required` list is checked here too, because this runs BEFORE +// the call walks and scans the repo: a missing `namePath` used to be reported +// only after a full walk (13.5 s for the first call on a 66k-file repo). +// Requirements a schema cannot express — `rules` or `configPath`, `lsp` needing +// `name` — stay with callTool and its tool-specific messages. export function validateArgs( - schema: { properties?: Record }, + schema: { properties?: Record; required?: readonly string[] }, args: Record, ): string | undefined { const props = (schema.properties ?? {}) as Record; + for (const key of schema.required ?? []) { + if (args[key] !== undefined && args[key] !== null) continue; + const description = props[key]?.description; + return description ? `\`${key}\` is required (${description})` : `\`${key}\` is required`; + } for (const [key, value] of Object.entries(args)) { if (value === undefined || value === null) continue; const spec = props[key]; diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index 0e4e90a..fd03b0f 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -1503,6 +1503,74 @@ describe("validateArgs", () => { it("ignores null, undefined and undeclared extras", () => { expect(validateArgs(schema, { limit: undefined, substring: null, future: "whatever" })).toBeUndefined(); }); + + it("checks the declared required list, naming the argument and what it is for", () => { + const required = { + properties: { repo: { type: "string", description: "Absolute path to the repository root" }, namePath: { type: "string" } }, + required: ["repo", "namePath"], + }; + expect(validateArgs(required, { namePath: "A" })).toBe("`repo` is required (Absolute path to the repository root)"); + expect(validateArgs(required, { repo: "/x", namePath: null })).toBe("`namePath` is required"); + expect(validateArgs(required, { repo: "/x", namePath: "A" })).toBeUndefined(); + }); +}); + +// Everything checkable from the request alone used to be checked only after +// callTool had walked and scanned the repo — 13.5 s to learn that `namePath` +// was missing on a 66k-file repo. A repository that does not exist proves the +// order: callTool's first act is to stat it, so any answer OTHER than "not a +// readable directory" was produced before the repo was touched. +describe("tools/call is validated before the repo is touched", () => { + const missingRepo = join(tmpdir(), "codeindex-never-created"); + + it("rejects unknown tools, missing required arguments and bad types without a scan", async () => { + const res = await mcpSession([ + { id: 1, method: "tools/call", params: { name: "nope", arguments: { repo: missingRepo } } }, + { id: 2, method: "tools/call", params: { name: "find_symbol", arguments: { repo: missingRepo } } }, + { id: 3, method: "tools/call", params: { name: "call_graph", arguments: { repo: missingRepo, symbol: "A", depth: "abc" } } }, + { id: 4, method: "tools/call", params: { name: "find_symbol", arguments: { repo: missingRepo, namePath: "A" } } }, + ]); + const text = (id: number) => res.get(id)!.result!.content![0]!.text; + for (const id of [1, 2, 3, 4]) expect(res.get(id)!.result!.isError, String(id)).toBe(true); + expect(text(1)).toBe("unknown tool: nope"); + expect(text(2)).toMatch(/^`namePath` is required/); + expect(text(3)).toMatch(/`depth` must be a number/); + // The control: a well-formed call does reach the repo check. + expect(text(4)).toMatch(/not a readable directory/); + }, 20_000); + + it("answers a tools/call whose params are malformed with -32602", async () => { + const res = await mcpSession([ + { id: 1, method: "tools/call", params: { name: "scan_summary", arguments: "xyz" } }, + { id: 2, method: "tools/call", params: { name: "scan_summary", arguments: [REPO] } }, + { id: 3, method: "tools/call", params: {} }, + { id: 4, method: "tools/call", params: { name: "scan_summary", arguments: { repo: REPO } } }, + ]); + for (const id of [1, 2, 3]) { + expect(res.get(id)!.result, String(id)).toBeUndefined(); + expect(res.get(id)!.error!.code, String(id)).toBe(-32602); + } + expect(res.get(4)!.result!.isError).toBeUndefined(); + }, 20_000); + + it("validates a tool left out of the active profile like any other", async () => { + const res = await mcpSession( + [ + { id: 1, method: "tools/call", params: { name: "call_graph", arguments: { repo: REPO, symbol: "HttpClient", depth: "abc" } } }, + { id: 2, method: "tools/call", params: { name: "call_graph", arguments: { repo: REPO, symbol: "HttpClient", depth: 9 } } }, + { id: 3, method: "tools/call", params: { name: "dead_code", arguments: { repo: REPO, limit: "x" } } }, + { id: 4, method: "tools/call", params: { name: "write_memory", arguments: { repo: REPO, content: "x" } } }, + ], + undefined, + [CLI, "mcp", "--tools", "find"], + ); + const text = (id: number) => res.get(id)!.result!.content![0]!.text; + for (const id of [1, 2, 3, 4]) expect(res.get(id)!.result!.isError, String(id)).toBe(true); + expect(text(1)).toMatch(/`depth` must be a number/); + expect(text(2)).toMatch(/`depth` must be at most 5/); + expect(text(3)).toMatch(/`limit` must be a number/); + expect(text(4)).toMatch(/`name` is required/); + }, 20_000); }); // The playground indexed socialgouv/egapro on a guessed `master` while the From 91f5edc6c2560f10501174622ae74c6d73dc87e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:23:14 +0000 Subject: [PATCH 049/130] fix(mcp): look symbols and profiles up by own key only `symbols {name}` indexed plain objects with `defs[lookup] ?? []`, so Object.prototype names answered from the prototype: `toString` dropped defs/refs (failing the outputSchema), `concise` threw `defs.map is not a function`, and `__proto__` returned `{}` where arrays belong. The same lookup made `--tools constructor` crash instead of naming the unknown profile. Both now use Object.hasOwn. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- src/mcp.ts | 8 ++++++-- src/mcp/tools.ts | 4 +++- tests/mcp-concise.test.ts | 10 ++++++++++ tests/mcp.test.ts | 3 +++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/mcp.ts b/src/mcp.ts index ab60744..506b65d 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -204,8 +204,12 @@ async function callTool(name: string, args: Record, defaultRepo const { symbols } = readArtifacts(); const lookup = str(args.name); if (lookup) { - const defs = symbols.defs[lookup] ?? []; - return JSON.stringify({ name: lookup, defs: args.concise === true ? defs.map((s) => symbolLocation(s, lookup)) : defs, refs: symbols.refs[lookup] ?? [] }, null, 2); + // Own keys only: the index is a plain object, so `toString`, + // `constructor` or `__proto__` read straight off Object.prototype + // (a function, or `{}`) instead of the empty answer. + const defs = Object.hasOwn(symbols.defs, lookup) ? symbols.defs[lookup]! : []; + const refs = Object.hasOwn(symbols.refs, lookup) ? symbols.refs[lookup]! : []; + return JSON.stringify({ name: lookup, defs: args.concise === true ? defs.map((s) => symbolLocation(s, lookup)) : defs, refs }, null, 2); } return JSON.stringify(args.concise === true ? conciseSymbolIndex(symbols) : symbols, null, 2); } diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index a3808a6..f06fceb 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -769,7 +769,9 @@ export function toolsInProfiles(spec: string): Set { const out = new Set(); for (const name of names) { if (name === "all") return new Set(TOOLS.map((t) => t.name)); - const profile = TOOL_PROFILES[name]; + // Own keys only — `--tools constructor` must be an unknown profile, not + // Object.prototype.constructor failing to iterate. + const profile = Object.hasOwn(TOOL_PROFILES, name) ? TOOL_PROFILES[name] : undefined; if (!profile) throw new Error(`unknown tool profile "${name}" — one of: ${profileNames().join(", ")}`); for (const tool of profile) out.add(tool); } diff --git a/tests/mcp-concise.test.ts b/tests/mcp-concise.test.ts index c2ae3ba..45f84a9 100644 --- a/tests/mcp-concise.test.ts +++ b/tests/mcp-concise.test.ts @@ -87,4 +87,14 @@ describe("concise MCP read answers", () => { const full = await call("find_references", { name: "greet", lsp: true }); expect(await call("find_references", { name: "greet", lsp: true, concise: true })).toEqual({ ...full, defs: full.defs.map(location) }); }); + it("answers Object.prototype names as absent symbols, not prototype members", async () => { + // The index is a plain object: `defs.toString` used to be the inherited + // function (serialized away, or `defs.map is not a function` under + // concise) and `__proto__` answered `{}` where arrays belong. + for (const name of ["toString", "constructor", "__proto__", "hasOwnProperty"]) { + for (const concise of [false, true]) { + expect(await call("symbols", { name, concise }), `${name} concise=${concise}`).toEqual({ name, defs: [], refs: [] }); + } + } + }); }); diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index fd03b0f..fb125e8 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -1646,6 +1646,9 @@ describe("tool profiles and onboarding", () => { it("rejects an unknown profile at startup rather than advertising everything", () => { expect(() => parseMcpFlags(["--tools", "nonsense"])).toThrow(/unknown tool profile/); + // Own keys only: an Object.prototype member is not a profile. + expect(() => parseMcpFlags(["--tools", "constructor"])).toThrow(/unknown tool profile/); + expect(() => parseMcpFlags(["--tools", "toString"])).toThrow(/unknown tool profile/); // "all" is the default and must stay expressible. expect(parseMcpFlags(["--tools", "all"]).profile).toBeUndefined(); expect(parseMcpFlags(["--tools", "find,impact"]).profile).toBe("find,impact"); From 4f25b5179bbc37f66e4f4825887531b5b33a7469 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:25:26 +0000 Subject: [PATCH 050/130] perf(preload): read a fresh index one artifact at a time `graph` and `symbols` on a fresh index read, sha1'd and JSON.parsed BOTH artifacts, then re-rendered the one they print into the bytes already on disk; `rules`, `impact`, `neighbors`, `mermaid` and `repomap` parsed an 80MB symbols.json they never looked at. On typescript-go that was 8.4s for `graph` and 7.6s for `symbols`. The preloaded session now exposes the persisted artifacts one file at a time (persistedArtifacts): `bytes(name)` returns the sha-verified on-disk bytes, which the guard proves equal to the render of a fresh build, and `graph()`/`symbols()` parse a single file. `graph`/`symbols` print those bytes as they are and graph-only commands parse only graph.json. Anything the index cannot vouch for falls back to the old path. typescript-go: `graph` 8.4s -> 5.0s, `symbols` 7.6s -> 5.3s, output byte-identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 12 +++-- src/engine-cli.ts | 58 ++++++++++++-------- src/preload.ts | 120 ++++++++++++++++++++++++++++++------------ tests/preload.test.ts | 57 ++++++++++++++++++++ 4 files changed, 188 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index beeb605..d491f6e 100644 --- a/README.md +++ b/README.md @@ -392,11 +392,13 @@ codeindex literals --repo . # values with no single source of `index` keeps a `cache.json` next to the artifacts, and every read command reuses whatever sits in `--index` (default `.codeindex`; relative to the repo, or absolute): unchanged files skip extraction, and when nothing changed the -artifacts load instead of being rebuilt. The index dir itself is never scanned, -and `--out .` at the repo root skips only the artifacts it writes. A record is -reused only if it was extracted the way this run would extract it — the same -`--no-ast`/`--max-calls` setting and the same grammar per language — so -switching either, or pulling a grammar, re-extracts exactly the files it +artifacts load instead of being rebuilt — one file at a time: `graph` and +`symbols` print the sha-verified bytes on disk as they are, and a command that +needs only the graph never reads `symbols.json`. The index dir itself is never +scanned, and `--out .` at the repo root skips only the artifacts it writes. A +record is reused only if it was extracted the way this run would extract it — +the same `--no-ast`/`--max-calls` setting and the same grammar per language — +so switching either, or pulling a grammar, re-extracts exactly the files it affects. Freshness is keyed on `(size, mtime)`; for an edit that preserves both, `--full-hash` re-hashes every file and `--no-index-cache` ignores the cache altogether (for `index` too). Artifacts are replaced atomically (a temp diff --git a/src/engine-cli.ts b/src/engine-cli.ts index c0ab5f7..6236c3e 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -1,6 +1,6 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; -import { SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord } from "./types.js"; +import { SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord, type Graph } from "./types.js"; import { ENGINE_VERSION } from "./types.js"; import { CORE_GRAMMARS, @@ -19,7 +19,15 @@ import { renderSymbolsJson } from "./render/symbols-json.js"; import { renderScip } from "./render/scip.js"; import { normalizeScope, scanSummary, scanWalkOptions, type RepoScan } from "./scan.js"; import { scanRepoParallel } from "./pool.js"; -import { indexDirPath, preloadSessionLazy, readPersistedIndex, INDEX_DIR, type PersistedMeta } from "./preload.js"; +import { + indexDirPath, + preloadSessionLazy, + readPersistedIndex, + INDEX_DIR, + type ArtifactName, + type PersistedMeta, + type PreloadedSession, +} from "./preload.js"; import { compatibleEntries, extractionProfile, sameExtractionProfile } from "./cache.js"; import { walk, type WalkResult } from "./walk.js"; import { buildTypeHierarchy, implementationsOf } from "./relations.js"; @@ -363,7 +371,7 @@ function parseFlags(args: string[]): CliFlags { return flags; } -function emit(content: string, out?: string): void { +function emit(content: string | Uint8Array, out?: string): void { if (out) writeFileSync(out, content); else process.stdout.write(content); } @@ -644,13 +652,10 @@ export async function runCli(rawArgv: string[]): Promise { // output is unchanged either way. Resolved lazily and at most once: a command // uses either the scan or the artifacts, never both. const indexDir = flags.indexDir ?? INDEX_DIR; + type Preloaded = Pick; let preloadTried = false; - let preloadPromise: Promise<{ - scan: RepoScan; - arts?: IndexArtifacts; - loadArtifacts?: () => IndexArtifacts | undefined; - } | undefined> | undefined; - let preloaded: { scan: RepoScan; arts?: IndexArtifacts; loadArtifacts?: () => IndexArtifacts | undefined } | undefined; + let preloadPromise: Promise | undefined; + let preloaded: Preloaded | undefined; // A read command answering from a scan that kept no file says so once, as // `index` and `scan` do: an empty answer otherwise looks like "no match". let warnedEmpty = false; @@ -672,7 +677,7 @@ export async function runCli(rawArgv: string[]): Promise { warmPresentGrammars, indexDir, ).then((p) => { - if (p) preloaded = { scan: noteEmpty(p.scan), arts: p.arts, loadArtifacts: p.loadArtifacts }; + if (p) preloaded = { scan: noteEmpty(p.scan), arts: p.arts, loadArtifacts: p.loadArtifacts, artifacts: p.artifacts }; // The default location being empty is the normal first run; an index the // user NAMED being unusable is a mistake worth one line (a typo'd path // otherwise just looks like a slow command). @@ -703,6 +708,20 @@ export async function runCli(rawArgv: string[]): Promise { if (p) return (p.arts ??= p.loadArtifacts?.() ?? buildArtifactsFromScan(p.scan, scanOptions(flags, precomputedWalk))); return (readArtifactsPromise ??= readScan().then((scan) => buildArtifactsFromScan(scan, scanOptions(flags, precomputedWalk)))); }; + // A command that needs ONE artifact reads only that file of a fresh index: + // readArtifacts loads both, so `rules` or `impact` parsed an 80MB + // symbols.json they never looked at. Anything the persisted index cannot + // vouch for falls back to readArtifacts, unchanged. + const readGraph = async (): Promise => { + const p = await tryPreload(); + return p?.arts?.graph ?? p?.artifacts?.graph() ?? (await readArtifacts()).graph; + }; + // An artifact the command prints whole: the verified on-disk bytes ARE the + // render of a fresh build (see PersistedArtifacts.bytes), so they are written + // out as they are instead of being parsed and re-rendered: about a second + // each on typescript-go's 80MB symbols.json, plus the GC, for the same bytes. + const readArtifactBytes = async (name: ArtifactName): Promise => + (await tryPreload())?.artifacts?.bytes(name); if (cmd === "index") { if (!flags.out) throw new Error("index needs --out "); @@ -858,11 +877,9 @@ export async function runCli(rawArgv: string[]): Promise { }; emit(JSON.stringify(summary, null, 2) + "\n", flags.out); } else if (cmd === "graph") { - const { graph } = await readArtifacts(); - emit(renderGraphJson(graph), flags.out); + emit((await readArtifactBytes("graph")) ?? renderGraphJson(await readGraph()), flags.out); } else if (cmd === "symbols") { - const { symbols } = await readArtifacts(); - emit(renderSymbolsJson(symbols), flags.out); + emit((await readArtifactBytes("symbols")) ?? renderSymbolsJson((await readArtifacts()).symbols), flags.out); } else if (cmd === "scip") { const scan = await readScan(); const bytes = renderScip(scan, { projectRoot: flags.projectRoot }); @@ -1140,7 +1157,7 @@ export async function runCli(rawArgv: string[]): Promise { } else if (cmd === "rules") { if (!flags.config) throw new Error("rules needs --config "); const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8"))); - const { graph } = await readArtifacts(); + const graph = await readGraph(); const violations = checkRules(graph, rules); const errors = violations.filter((v) => v.severity === "error").length; emit(JSON.stringify({ errors, warnings: violations.length - errors, violations }, null, 2) + "\n", flags.out); @@ -1161,8 +1178,8 @@ export async function runCli(rawArgv: string[]): Promise { for (const k of [...churn.keys()].sort()) sorted[k] = churn.get(k)!; emit(JSON.stringify({ ok, churn: sorted }, null, 2) + "\n", flags.out); } else if (cmd === "repomap") { - const { scan, graph } = await readArtifacts(); - emit(renderRepoMap(scan, graph, { budgetTokens: flags.budgetTokens }), flags.out); + const graph = await readGraph(); + emit(renderRepoMap(await readScan(), graph, { budgetTokens: flags.budgetTokens }), flags.out); } else if (cmd === "hotspots") { const scan = await readScan(); const { churn, ok } = gitChurn(flags.repo, { since: flags.since }); @@ -1197,20 +1214,19 @@ export async function runCli(rawArgv: string[]): Promise { emit(flags.json ? JSON.stringify(res, null, 2) + "\n" : formatDeltaPanel(res), flags.out); } else if (cmd === "impact") { if (!flags.positional) throw new Error("impact needs a target: cli.mjs impact --repo "); - const { graph } = await readArtifacts(); + const graph = await readGraph(); const res = impactOf(graph, flags.positional, flags.depth ?? Infinity); if (!res) throw new Error(`no such file or module in the index: ${flags.positional}`); emit(JSON.stringify(res, null, 2) + "\n", flags.out); } else if (cmd === "neighbors") { if (!flags.positional) throw new Error("neighbors needs a target: cli.mjs neighbors --repo "); - const { graph } = await readArtifacts(); + const graph = await readGraph(); const kinds = flags.kind ? new Set(flags.kind.split(",").map((k) => k.trim()).filter(Boolean)) : undefined; const res = neighborsOf(graph, flags.positional, flags.depth ?? 1, kinds); if (!res) throw new Error(`no such file or module in the index: ${flags.positional}`); emit(JSON.stringify(res, null, 2) + "\n", flags.out); } else if (cmd === "mermaid") { - const { graph } = await readArtifacts(); - emit(renderMermaid(graph, { module: flags.positional }), flags.out); + emit(renderMermaid(await readGraph(), { module: flags.positional }), flags.out); } else if (cmd === "grep") { if (!flags.positional) throw new Error("grep needs a pattern: cli.mjs grep --repo "); // `--scope ` is documented as global sugar for `--include '/**'`; diff --git a/src/preload.ts b/src/preload.ts index e6d72c8..fbe29f5 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -73,6 +73,32 @@ export interface PreloadedSession { arts?: IndexArtifacts; /** Async preload callers defer the large graph/symbol JSON read until needed. */ loadArtifacts?: () => IndexArtifacts | undefined; + /** + * The same on-disk artifacts one at a time, for a caller that needs only one + * of them (or only its bytes). Absent when the freshness guard fails. + */ + artifacts?: PersistedArtifacts; +} + +export type ArtifactName = "graph" | "symbols"; + +// graph.json/symbols.json as far as the freshness guard vouches for them, read +// one file at a time and only when asked. +// +// `loadArtifacts` reads, sha1s and JSON.parses BOTH files, which is what a +// command using both needs. `codeindex symbols` on typescript-go (55MB graph, +// 80MB symbols) paid for the graph it never used, then re-rendered a +// byte-identical symbols.json out of the parse: 8.3s, where streaming the +// verified bytes needs a read and a sha. Nothing is retained between calls, so +// a long-lived holder (the MCP session) keeps no 100MB buffer alive. +export interface PersistedArtifacts { + // The artifact's on-disk bytes, when they are EXACTLY what rendering a fresh + // build here prints (renderGraphJson / renderSymbolsJson): the guard proves + // the build equal, and the sha proves these bytes are its render. undefined + // otherwise. + bytes(name: ArtifactName): Buffer | undefined; + graph(): Graph | undefined; + symbols(): SymbolIndex | undefined; } // A scan re-expressed as the `ScanOptions.cache` shape (the exact map the CLI @@ -135,24 +161,50 @@ export function readPersistedIndex( }; } +// One artifact's on-disk bytes, or undefined when it is missing or they are not +// the bytes cache.json recorded (tampered, partial, rewritten since). sha over +// the raw bytes; sha1(string) hashes the same UTF-8 bytes writeFileSync put on +// disk, so this equals the meta sha the CLI computed over the render. +function verifiedBytes(dir: string, name: ArtifactName, sha: string | undefined): Buffer | undefined { + if (sha === undefined) return undefined; + let bytes: Buffer; + try { + bytes = readFileSync(join(dir, `${name}.json`)); + } catch { + return undefined; // a sha'd artifact went missing since cache.json — rebuild + } + return sha1(bytes) === sha ? bytes : undefined; +} + +function parsed(bytes: Buffer | undefined): T | undefined { + if (!bytes) return undefined; + try { + const value = JSON.parse(bytes.toString("utf8")) as T; + return value.schemaVersion === SCHEMA_VERSION ? value : undefined; + } catch { + // Unreachable once the sha matched (the bytes are valid JSON this engine + // wrote), but the contract is "never throw" — degrade to a rebuild. + return undefined; + } +} + // The freshness guard, applied to a scan seeded from cache.json: // contentUnchanged proves this scan's records are the ones that built the // on-disk artifacts; engineVersion pins the version stamp graph.json embeds and // commit the HEAD it embeds; the sha checks prove the on-disk bytes ARE that // build's output. All true ⇒ graph.json/symbols.json are byte-equal to -// buildArtifactsFromScan(scan) run here, so deserialize them instead of -// rebuilding. Graph/SymbolIndex are pure JSON POJOs (no Map/Set/typed fields), -// so JSON.parse is a lossless round-trip — a schemaVersion assert is the only -// reconstruction needed. ANY failure — a stale scan, a version/commit/sha -// mismatch, a missing/corrupt/partial artifact, an unexpected schemaVersion — -// returns undefined so the caller rebuilds. NEVER throws (a corrupt artifact -// must degrade, not crash the caller). -export function preloadArtifacts( +// buildArtifactsFromScan(scan) run here and rendered. Graph/SymbolIndex are pure +// JSON POJOs (no Map/Set/typed fields), so JSON.parse is a lossless round-trip +// — a schemaVersion assert is the only reconstruction needed. undefined when +// the guard fails; a missing/corrupt/partial artifact, or an unexpected +// schemaVersion, makes that artifact's reads undefined. NEVER throws (a +// corrupt artifact must degrade, not crash the caller). +export function persistedArtifacts( repo: string, scan: RepoScan, meta: PersistedMeta, indexDir: string = INDEX_DIR, -): IndexArtifacts | undefined { +): PersistedArtifacts | undefined { if ( !scan.contentUnchanged || meta.engineVersion !== ENGINE_VERSION || @@ -163,29 +215,29 @@ export function preloadArtifacts( return undefined; } const dir = indexDirPath(repo, indexDir); - let graphBytes: Buffer; - let symbolsBytes: Buffer; - try { - graphBytes = readFileSync(join(dir, "graph.json")); - symbolsBytes = readFileSync(join(dir, "symbols.json")); - } catch { - return undefined; // a sha'd artifact went missing since cache.json — rebuild - } - // sha over the raw bytes; sha1(string) hashes the same UTF-8 bytes writeFileSync - // put on disk, so this equals the meta sha the CLI computed over the render. - if (sha1(graphBytes) !== meta.graphSha1 || sha1(symbolsBytes) !== meta.symbolsSha1) { - return undefined; // tampered / partial / corrupt on-disk bytes — rebuild - } - try { - const graph = JSON.parse(graphBytes.toString("utf8")) as Graph; - const symbols = JSON.parse(symbolsBytes.toString("utf8")) as SymbolIndex; - if (graph.schemaVersion !== SCHEMA_VERSION || symbols.schemaVersion !== SCHEMA_VERSION) return undefined; - return { scan, graph, symbols }; - } catch { - // Unreachable once the shas matched (the bytes are valid JSON this engine - // wrote), but the contract is "never throw" — degrade to a rebuild. - return undefined; - } + const sha = (name: ArtifactName): string | undefined => (name === "graph" ? meta.graphSha1 : meta.symbolsSha1); + return { + bytes: (name) => verifiedBytes(dir, name, sha(name)), + graph: () => parsed(verifiedBytes(dir, "graph", meta.graphSha1)), + symbols: () => parsed(verifiedBytes(dir, "symbols", meta.symbolsSha1)), + }; +} + +// Both artifacts at once (see persistedArtifacts): value-equal to +// buildArtifactsFromScan(scan) run here, or undefined so the caller rebuilds. +export function preloadArtifacts( + repo: string, + scan: RepoScan, + meta: PersistedMeta, + indexDir: string = INDEX_DIR, +): IndexArtifacts | undefined { + return bothArtifacts(scan, persistedArtifacts(repo, scan, meta, indexDir)); +} + +function bothArtifacts(scan: RepoScan, onDisk: PersistedArtifacts | undefined): IndexArtifacts | undefined { + const graph = onDisk?.graph(); + const symbols = graph ? onDisk?.symbols() : undefined; + return graph && symbols ? { scan, graph, symbols } : undefined; } // Seed a scan from cache.json and, when the guard holds, the artifacts from @@ -266,6 +318,7 @@ export async function preloadSessionLazy( cache = compatible(grammarReady); } const scan = await scanRepoParallel(repo, { ...scanOpts, workers, cache, precomputedWalk: walked }); + const onDisk = persistedArtifacts(repo, scan, persisted.meta, indexDir); let artifactsTried = false; let artifacts: IndexArtifacts | undefined; return { @@ -274,9 +327,10 @@ export async function preloadSessionLazy( loadArtifacts: () => { if (!artifactsTried) { artifactsTried = true; - artifacts = preloadArtifacts(repo, scan, persisted.meta, indexDir); + artifacts = bothArtifacts(scan, onDisk); } return artifacts; }, + ...(onDisk ? { artifacts: onDisk } : {}), }; } diff --git a/tests/preload.test.ts b/tests/preload.test.ts index 51d2fdc..2e10541 100644 --- a/tests/preload.test.ts +++ b/tests/preload.test.ts @@ -8,6 +8,9 @@ import { preloadSessionLazy, readPersistedIndex } from "../src/preload.js"; import { ensureGrammars, grammarKeysForExts } from "../src/ast/loader.js"; import { scanRepo } from "../src/scan.js"; import { walk } from "../src/walk.js"; +import { sha1 } from "../src/hash.js"; +import { renderMermaid } from "../src/viz.js"; +import type { Graph } from "../src/types.js"; const REPO = fileURLToPath(new URL("./fixtures/mini-repo", import.meta.url)); const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); @@ -174,6 +177,60 @@ describe("persisted-index reuse — output-identical", { timeout: 60_000 }, () = }); }); +// Rewrite one artifact of the primed index and record its new sha in +// cache.json, as `index` would have: the index still vouches for it. Bytes that +// are NOT what a fresh render prints then show which path a command took — +// passing the verified bytes through, or parsing and re-rendering them. +function doctor(repo: string, name: "graph" | "symbols", bytes: string): void { + const dir = join(repo, ".codeindex"); + writeFileSync(join(dir, `${name}.json`), bytes); + const cache = JSON.parse(readFileSync(join(dir, "cache.json"), "utf8")) as Record; + cache[`${name}Sha1`] = sha1(bytes); + writeFileSync(join(dir, "cache.json"), JSON.stringify(cache) + "\n"); +} +const artifact = (repo: string, name: string): string => readFileSync(join(repo, ".codeindex", `${name}.json`), "utf8"); + +// graph/symbols parsed BOTH artifacts of a fresh index, then re-rendered the +// one they print: 8.4s/7.6s on typescript-go for bytes already on disk (now +// 5.0s/5.3s, the rest being cache.json and the walk). +describe("a fresh index is read one artifact at a time", { timeout: 60_000 }, () => { + it("graph and symbols print the verified on-disk bytes as they are", () => { + withRepo((repo) => { + prime(repo); + for (const name of ["graph", "symbols"] as const) { + const compact = JSON.stringify(JSON.parse(artifact(repo, name))) + "\n"; + doctor(repo, name, compact); + expect(run(repo, [name]), name).toBe(compact); + } + }); + }); + + it("symbols never reads graph.json", () => { + withRepo((repo) => { + prime(repo); + rmSync(join(repo, ".codeindex", "graph.json")); + const compact = JSON.stringify(JSON.parse(artifact(repo, "symbols"))) + "\n"; + doctor(repo, "symbols", compact); + expect(run(repo, ["symbols"])).toBe(compact); + }); + }); + + it("graph-only commands never read symbols.json", () => { + withRepo((repo) => { + prime(repo); + rmSync(join(repo, ".codeindex", "symbols.json")); + // A graph the pipeline would not build here, so the answer shows where + // it came from. + const graph = JSON.parse(artifact(repo, "graph")) as Graph; + expect(graph.moduleEdges.length).toBeGreaterThan(0); + const doctored: Graph = { ...graph, moduleEdges: [] }; + doctor(repo, "graph", JSON.stringify(doctored, null, 2) + "\n"); + expect(run(repo, ["mermaid"])).toBe(renderMermaid(doctored)); + expect(run(repo, ["mermaid"])).not.toBe(run(repo, ["mermaid", "--no-index-cache"])); + }); + }); +}); + describe("lazy grammar warm on persisted indexes", { timeout: 30_000 }, () => { it("does not warm tree-sitter when the persisted scan is unchanged", async () => { await withRepoAsync(async (repo) => { From f169fa0177b644169fbfb2344586fed6723772e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:26:30 +0000 Subject: [PATCH 051/130] fix(mcp): normalize file arguments and enforce enum-like options symbols_overview/complexity/the edits matched `file` against the index verbatim, so `./gin.go`, an absolute path or `src\a.ts` answered `[]` (indistinguishable from "declares nothing") or "no symbol matches". The indexed spelling is still taken as is; other spellings are relativized, and a file the index does not hold is now a tool error with same-name suggestions instead of an empty success. `direction` (call_graph) and `rank` (search) declare an enum that validateArgs enforces: `direction: "sideways"` silently became "both" and `rank: "pagerank"` lexical. An untyped array (`rules`) is no longer reported as needing to be "an array of strings". A test pins that every input property of every tool has a type validateArgs checks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 8 +++++ src/mcp.ts | 41 +++++++++++++++++---- src/mcp/protocol.ts | 20 +++++++---- src/mcp/tools.ts | 3 +- tests/mcp-output.test.ts | 12 +++++++ tests/mcp.test.ts | 78 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2a6a7d8..f58c0f0 100644 --- a/README.md +++ b/README.md @@ -758,6 +758,14 @@ claude mcp add codeindex -- codeindex mcp and persists it as the `onboarding` memory, so the second session reads instead of rebuilding. +Arguments are checked against each tool's schema before anything is walked or +scanned: types, required arguments and enums (`call_graph`'s `direction`, +`search`'s `rank`). A mistake comes back at once as a tool error that names the +argument, never as a default applied in silence. `file` arguments accept +`./src/a.ts`, an absolute path inside the repository or `src\a.ts`. A file the +index does not hold is an error suggesting indexed files with the same name, +not an empty answer. + ### Smaller read responses MCP `find_symbol`, `find_references`, `callers`, `symbols_overview` and `symbols` diff --git a/src/mcp.ts b/src/mcp.ts index 506b65d..617eb12 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -10,12 +10,14 @@ // with no main-module guard — see src/engine.ts — so that command does nothing. // The entrypoint is the `codeindex` bin, i.e. scripts/cli.mjs.) import { readFileSync, statSync, watch as watchFs, type FSWatcher } from "node:fs"; -import { isAbsolute, join } from "node:path"; +import { basename, isAbsolute, join, relative, resolve, sep } from "node:path"; import { createInterface } from "node:readline"; import { ENGINE_VERSION } from "./types.js"; import { renderGraphJson } from "./render/graph-json.js"; import { buildCallerIndex, lookupCallerEntry } from "./callers.js"; -import { callerIndexFor, hierarchyFor, symbolGraphFor } from "./derived.js"; +import { callerIndexFor, fileByRelFor, hierarchyFor, symbolGraphFor } from "./derived.js"; +import { byStr } from "./sort.js"; +import type { RepoScan } from "./scan.js"; import { implementationsOf } from "./relations.js"; import { neighborhood, type Direction } from "./symbolgraph.js"; import { detectWorkspaces } from "./workspaces.js"; @@ -133,6 +135,28 @@ function errMessage(e: unknown): string { return e instanceof Error ? e.message : String(e); } +// A `file` argument, as the index spells it: repo-relative, `/`-separated. +// +// Agents pass `./gin.go`, an absolute path they just read, or `src\a.ts`, and +// every spelling but the indexed one answered an empty `[]` — indistinguishable +// from "this file declares nothing" — or, for an edit, "no symbol matches". +// An exact indexed spelling is taken as is, so no working call changes. A path +// that names nothing indexed is an error with same-basename suggestions, since +// the empty answer is precisely what hid the mistake. +function indexedFile(scan: RepoScan, repo: string, file: string): string { + const byRel = fileByRelFor(scan); + if (byRel.has(file)) return file; + const rel = relative(repo, resolve(repo, file.replaceAll("\\", "/"))).split(sep).join("/"); + if (rel === ".." || rel.startsWith("../") || isAbsolute(rel)) throw new Error(`\`file\` is outside the repository: ${file}`); + if (byRel.has(rel)) return rel; + const name = basename(rel); + const near = scan.files.filter((f) => basename(f.rel) === name).map((f) => f.rel).sort(byStr).slice(0, 5); + throw new Error( + `file not in the index: ${file}` + + (near.length ? ` — did you mean ${near.join(", ")}?` : " (paths are repo-relative, as symbols_overview and find_symbol report them)"), + ); +} + // "There is no such symbol/type" from a lookup tool. Its text stays the // `{ "error": ... }` JSON it always was, but it travels as a tool execution // error (isError): it is not a result, and a declared outputSchema describes @@ -166,7 +190,8 @@ async function callTool(name: string, args: Record, defaultRepo throw new Error(`repository root is not a readable directory: ${repo}`); } const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) }; - // `search`'s optional structural prior; anything else falls back to the default. + // `search`'s optional structural prior. The schema's enum has already + // rejected anything else, so a typo no longer falls back to lexical in silence. const rankArg = str(args.rank); const rankOpt: { rank?: RankMode } = rankArg === "graph" || rankArg === "lexical" ? { rank: rankArg } : {}; // Scan-needing tools warm the present-language grammars (re-derived per call) @@ -249,7 +274,8 @@ async function callTool(name: string, args: Record, defaultRepo if (name === "symbols_overview") { const file = str(args.file); if (!file) throw new Error("`file` is required"); - const overview = symbolsOverview(readScan(), file); + const scan = readScan(); + const overview = symbolsOverview(scan, indexedFile(scan, repo, file)); return JSON.stringify(args.concise === true ? overview.map((s) => symbolLocation(s, s.name)) : overview, null, 2); } if (name === "find_symbol") { @@ -283,7 +309,8 @@ async function callTool(name: string, args: Record, defaultRepo if (!namePath || body === undefined) throw new Error("`namePath` and `body` are required"); const scan = readScan(); const fn = name === "replace_symbol_body" ? replaceSymbolBody : name === "insert_after_symbol" ? insertAfterSymbol : insertBeforeSymbol; - const result = fn(scan, namePath, body, str(args.file)); + const file = str(args.file); + const result = fn(scan, namePath, body, file === undefined ? undefined : indexedFile(scan, repo, file)); // A write WE just performed must not be trusted to the stat oracle: an // edit landing in the same mtime tick with the same byte count would pass // the (size, mtimeMs) fastpath and serve a stale scan. Drop the whole @@ -350,7 +377,9 @@ async function callTool(name: string, args: Record, defaultRepo const { churn, ok } = gitChurn(repo, { since: str(args.since) }); return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn, positiveNum(args.top)) }, null, 2); } - return JSON.stringify(symbolComplexity(scan, str(args.file), positiveNum(args.top)), null, 2); + const file = str(args.file); + const rel = file === undefined ? undefined : indexedFile(scan, repo, file); + return JSON.stringify(symbolComplexity(scan, rel, positiveNum(args.top)), null, 2); } if (name === "mermaid") { const { graph } = readArtifacts(); diff --git a/src/mcp/protocol.ts b/src/mcp/protocol.ts index 82b7cb3..479f120 100644 --- a/src/mcp/protocol.ts +++ b/src/mcp/protocol.ts @@ -33,8 +33,8 @@ export const RICH_TOOLS_SINCE = "2025-06-18"; // Tool.title, resource_link conte // with no way to tell why. // // Only the shapes these schemas actually use are checked (string / number / -// boolean / array-of-string) — this is a guard against silent misreads, not a -// JSON Schema implementation. The spec (2025-11-25) is explicit that input +// boolean / array / array-of-string, and a string `enum`) — this is a guard +// against silent misreads, not a JSON Schema implementation. The spec (2025-11-25) is explicit that input // validation failures belong in a Tool Execution Error, not a protocol error, // precisely so the model can read the message and retry. // @@ -52,6 +52,7 @@ export function validateArgs( items?: { type?: string }; minimum?: number; maximum?: number; + enum?: readonly unknown[]; description?: string; }>; for (const key of schema.required ?? []) { @@ -79,13 +80,20 @@ export function validateArgs( continue; } if (spec.type === "array") { - if (actual !== "array") return `\`${key}\` must be an array of strings, got ${actual}`; - if (spec.items?.type === "string" && !(value as unknown[]).every((x) => typeof x === "string")) { - return `\`${key}\` must be an array of strings`; - } + // "of strings" only where the schema says so: check_rules' `rules` is an + // array of objects, and telling the caller otherwise sent it astray. + const strings = spec.items?.type === "string"; + const expected = strings ? "an array of strings" : "an array"; + if (actual !== "array") return `\`${key}\` must be ${expected}, got ${actual}`; + if (strings && !(value as unknown[]).every((x) => typeof x === "string")) return `\`${key}\` must be ${expected}`; continue; } if (actual !== spec.type) return `\`${key}\` must be a ${spec.type}, got ${actual}`; + // `direction: "sideways"` used to become "both", and `rank: "pagerank"` + // lexical, with nothing in the answer to say the option was not understood. + if (spec.enum && !spec.enum.includes(value)) { + return `\`${key}\` must be one of ${spec.enum.map((v) => JSON.stringify(v)).join(", ")}, got ${JSON.stringify(value)}`; + } } return undefined; } diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index f06fceb..45d7d94 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -343,6 +343,7 @@ export const TOOLS = [ }, rank: { type: "string", + enum: ["lexical", "graph"], description: 'Structural prior: "graph" multiplies the lexical score by the file\'s PageRank over the resolved import graph; "lexical" (default) scores on text alone. Unproven on the judged corpus — see SearchOptions.rank.', }, @@ -418,7 +419,7 @@ export const TOOLS = [ ...repoProp, symbol: { type: "string", description: "Symbol name to centre on" }, depth: { type: "number", minimum: 1, maximum: 5, description: "Hops to follow (default 2, max 5)" }, - direction: { type: "string", description: "out | in | both (default both)" }, + direction: { type: "string", enum: ["out", "in", "both"], description: "out | in | both (default both)" }, }, required: ["repo", "symbol"], }, diff --git a/tests/mcp-output.test.ts b/tests/mcp-output.test.ts index 38cbb0f..4d7fcfa 100644 --- a/tests/mcp-output.test.ts +++ b/tests/mcp-output.test.ts @@ -236,6 +236,18 @@ describe("SDK conformance of the advertised tool list", () => { } }); + // validateArgs skips a property with no `type`, so an untyped argument + // would be read with no guard at all — the silent misreads it exists for. + it("types every input property with a shape validateArgs checks", () => { + const checked = new Set(["string", "number", "boolean", "array"]); + for (const tool of toolsFor(undefined, "2025-11-25") as { name: string; inputSchema: { properties?: Record } }[]) { + for (const [key, prop] of Object.entries(tool.inputSchema.properties ?? {})) { + expect(checked.has(prop.type ?? ""), `${tool.name}.${key}: ${prop.type}`).toBe(true); + if (prop.enum) expect(prop.enum.every((v) => typeof v === prop.type), `${tool.name}.${key} enum`).toBe(true); + } + } + }); + it("sends a capped response as a tool error without structuredContent, for every schema-declaring tool", async () => { // A cap this small withholds every payload, so each call exercises the // notice path. Before, the notice was a non-error result with no diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index fb125e8..ecd3554 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -1504,6 +1504,18 @@ describe("validateArgs", () => { expect(validateArgs(schema, { limit: undefined, substring: null, future: "whatever" })).toBeUndefined(); }); + it("enforces a declared enum instead of falling back to the default in silence", () => { + const enumerated = { properties: { direction: { type: "string", enum: ["out", "in", "both"] } } }; + expect(validateArgs(enumerated, { direction: "in" })).toBeUndefined(); + expect(validateArgs(enumerated, { direction: "sideways" })).toBe('`direction` must be one of "out", "in", "both", got "sideways"'); + }); + + it("says 'an array' when the items are not strings", () => { + const untyped = { properties: { rules: { type: "array" } } }; + expect(validateArgs(untyped, { rules: { from: "a" } })).toBe("`rules` must be an array, got object"); + expect(validateArgs(untyped, { rules: [{ from: "a" }] })).toBeUndefined(); + }); + it("checks the declared required list, naming the argument and what it is for", () => { const required = { properties: { repo: { type: "string", description: "Absolute path to the repository root" }, namePath: { type: "string" } }, @@ -1515,6 +1527,72 @@ describe("validateArgs", () => { }); }); +// `file` arguments are matched against the index's repo-relative spelling. +// `./src/util.ts`, an absolute path or `src\util.ts` used to answer an empty +// `[]` — indistinguishable from a file that declares nothing — and an edit +// qualified that way reported that no symbol matched. +describe("file arguments", () => { + it("accepts ./, absolute and backslash spellings of an indexed file", async () => { + const spellings = ["src/util.ts", "./src/util.ts", join(REPO, "src", "util.ts"), "src\\util.ts", "src//util.ts"]; + const res = await mcpSession([ + ...spellings.map((file, i) => ({ id: i + 1, method: "tools/call", params: { name: "symbols_overview", arguments: { repo: REPO, file } } })), + { id: 10, method: "tools/call", params: { name: "complexity", arguments: { repo: REPO, file: "./src/util.ts" } } }, + { id: 11, method: "tools/call", params: { name: "complexity", arguments: { repo: REPO, file: "src/util.ts" } } }, + ]); + const canonical = res.get(1)!.result!.content![0]!.text; + expect(JSON.parse(canonical).map((s: { name: string }) => s.name)).toContain("backoff"); + spellings.forEach((file, i) => { + expect(res.get(i + 1)!.result!.isError, file).toBeUndefined(); + expect(res.get(i + 1)!.result!.content![0]!.text, file).toBe(canonical); + }); + expect(res.get(10)!.result!.content![0]!.text).toBe(res.get(11)!.result!.content![0]!.text); + expect(JSON.parse(res.get(11)!.result!.content![0]!.text).length).toBeGreaterThan(0); + }, 20_000); + + it("reports a file the index does not hold, with same-name suggestions", async () => { + const res = await mcpSession([ + { id: 1, method: "tools/call", params: { name: "symbols_overview", arguments: { repo: REPO, file: "util.ts" } } }, + { id: 2, method: "tools/call", params: { name: "complexity", arguments: { repo: REPO, file: "nope.go" } } }, + { id: 3, method: "tools/call", params: { name: "symbols_overview", arguments: { repo: REPO, file: "../outside.ts" } } }, + { id: 4, method: "tools/call", params: { name: "insert_after_symbol", arguments: { repo: REPO, namePath: "backoff", body: "x", file: "nope.ts" } } }, + ]); + const text = (id: number) => res.get(id)!.result!.content![0]!.text; + for (const id of [1, 2, 3, 4]) expect(res.get(id)!.result!.isError, String(id)).toBe(true); + expect(text(1)).toBe("file not in the index: util.ts — did you mean src/util.ts?"); + expect(text(2)).toMatch(/^file not in the index: nope\.go \(paths are repo-relative/); + expect(text(3)).toBe("`file` is outside the repository: ../outside.ts"); + // Rejected before the edit could touch anything. + expect(text(4)).toMatch(/^file not in the index: nope\.ts/); + }, 20_000); + + it("resolves an edit's ./-qualified file to the indexed one", async () => { + const repo = tmpFixtureCopy("ci-edit-file-"); + const res = await mcpSession([ + { + id: 1, + method: "tools/call", + params: { name: "insert_after_symbol", arguments: { repo, namePath: "backoff", file: "./src/util.ts", body: "export const AFTER = 1;" } }, + }, + ]); + expect(res.get(1)!.result!.isError).toBeUndefined(); + expect(JSON.parse(res.get(1)!.result!.content![0]!.text).file).toBe("src/util.ts"); + expect(readFileSync(join(repo, "src", "util.ts"), "utf8")).toContain("export const AFTER = 1;"); + }, 20_000); + + it("rejects an enum value a tool does not understand", async () => { + const res = await mcpSession([ + { id: 1, method: "tools/call", params: { name: "call_graph", arguments: { repo: REPO, symbol: "HttpClient", direction: "sideways" } } }, + { id: 2, method: "tools/call", params: { name: "search", arguments: { repo: REPO, query: "client", rank: "pagerank" } } }, + { id: 3, method: "tools/call", params: { name: "search", arguments: { repo: REPO, query: "client", rank: "graph" } } }, + ]); + expect(res.get(1)!.result!.isError).toBe(true); + expect(res.get(1)!.result!.content![0]!.text).toMatch(/`direction` must be one of "out", "in", "both"/); + expect(res.get(2)!.result!.isError).toBe(true); + expect(res.get(2)!.result!.content![0]!.text).toMatch(/`rank` must be one of "lexical", "graph"/); + expect(res.get(3)!.result!.isError).toBeUndefined(); + }, 20_000); +}); + // Everything checkable from the request alone used to be checked only after // callTool had walked and scanned the repo — 13.5 s to learn that `namePath` // was missing on a 66k-file repo. A repository that does not exist proves the From 9017eae8a65b3035ac45b4dd44af4ff745d8a124 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:27:00 +0000 Subject: [PATCH 052/130] fix(rules): gate literals on the full list and reject configs that check nothing The literals builtin read graph.literalDuplications, a 24-entry headline sorted competing-first, so a `bypassed` gate passed while `codeindex literals` listed the violation. checkRules now takes the scan and computes the whole list, with the rule's own minFiles/minCount/includeTests. parseRules accepted configs that silently disabled a gate: `tiers` as a string or with a typo selected nothing, unknown keys (`sevrity`) were ignored, and `extends`/`implements` were refused although the graph emits them. Those now fail with a message naming the key; a forbidden rule whose globs match no indexed file is an `unmatched` warning; a malformed file is reported with its path and position. Over MCP, check_rules reads a configPath only inside the repository and never echoes its content. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 12 +++- src/engine-cli.ts | 12 ++-- src/mcp.ts | 42 ++++++++----- src/mcp/tools.ts | 4 +- src/rules.ts | 123 +++++++++++++++++++++++++++++++++--- tests/rules.test.ts | 148 +++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 310 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 6877cbc..4268a0b 100644 --- a/README.md +++ b/README.md @@ -412,7 +412,10 @@ codeindex literals --repo . --include-tests # count test files too ``` As a CI gate, via the `literals` builtin rule (defaults to the two actionable -tiers; `tiers` narrows it): +tiers; `tiers` narrows it, and `minFiles`/`minCount`/`includeTests` take the +command's thresholds). The rule computes the whole list, so it fails on exactly +what `codeindex literals` reports, not only on the 24-entry headline that +`graph.json` carries: ```json [{ "name": "no-uncentralized-routes", "builtin": "literals", "tiers": ["competing"] }] @@ -422,6 +425,13 @@ tiers; `tiers` narrows it): codeindex rules --repo . --config codeindex.rules.json # exit 1 on violations ``` +A rules config is validated strictly, because a gate that silently checks +nothing is worse than none. A key the rule does not read (`sevrity`), an unknown +tier, or an edge kind the graph does not emit fails with exit 2 and names the +file. A forbidden-edge rule whose `from` or `to` globs match no indexed file +can never fire, so it is reported as an `unmatched` warning. Over MCP, +`check_rules` reads a `configPath` only when it resolves inside the repository. + An arrow function returning a value (`export const getPath = () => "/a/b"`) is a *consumer*, not a source of truth, and is reported as a call site. A lookup table (`export const ROUTES = { … }`) genuinely is one, and is reported as a diff --git a/src/engine-cli.ts b/src/engine-cli.ts index 7994b82..b4b8867 100644 --- a/src/engine-cli.ts +++ b/src/engine-cli.ts @@ -38,7 +38,7 @@ import { renderMermaid } from "./viz.js"; import { impactOf, neighborsOf } from "./traverse.js"; import { deltaOfDiff, emptyDelta, formatDeltaPanel, readDeltaDiff } from "./delta.js"; import { explainQuery, searchIndex } from "./bm25.js"; -import { checkRules, parseRules } from "./rules.js"; +import { checkRules, parseRulesText } from "./rules.js"; import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, parseEmbedModel, resolveEmbedPullUrl, fetchEmbedModel } from "./embed/model.js"; import { buildEmbeddingIndex, serializeEmbeddings } from "./embed/index.js"; import { searchSemantic } from "./embed/search.js"; @@ -108,7 +108,9 @@ Commands: CODEINDEX_GRAMMARS_URL rules Architecture rules (forbidden edges, cycles, orphans, literals) validated against the link-graph: --config ; - exits 1 on any error-severity violation (a CI gate) + exits 1 on any error-severity violation (a CI gate), 2 on an + invalid config (unknown key, tier or edge kind); a forbidden + rule matching no file is an \`unmatched\` warning repomap Token-budgeted map of the highest-PageRank files (--budget-tokens) hotspots Churn × size ranking of the files where work concentrates: only files changed in the window, tests labelled (JSON; --since, --limit) @@ -1074,9 +1076,9 @@ export async function runCli(rawArgv: string[]): Promise { } } else if (cmd === "rules") { if (!flags.config) throw new Error("rules needs --config "); - const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8"))); - const { graph } = await readArtifacts(); - const violations = checkRules(graph, rules); + const rules = parseRulesText(readFileSync(flags.config, "utf8"), flags.config); + const { scan, graph } = await readArtifacts(); + const violations = checkRules(graph, rules, { scan }); const errors = violations.filter((v) => v.severity === "error").length; emit(JSON.stringify({ errors, warnings: violations.length - errors, violations }, null, 2) + "\n", flags.out); if (errors > 0) process.exitCode = 1; // the CI gate diff --git a/src/mcp.ts b/src/mcp.ts index 594f9b1..e657de0 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -9,8 +9,8 @@ // (NOT `node scripts/engine.mjs mcp`: engine.mjs is a side-effect-free library // with no main-module guard — see src/engine.ts — so that command does nothing. // The entrypoint is the `codeindex` bin, i.e. scripts/cli.mjs.) -import { readFileSync, statSync, watch as watchFs, type FSWatcher } from "node:fs"; -import { isAbsolute, join } from "node:path"; +import { readFileSync, realpathSync, statSync, watch as watchFs, type FSWatcher } from "node:fs"; +import { isAbsolute, join, sep } from "node:path"; import { createInterface } from "node:readline"; import { ENGINE_VERSION } from "./types.js"; import { renderGraphJson } from "./render/graph-json.js"; @@ -34,7 +34,7 @@ import { onboardBrief } from "./onboard.js"; import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit.js"; import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js"; import { explainQuery, searchIndex, type RankMode } from "./bm25.js"; -import { checkRules, parseRules } from "./rules.js"; +import { checkRules, parseRules, parseRulesText, type ArchRule } from "./rules.js"; import { deltaOfDiff, emptyDelta, formatDeltaPanel, readDeltaDiff } from "./delta.js"; import { EMBED_VERSION, resolveEmbedModelDir } from "./embed/model.js"; import { buildEmbeddingIndex } from "./embed/index.js"; @@ -526,19 +526,33 @@ async function callTool(name: string, args: Record, defaultRepo // which had no MCP equivalent, so a repo with a committed rules file had to // have it re-pasted into every call. const configPath = str(args.configPath); - let payload: unknown = args.rules; - if (payload === undefined && configPath) { - const abs = isAbsolute(configPath) ? configPath : join(repo, configPath); + let rules: ArchRule[]; + if (args.rules !== undefined) rules = parseRules(args.rules); // throws a descriptive error on a malformed payload + else if (configPath) { + // The path comes from the client, and the error below used to echo the + // start of whatever it named (`/etc/passwd` included): only a file + // inside the repository is read, symlinks resolved first. + const unreadable = new Error(`cannot read rules config ${configPath}`); + let abs: string; try { - payload = JSON.parse(readFileSync(abs, "utf8")); - } catch (e) { - throw new Error(`cannot read rules from ${abs}: ${errMessage(e)}`); + abs = realpathSync(isAbsolute(configPath) ? configPath : join(repo, configPath)); + } catch { + throw unreadable; } - } - if (payload === undefined) throw new Error("`rules` (or `configPath`) is required"); - const rules = parseRules(payload); // throws a descriptive error on a malformed payload - const { graph } = readArtifacts(); - return JSON.stringify(checkRules(graph, rules), null, 2); + const root = realpathSync(repo); + if (!abs.startsWith(root.endsWith(sep) ? root : root + sep)) { + throw new Error(`rules config must be a file inside the repository: ${configPath}`); + } + let text: string; + try { + text = readFileSync(abs, "utf8"); + } catch { + throw unreadable; + } + rules = parseRulesText(text, configPath); + } else throw new Error("`rules` (or `configPath`) is required"); + const { scan, graph } = readArtifacts(); + return JSON.stringify(checkRules(graph, rules, { scan }), null, 2); } if (name === "delta") { // The CLI's review panel, for an agent that just edited files over this diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index d5f9188..39f1100 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -442,7 +442,7 @@ export const TOOLS = [ { name: "check_rules", description: - 'Validate dependency-cruiser-style architecture rules against the link-graph. Rules (inline JSON array): forbidden edges {name, from, to, kind?, severity?, comment?} with glob paths, plus builtins {name, builtin: "cycles"|"orphans"} (module-level import cycles; edge-less code files). Returns deterministic violations with severity error|warn — a CI gate.', + 'Validate dependency-cruiser-style architecture rules against the link-graph. Rules (inline JSON array): forbidden edges {name, from, to, kind?, severity?, comment?} with glob paths, plus builtins {name, builtin: "cycles"|"orphans"|"literals"} (module-level import cycles; code files nothing connects to; values with no single source of truth, narrowed by tiers?/minFiles?/minCount?/includeTests?). Unknown keys are rejected; a forbidden rule whose globs match no indexed file comes back as an `unmatched` warning. Returns deterministic violations with severity error|warn — a CI gate.', inputSchema: { type: "object", properties: { @@ -452,7 +452,7 @@ export const TOOLS = [ configPath: { type: "string", description: - "Read the rules from this JSON file instead (repo-relative or absolute) — the CLI's --config. Ignored when `rules` is given.", + "Read the rules from this JSON file instead — the CLI's --config. Repo-relative, or absolute; either way it must resolve inside the repository. Ignored when `rules` is given.", }, }, required: ["repo"], diff --git a/src/rules.ts b/src/rules.ts index d7dce84..7763bb2 100644 --- a/src/rules.ts +++ b/src/rules.ts @@ -13,12 +13,16 @@ // entrypoint-looking basenames (index/main/cli/…); // literals — values with no single source of truth (see literals.ts): // a constant holds the value and other files rewrite it, or -// several constants hold the same one. Reads the duplications -// the pipeline stamped onto the graph, so it needs no rescan. +// several constants hold the same one. Computed from the scan +// (CheckRulesOptions.scan) with the rule's own thresholds. +// A forbidden rule whose `from` or `to` globs match no indexed file can never +// fire; it is reported as an `unmatched` warning instead of passing silently. // Violations are sorted deterministically (rule, from, to, kind) so two runs on // the same graph are byte-identical. import type { EdgeKind, Graph, LiteralDuplication } from "./types.js"; +import type { RepoScan } from "./scan.js"; import { compileGlobs } from "./glob.js"; +import { findLiteralDuplications } from "./literals.js"; import { byStr } from "./sort.js"; export type RuleSeverity = "error" | "warn"; @@ -40,24 +44,58 @@ export interface BuiltinRule { // a fix. "uncentralized" also flags values nothing owns yet, which is a // design decision rather than a defect and is noisy as a gate. tiers?: LiteralDuplication["tier"][]; + // `literals` only: the thresholds of `codeindex literals` (--min-files, + // --min-count, --include-tests), with the same defaults. + minFiles?: number; + minCount?: number; + includeTests?: boolean; severity?: RuleSeverity; comment?: string; } +export interface CheckRulesOptions { + // The scan the graph was built from. The `literals` builtin needs it to see + // every duplication: graph.json carries a 24-entry headline sorted + // competing-first, so a gate on `bypassed` read from it alone passed while + // `codeindex literals` listed violations. Without a scan the rule falls back + // to that headline, which is complete only when it holds fewer than 24. + scan?: RepoScan; +} + export type ArchRule = ForbiddenEdgeRule | BuiltinRule; export interface RuleViolation { rule: string; from: string; to: string; // for a cycle: the full path, "a -> b -> a" - kind: EdgeKind | "cycle" | "orphan" | "literal"; + kind: EdgeKind | "cycle" | "orphan" | "literal" | "unmatched"; severity: RuleSeverity; comment?: string; } -const EDGE_KINDS = new Set(["contains", "doc-link", "import", "call", "use", "mention"]); +// Every EdgeKind, checked both ways by the compiler: a kind added to the union +// and not here (as extends/implements once were) fails to build instead of +// being rejected in configs. +const EDGE_KIND_TABLE: Record = { + contains: true, + "doc-link": true, + import: true, + call: true, + extends: true, + implements: true, + use: true, + mention: true, +}; +const EDGE_KINDS = new Set(Object.keys(EDGE_KIND_TABLE)); const SEVERITIES = new Set(["error", "warn"]); const BUILTINS = new Set(["cycles", "orphans", "literals"]); +const TIERS = new Set(["competing", "bypassed", "uncentralized"]); +// The keys each rule shape reads. Anything else is a typo (`sevrity`, `tier`) +// that would otherwise leave a default in force without a word. +const COMMON_KEYS = ["name", "severity", "comment"]; +const FORBIDDEN_KEYS = new Set([...COMMON_KEYS, "from", "to", "kind"]); +const BUILTIN_KEYS = new Set([...COMMON_KEYS, "builtin"]); +const LITERALS_KEYS = new Set([...BUILTIN_KEYS, "tiers", "minFiles", "minCount", "includeTests"]); // Gate default: the two tiers that name a concrete fix. const GATED_TIERS: LiteralDuplication["tier"][] = ["competing", "bypassed"]; @@ -90,6 +128,29 @@ function toList(v: string | string[]): string[] { return Array.isArray(v) ? v : [v]; } +// Parse a rules file's text. The error names the file and the position but +// never echoes its content: over MCP the path comes from the client. +export function parseRulesText(text: string, source: string): ArchRule[] { + let payload: unknown; + try { + payload = JSON.parse(text); + } catch (e) { + const pos = /position (\d+)/.exec(e instanceof Error ? e.message : ""); + let where = ""; + if (pos) { + const before = text.slice(0, Number(pos[1])); + const line = before.split("\n").length; + where = ` (line ${line}, column ${before.length - before.lastIndexOf("\n")})`; + } + throw new Error(`rules config ${source} is not valid JSON${where}`); + } + try { + return parseRules(payload); + } catch (e) { + throw new Error(`rules config ${source}: ${e instanceof Error ? e.message : String(e)}`); + } +} + // Validate an untrusted rules payload (CLI --config file, MCP inline JSON) into // a typed rules array. Accepts either a bare array or a `{ rules: [...] }` // wrapper. Throws a descriptive error on the first malformed entry. @@ -98,9 +159,15 @@ export function parseRules(input: unknown): ArchRule[] { if (!Array.isArray(raw)) throw new Error("rules config must be an array (or an object with a `rules` array)"); return raw.map((entry, i) => { const at = `rules[${i}]`; - if (typeof entry !== "object" || entry === null) throw new Error(`${at}: must be an object`); + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new Error(`${at}: must be an object`); const r = entry as Record; if (typeof r.name !== "string" || !r.name) throw new Error(`${at}: \`name\` (non-empty string) is required`); + const allowed = r.builtin === undefined ? FORBIDDEN_KEYS : r.builtin === "literals" ? LITERALS_KEYS : BUILTIN_KEYS; + const unknown = Object.keys(r).filter((k) => !allowed.has(k)).sort(byStr); + if (unknown.length) { + const shape = r.builtin === undefined ? "a forbidden-edge rule" : `builtin "${String(r.builtin)}"`; + throw new Error(`${at} (${r.name}): unknown key${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `\`${k}\``).join(", ")} — ${shape} takes ${[...allowed].join(", ")}`); + } if (r.severity !== undefined && !SEVERITIES.has(r.severity as string)) throw new Error(`${at} (${r.name}): \`severity\` must be "error" or "warn"`); if (r.comment !== undefined && typeof r.comment !== "string") @@ -108,12 +175,27 @@ export function parseRules(input: unknown): ArchRule[] { if (r.builtin !== undefined) { if (!BUILTINS.has(r.builtin as string)) throw new Error(`${at} (${r.name}): \`builtin\` must be "cycles", "orphans" or "literals"`); + // A tier typo, or a bare string, used to select nothing: the gate passed. + if (r.tiers !== undefined) { + const ok = Array.isArray(r.tiers) && r.tiers.length > 0 && r.tiers.every((t) => TIERS.has(t as string)); + if (!ok) throw new Error(`${at} (${r.name}): \`tiers\` must be a non-empty array of ${[...TIERS].join(", ")}`); + } + for (const key of ["minFiles", "minCount"] as const) { + const v = r[key]; + if (v !== undefined && !(typeof v === "number" && Number.isInteger(v) && v >= 1)) + throw new Error(`${at} (${r.name}): \`${key}\` must be a positive integer`); + } + if (r.includeTests !== undefined && typeof r.includeTests !== "boolean") + throw new Error(`${at} (${r.name}): \`includeTests\` must be a boolean`); return { name: r.name, builtin: r.builtin, severity: r.severity, comment: r.comment, ...(r.tiers !== undefined ? { tiers: r.tiers } : {}), + ...(r.minFiles !== undefined ? { minFiles: r.minFiles } : {}), + ...(r.minCount !== undefined ? { minCount: r.minCount } : {}), + ...(r.includeTests !== undefined ? { includeTests: r.includeTests } : {}), } as BuiltinRule; } const glob = (field: "from" | "to"): string | string[] => { @@ -222,7 +304,7 @@ function findImportCycles(graph: Graph): { start: string; path: string[] }[] { // Validate `rules` against the built graph. Pure and deterministic: violations // are fully sorted; severity defaults to "error"; a rule's `comment` (when set) // is echoed onto each of its violations. -export function checkRules(graph: Graph, rules: ArchRule[]): RuleViolation[] { +export function checkRules(graph: Graph, rules: ArchRule[], opts: CheckRulesOptions = {}): RuleViolation[] { const out: RuleViolation[] = []; const emit = (rule: ArchRule, v: Omit): void => { out.push({ @@ -233,6 +315,22 @@ export function checkRules(graph: Graph, rules: ArchRule[]): RuleViolation[] { }); }; const fileSet = new Set(graph.files.map((f) => f.rel)); + // One literals pass per distinct threshold set, shared by the rules using it. + const literalRuns = new Map(); + const duplicationsFor = (rule: BuiltinRule): LiteralDuplication[] => { + if (!opts.scan) return graph.literalDuplications ?? []; + const key = `${rule.minFiles ?? ""}\0${rule.minCount ?? ""}\0${rule.includeTests === true}`; + let dups = literalRuns.get(key); + if (!dups) { + dups = findLiteralDuplications(opts.scan, { + minFiles: rule.minFiles, + minCount: rule.minCount, + includeTests: rule.includeTests, + }).duplications; + literalRuns.set(key, dups); + } + return dups; + }; for (const rule of rules) { if ("builtin" in rule) { @@ -242,7 +340,7 @@ export function checkRules(graph: Graph, rules: ArchRule[]): RuleViolation[] { } } else if (rule.builtin === "literals") { const wanted = new Set(rule.tiers?.length ? rule.tiers : GATED_TIERS); - for (const d of graph.literalDuplications ?? []) { + for (const d of duplicationsFor(rule)) { if (!wanted.has(d.tier)) continue; // `from` is where the value is DEFINED (or the first site rewriting // it, when nothing defines it) and `to` names the value, so a CI log @@ -266,6 +364,17 @@ export function checkRules(graph: Graph, rules: ArchRule[]): RuleViolation[] { const fromMatch = compileGlobs(toList(rule.from)); const toMatch = compileGlobs(toList(rule.to)); if (!fromMatch || !toMatch) continue; // empty glob list — matches nothing + // A glob side matching no indexed file (a typo, a moved directory) makes + // the rule vacuous: it passes forever. Always a warning, whatever the + // rule's severity — the rule is misconfigured, the architecture is not + // shown to be wrong. + let vacuous = false; + for (const [side, match] of [["from", fromMatch], ["to", toMatch]] as const) { + if (graph.files.some((f) => match(f.rel))) continue; + vacuous = true; + out.push({ rule: rule.name, from: toList(rule[side]).join(", "), to: `\`${side}\` matches no indexed file`, kind: "unmatched", severity: "warn" }); + } + if (vacuous) continue; const kinds = rule.kind?.length ? new Set(rule.kind) : null; for (const e of graph.fileEdges) { if (e.dangling || !fileSet.has(e.to)) continue; diff --git a/tests/rules.test.ts b/tests/rules.test.ts index c20d102..8e8f42d 100644 --- a/tests/rules.test.ts +++ b/tests/rules.test.ts @@ -1,14 +1,15 @@ import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; -import { checkRules, parseRules, type ArchRule } from "../src/rules.js"; +import { checkRules, parseRules, parseRulesText, type ArchRule } from "../src/rules.js"; import { buildIndexArtifacts } from "../src/pipeline.js"; import type { Edge, FileNode, Graph } from "../src/types.js"; const CLI = fileURLToPath(new URL("../scripts/cli.mjs", import.meta.url)); +const clientModule = new URL("../scripts/bench/mcp-client.mjs", import.meta.url).href; // Hand-built graphs: checkRules only reads files (rel/fileKind/degIn/degOut), // fileEdges and moduleEdges, so a minimal Graph isolates each rule semantics. @@ -136,6 +137,63 @@ describe("parseRules", () => { expect(() => parseRules([{ name: "r", from: "a", to: "b", severity: "fatal" }])).toThrow(/severity/); expect(() => parseRules([{ name: "r", from: "a", to: "b", kind: ["teleport"] }])).toThrow(/kind/); }); + + it("accepts every edge kind the graph emits, extends and implements included", () => { + const kinds = ["contains", "doc-link", "import", "call", "extends", "implements", "use", "mention"] as const; + for (const k of kinds) expect(parseRules([{ name: "r", from: "a", to: "b", kind: [k] }])[0]).toMatchObject({ kind: [k] }); + }); + + it("rejects a literals `tiers` that would select nothing, instead of disabling the gate", () => { + const lit = { name: "gate", builtin: "literals" }; + expect(() => parseRules([{ ...lit, tiers: "competing" }])).toThrow(/`tiers` must be a non-empty array of competing, bypassed, uncentralized/); + expect(() => parseRules([{ ...lit, tiers: ["competng"] }])).toThrow(/`tiers`/); + expect(() => parseRules([{ ...lit, tiers: [] }])).toThrow(/`tiers`/); + expect(() => parseRules([{ ...lit, minFiles: 0 }])).toThrow(/`minFiles` must be a positive integer/); + expect(() => parseRules([{ ...lit, includeTests: "yes" }])).toThrow(/`includeTests` must be a boolean/); + expect(parseRules([{ ...lit, tiers: ["bypassed"], minFiles: 3, minCount: 4, includeTests: true }])[0]).toEqual({ + ...lit, + severity: undefined, + comment: undefined, + tiers: ["bypassed"], + minFiles: 3, + minCount: 4, + includeTests: true, + }); + }); + + it("rejects keys the rule shape does not read, so a typo cannot leave a default in force", () => { + expect(() => parseRules([{ name: "r", from: "a", to: "b", sevrity: "warn" }])).toThrow( + /rules\[0\] \(r\): unknown key `sevrity` — a forbidden-edge rule takes name, severity, comment, from, to, kind/, + ); + expect(() => parseRules([{ name: "c", builtin: "cycles", tiers: ["competing"] }])).toThrow(/unknown key `tiers` — builtin "cycles"/); + expect(() => parseRules([{ name: "l", builtin: "literals", tier: ["competing"] }])).toThrow(/unknown key `tier`/); + }); + + it("names the rules file in a JSON error, without echoing its content", () => { + expect(() => parseRulesText('{\n bad: 1 }', "codeindex.rules.json")).toThrow( + /^rules config codeindex\.rules\.json is not valid JSON \(line 2, column 3\)$/, + ); + let msg = ""; + try { + parseRulesText("root:x:0:0:root:/root:/bin/bash\n", "/etc/passwd"); + } catch (e) { + msg = (e as Error).message; + } + expect(msg).toMatch(/^rules config \/etc\/passwd is not valid JSON/); + expect(msg).not.toContain("root:x"); + expect(() => parseRulesText('[{"name":"r","from":"a"}]', "x.json")).toThrow(/^rules config x\.json: rules\[0\] \(r\): `to`/); + }); +}); + +describe("checkRules — vacuous rules", () => { + it("reports a forbidden rule whose globs match no indexed file as an unmatched warning", () => { + const g = graphOf([fileNode("render/a.ts", { degOut: 1 }), fileNode("lib/b.ts", { degIn: 1 })], [edge("render/a.ts", "lib/b.ts")]); + expect(checkRules(g, [{ name: "typo", from: "rendr/**", to: ["lib/**"] }])).toEqual([ + { rule: "typo", from: "rendr/**", to: "`from` matches no indexed file", kind: "unmatched", severity: "warn" }, + ]); + // Both sides matching and no offending edge is a passing rule, not a vacuous one. + expect(checkRules(g, [{ name: "ok", from: "lib/**", to: "render/**" }])).toEqual([]); + }); }); // A synthetic monorepo written to a temp dir and run through the REAL pipeline: @@ -220,3 +278,89 @@ describe("rules on a synthetic monorepo (real pipeline)", () => { expect(warnOut.warnings).toBe(1); }); }); + +// 25 values each held by constants in three files (competing), plus one route +// that a constant holds and two other files rewrite (bypassed). graph.json +// carries 24 duplications, competing first, so the bypassed one is not on it. +function writeLiteralsRepo(): string { + const repo = mkdtempSync(join(tmpdir(), "ci-rules-lit-")); + for (const f of ["a", "b", "c"]) { + const lines = Array.from({ length: 25 }, (_, i) => `export const K${f.toUpperCase()}${i} = "value-number-${String(i).padStart(2, "0")}";`); + writeFileSync(join(repo, `${f}.ts`), lines.join("\n") + "\n"); + } + writeFileSync(join(repo, "route.ts"), 'export const ROUTE = "/api/bypassed/path";\n'); + writeFileSync(join(repo, "use1.ts"), 'export function u1() {\n return fetch("/api/bypassed/path");\n}\n'); + writeFileSync(join(repo, "use2.ts"), 'export function u2() {\n return fetch("/api/bypassed/path");\n}\n'); + return repo; +} + +describe("checkRules — literals reads the whole duplication list", () => { + const BYPASS: ArchRule[] = [{ name: "bypass-gate", builtin: "literals", tiers: ["bypassed"] }]; + + it("finds a violation past the 24-entry graph headline when given the scan", () => { + const repo = writeLiteralsRepo(); + try { + const { scan, graph } = buildIndexArtifacts(repo); + expect(graph.literalDuplications).toHaveLength(24); + expect(graph.literalDuplications!.every((d) => d.tier === "competing")).toBe(true); + expect(checkRules(graph, BYPASS)).toEqual([]); // the headline alone cannot see it + expect(checkRules(graph, BYPASS, { scan })).toEqual([ + { + rule: "bypass-gate", + from: "route.ts:1", + to: 'bypassed "/api/bypassed/path" (3 sites, 3 files)', + kind: "literal", + severity: "error", + }, + ]); + expect(checkRules(graph, [{ name: "c", builtin: "literals", tiers: ["competing"] }], { scan })).toHaveLength(25); + // The rule's own thresholds apply, as `codeindex literals --min-files 4` would. + expect(checkRules(graph, [{ ...BYPASS[0]!, minFiles: 4 } as ArchRule], { scan })).toEqual([]); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + it("CLI `rules` fails the gate on it", () => { + const repo = writeLiteralsRepo(); + try { + const config = join(repo, "codeindex.rules.json"); + writeFileSync(config, JSON.stringify(BYPASS)); + const res = spawnSync(process.execPath, [CLI, "rules", "--repo", repo, "--config", config], { encoding: "utf8" }); + expect(res.status).toBe(1); + expect(JSON.parse(res.stdout)).toMatchObject({ errors: 1, warnings: 0 }); + writeFileSync(config, "[{ oops }]"); + const bad = spawnSync(process.execPath, [CLI, "rules", "--repo", repo, "--config", config], { encoding: "utf8" }); + expect(bad.status).not.toBe(0); + expect(bad.stderr).toContain(`rules config ${config} is not valid JSON (line 1, column 4)`); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + it("MCP check_rules reads a configPath inside the repository only", async () => { + const repo = writeLiteralsRepo(); + const outside = mkdtempSync(join(tmpdir(), "ci-rules-out-")); + const { startMcpClient } = await import(/* @vite-ignore */ clientModule); + const client = startMcpClient(process.execPath, [CLI, "mcp", "--repo", repo], { timeoutMs: 30_000 }); + try { + expect((await client.handshake()).ok).toBe(true); + writeFileSync(join(repo, "rules.json"), JSON.stringify(BYPASS)); + writeFileSync(join(outside, "secret.txt"), "root:x:0:0:root:/root:/bin/bash\n"); + const call = (args: Record) => client.request("tools/call", { name: "check_rules", arguments: args }); + const inside = await call({ configPath: "rules.json" }); + expect(inside.result.isError).not.toBe(true); + expect(JSON.parse(inside.result.content[0].text)).toHaveLength(1); + const escaped = await call({ configPath: join(outside, "secret.txt") }); + expect(escaped.result.isError).toBe(true); + expect(escaped.result.content[0].text).toMatch(/must be a file inside the repository/); + expect(escaped.result.content[0].text).not.toContain("root:x"); + const dotdot = await call({ configPath: "../" + outside.split("/").pop() + "/secret.txt" }); + expect(dotdot.result.content[0].text).toMatch(/must be a file inside the repository/); + } finally { + await client.close(); + rmSync(repo, { recursive: true, force: true }); + rmSync(outside, { recursive: true, force: true }); + } + }, 60_000); +}); From 868a9001e8cce01494851f73fd3dbf962fb46a65 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:28:21 +0000 Subject: [PATCH 053/130] feat(mcp): keep a member's parent in concise declarations concise:true reduced declarations to name/kind/file/line, which made same-named members indistinguishable: on gin's response_writer.go, 28 declarations with 28 distinct Parent/name pairs collapsed to 22 names (ResponseWriter/Flush and responseWriter/Flush both read `Flush`). The Parent/name path that find_symbol and the edit tools take could not be formed without a second, full call. `parent` now rides along when the declaration has one; top-level declarations are byte-identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 3 ++- src/mcp/concise.ts | 10 +++++++--- src/mcp/tools.ts | 6 +++--- src/query.ts | 4 +++- tests/mcp-concise.test.ts | 18 +++++++++++++++++- tests/phase2.test.ts | 5 +++++ 6 files changed, 37 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f58c0f0..26d56bd 100644 --- a/README.md +++ b/README.md @@ -769,7 +769,8 @@ not an empty answer. ### Smaller read responses MCP `find_symbol`, `find_references`, `callers`, `symbols_overview` and `symbols` -accept `concise: true`. Declarations are reduced to `name/kind/file/line` while +accept `concise: true`. Declarations are reduced to `name/kind/file/line`, plus +`parent` for a member so its `Parent/name` path stays formable, while result membership, order, reference groups, call-site locations, confidence labels and LSP metadata stay intact. Defaults retain their full existing shape. `symbols` keeps its name-keyed groups and references for full-index requests. diff --git a/src/mcp/concise.ts b/src/mcp/concise.ts index fe8add3..eef6cc9 100644 --- a/src/mcp/concise.ts +++ b/src/mcp/concise.ts @@ -4,10 +4,14 @@ import type { CodeSymbol, SymbolIndex } from "../types.js"; import type { CallerEntry } from "../callers.js"; import type { SymbolReferences } from "../query.js"; -export type SymbolLocation = Pick; +// `parent` rides along when there is one: it is what makes a member +// addressable. Without it `ResponseWriter/Flush` and `responseWriter/Flush` +// both read as `Flush`, and the Parent/name path the edit tools and +// find_symbol take could not be formed without a second, full call. +export type SymbolLocation = Pick; -export function symbolLocation(symbol: Pick, name: string): SymbolLocation { - return { name, kind: symbol.kind, file: symbol.file, line: symbol.line }; +export function symbolLocation(symbol: Pick, name: string): SymbolLocation { + return { name, kind: symbol.kind, file: symbol.file, line: symbol.line, ...(symbol.parent ? { parent: symbol.parent } : {}) }; } export function conciseCaller(entry: T): Omit & { def: SymbolLocation } { diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 45d7d94..310677a 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -7,7 +7,7 @@ import { ANNOTATIONS_SINCE, PROTOCOL_VERSIONS, RICH_TOOLS_SINCE } from "./protocol.js"; const repoProp = { repo: { type: "string", description: "Absolute path to the repository root" } }; -const conciseProp = { concise: { type: "boolean", description: "Return declaration locations (name/kind/file/line) without full symbol metadata. Keeps every result, reference tier and confidence label (default false)." } }; +const conciseProp = { concise: { type: "boolean", description: "Return declaration locations (name/kind/file/line, plus parent for a member) without full symbol metadata. Keeps every result, reference tier and confidence label (default false)." } }; const scopeProps = { scope: { type: "string", description: "Restrict to one directory (repo-relative)" }, include: { type: "array", items: { type: "string" }, description: "Include globs" }, @@ -85,7 +85,7 @@ export const TOOLS = [ { name: "find_symbol", description: - "Find symbol declarations by name or name path ('Class/method' matches a method inside Class). Each match carries its COMPLETE SIGNATURE (parameters and return type) by default, because \"what shape is it\" is the question that follows \"where is it\" almost every time and one round trip beats two. Options: substring matching, includeBody for the declaration's source, concise to drop everything but name/kind/file/line when you genuinely only want a location. Exact-name matches rank first.", + "Find symbol declarations by name or name path ('Class/method' matches a method inside Class). Each match carries its COMPLETE SIGNATURE (parameters and return type) by default, because \"what shape is it\" is the question that follows \"where is it\" almost every time and one round trip beats two. Options: substring matching, includeBody for the declaration's source, concise to drop everything but name/kind/file/line (and a member's parent) when you genuinely only want a location. Exact-name matches rank first.", inputSchema: { type: "object", properties: { @@ -96,7 +96,7 @@ export const TOOLS = [ concise: { type: "boolean", description: - "Return only name/kind/file/line — drop the signature, line span, visibility and language. Roughly 2.5x smaller; use it when you are resolving a path and nothing more (default false).", + "Return only name/kind/file/line (plus parent for a member) — drop the signature, line span, visibility and language. Roughly 2.5x smaller; use it when you are resolving a path and nothing more (default false).", }, maxResults: { type: "number", minimum: 1, description: "Cap matches (default 50)" }, }, diff --git a/src/query.ts b/src/query.ts index d7a8caf..ee7b80d 100644 --- a/src/query.ts +++ b/src/query.ts @@ -107,13 +107,15 @@ export function findSymbol(scan: RepoScan, namePath: string, opts: FindSymbolOpt } // Applied LAST so it composes predictably: `concise` with `includeBody` keeps // the body, because a caller that asked for both wants the source without the - // metadata around it. + // metadata around it. `parent` stays: a member's Parent/name path is how it + // is addressed, and two same-named methods are otherwise indistinguishable. if (opts.concise) { return capped.map((m) => ({ name: m.name, kind: m.kind, file: m.file, line: m.line, + ...(m.parent ? { parent: m.parent } : {}), ...(m.body !== undefined ? { body: m.body } : {}), })) as SymbolMatch[]; } diff --git a/tests/mcp-concise.test.ts b/tests/mcp-concise.test.ts index 45f84a9..edc5a4f 100644 --- a/tests/mcp-concise.test.ts +++ b/tests/mcp-concise.test.ts @@ -15,6 +15,7 @@ beforeAll(async () => { repo = mkdtempSync(join(tmpdir(), "ci-concise-")); writeFileSync(join(repo, "lib.ts"), '/** Return a friendly greeting. */\nexport function greet(name: string): string {\n return "hello " + name;\n}\nexport function unused(): void {}\n'); writeFileSync(join(repo, "app.ts"), 'import { greet } from "./lib";\nexport function main(): void {\n greet("world");\n}\n'); + writeFileSync(join(repo, "shapes.ts"), "export class Square {\n area(): number { return 1; }\n}\nexport class Circle {\n area(): number { return 3; }\n}\n"); const { startMcpClient } = await import(/* @vite-ignore */ clientModule); client = startMcpClient(process.execPath, [CLI, "mcp", "--repo", repo, "--tools", "find,impact"], { timeoutMs: 10_000 }); handshake = await client.handshake(); @@ -28,7 +29,7 @@ async function call(name: string, args: Record = {}) { expect(response.result.isError, JSON.stringify(response.result)).not.toBe(true); return JSON.parse(response.result.content[0].text); } -const location = ({ name, kind, file, line }: any) => ({ name, kind, file, line }); +const location = ({ name, kind, file, line, parent }: any) => ({ name, kind, file, line, ...(parent ? { parent } : {}) }); describe("concise MCP read answers", () => { it("advertises each concise option and the available/active profiles", async () => { @@ -87,6 +88,21 @@ describe("concise MCP read answers", () => { const full = await call("find_references", { name: "greet", lsp: true }); expect(await call("find_references", { name: "greet", lsp: true, concise: true })).toEqual({ ...full, defs: full.defs.map(location) }); }); + it("keeps a member's parent, so same-named methods stay addressable", async () => { + const overview = await call("symbols_overview", { file: "shapes.ts", concise: true }); + expect(overview.filter((s: any) => s.name === "area")).toEqual([ + { name: "area", kind: "method", file: "shapes.ts", line: 2, parent: "Square" }, + { name: "area", kind: "method", file: "shapes.ts", line: 5, parent: "Circle" }, + ]); + // Top-level declarations carry no parent key at all. + expect(overview.find((s: any) => s.name === "Square")).toEqual({ name: "Square", kind: "class", file: "shapes.ts", line: 1 }); + const found = await call("find_symbol", { namePath: "area", concise: true }); + expect(found.map((s: any) => `${s.parent}/${s.name}`)).toEqual(["Square/area", "Circle/area"]); + // The concise path round-trips into a namePath lookup. + expect(await call("find_symbol", { namePath: `${found[1].parent}/${found[1].name}`, concise: true })).toEqual([found[1]]); + const indexed = await call("symbols", { name: "area", concise: true }); + expect(indexed.defs.map((d: any) => d.parent)).toEqual(["Square", "Circle"]); + }); it("answers Object.prototype names as absent symbols, not prototype members", async () => { // The index is a plain object: `defs.toString` used to be the inherited // function (serialized away, or `defs.map is not a function` under diff --git a/tests/phase2.test.ts b/tests/phase2.test.ts index 7a9507e..31108bf 100644 --- a/tests/phase2.test.ts +++ b/tests/phase2.test.ts @@ -362,6 +362,11 @@ describe("symbol query API", () => { const withBody = findSymbol(scan, "makeWidget", { concise: true, includeBody: true })[0]!; expect(withBody.body).toContain("return new Widget()"); expect(withBody.signature).toBeUndefined(); + + // A member keeps its parent: `Widget/size` is how it is addressed. + const member = findSymbol(scan, "size", { concise: true }); + expect(member.map((m) => Object.keys(m).sort())).toEqual([["file", "kind", "line", "name", "parent"]]); + expect(member[0]!.parent).toBe("Widget"); }); it("findReferences merges precise call sites with file-level references", () => { From 57ad9359c90dbabc4a2c81327d81ce95e2fd9150 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 05:28:38 +0000 Subject: [PATCH 054/130] fix(extract): summarize a file from its description, not its license MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File summaries were boilerplate for whole ecosystems: 88 of gin's 100 files read "Use of this source code is governed by a MIT style license…", C files "include ", headers "pragma once Widget API.", a Rust crate root "!", Ruby files "frozen_string_literal: true", and a Python coding line also became the doc of the module's first function. The summary feeds BM25, repomap and onboard, so one license sentence made "license" match every file. topDocComment is replaced by fileSummary in doc-text.ts, built on the same stripCommentMarkers/summarizeDocLines the symbol docs use. It reads `#` as a comment only where the language says so, looks past a shebang, include guard, `#pragma once`, `#![…]`, ` Claude-Session: https://claude.ai/code/session_01XWLTLwAs9kt9Ac2YvAXV1q --- README.md | 6 + src/extract/code.ts | 69 +---------- src/extract/doc-text.ts | 237 ++++++++++++++++++++++++++++++++++--- tests/file-summary.test.ts | 213 +++++++++++++++++++++++++++++++++ 4 files changed, 440 insertions(+), 85 deletions(-) create mode 100644 tests/file-summary.test.ts diff --git a/README.md b/README.md index b184c3e..27b84a3 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,12 @@ compares](#how-it-compares). symbols, imports, and calls from both the script and the template, bound in the JS/TS call family. A Svelte prop (`export let`) and an Astro frontmatter export are not module exports, so they are never reported as dead code. + Each file's **summary** is the first leading comment that describes + something: license and copyright text (MIT, BSD, Apache, GPL, MPL, the Go + "governed by" line), linter and editor magic comments (`frozen_string_literal`, + `-*- coding -*-`, `go:build`), Xcode's file stamp and bundler region markers + are skipped, and `#` reads as a comment only in languages where it is one — + never a C `#include` or a Rust `#[attribute]`. - **Resolve imports** across languages: tsconfig paths, package `exports`, go.mod, Cargo, Java packages, PSR-4, C# namespaces. - **Build a typed link-graph**: `import` / `call` / `extends` / `implements` / diff --git a/src/extract/code.ts b/src/extract/code.ts index f6154dc..4e4d706 100644 --- a/src/extract/code.ts +++ b/src/extract/code.ts @@ -6,7 +6,7 @@ import { extractReexports, extToLang, MAX_REEXPORTS } from "../lang/common.js"; import { extractImports, extractPackage } from "./imports.js"; import { sfcParts } from "./sfc.js"; import { isMinified } from "./minified.js"; -import { isBanner, isDirective, stripCommentMarkers } from "./doc-text.js"; +import { fileSummary, stripCommentMarkers } from "./doc-text.js"; import { subtokens } from "../util.js"; // Per-file symbol ceiling. Raised from 400: a real 3000-line generated client or @@ -40,69 +40,6 @@ export interface CodeInfo { relations?: RawRelation[]; } -// The leading comment block of a file, turned into one summary line. Handles -// `//`, `#`, and `/* … */` / `""" … """` openers. Stops at the first code line. -function topDocComment(content: string): string | undefined { - const lines = content.split(/\r?\n/); - const collected: string[] = []; - let inBlock: "c" | "py" | null = null; - for (let i = 0; i < Math.min(lines.length, 40); i++) { - const raw = lines[i]!; - const line = raw.trim(); - if (inBlock === "c") { - // Strip the closing `*/` BEFORE the leading `*`s, so a lone `*/` (or a line - // ending in `*/`) doesn't leave a stray "/" once the leading star is gone. - collected.push(line.replace(/\*+\/\s*$/, "").replace(/^\*+/, "").trim()); - if (line.includes("*/")) inBlock = null; - continue; - } - if (inBlock === "py") { - if (line.includes('"""') || line.includes("'''")) { - collected.push(line.replace(/['"]{3}.*$/, "").trim()); - inBlock = null; - } else collected.push(line); - continue; - } - if (line === "" && collected.length === 0) continue; // skip leading blanks - if (line.startsWith("#!")) continue; // shebang - if (line.startsWith("//")) { - collected.push(line.replace(/^\/+/, "").trim()); - continue; - } - if (line.startsWith("#")) { - collected.push(line.replace(/^#+/, "").trim()); - continue; - } - if (line.startsWith("/*")) { - // Drop the opener, INCLUDING the `!` of a `/*!` "preserve" banner — else the - // stripped text is just "!", which the first-sentence regex then treats as a - // whole sentence, yielding the garbage summary "!". - collected.push(line.replace(/^\/\*+!?/, "").replace(/\*+\/\s*$/, "").trim()); - if (!line.includes("*/")) inBlock = "c"; - continue; - } - if (line.startsWith('"""') || line.startsWith("'''")) { - const rest = line.slice(3); - if (rest.includes('"""') || rest.includes("'''")) collected.push(rest.replace(/['"]{3}.*$/, "").trim()); - else { - collected.push(rest.trim()); - inBlock = "py"; - } - continue; - } - break; // first real code line - } - const text = collected - .filter((l) => l && !isDirective(l) && !isBanner(l)) - .join(" ") - .replace(/\s+/g, " ") - .trim(); - if (text.length < 8) return undefined; - // First sentence, capped. - const sentence = /^(.*?[.!?])(\s|$)/.exec(text); - return (sentence ? sentence[1]! : text).slice(0, 200); -} - // Control-flow and declaration keywords that syntactically precede `(` but are // never call targets — the union across supported languages. Deliberately does // NOT list real builtins (python's `print`, go's `make`…): a false call to a @@ -316,7 +253,7 @@ export function extractCode(rel: string, ext: string, content: string, opts: { m // (real edges, whoever wrote them) — and nothing else: its symbols and call // sites are one-letter noise (see extract/minified.ts). The flag says so. if (isMinified(ext, content)) { - return { symbols: [], minified: true, summary: topDocComment(content), refs: extractImports(ext, content) }; + return { symbols: [], minified: true, summary: fileSummary(ext, content), refs: extractImports(ext, content) }; } // A single-file component (.vue/.svelte/.astro) is extracted as its script: // the JS/TS tier runs over a copy with the markup blanked, lines unchanged @@ -373,7 +310,7 @@ export function extractCode(rel: string, ext: string, content: string, opts: { m ...(ast?.truncated || raw.length > symbols.length || reexports.length >= MAX_REEXPORTS ? { truncated: true as const } : {}), - summary: topDocComment(content), + summary: fileSummary(ext, content), refs, pkg: extractPackage(ext, content), idents: ast?.idents, diff --git a/src/extract/doc-text.ts b/src/extract/doc-text.ts index d031bef..6335a89 100644 --- a/src/extract/doc-text.ts +++ b/src/extract/doc-text.ts @@ -1,6 +1,6 @@ // Turning raw comment text into one useful sentence — the logic shared by the -// FILE summary (extract/code.ts topDocComment, line-based over raw content) and -// the per-SYMBOL doc comment (ast/doc.ts, node-based over comment siblings). +// FILE summary (fileSummary below, line-based over raw content) and the +// per-SYMBOL doc comment (ast/doc.ts, node-based over comment siblings). // // Both need the same three things: strip whatever markers the language uses, // discard lines that are tooling noise rather than prose, and reduce what's left @@ -8,25 +8,73 @@ // disagreeing about whether `/*! jQuery */` is a description. // Tooling pragmas and boilerplate that are technically the first comment but say -// nothing about what the code does — never use them as a summary. +// nothing about what the code does — never use them as a summary. Includes the +// magic comments that open whole ecosystems' files: Ruby's +// `frozen_string_literal`, Sorbet's `typed:`, an Emacs `-*- coding: utf-8 -*-` +// line (which was also becoming the doc of a module's first function), Go build +// constraints, a copyright line. An ESLint `global a, b` list is matched whole, +// so a doc that opens "Global registry of handlers." is still prose. const DIRECTIVE_RE = - /^(eslint\b|eslint-|prettier\b|prettier-|tslint\b|jshint\b|jslint\b|globals?\b|istanbul\b|c8\s|v8\s|@ts-|ts-|@flow\b|@jsx\b|@jsxRuntime\b|@jest-environment\b|@vitest-environment\b|@license\b|@preserve\b|@copyright\b|copyright\b|spdx-|" and headers as "pragma once Widget API.". +const HASH_COMMENT = new Set([ + ".py", ".pyi", ".rb", ".rake", ".sh", ".bash", ".zsh", ".ksh", ".fish", + ".ex", ".exs", ".tf", ".tfvars", ".hcl", ".graphql", ".gql", +]); +const DASH_COMMENT = new Set([".lua", ".sql", ".hs", ".elm"]); +const DOCSTRING = new Set([".py", ".pyi"]); +const MARKUP = new Set([".vue", ".svelte", ".astro"]); + +// Lines that may precede a file's opening comment without ending the search: +// a shebang, a Rust inner attribute (`#![…]`, above the `//!` crate docs), a +// header's `#pragma once` or include guard, a PHP open tag and its +// strict-types declaration, a JS/TS directive prologue (`"use client";`), a +// Haskell `{-# LANGUAGE … #-}` pragma, a component's `