From 302606ee5e2da142d0944cc14f9a6dc7285d993b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Mon, 26 Oct 2026 08:12:13 +0100 Subject: [PATCH 1/7] feat(vocabulary): drift against business terms `parseBusinessTerms(text)` reads a plain glossary, one term per line, each split with no lexicon so `Order Line` meets `order` and `line`. Blank lines and `#` comments are skipped; a line naming no word (`2024`) is reported by its number; a term whose words repeat an earlier one's is that term. `vocabularyDrift(artifact, terms, {level})` says, per business term, which scopes of the level speak every one of its words (whatever class the code gives them) and which words no scope of the level says. The other half is the code-only words: `domain` terms some scope of the level speaks that no business term contains, summed over those scopes, by weight then term. Nothing is stemmed: `lines` is not `line`. Tested on the hand-counted corpus at the three levels, and by properties over the java and ruby fixtures. Co-Authored-By: Claude Opus 5.5 --- packages/vocabulary/src/drift.ts | 119 +++++++++++++ packages/vocabulary/src/index.ts | 1 + packages/vocabulary/test/drift.test.ts | 237 +++++++++++++++++++++++++ 3 files changed, 357 insertions(+) create mode 100644 packages/vocabulary/src/drift.ts create mode 100644 packages/vocabulary/test/drift.test.ts diff --git a/packages/vocabulary/src/drift.ts b/packages/vocabulary/src/drift.ts new file mode 100644 index 0000000..dcf3293 --- /dev/null +++ b/packages/vocabulary/src/drift.ts @@ -0,0 +1,119 @@ +import type { ScopeLevel, VocabularyArtifact } from "./artifact.js"; +import { splitIdentifier } from "./split.js"; + +/** + * DRIFT (PLAN.md §20.1): a business glossary against the words the code + * speaks. A business term is spoken in a scope when every one of its words is, + * whatever class the code gives the word; the other half is the `domain` + * words the code speaks that no business term contains. Words match as + * written: `lines` is not `line`, nothing is stemmed. + */ + +/** One glossary line: the text as written and the words the universal rules make of it. */ +export interface BusinessTerm { + readonly text: string; + readonly words: readonly string[]; +} + +export interface BusinessTerms { + /** In the file's order; a term whose words repeat an earlier one's is that term. */ + readonly terms: readonly BusinessTerm[]; + /** Lines that name no word (`2024`, `—`): no term, said so. */ + readonly wordless: readonly { readonly line: number; readonly text: string }[]; +} + +/** + * One business term per line, split with no lexicon, so `Order Line` meets + * `order` and `line`. Lines are trimmed; blank lines and `#` comments are not + * terms. + */ +export function parseBusinessTerms(text: string): BusinessTerms { + const terms: BusinessTerm[] = []; + const wordless: { line: number; text: string }[] = []; + const seen = new Set(); + text.split("\n").forEach((raw, i) => { + const line = raw.trim(); + if (line.length === 0 || line.startsWith("#")) return; + const words = [...new Set(splitIdentifier(line, "", undefined).tokens)]; + if (words.length === 0) { + wordless.push({ line: i + 1, text: line }); + return; + } + const key = words.join(" "); + if (seen.has(key)) return; + seen.add(key); + terms.push({ text: line, words }); + }); + return { terms, wordless }; +} + +export interface SpokenTerm extends BusinessTerm { + /** The scopes of the level that say every word, by index into the artifact's `scopes`. */ + readonly scopes: readonly number[]; + /** Its words no scope of the level says at all; empty with no scope when they are never said together. */ + readonly missing: readonly string[]; +} + +export interface CodeOnlyWord { + /** Index into the artifact's `terms`; always a `domain` term. */ + readonly term: number; + /** Summed over the scopes of the level that speak it. */ + readonly weight: number; + readonly occurrences: number; + readonly scopes: number; +} + +export interface DriftReport { + readonly level: ScopeLevel; + /** Every business term, in the glossary's order. */ + readonly business: readonly SpokenTerm[]; + /** By weight descending, then term. */ + readonly codeOnly: readonly CodeOnlyWord[]; +} + +export function vocabularyDrift( + artifact: VocabularyArtifact, + terms: readonly BusinessTerm[], + options: { readonly level?: ScopeLevel } = {}, +): DriftReport { + const level = options.level ?? "corpus"; + + // Which scopes of the level say each word (any class), in scope order. + const speakers = new Map(); + const codeOnly = new Map(); + artifact.scopes.forEach((scope, index) => { + if (scope.level !== level) return; + for (const [term, weight, occurrences] of scope.terms) { + const entry = artifact.terms[term]; + if (entry === undefined) continue; + const list = speakers.get(entry.text) ?? []; + if (list[list.length - 1] !== index) list.push(index); + speakers.set(entry.text, list); + if (entry.class !== "domain") continue; + const sum = codeOnly.get(term) ?? { weight: 0, occurrences: 0, scopes: 0 }; + sum.weight += weight; + sum.occurrences += occurrences; + sum.scopes += 1; + codeOnly.set(term, sum); + } + }); + + const business = terms.map((term): SpokenTerm => { + const lists = term.words.map((word) => speakers.get(word) ?? []); + const missing = term.words.filter((_word, i) => lists[i]?.length === 0); + const [first = [], ...others] = lists; + const sets = others.map((list) => new Set(list)); + const scopes = first.filter((scope) => sets.every((set) => set.has(scope))); + return { text: term.text, words: term.words, scopes, missing }; + }); + + const named = new Set(terms.flatMap((term) => term.words)); + return { + level, + business, + codeOnly: [...codeOnly] + .filter(([term]) => !named.has(artifact.terms[term]?.text ?? "")) + .map(([term, sum]): CodeOnlyWord => ({ term, ...sum })) + .sort((a, b) => b.weight - a.weight || a.term - b.term), + }; +} diff --git a/packages/vocabulary/src/index.ts b/packages/vocabulary/src/index.ts index 68675fb..a919578 100644 --- a/packages/vocabulary/src/index.ts +++ b/packages/vocabulary/src/index.ts @@ -11,3 +11,4 @@ export * from "./classify.js"; export * from "./weight.js"; export * from "./artifact.js"; export * from "./aggregate.js"; +export * from "./drift.js"; diff --git a/packages/vocabulary/test/drift.test.ts b/packages/vocabulary/test/drift.test.ts new file mode 100644 index 0000000..353ec6c --- /dev/null +++ b/packages/vocabulary/test/drift.test.ts @@ -0,0 +1,237 @@ +import { fileURLToPath } from "node:url"; +import { readModelFileSync } from "@codegraph/core"; +import { buildGraph, loadModels } from "@codegraph/analyzer"; +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { buildVocabulary } from "../src/aggregate.js"; +import type { VocabularyArtifact } from "../src/artifact.js"; +import { parseBusinessTerms, vocabularyDrift, type BusinessTerm, type DriftReport } from "../src/drift.js"; +import { graphOf, orderCorpus } from "./toy.js"; + +/** + * Drift (PLAN.md §20.1): a business glossary against the words the code + * speaks. A business term is SPOKEN in a scope when every one of its words is; + * the code's own `domain` words no business term contains are the other half. + */ + +const toy = buildVocabulary(graphOf([{ lang: "java", entities: orderCorpus() }])); +const terms = (...lines: string[]): readonly BusinessTerm[] => parseBusinessTerms(lines.join("\n")).terms; + +/** `[text, scopes, missing]` per business term; `[word, weight, occurrences, scopes]` per code-only word. */ +function summary(artifact: VocabularyArtifact, report: DriftReport) { + return { + business: report.business.map(({ text, scopes, missing }) => [text, scopes, missing]), + codeOnly: report.codeOnly.map(({ term, weight, occurrences, scopes }) => [ + artifact.terms[term]?.text, + weight, + occurrences, + scopes, + ]), + }; +} + +describe("parseBusinessTerms", () => { + it("reads one term per line, split by the universal rules", () => { + expect(parseBusinessTerms("Order\nOrder Line\nbon de commande\nHTTPServer\n").terms).toEqual([ + { text: "Order", words: ["order"] }, + { text: "Order Line", words: ["order", "line"] }, + { text: "bon de commande", words: ["bon", "de", "commande"] }, + { text: "HTTPServer", words: ["http", "server"] }, + ]); + }); + + it("skips blank lines and # comments, and trims every line", () => { + expect(parseBusinessTerms("# the glossary\n\n Order Line \r\n\t\n#Customer\n").terms).toEqual([ + { text: "Order Line", words: ["order", "line"] }, + ]); + }); + + it("keeps the first spelling of a term whose words repeat an earlier one's", () => { + expect(parseBusinessTerms("Order Line\norder-line\nORDER_LINE\nLine Order\nOrder Order\n").terms).toEqual([ + { text: "Order Line", words: ["order", "line"] }, + { text: "Line Order", words: ["line", "order"] }, + { text: "Order Order", words: ["order"] }, + ]); + }); + + it("reports a line that names no word, by its line number", () => { + expect(parseBusinessTerms("Order\n2024\n— —\nForm 1099\n")).toEqual({ + terms: [ + { text: "Order", words: ["order"] }, + { text: "Form 1099", words: ["form"] }, + ], + wordless: [ + { line: 2, text: "2024" }, + { line: 3, text: "— —" }, + ], + }); + }); +}); + +describe("vocabularyDrift on the hand-counted corpus", () => { + // Scopes: 0 the corpus; 1 com.acme.order, 2 com.acme.order.billing; + // 3 billing/Invoice, 4 order/Order, 5 order/Order$Line. + const glossary = terms("Order", "Order Line", "Invoice", "Invoice Line", "Billing Invoice", "Customer"); + + it("at the corpus: a term is spoken when the corpus says every word", () => { + expect(summary(toy, vocabularyDrift(toy, glossary))).toEqual({ + business: [ + ["Order", [0], []], + ["Order Line", [0], []], + ["Invoice", [0], []], + ["Invoice Line", [0], []], + ["Billing Invoice", [0], []], + ["Customer", [], ["customer"]], + ], + // `billing` is the business's word too, in "Billing Invoice". + codeOnly: [ + ["lines", 4, 1, 1], + ["qty", 4, 1, 1], + ["total", 4, 2, 1], + ["acme", 2, 2, 1], + ], + }); + }); + + it("by module: every word in ONE module, or the term is not spoken there", () => { + expect(summary(toy, vocabularyDrift(toy, glossary, { level: "module" }))).toEqual({ + business: [ + // The billing package says `order` through its qualifier. + ["Order", [1, 2], []], + ["Order Line", [1], []], + ["Invoice", [2], []], + // Both words are spoken, never in one module: nothing is missing. + ["Invoice Line", [], []], + ["Billing Invoice", [2], []], + ["Customer", [], ["customer"]], + ], + codeOnly: [ + ["lines", 4, 1, 1], + ["qty", 4, 1, 1], + ["total", 4, 2, 1], + ["acme", 2, 2, 2], + ], + }); + }); + + it("by type: a word only a module's name says is missing from every type", () => { + expect(summary(toy, vocabularyDrift(toy, glossary, { level: "type" }))).toEqual({ + business: [ + ["Order", [4], []], + // `line` from the parameter orderLine, inside Order. + ["Order Line", [4], []], + ["Invoice", [3], []], + ["Invoice Line", [], []], + ["Billing Invoice", [], ["billing"]], + ["Customer", [], ["customer"]], + ], + codeOnly: [ + ["lines", 4, 1, 1], + ["qty", 4, 1, 1], + ["total", 4, 2, 1], + ], + }); + }); + + it("matches a business word to the code's word whatever its class", () => { + // `get` is technical, `com` too: the business still says them. + expect(vocabularyDrift(toy, terms("Get Total", "Com")).business.map(({ scopes }) => scopes)).toEqual([[0], [0]]); + }); + + it("lists as code-only the domain words alone, never a technical or keyword one", () => { + const report = vocabularyDrift(toy, []); + expect(report.codeOnly.map(({ term }) => toy.terms[term]?.class)).toEqual( + report.codeOnly.map(() => "domain"), + ); + expect(report.codeOnly.map(({ term }) => toy.terms[term]?.text)).not.toContain("get"); + }); + + it("never stems: `lines` is not the business's `line`", () => { + expect(summary(toy, vocabularyDrift(toy, terms("Line"))).codeOnly.map(([word]) => word)).toEqual([ + "order", + "billing", + "invoice", + "lines", + "qty", + "total", + "acme", + ]); + }); + + it("speaks a term with no word nowhere", () => { + expect(vocabularyDrift(toy, [{ text: "—", words: [] }]).business).toEqual([ + { text: "—", words: [], scopes: [], missing: [] }, + ]); + }); +}); + +const fixture = (name: string): VocabularyArtifact => { + const path = fileURLToPath(new URL(`../../../fixtures/${name}/expected/model.jsonl`, import.meta.url)); + return buildVocabulary(buildGraph(loadModels(readModelFileSync(path), { sources: [name] }).union)); +}; + +describe.each(["java", "ruby"])("vocabularyDrift's properties on fixtures/%s", (name) => { + const artifact = fixture(name); + const root = new Set(artifact.scopes[0]?.terms.map(([term]) => artifact.terms[term]?.text ?? "")); + const spoken = [...root].sort(); + const level = fc.constantFrom("corpus" as const, "module" as const, "type" as const); + + it("speaks at the corpus every term made of words the corpus says", () => { + fc.assert( + fc.property(fc.array(fc.subarray(spoken, { minLength: 1, maxLength: 3 }), { maxLength: 5 }), (lines) => { + const report = vocabularyDrift(artifact, terms(...lines.map((words) => words.join(" ")))); + return report.business.every(({ scopes, missing }) => scopes.length === 1 && scopes[0] === 0 && missing.length === 0); + }), + ); + }); + + it("speaks a term at a level only where the corpus speaks it, in scopes of that level", () => { + fc.assert( + fc.property(fc.array(fc.subarray(spoken, { minLength: 1, maxLength: 2 }), { maxLength: 5 }), level, (lines, at) => { + const report = vocabularyDrift(artifact, terms(...lines.map((words) => words.join(" "))), { level: at }); + return report.business.every(({ scopes }) => + scopes.every((scope) => artifact.scopes[scope]?.level === at) && + scopes.every((scope, i) => i === 0 || (scopes[i - 1] ?? -1) < scope), + ); + }), + ); + }); + + it("names as missing exactly the words no scope of the level says", () => { + fc.assert( + fc.property(fc.array(fc.stringMatching(/^[a-z]{3,8}$/), { minLength: 1, maxLength: 3 }), level, (words, at) => { + const [term] = terms(words.join(" ")); + if (term === undefined) return true; + const [found] = vocabularyDrift(artifact, [term], { level: at }).business; + const said = new Set( + artifact.scopes + .filter((scope) => scope.level === at) + .flatMap((scope) => scope.terms.map(([index]) => artifact.terms[index]?.text ?? "")), + ); + return JSON.stringify(found?.missing) === JSON.stringify(term.words.filter((word) => !said.has(word))); + }), + ); + }); + + it("keeps every code-only word out of every business term, and sums it over its level", () => { + fc.assert( + fc.property(fc.subarray(spoken, { maxLength: 20 }), level, (words, at) => { + const report = vocabularyDrift(artifact, terms(...words), { level: at }); + const named = new Set(words); + return report.codeOnly.every(({ term, weight, occurrences, scopes }) => { + const rows = artifact.scopes + .filter((scope) => scope.level === at) + .flatMap((scope) => scope.terms.filter(([index]) => index === term)); + return ( + artifact.terms[term]?.class === "domain" && + !named.has(artifact.terms[term]?.text ?? "") && + rows.length === scopes && + rows.reduce((sum, row) => sum + row[1], 0) === weight && + rows.reduce((sum, row) => sum + row[2], 0) === occurrences + ); + }); + }), + ); + }); +}); + From 274d3949229399a481ae72c19b2a8f864dee2f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Mon, 26 Oct 2026 08:15:39 +0100 Subject: [PATCH 2/7] feat(vocabulary): an operation's words on demand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `operationScope(graph, artifact, id, options)` counts one operation's words as the artifact counts a scope: the operation and every kept entity written inside it, lambdas and their parameters included, short of a type or module, which are scopes of their own. Its rows index the artifact's terms and its parent is the nearest stored scope, so a reader shows it beside the stored ones. PLAN.md §20.5 keeps these out of the artifact; `codegraph vocabulary --scope ID` will ask for one. The counting rule moves into one function both paths call, so the fixture snapshots are unchanged. An artifact built under other options is refused rather than mis-indexed. A C# delegate, invocable but a type, stays a type scope. Co-Authored-By: Claude Opus 5.5 --- packages/vocabulary/src/aggregate.ts | 197 +++++++++++++++------ packages/vocabulary/src/artifact.ts | 12 ++ packages/vocabulary/test/operation.test.ts | 166 +++++++++++++++++ 3 files changed, 324 insertions(+), 51 deletions(-) create mode 100644 packages/vocabulary/test/operation.test.ts diff --git a/packages/vocabulary/src/aggregate.ts b/packages/vocabulary/src/aggregate.ts index 6040b23..b704fb5 100644 --- a/packages/vocabulary/src/aggregate.ts +++ b/packages/vocabulary/src/aggregate.ts @@ -12,6 +12,7 @@ import { import { VOCABULARY_ARTEFACT_KIND, VOCABULARY_GENERATOR, + type OperationScope, type ScopeLevel, type TermRow, type VocabularyArtifact, @@ -55,8 +56,6 @@ export function buildVocabulary(graph: CodeGraph, options: VocabularyOptions = { const weights = options.weights ?? DEFAULT_WEIGHTS; const technical = options.technical ?? DEFAULT_TECHNICAL; const folder = folderFor(graph); - const slots = structuralSlots(graph.entities.values()); - const langOf = languagesOf(graph); const corpus = corpusOf(graph, options.name); const kept: Entity[] = []; @@ -76,57 +75,13 @@ export function buildVocabulary(graph: CodeGraph, options: VocabularyOptions = { // --- the words: one entity at a time, into every scope it folds into --- const terms = new Interner(); - const classifiers = new Map(); - const classifierOf = (lang: string): Classify => { - let classify = classifiers.get(lang); - if (classify === undefined) { - classify = createClassifier(getProfile(lang)?.lexicon, technical); - classifiers.set(lang, classify); - } - return classify; - }; + const speak = speakerFor(graph, terms, weights, technical); const unplaced: EntityId[] = []; for (const entity of kept) { const module = modules.get(folder.containingModule(entity.id) ?? ""); const type = types.get(folder.containingType(entity.id) ?? ""); if (module === undefined) unplaced.push(entity.id); - const into = [root, ...(module === undefined ? [] : [module]), ...(type === undefined ? [] : [type])]; - for (const scope of into) scope.entities += 1; - - const name = entityName(entity); - if (name === undefined) { - for (const scope of into) scope.skipped.unnamed += 1; - continue; - } - const lang = langOf.get(entity.id) ?? ""; - const split = splitIdentifier(name, entity.kind, getProfile(lang)?.lexicon); - if (split.conventional) { - for (const scope of into) scope.skipped.conventional += 1; - continue; - } - const role = roleOf(entity, slots); - const classify = classifierOf(lang); - - // This entity's own table first: a word it says twice has one carrier. - const own = new Map(); - split.tokens.forEach((token, i) => { - const term = terms.intern(token, classify(token)); - const row = own.get(term) ?? [0, 0]; - row[0] += i < split.qualifier ? weights.qualifier : weights[role]; - row[1] += 1; - own.set(term, row); - }); - for (const scope of into) { - scope.skipped.numeric += split.numeric; - scope.roles[role] += 1; - for (const [term, [weight, occurrences]] of own) { - const row = scope.table.get(term) ?? [0, 0, 0]; - row[0] += weight; - row[1] += occurrences; - row[2] += 1; - scope.table.set(term, row); - } - } + speak(entity, [root, ...(module === undefined ? [] : [module]), ...(type === undefined ? [] : [type])]); } // --- the artifact: terms sorted, scopes indexed, rows remapped -------- @@ -140,9 +95,7 @@ export function buildVocabulary(graph: CodeGraph, options: VocabularyOptions = { const spread = order.map(() => ({ modules: 0, types: 0, exclusive: -1 })); const scopes: VocabularyScope[] = tallies.map((scope, i) => { - const rows: TermRow[] = [...scope.table] - .map(([term, [weight, occurrences, carriers]]): TermRow => [index.get(term) ?? -1, weight, occurrences, carriers]) - .sort((a, b) => b[1] - a[1] || a[0] - b[0]); + const rows = rowsOf(scope, (term) => index.get(term) ?? -1); for (const [term] of rows) { const counts = spread[term]; if (counts === undefined || scope.level === "corpus") continue; @@ -177,6 +130,139 @@ export function buildVocabulary(graph: CodeGraph, options: VocabularyOptions = { }; } +/** + * AN OPERATION'S WORDS, computed on demand (PLAN.md §20.1; §20.5 keeps them + * out of the artifact): the operation and every kept entity written inside it, + * lambdas and their parameters included, short of a type or module, which are + * scopes of their own. Counted as `buildVocabulary` counts a scope, under the + * same options, and indexed by the artifact's terms; `undefined` when `id` is + * no kept operation. An artifact built under other options is refused: its + * terms would not be this count's. + */ +export function operationScope( + graph: CodeGraph, + artifact: VocabularyArtifact, + id: EntityId, + options: VocabularyOptions = {}, +): OperationScope | undefined { + const view = options.view ?? identityView; + const operation = graph.entity(id); + if (operation === undefined || !isOperation(operation) || graph.isStub(id) || !view.entity(operation, graph)) { + return undefined; + } + + const terms = new Interner(artifact.terms); + const speak = speakerFor(graph, terms, options.weights ?? DEFAULT_WEIGHTS, options.technical ?? DEFAULT_TECHNICAL); + const scope = tally("corpus", operation); + const pending: EntityId[] = [id]; + for (let cursor = pending.pop(); cursor !== undefined; cursor = pending.pop()) { + const entity = graph.entity(cursor); + if (entity === undefined) continue; + if (!graph.isStub(cursor) && view.entity(entity, graph)) speak(entity, [scope]); + for (const child of graph.childrenOf(cursor)) { + const inner = graph.entity(child); + if (inner !== undefined && !hasTrait(inner, "TModule") && !hasTrait(inner, "TType")) pending.push(child); + } + } + if (terms.size() !== artifact.terms.length) { + throw new Error(`operationScope: ${id} speaks words the artifact does not hold — it was built under other options`); + } + + const scopeIndex = new Map(); + artifact.scopes.forEach((stored, i) => { + if (stored.id !== undefined) scopeIndex.set(stored.id, i); + }); + const { level: _level, parent: _parent, ...counts } = scopeOf( + scope, + rowsOf(scope, (term) => term), + artifact.terms, + undefined, + artifact.corpus.name, + ); + return { + ...counts, + level: "operation", + id, + kind: operation.kind, + parent: parentScope(graph, operation, scopeIndex) ?? 0, + }; +} + +/** An invocable that is no scope of its own: a C# delegate, TType + TInvocable, is a type. */ +function isOperation(entity: Entity): boolean { + return hasTrait(entity, "TInvocable") && !hasTrait(entity, "TModule") && !hasTrait(entity, "TType"); +} + +/** + * Counts one entity's words into every scope given: its role's weight per + * word, `qualifier` before the last separator, one carrier per word however + * often the name says it. The one counting rule of the artifact and of an + * operation's scope. + */ +function speakerFor( + graph: CodeGraph, + terms: Interner, + weights: Weights, + technical: readonly string[], +): (entity: Entity, into: readonly Tally[]) => void { + const slots = structuralSlots(graph.entities.values()); + const langOf = languagesOf(graph); + const classifiers = new Map(); + const classifierOf = (lang: string): Classify => { + let classify = classifiers.get(lang); + if (classify === undefined) { + classify = createClassifier(getProfile(lang)?.lexicon, technical); + classifiers.set(lang, classify); + } + return classify; + }; + + return (entity, into) => { + for (const scope of into) scope.entities += 1; + const name = entityName(entity); + if (name === undefined) { + for (const scope of into) scope.skipped.unnamed += 1; + return; + } + const lang = langOf.get(entity.id) ?? ""; + const split = splitIdentifier(name, entity.kind, getProfile(lang)?.lexicon); + if (split.conventional) { + for (const scope of into) scope.skipped.conventional += 1; + return; + } + const role = roleOf(entity, slots); + const classify = classifierOf(lang); + + // This entity's own table first: a word it says twice has one carrier. + const own = new Map(); + split.tokens.forEach((token, i) => { + const term = terms.intern(token, classify(token)); + const row = own.get(term) ?? [0, 0]; + row[0] += i < split.qualifier ? weights.qualifier : weights[role]; + row[1] += 1; + own.set(term, row); + }); + for (const scope of into) { + scope.skipped.numeric += split.numeric; + scope.roles[role] += 1; + for (const [term, [weight, occurrences]] of own) { + const row = scope.table.get(term) ?? [0, 0, 0]; + row[0] += weight; + row[1] += occurrences; + row[2] += 1; + scope.table.set(term, row); + } + } + }; +} + +/** A scope's table as rows, by weight descending, then term. */ +function rowsOf(scope: Tally, termOf: (interned: number) => number): TermRow[] { + return [...scope.table] + .map(([term, [weight, occurrences, carriers]]): TermRow => [termOf(term), weight, occurrences, carriers]) + .sort((a, b) => b[1] - a[1] || a[0] - b[0]); +} + function tally(level: ScopeLevel, entity?: Entity): Tally { return { level, @@ -247,6 +333,15 @@ class Interner { private readonly ids = new Map>(); private readonly entries: { readonly id: number; readonly text: string; readonly class: TermClass }[] = []; + /** Seeded with an artifact's terms, an interned word IS its index there. */ + constructor(seed: readonly { readonly text: string; readonly class: TermClass }[] = []) { + for (const { text, class: termClass } of seed) this.intern(text, termClass); + } + + size(): number { + return this.entries.length; + } + intern(text: string, termClass: TermClass): number { let byClass = this.ids.get(text); if (byClass === undefined) { diff --git a/packages/vocabulary/src/artifact.ts b/packages/vocabulary/src/artifact.ts index 7f41e63..eacd413 100644 --- a/packages/vocabulary/src/artifact.ts +++ b/packages/vocabulary/src/artifact.ts @@ -67,6 +67,18 @@ export interface VocabularyScope { readonly terms: readonly TermRow[]; } +/** + * An operation's scope, computed on demand and never stored (PLAN.md §20.5): + * its rows index the artifact's `terms`, its parent the artifact's `scopes`. + */ +export interface OperationScope extends Omit { + readonly level: "operation"; + readonly id: string; + readonly kind: string; + /** The nearest stored scope around it; the corpus when there is none. */ + readonly parent: number; +} + export interface VocabularyDiagnostics { /** Kept entities no module contains, sorted: counted at the corpus alone. */ readonly unplaced: readonly string[]; diff --git a/packages/vocabulary/test/operation.test.ts b/packages/vocabulary/test/operation.test.ts new file mode 100644 index 0000000..8715f46 --- /dev/null +++ b/packages/vocabulary/test/operation.test.ts @@ -0,0 +1,166 @@ +import { fileURLToPath } from "node:url"; +import { readModelFileSync, type Entity } from "@codegraph/core"; +import { buildGraph, hasTrait, loadModels, makeView } from "@codegraph/analyzer"; +import { describe, expect, it } from "vitest"; +import { buildVocabulary, operationScope } from "../src/aggregate.js"; +import type { VocabularyArtifact } from "../src/artifact.js"; +import { ROLES } from "../src/weight.js"; +import { cls, graphOf, method, orderCorpus, structural } from "./toy.js"; + +/** + * An operation's words, computed on demand (PLAN.md §20.1; §20.5 keeps them + * out of the artifact): the operation and every kept entity written inside it, + * short of a type or module, which are scopes of their own. + */ + +const NO_ROLES = Object.fromEntries(ROLES.map((role) => [role, 0])); +const Order = "java:com.acme.order/Order"; +const getTotal = `${Order}.getTotal(Line)`; + +/** A scope's rows as `[text, weight, occurrences, carriers]`. */ +const words = (artifact: VocabularyArtifact, rows: readonly (readonly [number, number, number, number])[]) => + rows.map(([term, w, n, c]) => [artifact.terms[term]?.text, w, n, c]); + +describe("operationScope on the hand-counted corpus", () => { + const graph = graphOf([{ lang: "java", entities: orderCorpus() }]); + const artifact = buildVocabulary(graph); + + it("counts the operation, its parameters and its locals, as the artifact counts a scope", () => { + const scope = operationScope(graph, artifact, getTotal); + expect(scope).toMatchObject({ + level: "operation", + id: getTotal, + name: "getTotal", + kind: "method", + // order/Order, the nearest stored scope around it. + parent: 4, + entities: 3, + roles: { ...NO_ROLES, operation: 1, parameter: 1, local: 1 }, + skipped: { unnamed: 0, conventional: 0, numeric: 0 }, + distinct: 4, + distinctDomain: 3, + weight: 9, + occurrences: 5, + }); + expect(words(artifact, scope?.terms ?? [])).toEqual([ + ["total", 4, 2, 2], + ["get", 3, 1, 1], + ["line", 1, 1, 1], + ["order", 1, 1, 1], + ]); + }); + + it("indexes the artifact's own terms", () => { + const scope = operationScope(graph, artifact, getTotal); + for (const [term] of scope?.terms ?? []) expect(artifact.scopes[0]?.terms.map(([index]) => index)).toContain(term); + }); + + it("says what a conventional or unnamed operation skipped", () => { + expect(operationScope(graph, artifact, `${Order}.toString()`)).toMatchObject({ + entities: 1, + skipped: { unnamed: 0, conventional: 1, numeric: 0 }, + terms: [], + }); + expect(operationScope(graph, artifact, `${Order}.()`)).toMatchObject({ + name: `${Order}.()`, + entities: 1, + skipped: { unnamed: 1, conventional: 0, numeric: 0 }, + terms: [], + }); + }); + + it("is no operation scope for a module, a type, a member that is no operation, or an unknown id", () => { + for (const id of ["java:com.acme.order", Order, `${Order}#lines`, "java:nowhere/Nothing"]) { + expect(operationScope(graph, artifact, id), id).toBeUndefined(); + } + }); + + it("follows the view: what the view drops is not counted", () => { + const noParameters = makeView("noParameters", (entity) => entity.kind !== "parameter", () => true); + const filtered = buildVocabulary(graph, { view: noParameters }); + const scope = operationScope(graph, filtered, getTotal, { view: noParameters }); + expect(scope?.entities).toBe(2); + expect(words(filtered, scope?.terms ?? [])).toEqual([ + ["total", 4, 2, 2], + ["get", 3, 1, 1], + ]); + expect(operationScope(graph, filtered, `${Order}.()`, { view: makeView("none", () => false, () => true) })).toBeUndefined(); + }); + + it("refuses an artifact built under other options than its own", () => { + expect(() => operationScope(graph, artifact, getTotal, { technical: [] })).toThrow(/other options/); + }); +}); + +describe("operationScope stops at a scope of its own", () => { + const lambda = `${getTotal}#lambda$0`; + const local = `${getTotal}/Local`; + const extra: Entity[] = [ + { + id: lambda, + kind: "lambda", + traits: ["TInvocable", "TWithParameters", "TChildOf"], + signature: "(discount) -> …", + parameters: [`${lambda}#param:discount`], + parent: getTotal, + } as Entity, + structural(`${lambda}#param:discount`, "parameter", "discount", lambda), + cls(local, "Rebate", getTotal), + structural(`${local}#amount`, "attribute", "amount", local), + method(`${local}.apply()`, "apply", local), + ]; + const graph = graphOf([{ lang: "java", entities: [...orderCorpus(), ...extra] }]); + const artifact = buildVocabulary(graph); + + it("counts a lambda and its parameter, never a local class or its members", () => { + const scope = operationScope(graph, artifact, getTotal); + expect(scope?.entities).toBe(5); + expect(scope?.skipped.unnamed).toBe(1); + expect(words(artifact, scope?.terms ?? []).map(([text]) => text)).toEqual(["total", "get", "discount", "line", "order"]); + }); +}); + +describe.each(["java", "csharp", "typescript", "elixir", "ruby"])("operationScope over fixtures/%s", (name) => { + const path = fileURLToPath(new URL(`../../../fixtures/${name}/expected/model.jsonl`, import.meta.url)); + const graph = buildGraph(loadModels(readModelFileSync(path), { sources: [name] }).union); + const artifact = buildVocabulary(graph); + const operations = graph + .ids() + .map((id) => graph.entity(id)) + .filter((entity): entity is Entity => entity !== undefined && !graph.isStub(entity.id)) + .filter((entity) => hasTrait(entity, "TInvocable") && !hasTrait(entity, "TType") && !hasTrait(entity, "TModule")); + + it("has operations to check", () => { + expect(operations.length).toBeGreaterThan(0); + }); + + it("adds up, and never says more than the stored scope around it", () => { + for (const operation of operations) { + const scope = operationScope(graph, artifact, operation.id); + expect(scope, operation.id).toBeDefined(); + if (scope === undefined) continue; + const roles = Object.values(scope.roles).reduce((sum, n) => sum + n, 0); + expect(scope.entities).toBe(roles + scope.skipped.unnamed + scope.skipped.conventional); + expect(scope.weight).toBe(scope.terms.reduce((sum, row) => sum + row[1], 0)); + expect(scope.occurrences).toBe(scope.terms.reduce((sum, row) => sum + row[2], 0)); + const around = new Map(artifact.scopes[scope.parent]?.terms.map((row) => [row[0], row] as const)); + for (const [term, weight, occurrences, carriers] of scope.terms) { + const row = around.get(term); + expect(row, `${operation.id}: ${artifact.terms[term]?.text} is said around it`).toBeDefined(); + expect(weight).toBeLessThanOrEqual(row?.[1] ?? 0); + expect(occurrences).toBeLessThanOrEqual(row?.[2] ?? 0); + expect(carriers).toBeLessThanOrEqual(row?.[3] ?? 0); + } + } + }); +}); + +it("csharp: a delegate is invocable, and a type scope rather than an operation's", () => { + const path = fileURLToPath(new URL("../../../fixtures/csharp/expected/model.jsonl", import.meta.url)); + const graph = buildGraph(loadModels(readModelFileSync(path), { sources: ["csharp"] }).union); + const artifact = buildVocabulary(graph); + const delegate = [...graph.entities.values()].find((entity) => entity.kind === "delegate"); + expect(delegate === undefined ? false : hasTrait(delegate, "TInvocable")).toBe(true); + expect(artifact.scopes.find((scope) => scope.id === delegate?.id)?.level).toBe("type"); + expect(operationScope(graph, artifact, delegate?.id ?? "")).toBeUndefined(); +}); From 1fb328f28b9e3ca71529bd54e3c5bec61b0f8741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Mon, 26 Oct 2026 08:19:40 +0100 Subject: [PATCH 3/7] feat(vocabulary): read a technical list from text `parseTechnicalWords(text)` reads the list `--technical FILE` will replace the shared one with: one word per line, lowercased as the splitter lowers a token, blank lines and `#` comments skipped. A line that is not one word (`HTTPServer`, `value object`, `2024`) could never match a token, so it is refused by its line number instead of kept. Co-Authored-By: Claude Opus 5.5 --- packages/vocabulary/src/classify.ts | 24 ++++++++++++++++++ packages/vocabulary/test/classify.test.ts | 30 ++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/vocabulary/src/classify.ts b/packages/vocabulary/src/classify.ts index 738bb2d..0950c0e 100644 --- a/packages/vocabulary/src/classify.ts +++ b/packages/vocabulary/src/classify.ts @@ -1,4 +1,5 @@ import type { Lexicon } from "@codegraph/core"; +import { splitIdentifier } from "./split.js"; /** * The classes (PLAN.md §20.1): every word is kept and tagged, never filtered — @@ -64,3 +65,26 @@ function isShort(token: string): boolean { for (const char of token) if (!MARK.test(char) && ++letters > 1) return false; return true; } + +/** + * A technical list as a file holds it (`--technical FILE`): one word per line, + * lowercased as the splitter lowers a token; blank lines and `#` comments + * skipped. A line that is not one word could never match a token, so it is + * refused by its line number rather than kept. + */ +export function parseTechnicalWords(text: string): { + readonly words: readonly string[]; + readonly invalid: readonly { readonly line: number; readonly text: string }[]; +} { + const words: string[] = []; + const invalid: { line: number; text: string }[] = []; + text.split("\n").forEach((raw, i) => { + const line = raw.trim(); + if (line.length === 0 || line.startsWith("#")) return; + const { tokens } = splitIdentifier(line, "", undefined); + const [word] = tokens; + if (tokens.length === 1 && word !== undefined && WORD.test(line)) words.push(word); + else invalid.push({ line: i + 1, text: line }); + }); + return { words, invalid }; +} diff --git a/packages/vocabulary/test/classify.test.ts b/packages/vocabulary/test/classify.test.ts index ed284d3..0b7ab76 100644 --- a/packages/vocabulary/test/classify.test.ts +++ b/packages/vocabulary/test/classify.test.ts @@ -1,7 +1,7 @@ import { PROFILES, getProfile, type Lexicon } from "@codegraph/core"; import fc from "fast-check"; import { describe, expect, it } from "vitest"; -import { DEFAULT_TECHNICAL, TERM_CLASSES, createClassifier, type TermClass } from "../src/classify.js"; +import { DEFAULT_TECHNICAL, TERM_CLASSES, createClassifier, parseTechnicalWords, type TermClass } from "../src/classify.js"; /** * Classification, not filtering (PLAN.md §20 principle 3): every word is kept @@ -152,3 +152,31 @@ describe("properties", () => { } }); }); + +describe("parseTechnicalWords", () => { + it("reads one word per line, lowercased as the splitter lowers a token", () => { + expect(parseTechnicalWords("dto\nDAO\n Bean \r\nutf8\n")).toEqual({ words: ["dto", "dao", "bean", "utf8"], invalid: [] }); + }); + + it("skips blank lines and # comments", () => { + expect(parseTechnicalWords("# plumbing\n\ndto\n#dao\n").words).toEqual(["dto"]); + }); + + it("refuses a line that is not one word, by its line number: it could never match a token", () => { + expect(parseTechnicalWords("dto\nHTTPServer\nvalue object\nfoo-bar\n2024\n")).toEqual({ + words: ["dto"], + invalid: [ + { line: 2, text: "HTTPServer" }, + { line: 3, text: "value object" }, + { line: 4, text: "foo-bar" }, + { line: 5, text: "2024" }, + ], + }); + }); + + it("classifies by the words it read", () => { + const { words } = parseTechnicalWords("order\n"); + expect(createClassifier(undefined, words)("order")).toBe("technical"); + expect(createClassifier(undefined, words)("dto")).toBe("domain"); + }); +}); From 8ae0aaf7de8a0018ca0b47715cf77a99794975c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Mon, 26 Oct 2026 08:28:59 +0100 Subject: [PATCH 4/7] refactor(cli): share the view label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `analyze` printed its view as `name (filters)`, or `all (nothing filtered …)`, through a private helper. It moves into view.ts beside `resolveView`, so the next report that states its view prints it the same way. analyze's output is unchanged. Co-Authored-By: Claude Opus 5.5 --- packages/cli/src/commands/analyze.ts | 8 +------- packages/cli/src/view.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/commands/analyze.ts b/packages/cli/src/commands/analyze.ts index f5044a4..07efe94 100644 --- a/packages/cli/src/commands/analyze.ts +++ b/packages/cli/src/commands/analyze.ts @@ -28,7 +28,7 @@ import { EXIT, type ExitCode } from "../exit.js"; import { errLine, errLines, outLines, type IoSink } from "../io.js"; import { benignDuplicateIds } from "../load.js"; import { openAnalysis, type AnalysisSource } from "../source.js"; -import { resolveView } from "../view.js"; +import { resolveView, viewLabel } from "../view.js"; /** * `codegraph analyze --report deps|cycles|coupling`. @@ -350,12 +350,6 @@ function rankingLine(ranking: Ranking, noun: string): string { /* ------------------------------------------------------------- shared text */ -function viewLabel(view: ViewDescriptor): string { - return view.filters.length === 0 - ? `${view.name} (nothing filtered — stubs and inferred edges included)` - : `${view.name} (${view.filters.join(", ")})`; -} - /** The header IS part of the artifact: a number without its view is not a fact. */ function headerLines(context: ReportContext): readonly string[] { return [ diff --git a/packages/cli/src/view.ts b/packages/cli/src/view.ts index 608053a..189ae80 100644 --- a/packages/cli/src/view.ts +++ b/packages/cli/src/view.ts @@ -4,6 +4,7 @@ import { identityView, internalOnly, type View, + type ViewDescriptor, } from "@codegraph/analyzer"; import type { ViewOptions } from "./args.js"; @@ -24,3 +25,10 @@ export function resolveView(options: ViewOptions): View { if (views.length === 1) return views[0] as View; return composeViews(...views); } + +/** A view as a report's header states it: a number without its view is not a fact. */ +export function viewLabel(view: ViewDescriptor): string { + return view.filters.length === 0 + ? `${view.name} (nothing filtered — stubs and inferred edges included)` + : `${view.name} (${view.filters.join(", ")})`; +} From 92576576d67b8f779843994466b7afde5882a543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Mon, 26 Oct 2026 08:28:59 +0100 Subject: [PATCH 5/7] feat(cli): codegraph vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `codegraph vocabulary [model.jsonl...]` reports the words the model's names are made of, as `@codegraph/vocabulary` counts them (PLAN.md §20.1). Every report states its view, weights and technical list. - No flag: the corpus's 40 heaviest domain words. Each row gives weight, occurrences, carriers, and how many modules and types say it. - `--level module|type`: one block per scope, ranked by distinct terms (the cognitive-load report), 10 words each by default. - `--scope ID`: one stored module or type, or an operation counted on demand. An unknown id, a stub, or a member that is no operation is a usage error that says what the id is. - `--class domain|technical|keyword|short|all` and `--top N` pick the rows. - `--weights KEY=N,…` overrides weights over the defaults. An unknown key, a repeated key or a negative weight is a usage error. - `--technical FILE` replaces the shared list whole; a line that is not one word is refused by its number. - `--drift FILE` compares with a glossary at `--level`. It lists the business terms no scope says, with their missing words, the ones it says, and the domain words no term contains. It exits 3 when a term goes unsaid, as `analyze --report cycles` does for a cycle. - `--out FILE` writes vocabulary.json beside the report. - `--json`, `--internal-only`, `--declared-only` and `--no-cache` work as on the other commands. The cache path counts the same words. Tests cover each flag, text/JSON parity, determinism, stream purity and the exit codes end to end. Co-Authored-By: Claude Opus 5.5 --- packages/cli/package.json | 3 +- packages/cli/src/args.ts | 152 ++++++ packages/cli/src/commands/vocabulary.ts | 505 ++++++++++++++++++++ packages/cli/src/main.ts | 3 + packages/cli/test/args.test.ts | 93 ++++ packages/cli/test/cache.test.ts | 32 ++ packages/cli/test/e2e-determinism.test.ts | 4 + packages/cli/test/e2e-exit-codes.test.ts | 11 + packages/cli/test/e2e-json-parity.test.ts | 25 + packages/cli/test/e2e-stream-purity.test.ts | 7 + packages/cli/test/vocabulary.test.ts | 405 ++++++++++++++++ packages/cli/vitest.config.ts | 1 + pnpm-lock.yaml | 3 + 13 files changed, 1243 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/commands/vocabulary.ts create mode 100644 packages/cli/test/vocabulary.test.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index d6df520..17e8ab5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -24,7 +24,8 @@ "@codegraph/navigator": "workspace:*", "@codegraph/navigator-ui": "workspace:*", "@codegraph/scm": "workspace:*", - "@codegraph/viz": "workspace:*" + "@codegraph/viz": "workspace:*", + "@codegraph/vocabulary": "workspace:*" }, "devDependencies": { "@types/node": "catalog:", diff --git a/packages/cli/src/args.ts b/packages/cli/src/args.ts index 5911ac6..bb55e7b 100644 --- a/packages/cli/src/args.ts +++ b/packages/cli/src/args.ts @@ -3,6 +3,7 @@ import { basename, resolve } from "node:path"; import { parseArgs } from "node:util"; import { FOLD_LEVELS, FRAMEWORK_PROFILES, type FoldLevel } from "@codegraph/analyzer"; import { METRIC_PREFIXES, SCALES, metricNames } from "@codegraph/city"; +import { SCOPE_LEVELS, TERM_CLASSES, WEIGHT_KEYS, type ScopeLevel, type TermClass, type WeightKey } from "@codegraph/vocabulary"; import { defaultDataDir } from "./app/store.js"; import { UsageError } from "./exit.js"; @@ -26,6 +27,7 @@ export const COMMAND_NAMES = [ "export", "city", "navigator", + "vocabulary", "serve", "domain-facts", "explain", @@ -372,6 +374,78 @@ export const NAVIGATOR_SPEC: CommandSpec = { ], }; +/** `vocabulary --class`: one of the package's classes, or every one. */ +export const VOCABULARY_CLASSES = [...TERM_CLASSES, "all"] as const; +export type VocabularyClass = (typeof VOCABULARY_CLASSES)[number]; + +/** + * `codegraph vocabulary`: the words the model's names are made of (PLAN.md + * §20.1). The counts are `@codegraph/vocabulary`'s; this command picks the + * scope, the class and the report, and writes the artifact on `--out`. + */ +export const VOCABULARY_SPEC: CommandSpec = { + name: "vocabulary", + summary: "Report the words the model's names are made of, per corpus, module or type; or drift against a glossary.", + positional: MODELS_POSITIONAL_DEFAULTED, + options: [ + { + name: "level", + type: "string", + describe: "The corpus's words, or one block per module or type, ranked by distinct terms (the cognitive-load report).", + choices: SCOPE_LEVELS, + defaultValue: "corpus", + }, + { + name: "scope", + type: "string", + describe: "Report one scope: a module or type by its entity id, or an operation, counted on demand.", + placeholder: "ID", + }, + { + name: "top", + type: "string", + describe: "Show the N heaviest words: 40 by default, 10 per block under --level module|type.", + placeholder: "N", + integer: true, + }, + { + name: "class", + type: "string", + describe: "Which words to show.", + choices: VOCABULARY_CLASSES, + defaultValue: "domain", + }, + { + name: "weights", + type: "string", + describe: `Override weights, e.g. type=6,qualifier=0. Keys: ${WEIGHT_KEYS.join(", ")}.`, + placeholder: "KEY=N,…", + }, + { + name: "technical", + type: "string", + describe: "Replace the shared technical words with this file's, one word per line; each language's own still apply.", + placeholder: "FILE", + }, + { + name: "drift", + type: "string", + describe: + "Compare with a glossary, one business term per line: the terms no scope of --level says (exit 3), and the domain words no term contains.", + placeholder: "FILE", + }, + { + name: "out", + type: "string", + describe: "Also write the vocabulary.json artifact to this file.", + placeholder: "FILE", + }, + JSON_OPTION, + ...VIEW_OPTIONS, + NO_CACHE_OPTION, + ], +}; + /** * `codegraph serve`: ONE PAGE over a model — the navigator (tree, evidence, * graph, cycles, coupling) with the 3D city as one of its tabs. Both @@ -1063,6 +1137,7 @@ export const COMMAND_SPECS: readonly CommandSpec[] = [ EXPORT_SPEC, CITY_SPEC, NAVIGATOR_SPEC, + VOCABULARY_SPEC, SERVE_SPEC, DOMAIN_FACTS_SPEC, EXPLAIN_SPEC, @@ -1146,6 +1221,24 @@ export interface NavigatorOptions extends ModelInputOptions, ViewOptions, CacheO readonly out: string | undefined; } +export interface VocabularyOptions extends ModelInputOptions, ViewOptions, CacheOptions { + readonly level: ScopeLevel; + /** `--scope ID`: one module, type or operation; undefined = every scope of `level`. */ + readonly scope: string | undefined; + /** `--top N`; undefined lets each report pick its own default. */ + readonly top: number | undefined; + readonly termClass: TermClass | "all"; + /** `--weights`: the keys typed, laid over the defaults by the command. */ + readonly weights: Readonly>>; + /** `--technical FILE`; undefined = the shared list. */ + readonly technical: string | undefined; + /** `--drift FILE`: the glossary to compare with. */ + readonly drift: string | undefined; + /** `--out FILE`: the artifact, beside the report. */ + readonly out: string | undefined; + readonly json: boolean; +} + export interface ServeOptions extends ModelInputOptions, CityBuildOptions, CacheOptions { /** `--port N`; 0 = an ephemeral port (the default under `--app`). */ readonly port: number; @@ -1316,6 +1409,7 @@ export type Invocation = | { readonly kind: "run"; readonly command: "export"; readonly options: ExportOptions } | { readonly kind: "run"; readonly command: "city"; readonly options: CityOptions } | { readonly kind: "run"; readonly command: "navigator"; readonly options: NavigatorOptions } + | { readonly kind: "run"; readonly command: "vocabulary"; readonly options: VocabularyOptions } | { readonly kind: "run"; readonly command: "serve"; readonly options: ServeOptions } | { readonly kind: "run"; readonly command: "insights"; readonly options: InsightsOptions } | { readonly kind: "run"; readonly command: "domain-facts"; readonly options: DomainFactsOptions } @@ -1811,6 +1905,8 @@ export function parseInvocation(argv: readonly string[]): Invocation { out: stringOf(values, "out"), }, }; + case "vocabulary": + return { kind: "run", command: "vocabulary", options: { models, ...vocabularyOf(values) } }; case "serve": { const app = appOf(values, spec, models); return { @@ -2022,6 +2118,62 @@ export function parseInvocation(argv: readonly string[]): Invocation { } } +function vocabularyOf(values: ParsedValues): Omit { + const scope = stringOf(values, "scope"); + const drift = stringOf(values, "drift"); + // A flag the chosen report would not read is refused, never silently dropped. + const unread: readonly (readonly [boolean, string, string])[] = [ + [scope !== undefined && stringOf(values, "level") !== undefined, "--level", "--scope"], + [scope !== undefined && drift !== undefined, "--drift", "--scope"], + [drift !== undefined && stringOf(values, "class") !== undefined, "--class", "--drift"], + ]; + for (const [conflict, flag, report] of unread) { + if (!conflict) continue; + throw new UsageError( + `${flag} does not apply with ${report}`, + report === "--scope" + ? "--scope reports one scope; drop --level and --drift, or drop --scope." + : "--drift compares the business terms with every word, and lists the domain words no term contains.", + ); + } + return { + level: (stringOf(values, "level") ?? "corpus") as ScopeLevel, + scope, + top: integerOf(values, "top"), + termClass: (stringOf(values, "class") ?? "domain") as TermClass | "all", + weights: weightsOf(stringOf(values, "weights")), + technical: stringOf(values, "technical"), + drift, + out: stringOf(values, "out"), + json: flagOf(values, "json"), + ...viewOf(values), + noCache: flagOf(values, "no-cache"), + }; +} + +/** `--weights type=6,qualifier=0`: a known key, once, weighing a number >= 0. */ +function weightsOf(raw: string | undefined): Partial> { + if (raw === undefined) return {}; + const weights: Partial> = {}; + for (const entry of raw.split(",")) { + const [key = "", value, ...rest] = entry.split("=").map((part) => part.trim()); + const weight = Number(value); + if (value === undefined || value === "" || rest.length > 0 || key === "" || !Number.isFinite(weight) || weight < 0) { + throw new UsageError( + `invalid entry '${entry.trim()}' in --weights`, + "Each entry is KEY=N, N a number >= 0: --weights type=6,qualifier=0", + ); + } + if (!(WEIGHT_KEYS as readonly string[]).includes(key)) { + throw new UsageError(`unknown role '${key}' in --weights`, `Valid keys: ${WEIGHT_KEYS.join(", ")}.`); + } + const role = key as WeightKey; + if (weights[role] !== undefined) throw new UsageError(`--weights weighs ${role} twice`, "Name each key once."); + weights[role] = weight; + } + return weights; +} + /** `--time`: unix seconds, or anything `Date.parse` reads (ISO dates). */ function timeOf(raw: string | undefined): number | undefined { if (raw === undefined) return undefined; diff --git a/packages/cli/src/commands/vocabulary.ts b/packages/cli/src/commands/vocabulary.ts new file mode 100644 index 0000000..9df86e4 --- /dev/null +++ b/packages/cli/src/commands/vocabulary.ts @@ -0,0 +1,505 @@ +import { Buffer } from "node:buffer"; +import { readFileSync } from "node:fs"; +import { type CodeGraph, type ViewDescriptor } from "@codegraph/analyzer"; +import { + DEFAULT_WEIGHTS, + WEIGHT_KEYS, + buildVocabulary, + operationScope, + parseBusinessTerms, + parseTechnicalWords, + vocabularyDrift, + vocabularyToJsonString, + type BusinessTerms, + type DriftReport, + type OperationScope, + type ScopeLevel, + type VocabularyArtifact, + type VocabularyOptions as CountOptions, + type VocabularyScope, + type Weights, +} from "@codegraph/vocabulary"; +import type { VocabularyOptions } from "../args.js"; +import { EXIT, UsageError, type ExitCode } from "../exit.js"; +import { errLine, errLines, outLines, type IoSink } from "../io.js"; +import { openAnalysis } from "../source.js"; +import { resolveView, viewLabel } from "../view.js"; + +/** + * `codegraph vocabulary `: the words the model's names are + * made of (PLAN.md §20.1). + * + * A SHELL over `@codegraph/vocabulary`, like `analyze` over the analyzer: + * every count printed here is the artifact's, or an operation's counted by + * the package on demand. This command picks the scope, the class and the + * report, and writes the artifact on `--out` beside the report. + * + * The header IS part of the report: a count without its view, its weights and + * its technical list is not a fact, so all three are stated on stdout. + * + * `--drift` exits 3 when a business term is said by no scope of the level, + * as `analyze --report cycles` does when a cycle exists: the tool worked, and + * the answer is bad news a CI job may gate on. + */ +export function vocabularyCommand(options: VocabularyOptions, io: IoSink): ExitCode { + // The flag files first: a bad one must fail before the model is read. + const technical = options.technical === undefined ? undefined : technicalOf(options.technical); + const glossary = options.drift === undefined ? undefined : glossaryOf(options.drift); + const weights: Weights = { ...DEFAULT_WEIGHTS, ...options.weights }; + + const source = openAnalysis(options.models, options, io); + try { + const graph = source.graph(); + const view = resolveView(options); + const counting: CountOptions = { view, weights, ...(technical === undefined ? {} : { technical: technical.words }) }; + const artifact = buildVocabulary(graph, counting); + errLines(io, vocabularyWarnings(source, artifact)); + + const header: Header = { + models: source.paths, + view: artifact.view, + weights: artifact.weights, + technical: artifact.technical, + technicalFile: options.technical, + }; + let code: ExitCode = source.clean ? EXIT.OK : EXIT.FINDINGS; + if (glossary !== undefined) { + const report = vocabularyDrift(artifact, glossary.terms, { level: options.level }); + errLines( + io, + glossary.wordless.map(({ line, text }) => `warning: line ${line} of ${glossary.path} names no word, skipped: ${text}`), + ); + outLines(io, driftReport(artifact, report, header, options, glossary)); + const unsaid = report.business.filter((term) => term.scopes.length === 0).length; + if (unsaid > 0) { + errLine(io, `${plural(unsaid, "business term")} ${neverSays(report.level)} — exiting 3 (findings).`); + code = EXIT.FINDINGS; + } + } else { + const scopes = + options.scope === undefined + ? scopesAt(artifact, options.level) + : [scopeOf(graph, artifact, options.scope, counting)]; + const top = options.top ?? (options.scope === undefined && options.level !== "corpus" ? 10 : 40); + outLines(io, wordsReport(artifact, scopes, header, options, top)); + } + + if (options.out !== undefined) { + const text = vocabularyToJsonString(artifact); + io.writeFile(options.out, text); + errLine( + io, + `wrote ${plural(Buffer.byteLength(text, "utf8"), "byte")} to ${options.out} ` + + `(vocabulary, view ${artifact.view.name}, ${plural(artifact.terms.length, "term")}, ` + + `${plural(artifact.scopes.length, "scope")}).`, + ); + } + return code; + } finally { + source.close(); + } +} + +/* ------------------------------------------------------------- flag files */ + +function readText(path: string, flag: string): string { + try { + return readFileSync(path, "utf8"); + } catch (error) { + throw new UsageError( + `cannot read ${path}: ${error instanceof Error ? error.message : String(error)}`, + `${flag} takes a plain text file, one entry per line.`, + { cause: error }, + ); + } +} + +function technicalOf(path: string): { readonly words: readonly string[] } { + const parsed = parseTechnicalWords(readText(path, "--technical")); + if (parsed.invalid.length > 0) { + throw new UsageError( + `${plural(parsed.invalid.length, "line")} of ${path} ${parsed.invalid.length === 1 ? "is" : "are"} not one word`, + [ + ...parsed.invalid.slice(0, 10).map(({ line, text }) => ` line ${line}: ${text}`), + "A technical entry is one word, as the splitter yields it: `dto`, not `HTTPServer` or `value object`.", + ].join("\n"), + ); + } + return parsed; +} + +/** A glossary and the path it was read from, which the report names. */ +interface Glossary extends BusinessTerms { + readonly path: string; +} + +function glossaryOf(path: string): Glossary { + const parsed = parseBusinessTerms(readText(path, "--drift")); + if (parsed.terms.length === 0) { + throw new UsageError( + `${path} holds no business term`, + "--drift takes one business term per line: Order Line, Customer, …", + ); + } + return { path, ...parsed }; +} + +/* ------------------------------------------------------------- the scopes */ + +/** A stored scope, or an operation's: what a block prints. */ +type AnyScope = VocabularyScope | OperationScope; + +/** The level's scopes, by distinct terms descending, ties by artifact order (which is id order). */ +function scopesAt(artifact: VocabularyArtifact, level: ScopeLevel): readonly AnyScope[] { + return artifact.scopes + .map((scope, index) => ({ scope, index })) + .filter(({ scope }) => scope.level === level) + .sort((a, b) => b.scope.distinct - a.scope.distinct || a.index - b.index) + .map(({ scope }) => scope); +} + +/** + * `--scope ID`: a stored module or type, else an operation counted on demand. + * Anything else is a usage error that says what the id IS — an unknown id is + * a typo, a stub speaks no words, a member's words are its scope's. + */ +function scopeOf( + graph: CodeGraph, + artifact: VocabularyArtifact, + id: string, + counting: CountOptions, +): AnyScope { + const stored = artifact.scopes.find((scope) => scope.id === id); + if (stored !== undefined) return stored; + const entity = graph.entity(id); + if (entity === undefined) { + throw new UsageError( + `unknown scope '${id}'`, + "Pass a module, type or operation id as the model writes it (`codegraph navigator` and `analyze` print them).", + ); + } + if (graph.isStub(id)) { + throw new UsageError(`${id} is external (a stub): its name was chosen elsewhere, so it speaks no words`); + } + const operation = operationScope(graph, artifact, id, counting); + if (operation !== undefined) return operation; + throw new UsageError( + `${id} is a ${entity.kind}, not a module, type or operation`, + `Its words are counted in ${aroundOf(graph, artifact, id)}.`, + ); +} + +/** The nearest stored scope up the containment chain, by id; the corpus when there is none. */ +function aroundOf(graph: CodeGraph, artifact: VocabularyArtifact, id: string): string { + const ids = new Set(artifact.scopes.map((scope) => scope.id)); + const seen = new Set([id]); + for (let cursor = graph.parentOf(id); cursor !== undefined && !seen.has(cursor); cursor = graph.parentOf(cursor)) { + if (ids.has(cursor)) return cursor; + seen.add(cursor); + } + return `the corpus ${artifact.corpus.name}`; +} + +/* ------------------------------------------------------------- the header */ + +/** Both reports' `--json` kind; `report` says which. */ +const REPORT_KIND = "codegraph.vocabularyReport/1"; + +interface Header { + readonly models: readonly string[]; + readonly view: ViewDescriptor; + readonly weights: Weights; + readonly technical: VocabularyArtifact["technical"]; + readonly technicalFile: string | undefined; +} + +function headerLines(report: "words" | "drift", header: Header, where: string): readonly string[] { + const list = header.technicalFile === undefined ? "" : ` from ${header.technicalFile}`; + return [ + `codegraph vocabulary — ${report}`, + `models: ${header.models.join(", ")}`, + where, + `view: ${viewLabel(header.view)}`, + `weights: ${WEIGHT_KEYS.map((key) => `${key} ${header.weights[key]}`).join(" · ")}`, + `technical: ${plural(header.technical.words, "shared word")}${list} ` + + `(sha256 ${header.technical.sha256.slice(0, 12)}) + each language's own`, + ]; +} + +/* ------------------------------------------------------------ words report */ + +const WORD_COLUMNS = ["WEIGHT", "OCC", "CARRIERS", "MODULES", "TYPES"] as const; + +function wordsReport( + artifact: VocabularyArtifact, + scopes: readonly AnyScope[], + header: Header, + options: VocabularyOptions, + top: number, +): readonly string[] { + if (options.json) { + return [ + JSON.stringify({ + kind: REPORT_KIND, + report: "words", + models: header.models, + level: options.scope === undefined ? options.level : (scopes[0]?.level ?? options.level), + scope: options.scope ?? null, + view: header.view, + weights: header.weights, + technical: header.technical, + class: options.termClass, + top, + scopes: scopes.map((scope) => scopeJson(artifact, scope, options.termClass, top)), + }), + ]; + } + + const where = options.scope === undefined ? `level: ${options.level}` : `scope: ${options.scope}`; + const lines = [...headerLines("words", header, where), ""]; + if (options.scope === undefined && options.level !== "corpus") { + lines.push(`${plural(scopes.length, options.level)}, ranked by distinct terms descending, ties by id`, ""); + } + for (const scope of scopes) { + lines.push(...scopeBlock(artifact, scope, options.termClass, top), ""); + } + lines.pop(); + return lines; +} + +function scopeTitle(artifact: VocabularyArtifact, scope: AnyScope): string { + if (scope.level === "corpus") return `corpus ${scope.name}`; + const what = `${scope.level} ${scope.id ?? ""} (${scope.kind ?? ""} ${scope.name})`; + if (scope.level !== "operation") return what; + const around = artifact.scopes[scope.parent]; + return `${what}, counted on demand, inside ${around?.id ?? `the corpus ${artifact.corpus.name}`}`; +} + +function scopeBlock( + artifact: VocabularyArtifact, + scope: AnyScope, + termClass: VocabularyOptions["termClass"], + top: number, +): readonly string[] { + const spoke = Object.entries(scope.roles).filter(([, count]) => count > 0); + const speakers = spoke.reduce((sum, [, count]) => sum + count, 0); + const skipped = [ + ...(scope.skipped.unnamed > 0 ? [`${scope.skipped.unnamed} unnamed`] : []), + ...(scope.skipped.conventional > 0 ? [`${scope.skipped.conventional} conventional`] : []), + ...(scope.skipped.numeric > 0 ? [`${plural(scope.skipped.numeric, "numeric piece")} dropped`] : []), + ]; + const roles = spoke.length === 0 ? "" : ` (${spoke.map(([role, count]) => `${role} ${count}`).join(" · ")})`; + const lines = [ + scopeTitle(artifact, scope), + ` ${plural(scope.entities, "entity", "entities")}: ${speakers} ${speakers === 1 ? "speaks" : "speak"}${roles}` + + (skipped.length === 0 ? "" : `, ${skipped.join(", ")}`), + ` ${plural(scope.distinct, "distinct term")}, ${scope.distinctDomain} domain · weight ${scope.weight} · ` + + plural(scope.occurrences, "occurrence"), + ]; + + const { rows, total } = rowsOfClass(artifact, scope, termClass, top); + const what = termClass === "all" ? "all classes" : termClass; + if (total === 0) { + lines.push(` ${what}: no term`); + return lines; + } + const count = rows.length === total ? plural(total, "term") : `${rows.length} of ${total} terms (--top ${top})`; + lines.push(` ${what}: ${count}, by weight descending, ties by term`, ""); + + const cells = rows.map((row) => [ + row.text, + row.class, + ...[row.weight, row.occurrences, row.carriers, row.modules, row.types].map(String), + ]); + lines.push(...table(["TERM", "CLASS", ...WORD_COLUMNS], cells, 2)); + return lines; +} + +interface WordRow { + readonly text: string; + readonly class: string; + readonly weight: number; + readonly occurrences: number; + readonly carriers: number; + readonly modules: number; + readonly types: number; +} + +function rowsOfClass( + artifact: VocabularyArtifact, + scope: AnyScope, + termClass: VocabularyOptions["termClass"], + top: number, +): { readonly rows: readonly WordRow[]; readonly total: number } { + const matching = scope.terms.filter(([term]) => termClass === "all" || artifact.terms[term]?.class === termClass); + const rows = matching.slice(0, top).map(([term, weight, occurrences, carriers]): WordRow => { + const entry = artifact.terms[term]; + return { + text: entry?.text ?? "", + class: entry?.class ?? "", + weight, + occurrences, + carriers, + modules: entry?.modules ?? 0, + types: entry?.types ?? 0, + }; + }); + return { rows, total: matching.length }; +} + +function scopeJson( + artifact: VocabularyArtifact, + scope: AnyScope, + termClass: VocabularyOptions["termClass"], + top: number, +): unknown { + const { rows, total } = rowsOfClass(artifact, scope, termClass, top); + const { terms: _terms, parent, ...counts } = scope; + const around = parent === undefined ? undefined : (artifact.scopes[parent]?.id ?? null); + return { ...counts, ...(around === undefined ? {} : { parent: around }), shown: rows.length, total, terms: rows }; +} + +/* ------------------------------------------------------------ drift report */ + +function neverSays(level: ScopeLevel): string { + return level === "corpus" ? "the corpus never says" : `no ${level} says`; +} + +function driftReport( + artifact: VocabularyArtifact, + report: DriftReport, + header: Header, + options: VocabularyOptions, + glossary: Glossary, +): readonly string[] { + const top = options.top ?? 40; + const shown = report.codeOnly.slice(0, top); + const idOf = (index: number): string => artifact.scopes[index]?.id ?? ""; + + if (options.json) { + return [ + JSON.stringify({ + kind: REPORT_KIND, + report: "drift", + models: header.models, + level: report.level, + view: header.view, + weights: header.weights, + technical: header.technical, + glossary: { path: glossary.path, terms: glossary.terms.length, wordless: glossary.wordless }, + business: report.business.map(({ text, words, scopes, missing }) => ({ + text, + words, + said: scopes.length > 0, + ...(report.level === "corpus" ? {} : { scopes: scopes.map(idOf) }), + missing, + })), + codeOnly: { + shown: shown.length, + total: report.codeOnly.length, + top, + words: shown.map(({ term, weight, occurrences, scopes }) => ({ + text: artifact.terms[term]?.text ?? "", + weight, + occurrences, + scopes, + })), + }, + }), + ]; + } + + const lines = [ + ...headerLines("drift", header, `level: ${report.level}`), + `glossary: ${glossary.path}, ${plural(glossary.terms.length, "business term")}`, + "", + ]; + const total = report.business.length; + const unsaid = report.business.filter((term) => term.scopes.length === 0); + const said = report.business.filter((term) => term.scopes.length > 0); + const width = Math.max(...report.business.map((term) => term.text.length)); + + if (unsaid.length === 0) lines.push(`never said: none of ${plural(total, "business term")}`); + else { + const why = report.level === "corpus" ? "a word the corpus never says" : `no ${report.level} says all their words`; + lines.push(`never said: ${unsaid.length} of ${plural(total, "business term")}, ${why}`); + for (const term of unsaid) { + const reason = + term.missing.length > 0 ? `missing ${term.missing.join(", ")}` : `every word is said, never in one ${report.level}`; + lines.push(` ${pad(term.text, width)} ${reason}`); + } + } + lines.push("", `said: ${said.length} of ${plural(total, "business term")}`); + for (const term of said) { + if (report.level === "corpus") lines.push(` ${term.text}`); + else { + const where = + term.scopes.length === 1 ? `1 ${report.level}: ${idOf(term.scopes[0] ?? 0)}` : plural(term.scopes.length, report.level); + lines.push(` ${pad(term.text, width)} ${where}`); + } + } + + const count = shown.length === report.codeOnly.length ? `${shown.length}` : `${shown.length} of ${report.codeOnly.length}`; + const cut = shown.length === report.codeOnly.length ? "" : ` (--top ${top})`; + lines.push("", `code only: ${count} domain words no business term contains${cut}, by weight descending, ties by term`); + if (shown.length > 0) { + const perLevel = report.level === "corpus" ? [] : [`${report.level.toUpperCase()}S`]; + const cells = shown.map(({ term, weight, occurrences, scopes }) => [ + artifact.terms[term]?.text ?? "", + String(weight), + String(occurrences), + ...(report.level === "corpus" ? [] : [String(scopes)]), + ]); + lines.push("", ...table(["TERM", "WEIGHT", "OCC", ...perLevel], cells, 1)); + } + return lines; +} + +/* ---------------------------------------------------------------- shared */ + +/** A left-aligned text column (the first `left` ones), the numbers right-aligned. */ +function table(columns: readonly string[], cells: readonly (readonly string[])[], left: number): readonly string[] { + const widths = columns.map((column, i) => Math.max(column.length, ...cells.map((row) => (row[i] ?? "").length))); + const line = (row: readonly string[]): string => + ` ${row + .map((cell, i) => (i < left ? pad(cell, widths[i] ?? 0) : cell.padStart(widths[i] ?? 0))) + .join(" ") + .trimEnd()}`; + return [line(columns), ...cells.map(line)]; +} + +function pad(text: string, width: number): string { + return text.length >= width ? text : text + " ".repeat(width - text.length); +} + +function plural(count: number, noun: string, plural_ = `${noun}s`): string { + return `${count} ${count === 1 ? noun : plural_}`; +} + +/** + * What makes the counts smaller or vaguer than the model, on stderr: an + * unclean load, a language split with no lexicon, entities no module holds. + */ +function vocabularyWarnings( + source: { readonly clean: boolean; readonly paths: readonly string[] }, + artifact: VocabularyArtifact, +): readonly string[] { + const lines: string[] = []; + if (!source.clean) { + lines.push( + "warning: the models are not clean; the vocabulary was counted anyway.", + `Run 'codegraph validate ${source.paths.join(" ")}' for the detail.`, + ); + } + for (const lang of artifact.diagnostics.withoutLexicon) { + lines.push( + `note: no lexicon for ${lang}: its names are split by the universal rules alone, ` + + "no keyword or conventional name known.", + ); + } + const unplaced = artifact.diagnostics.unplaced.length; + if (unplaced > 0) { + lines.push(`note: ${plural(unplaced, "entity", "entities")} in no module: counted in the corpus alone.`); + } + return lines; +} diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 986fd18..dddaf1c 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -14,6 +14,7 @@ import { timelineCommand } from "./commands/timeline.js"; import { replayCommand } from "./commands/replay.js"; import { profilesCommand } from "./commands/profiles.js"; import { validateCommand } from "./commands/validate.js"; +import { vocabularyCommand } from "./commands/vocabulary.js"; import { EXIT, isUsageError, type ExitCode } from "./exit.js"; import { errLine, outLine, type IoSink } from "./io.js"; import { cliVersion } from "./version.js"; @@ -101,6 +102,8 @@ function dispatch(invocation: Invocation, io: IoSink): ExitCode | Promise { }); }); +describe("vocabulary", () => { + it("defaults to the corpus's domain words, every flag unset", () => { + expect(parse(["vocabulary", "a.json"])).toEqual({ + kind: "run", + command: "vocabulary", + options: { + models: ["a.json"], + level: "corpus", + scope: undefined, + top: undefined, + termClass: "domain", + weights: {}, + technical: undefined, + drift: undefined, + out: undefined, + json: false, + internalOnly: false, + declaredOnly: false, + noCache: false, + }, + }); + }); + + it("takes every flag", () => { + const argv = ["vocabulary", "a.json", "--level", "module", "--top", "5", "--weights", "type=6, qualifier=0,local=0.5"]; + const flags = ["--technical", "tech.txt", "--drift", "glossary.txt", "--out", "v.json", "--json", "--internal-only", "--declared-only", "--no-cache"]; + expect(parse([...argv, ...flags])).toMatchObject({ + command: "vocabulary", + options: { + level: "module", + top: 5, + termClass: "domain", + weights: { type: 6, qualifier: 0, local: 0.5 }, + technical: "tech.txt", + drift: "glossary.txt", + out: "v.json", + json: true, + internalOnly: true, + declaredOnly: true, + noCache: true, + }, + }); + expect(parse(["vocabulary", "a.json", "--scope", "java:com.acme/Order", "--class", "all"])).toMatchObject({ + options: { scope: "java:com.acme/Order", termClass: "all" }, + }); + }); + + it("offers the package's levels and classes, not a copy", () => { + const help = renderHelp(VOCABULARY_SPEC); + expect(help).toContain(`--level <${SCOPE_LEVELS.join("|")}>`); + expect(help).toContain(`--class <${[...TERM_CLASSES, "all"].join("|")}>`); + for (const key of WEIGHT_KEYS) expect(help).toContain(key); + }); + + it("rejects an unknown class, naming the valid ones", () => { + const error = usageErrorFor(["vocabulary", "a.json", "--class", "nouns"]); + expect(error.message).toContain("--class"); + expect(error.hint ?? "").toContain("domain, technical, keyword, short, all"); + }); + + it("rejects an unknown role in --weights, naming the valid ones", () => { + const error = usageErrorFor(["vocabulary", "a.json", "--weights", "class=8"]); + expect(error.message).toBe("unknown role 'class' in --weights"); + expect(error.hint ?? "").toContain(WEIGHT_KEYS.join(", ")); + }); + + it("rejects a --weights entry that is no role=number, or a number below 0", () => { + for (const bad of ["type", "type=", "=8", "type=heavy", "type=-1", "type=Infinity", "", "type=8,,module=8"]) { + const error = usageErrorFor(["vocabulary", "a.json", "--weights", bad]); + expect(error.message, bad).toContain("--weights"); + } + }); + + it("rejects a role weighed twice", () => { + expect(usageErrorFor(["vocabulary", "a.json", "--weights", "type=8,type=4"]).message).toBe( + "--weights weighs type twice", + ); + }); + + it("refuses the flags --scope or --drift would leave unread", () => { + for (const [argv, flag] of [ + [["--scope", "x", "--level", "module"], "--level"], + [["--scope", "x", "--drift", "g.txt"], "--drift"], + [["--drift", "g.txt", "--class", "all"], "--class"], + ] as const) { + const error = usageErrorFor(["vocabulary", "a.json", ...argv]); + expect(error.message, argv.join(" ")).toContain(flag); + } + }); +}); + describe("parseArgs errors are re-said in codegraph's terms", () => { it("keeps the problem and drops node's advice about '--'", () => { const error = usageErrorFor(["analyze", "a.json", "--repot", "deps"]); diff --git a/packages/cli/test/cache.test.ts b/packages/cli/test/cache.test.ts index c499dad..9e53223 100644 --- a/packages/cli/test/cache.test.ts +++ b/packages/cli/test/cache.test.ts @@ -8,6 +8,7 @@ import { STORE_OPEN_OPTIONS, cacheStatus, loadSqlite, storePathFor } from "@code import type { AnalyzeOptions, ExportOptions } from "../src/args.js"; import { analyzeCommand } from "../src/commands/analyze.js"; import { exportCommand } from "../src/commands/export.js"; +import { vocabularyCommand } from "../src/commands/vocabulary.js"; import { EXIT } from "../src/exit.js"; import { captureIo, type CapturedIo } from "../src/io.js"; @@ -246,3 +247,34 @@ describe("a store that must not be reused", () => { expect(cacheStatus(model).state).toBe("fresh"); }); }); + +describe("vocabulary through the store", () => { + it("counts the same words from the hydrated graph as from the file", () => { + const path = corpus("vocabulary"); + const words = (noCache: boolean): CapturedIo => { + const io = captureIo(); + vocabularyCommand( + { + models: [path], + level: "type", + scope: undefined, + top: undefined, + termClass: "all", + weights: {}, + technical: undefined, + drift: undefined, + out: undefined, + json: true, + internalOnly: false, + declaredOnly: false, + noCache, + }, + io, + ); + return io; + }; + const cached = words(false); + expect(cached.stderr()).toContain(`cache: ${storePathFor(path)}`); + expect(cached.stdout()).toBe(words(true).stdout()); + }); +}); diff --git a/packages/cli/test/e2e-determinism.test.ts b/packages/cli/test/e2e-determinism.test.ts index 1da69d8..5629be8 100644 --- a/packages/cli/test/e2e-determinism.test.ts +++ b/packages/cli/test/e2e-determinism.test.ts @@ -40,6 +40,9 @@ const INVOCATIONS: readonly (readonly string[])[] = [ // The explain plan: no model call, no key — the walk order and estimates alone. ["explain", FIXTURE, "--dry-run"], ["explain", FIXTURE, "--dry-run", "--json"], + ["vocabulary", FIXTURE], + ["vocabulary", FIXTURE, "--level", "module", "--class", "all", "--json"], + ["vocabulary", FIXTURE, "--scope", "java:com.acme.order/AbstractOrder.price()"], // The collation fixture: its identifiers order differently under UTF-16 code // units (what codegraph uses) than under UTF-8 bytes (what a database would), // so it is the one input where "sorted" is not a single answer. @@ -47,6 +50,7 @@ const INVOCATIONS: readonly (readonly string[])[] = [ ["analyze", UNICODE, "--report", "deps", "--level", "type"], ["export", UNICODE, "--format", "csv", "--level", "type"], ["export", UNICODE, "--format", "json", "--level", "type"], + ["vocabulary", UNICODE, "--class", "all"], ]; describe("stdout is byte-identical across runs", () => { diff --git a/packages/cli/test/e2e-exit-codes.test.ts b/packages/cli/test/e2e-exit-codes.test.ts index 781b204..b6b8328 100644 --- a/packages/cli/test/e2e-exit-codes.test.ts +++ b/packages/cli/test/e2e-exit-codes.test.ts @@ -69,6 +69,12 @@ const SUCCESS_CASES: readonly Case[] = [ argv: ["export", FIXTURE, "--format", "csv", "--level", "type", "--declared-only"], expected: EXIT.OK, }, + { why: "the corpus's words", argv: ["vocabulary", FIXTURE], expected: EXIT.OK }, + { + why: "the words per type, as json", + argv: ["vocabulary", FIXTURE, "--level", "type", "--json"], + expected: EXIT.OK, + }, ]; const USAGE_CASES: readonly Case[] = [ @@ -120,6 +126,10 @@ const USAGE_CASES: readonly Case[] = [ }, { why: "a positional profiles does not take", argv: ["profiles", FIXTURE], expected: EXIT.USAGE }, { why: "an unknown language", argv: ["profiles", "--lang", "klingon"], expected: EXIT.USAGE }, + { why: "an unknown word class", argv: ["vocabulary", FIXTURE, "--class", "nouns"], expected: EXIT.USAGE }, + { why: "an unknown role to weigh", argv: ["vocabulary", FIXTURE, "--weights", "class=8"], expected: EXIT.USAGE }, + // Raised inside the command, once the model says the id is not there. + { why: "an unknown scope", argv: ["vocabulary", FIXTURE, "--scope", "java:nowhere/Nothing"], expected: EXIT.USAGE }, ]; const FINDING_CASES: readonly Case[] = [ @@ -158,6 +168,7 @@ const FINDING_CASES: readonly Case[] = [ argv: ["export", models.danglingReference, "--format", "dot"], expected: EXIT.FINDINGS, }, + { why: "the words of a model with a self edge", argv: ["vocabulary", models.selfEdge], expected: EXIT.FINDINGS }, ]; /** diff --git a/packages/cli/test/e2e-json-parity.test.ts b/packages/cli/test/e2e-json-parity.test.ts index eae3db5..37fa1d5 100644 --- a/packages/cli/test/e2e-json-parity.test.ts +++ b/packages/cli/test/e2e-json-parity.test.ts @@ -208,3 +208,28 @@ describe("profiles says the same thing in both forms", () => { expect(text.stdout).toContain("package"); }); }); + +describe("vocabulary says the same thing in both forms", () => { + /** `[term, weight, occurrences]` per table row of the text form, in order. */ + const textRows = (stdout: string): readonly string[] => + stdout + .split("\n") + .filter((line) => /^ {2}\S+ +(domain|technical|keyword|short) /u.test(line)) + .map((line) => line.trim().split(/ +/u).slice(0, 4).filter((_cell, i) => i !== 1).join(" ")); + + it.each([ + [["--top", "8"]], + [["--level", "module", "--class", "all"]], + [["--scope", "java:com.acme.order/AbstractOrder.price()"]], + ])("lists the same words with the same counts: %s", (flags) => { + const { text, json } = bothForms(["vocabulary", FIXTURE, ...flags]); + expect(text.code, describeResult(text)).toBe(EXIT.OK); + expect(json.code).toBe(text.code); + const parsed = parseJsonArtifact(json.stdout, "vocabulary --json") as { + scopes: { terms: { text: string; weight: number; occurrences: number }[] }[]; + }; + const rows = parsed.scopes.flatMap((scope) => scope.terms.map((term) => `${term.text} ${term.weight} ${term.occurrences}`)); + expect(rows.length).toBeGreaterThan(0); + expect(textRows(text.stdout)).toEqual(rows); + }); +}); diff --git a/packages/cli/test/e2e-stream-purity.test.ts b/packages/cli/test/e2e-stream-purity.test.ts index 53632b8..df14deb 100644 --- a/packages/cli/test/e2e-stream-purity.test.ts +++ b/packages/cli/test/e2e-stream-purity.test.ts @@ -131,6 +131,7 @@ describe("no ANSI colour anywhere (decision 4)", () => { ["export", FIXTURE, "--format", "dot"], ["export", FIXTURE, "--format", "plantuml"], ["explain", FIXTURE, "--dry-run"], + ["vocabulary", FIXTURE], ["frobnicate"], ]; @@ -161,6 +162,12 @@ describe("reporting commands keep their framing off stdout", () => { expect(() => parseJsonArtifact(result.stdout, "validate --json stdout")).not.toThrow(); }); + it("vocabulary --json puts exactly one JSON object on stdout", () => { + const result = runCli(["vocabulary", FIXTURE, "--level", "module", "--json"]); + expect(result.code, describeResult(result)).toBe(EXIT.OK); + expect(() => parseJsonArtifact(result.stdout, "vocabulary --json stdout")).not.toThrow(); + }); + it("profiles --json puts exactly one JSON object on stdout", () => { const result = runCli(["profiles", "--json"]); expect(result.code, describeResult(result)).toBe(EXIT.OK); diff --git a/packages/cli/test/vocabulary.test.ts b/packages/cli/test/vocabulary.test.ts new file mode 100644 index 0000000..cf89da6 --- /dev/null +++ b/packages/cli/test/vocabulary.test.ts @@ -0,0 +1,405 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { readModelFileSync } from "@codegraph/core"; +import { buildGraph, loadModels } from "@codegraph/analyzer"; +import { buildVocabulary, vocabularyToJsonString } from "@codegraph/vocabulary"; +import { describe, expect, it } from "vitest"; +import type { VocabularyOptions } from "../src/args.js"; +import { vocabularyCommand } from "../src/commands/vocabulary.js"; +import { EXIT, UsageError } from "../src/exit.js"; +import { captureIo, type CapturedIo } from "../src/io.js"; +import { run } from "../src/main.js"; + +/** + * `codegraph vocabulary` over the committed Spoon output. The counts are + * `@codegraph/vocabulary`'s (its own suite); what is checked here is the + * command: which scope, class and report each flag picks, the streams, the + * usage errors and the exit codes. + */ +const FIXTURE = fileURLToPath(new URL("../../../fixtures/java/expected/model.jsonl", import.meta.url)); +const ABSTRACT_ORDER = "java:com.acme.order/AbstractOrder"; + +function options(overrides: Partial = {}): VocabularyOptions { + return { + models: [FIXTURE], + level: "corpus", + scope: undefined, + top: undefined, + termClass: "domain", + weights: {}, + technical: undefined, + drift: undefined, + out: undefined, + json: false, + internalOnly: false, + declaredOnly: false, + noCache: true, + ...overrides, + }; +} + +function vocabularyTo(overrides: Partial = {}): { io: CapturedIo; code: number } { + const io = captureIo(); + const code = vocabularyCommand(options(overrides), io); + return { io, code }; +} + +function usageErrorOf(overrides: Partial): UsageError { + try { + vocabularyTo(overrides); + } catch (error) { + expect(error).toBeInstanceOf(UsageError); + return error as UsageError; + } + throw new Error("expected a usage error"); +} + +/** A file in a fresh temporary directory. */ +function file(name: string, text: string): string { + const path = join(mkdtempSync(join(tmpdir(), "codegraph-cli-vocabulary-")), name); + writeFileSync(path, text); + return path; +} + +/** The table rows of a text report, each as its cells. */ +function rows(stdout: string): readonly (readonly string[])[] { + return stdout + .split("\n") + .filter((line) => /^ {2}\S+ +(domain|technical|keyword|short) /u.test(line)) + .map((line) => line.trim().split(/ +/u)); +} + +interface WordsJson { + kind: string; + report: string; + level: string; + scope: string | null; + view: { name: string }; + weights: Record; + technical: { words: number; sha256: string }; + class: string; + top: number; + scopes: { + level: string; + id?: string; + name: string; + kind?: string; + parent?: string | null; + distinct: number; + distinctDomain: number; + shown: number; + total: number; + terms: { text: string; class: string; weight: number; occurrences: number; carriers: number; modules: number; types: number }[]; + }[]; +} + +describe("vocabulary: the corpus's words", () => { + const { io, code } = vocabularyTo(); + + it("prints the corpus's heaviest domain words as a table, and nothing else on stdout", () => { + expect(code).toBe(EXIT.OK); + expect(rows(io.stdout()).slice(0, 3)).toEqual([ + ["order", "domain", "40", "12", "12", "3", "4"], + ["price", "domain", "18", "6", "6", "1", "6"], + ["basket", "domain", "16", "2", "2", "1", "2"], + ]); + expect(io.stdout()).toContain("TERM"); + expect(io.stdout()).not.toContain("cache:"); + expect(io.stderr()).toContain("cache: not used"); + }); + + it("states the view, the weights and the technical list it counted under", () => { + const lines = io.stdoutLines(); + expect(lines[0]).toBe("codegraph vocabulary — words"); + expect(lines).toContain(`models: ${FIXTURE}`); + expect(lines).toContain("level: corpus"); + expect(lines).toContain("view: all (nothing filtered — stubs and inferred edges included)"); + expect(lines).toContain( + "weights: module 8 · type 8 · operation 3 · attribute 4 · parameter 1 · local 1 · other 1 · qualifier 1", + ); + expect(lines.find((line) => line.startsWith("technical: "))).toMatch( + /^technical: 125 shared words \(sha256 [0-9a-f]{12}\) \+ each language's own$/u, + ); + }); + + it("says what the corpus counted, and how much of the table it shows", () => { + const lines = io.stdoutLines(); + expect(lines).toContain("corpus src"); + expect(lines).toContain( + " 152 entities: 127 speak (module 3 · type 19 · operation 43 · attribute 21 · parameter 35 · local 6), 25 unnamed", + ); + expect(lines).toContain(" 85 distinct terms, 75 domain · weight 503 · 146 occurrences"); + expect(lines).toContain(" domain: 40 of 75 terms (--top 40), by weight descending, ties by term"); + expect(rows(io.stdout())).toHaveLength(40); + }); + + it("shows another class, or every class, with --class", () => { + const all = vocabularyTo({ termClass: "all" }).io.stdout(); + expect(all).toContain(" all classes: 40 of 85 terms (--top 40), by weight descending, ties by term"); + const technical = rows(vocabularyTo({ termClass: "technical" }).io.stdout()); + expect(technical.map(([, termClass]) => termClass)).toEqual(technical.map(() => "technical")); + expect(technical.length).toBeGreaterThan(0); + }); + + it("shows the N heaviest with --top", () => { + const stdout = vocabularyTo({ top: 2 }).io.stdout(); + expect(rows(stdout).map(([term]) => term)).toEqual(["order", "price"]); + expect(stdout).toContain(" domain: 2 of 75 terms (--top 2), by weight descending, ties by term"); + }); + + it("is byte-identical across runs", () => { + expect(vocabularyTo().io.stdout()).toBe(io.stdout()); + }); + + it("counts under the view", () => { + const internal = vocabularyTo({ internalOnly: true }).io; + expect(internal.stdoutLines()).toContain("view: internalOnly (internalOnly)"); + }); +}); + +describe("vocabulary: --json", () => { + it("says the same thing as one JSON object", () => { + const { io, code } = vocabularyTo({ json: true, top: 2 }); + expect(code).toBe(EXIT.OK); + const json = JSON.parse(io.stdout()) as WordsJson; + expect(json).toMatchObject({ + kind: "codegraph.vocabularyReport/1", + report: "words", + level: "corpus", + scope: null, + view: { name: "all" }, + weights: { module: 8, qualifier: 1 }, + technical: { words: 125 }, + class: "domain", + top: 2, + }); + expect(json.scopes).toHaveLength(1); + expect(json.scopes[0]).toMatchObject({ level: "corpus", name: "src", distinct: 85, distinctDomain: 75, shown: 2, total: 75 }); + expect(json.scopes[0]?.terms).toEqual([ + { text: "order", class: "domain", weight: 40, occurrences: 12, carriers: 12, modules: 3, types: 4 }, + { text: "price", class: "domain", weight: 18, occurrences: 6, carriers: 6, modules: 1, types: 6 }, + ]); + }); +}); + +describe("vocabulary: --level module|type, the cognitive-load report", () => { + it("prints one block per module, by distinct terms descending, ties by id", () => { + const { io, code } = vocabularyTo({ level: "module" }); + expect(code).toBe(EXIT.OK); + const titles = io.stdoutLines().filter((line) => line.startsWith("module ")); + expect(titles).toEqual([ + "module java:com.acme.order (package com.acme.order)", + "module java:com.acme.order.adapter (package com.acme.order.adapter)", + "module java:com.acme.order.legacy (package com.acme.order.legacy)", + ]); + expect(io.stdoutLines()).toContain("3 modules, ranked by distinct terms descending, ties by id"); + expect(io.stdoutLines()).toContain(" 77 distinct terms, 68 domain · weight 433 · 126 occurrences"); + }); + + it("shows 10 words per block by default, --top N otherwise", () => { + const blocks = (stdout: string) => stdout.split("\nmodule ").slice(1).map((block) => rows(`\n${block}`).length); + expect(blocks(vocabularyTo({ level: "module" }).io.stdout())).toEqual([10, 6, 7]); + expect(blocks(vocabularyTo({ level: "module", top: 3 }).io.stdout())).toEqual([3, 3, 3]); + }); + + it("ranks the types the same way, and says so in JSON", () => { + const json = JSON.parse(vocabularyTo({ level: "type", json: true }).io.stdout()) as WordsJson; + expect(json.scopes).toHaveLength(19); + expect(json.top).toBe(10); + const distinct = json.scopes.map((scope) => scope.distinct); + expect(distinct).toEqual([...distinct].sort((a, b) => b - a)); + expect(json.scopes.every((scope) => scope.level === "type" && scope.parent !== undefined)).toBe(true); + }); +}); + +describe("vocabulary: --scope", () => { + it("prints one stored scope, a type or a module", () => { + const { io } = vocabularyTo({ scope: ABSTRACT_ORDER, termClass: "all" }); + expect(io.stdoutLines()).toContain("scope: java:com.acme.order/AbstractOrder"); + expect(io.stdoutLines()).toContain("type java:com.acme.order/AbstractOrder (class AbstractOrder)"); + expect(rows(io.stdout()).map(([term, termClass, weight]) => [term, termClass, weight])).toEqual([ + ["abstract", "keyword", "8"], + ["order", "domain", "8"], + ["reference", "domain", "8"], + ["total", "domain", "4"], + ["price", "domain", "3"], + ]); + expect(vocabularyTo({ scope: "java:com.acme.order.adapter" }).io.stdoutLines()).toContain( + "module java:com.acme.order.adapter (package com.acme.order.adapter)", + ); + }); + + it("counts an operation on demand, inside its stored scope", () => { + const { io, code } = vocabularyTo({ scope: `${ABSTRACT_ORDER}.price()` }); + expect(code).toBe(EXIT.OK); + expect(io.stdoutLines()).toContain( + `operation ${ABSTRACT_ORDER}.price() (method price), counted on demand, inside ${ABSTRACT_ORDER}`, + ); + expect(rows(io.stdout())).toEqual([["price", "domain", "3", "1", "1", "1", "6"]]); + const json = JSON.parse(vocabularyTo({ scope: `${ABSTRACT_ORDER}.price()`, json: true }).io.stdout()) as WordsJson; + expect(json.scope).toBe(`${ABSTRACT_ORDER}.price()`); + expect(json.scopes[0]).toMatchObject({ level: "operation", kind: "method", parent: ABSTRACT_ORDER }); + }); + + it("refuses an id the model does not declare", () => { + const error = usageErrorOf({ scope: "java:com.acme/Nothing" }); + expect(error.message).toBe("unknown scope 'java:com.acme/Nothing'"); + expect(error.hint ?? "").toContain("module, type or operation"); + }); + + it("refuses an external entity: a stub speaks no words", () => { + expect(usageErrorOf({ scope: "java:com.megacorp.ledger" }).message).toBe( + "java:com.megacorp.ledger is external (a stub): its name was chosen elsewhere, so it speaks no words", + ); + }); + + it("refuses a member that is no operation, naming the scope that counts it", () => { + const model = readModelFileSync(FIXTURE); + const member = model.entities.find((entity) => entity.traits.includes("TStructural") && !entity.isStub); + const error = usageErrorOf({ scope: member?.id ?? "" }); + expect(error.message).toBe(`${member?.id} is a ${member?.kind}, not a module, type or operation`); + expect(error.hint ?? "").toMatch(/^Its words are counted in /u); + }); +}); + +describe("vocabulary: --weights and --technical", () => { + it("lays the typed weights over the defaults, and says so", () => { + const { io } = vocabularyTo({ weights: { type: 0, qualifier: 0 } }); + expect(io.stdoutLines()).toContain( + "weights: module 8 · type 0 · operation 3 · attribute 4 · parameter 1 · local 1 · other 1 · qualifier 0", + ); + // `order` names types and qualifies the packages; `price` does neither. + expect(rows(io.stdout()).slice(0, 2)).toEqual([ + ["price", "domain", "18", "6", "6", "1", "6"], + ["order", "domain", "14", "12", "12", "3", "4"], + ]); + }); + + it("replaces the shared technical list with the file's words", () => { + const path = file("technical.txt", "# ours\norder\nprice\n"); + const { io } = vocabularyTo({ technical: path }); + // Replaced whole: `adapter`, shared plumbing, is a domain word now. + expect(rows(io.stdout()).map(([term]) => term).slice(0, 3)).toEqual(["adapter", "basket", "channel"]); + expect(io.stdoutLines().find((line) => line.startsWith("technical: "))).toMatch( + /^technical: 2 shared words from .*technical\.txt \(sha256 [0-9a-f]{12}\) \+ each language's own$/u, + ); + expect(rows(vocabularyTo({ technical: path, termClass: "technical" }).io.stdout()).map(([term]) => term)).toContain("order"); + }); + + it("refuses a technical line that is not one word, by its line number", () => { + const error = usageErrorOf({ technical: file("technical.txt", "dto\nvalue object\nHTTPServer\n") }); + expect(error.message).toContain("2 lines of"); + expect(error.hint ?? "").toContain("line 2: value object"); + expect(error.hint ?? "").toContain("line 3: HTTPServer"); + }); + + it("refuses a technical file it cannot read", () => { + expect(usageErrorOf({ technical: "/nonexistent/technical.txt" }).message).toContain("cannot read /nonexistent/technical.txt"); + }); +}); + +describe("vocabulary: --drift", () => { + const glossary = file("glossary.txt", "# the business\nOrder\nOrder Line\nCustomer\nPayment Schedule\n2024\n"); + + it("lists the business terms no scope says, then the ones it says, then the code's own words", () => { + const { io, code } = vocabularyTo({ drift: glossary }); + const lines = io.stdoutLines(); + expect(lines[0]).toBe("codegraph vocabulary — drift"); + expect(lines).toContain(`glossary: ${glossary}, 4 business terms`); + expect(lines).toContain("never said: 2 of 4 business terms, a word the corpus never says"); + expect(lines.find((line) => line.trimStart().startsWith("Customer"))).toMatch(/^ {2}Customer +missing customer$/u); + expect(lines.find((line) => line.trimStart().startsWith("Payment Schedule"))).toMatch( + /^ {2}Payment Schedule +missing payment, schedule$/u, + ); + expect(lines).toContain("said: 2 of 4 business terms"); + expect(lines.some((line) => /^ {2}Order Line$/u.test(line))).toBe(true); + expect(lines.find((line) => line.startsWith("code only: "))).toBe( + "code only: 40 of 73 domain words no business term contains (--top 40), by weight descending, ties by term", + ); + expect(lines.find((line) => /^ {2}price /u.test(line))).toMatch(/^ {2}price +18 +6$/u); + // A finding: the code does not say what the business says. + expect(code).toBe(EXIT.FINDINGS); + expect(io.stderr()).toContain("2 business terms the corpus never says — exiting 3 (findings)."); + expect(io.stderr()).toContain("warning: line 6 of"); + expect(io.stderr()).toContain("names no word, skipped: 2024"); + }); + + it("names, by module, the one module saying a term, or how many do", () => { + const lines = vocabularyTo({ drift: glossary, level: "module" }).io.stdoutLines(); + expect(lines).toContain("level: module"); + expect(lines).toContain("never said: 2 of 4 business terms, no module says all their words"); + expect(lines.find((line) => line.trimStart().startsWith("Order Line"))).toMatch( + /^ {2}Order Line +1 module: java:com\.acme\.order$/u, + ); + expect(lines.find((line) => /^ {2}Order {2,}/u.test(line))).toMatch(/^ {2}Order +3 modules$/u); + expect(lines.find((line) => line.startsWith(" TERM"))).toMatch(/^ {2}TERM +WEIGHT +OCC +MODULES$/u); + }); + + it("exits 0 when every business term is said", () => { + const { code, io } = vocabularyTo({ drift: file("glossary.txt", "Order\nOrder Line\n") }); + expect(code).toBe(EXIT.OK); + expect(io.stdoutLines()).toContain("never said: none of 2 business terms"); + }); + + it("says the same thing as one JSON object", () => { + const json = JSON.parse(vocabularyTo({ drift: glossary, level: "module", json: true, top: 1 }).io.stdout()) as { + kind: string; + report: string; + level: string; + glossary: { path: string; terms: number; wordless: { line: number; text: string }[] }; + business: { text: string; words: string[]; said: boolean; scopes?: string[]; missing: string[] }[]; + codeOnly: { shown: number; total: number; top: number; words: { text: string; weight: number; occurrences: number; scopes: number }[] }; + }; + expect(json).toMatchObject({ kind: "codegraph.vocabularyReport/1", report: "drift", level: "module" }); + expect(json.glossary).toEqual({ path: glossary, terms: 4, wordless: [{ line: 6, text: "2024" }] }); + expect(json.business[1]).toEqual({ + text: "Order Line", + words: ["order", "line"], + said: true, + scopes: ["java:com.acme.order"], + missing: [], + }); + expect(json.business[2]).toMatchObject({ text: "Customer", said: false, scopes: [], missing: ["customer"] }); + expect(json.codeOnly).toMatchObject({ shown: 1, top: 1 }); + expect(json.codeOnly.words).toHaveLength(1); + }); + + it("refuses a glossary that holds no business term", () => { + expect(usageErrorOf({ drift: file("glossary.txt", "# nothing yet\n\n") }).message).toContain("holds no business term"); + }); +}); + +describe("vocabulary: --out", () => { + it("writes the artifact beside the report, and confirms on stderr", () => { + const path = join(mkdtempSync(join(tmpdir(), "codegraph-cli-vocabulary-")), "vocabulary.json"); + const { io, code } = vocabularyTo({ out: path, weights: { qualifier: 0 } }); + expect(code).toBe(EXIT.OK); + expect(rows(io.stdout()).length).toBeGreaterThan(0); + const graph = buildGraph(loadModels(readModelFileSync(FIXTURE), { sources: [FIXTURE] }).union); + const weights = { module: 8, type: 8, attribute: 4, operation: 3, parameter: 1, local: 1, other: 1, qualifier: 0 }; + expect(io.files().get(path)).toBe(vocabularyToJsonString(buildVocabulary(graph, { weights }))); + expect(io.stderr()).toMatch(/wrote \d+ bytes to .*vocabulary\.json \(vocabulary, view all, 85 terms, 23 scopes\)\./u); + }); +}); + +describe("vocabulary: through the real dispatcher", () => { + it("runs from argv", () => { + const io = captureIo(); + expect(run(["vocabulary", FIXTURE, "--no-cache", "--top", "1"], io)).toBe(EXIT.OK); + expect(rows(io.stdout())).toEqual([["order", "domain", "40", "12", "12", "3", "4"]]); + }); + + it("reports a usage error raised by the command with exit 2", () => { + const io = captureIo(); + expect(run(["vocabulary", FIXTURE, "--no-cache", "--scope", "nope"], io)).toBe(EXIT.USAGE); + expect(io.stderr()).toContain("codegraph: unknown scope 'nope'"); + }); + + it("appears in the global help", () => { + const io = captureIo(); + expect(run(["--help"], io)).toBe(EXIT.OK); + expect(io.stdout()).toContain("vocabulary"); + }); +}); diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index e9736e7..367c9f9 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -17,6 +17,7 @@ export default defineConfig({ "@codegraph/llm": fileURLToPath(new URL("../llm/src/index.ts", import.meta.url)), "@codegraph/navigator": fileURLToPath(new URL("../navigator/src/index.ts", import.meta.url)), "@codegraph/scm": fileURLToPath(new URL("../scm/src/index.ts", import.meta.url)), + "@codegraph/vocabulary": fileURLToPath(new URL("../vocabulary/src/index.ts", import.meta.url)), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bd25a8..0d4461c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,6 +161,9 @@ importers: '@codegraph/viz': specifier: workspace:* version: link:../viz + '@codegraph/vocabulary': + specifier: workspace:* + version: link:../vocabulary devDependencies: '@types/node': specifier: 'catalog:' From 6a0ebcd5f64deeb62b8a32bc790663ba96669460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Mon, 26 Oct 2026 08:31:42 +0100 Subject: [PATCH 6/7] docs: codegraph vocabulary and vocabulary.json The command gets its section and synopsis in docs/cli.md and a website page: the flags, what a scope is, how drift matches a glossary, the exit codes, and two examples taken from the java fixture's real output. The artifact gets a website page: every field of `codegraph.vocabulary/1` and the weights table. The CLI index, the artifacts index and the exit-codes table list both, and `vocabulary` joins the commands that build and reuse a sibling model.db. Co-Authored-By: Claude Opus 5.5 --- docs/cli.md | 25 +++- .../docs/reference/artifacts/_index.md | 1 + .../reference/artifacts/vocabulary-json.md | 69 +++++++++++ website/content/docs/reference/cli/_index.md | 5 +- .../content/docs/reference/cli/vocabulary.md | 109 ++++++++++++++++++ website/content/docs/reference/exit-codes.md | 1 + 6 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 website/content/docs/reference/artifacts/vocabulary-json.md create mode 100644 website/content/docs/reference/cli/vocabulary.md diff --git a/docs/cli.md b/docs/cli.md index e1d4fdc..5594913 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -25,8 +25,9 @@ share. Task-oriented walkthroughs live in the [README](../README.md). worked; the input did not). A CI job gating on model quality checks for `3`. - **`--json`** prints the same information as a machine-readable object. - **A sibling `model.db`** (the SQLite analysis store) is built on first use and - reused afterwards by `analyze`, `export`, `navigator`, `domain-facts` and - `explain`; on those, `--no-cache` reads the JSONL directly. + reused afterwards by `analyze`, `export`, `navigator`, `vocabulary`, + `domain-facts` and `explain`; on those, `--no-cache` reads the JSONL + directly. ## Synopsis @@ -65,6 +66,11 @@ codegraph city [model.jsonl] [--layout] [--out FILE] codegraph navigator [model.jsonl] [--out FILE] [--name STR] [--internal-only] [--declared-only] +codegraph vocabulary [model.jsonl] [--level corpus|module|type] [--scope ID] + [--top N] [--class domain|technical|keyword|short|all] + [--weights KEY=N,…] [--technical FILE] [--drift FILE] + [--out FILE] [--json] [--internal-only] [--declared-only] + codegraph domain-facts [model.jsonl] [--framework spring] [--out FILE] [--internal-only] [--declared-only] @@ -213,6 +219,21 @@ Writes the navigator artifact (`navigator.json`): the tree, one classified dependency row per base edge, and the cycle/coupling reports. Type and module nodes carry their entity id, which is how the city's buildings address them. +### `vocabulary` +The words the model's names are made of (never source): each `TNamed` +entity's name split into words, classed `domain`, `technical`, `keyword` or +`short`, and weighted by the role of what carries it. A type's or module's +name weighs 8, an attribute's 4, an operation's 3, a parameter's or local's +1, and the qualifier of a qualified name (`com.acme` of `com.acme.order`) 1. +No flag prints the corpus's 40 heaviest domain words; `--level module|type` +one block per scope, ranked by distinct terms (the cognitive-load report); +`--scope ID` one module or type, or an operation counted on demand. +`--weights` and `--technical FILE` change the scheme, and the header says +which one counted. `--drift FILE` compares with a glossary, one business term +per line: the terms no scope of `--level` says (exit `3`), the ones it says +and where, and the domain words no term contains. `--out FILE` writes the +`vocabulary.json` artifact (`codegraph.vocabulary/1`) beside the report. + ### `domain-facts` One JSON artifact (`codegraph.domainFacts/1`) with one dossier per corpus type, every fact the model holds about it pre-joined: annotations with their diff --git a/website/content/docs/reference/artifacts/_index.md b/website/content/docs/reference/artifacts/_index.md index 21adfda..9f21425 100644 --- a/website/content/docs/reference/artifacts/_index.md +++ b/website/content/docs/reference/artifacts/_index.md @@ -10,6 +10,7 @@ The files codegraph writes that are not the interchange model. Each is a **deriv |---|---|---|---| | [`city.json`](/docs/reference/artifacts/city-json/) | [`city`](/docs/reference/cli/city/), [`replay`](/docs/reference/cli/replay/), [`history --city`](/docs/reference/cli/history/) | `codegraph.city/1` | one JSON document | | [`navigator.json`](/docs/reference/artifacts/navigator-json/) | [`navigator`](/docs/reference/cli/navigator/) | `codegraph.navigator/1` | one JSON document | +| [`vocabulary.json`](/docs/reference/artifacts/vocabulary-json/) | [`vocabulary --out`](/docs/reference/cli/vocabulary/) | `codegraph.vocabulary/1` | one JSON document | | [`history.jsonl`](/docs/reference/artifacts/history-jsonl/) | [`scm`](/docs/reference/cli/scm/) | `"artifact": "history"` | JSONL, section-ordered | | [`.history-metrics.jsonl`](/docs/reference/artifacts/history-metrics-jsonl/) | [`codegraph-metrics --history`](/docs/reference/metrics-pass/) | `codegraph.history-metrics/1` | JSONL, section-ordered | | [`domain-facts.json`](/docs/reference/artifacts/domain-facts/) | [`domain-facts`](/docs/reference/cli/domain-facts/) | `codegraph.domainFacts/1` | one JSON document | diff --git a/website/content/docs/reference/artifacts/vocabulary-json.md b/website/content/docs/reference/artifacts/vocabulary-json.md new file mode 100644 index 0000000..6ff79fc --- /dev/null +++ b/website/content/docs/reference/artifacts/vocabulary-json.md @@ -0,0 +1,69 @@ +--- +title: vocabulary.json +weight: 7 +--- + +The words a corpus's names are made of, counted per scope: the corpus, each module, each type. Written by [`vocabulary --out`](/docs/reference/cli/vocabulary/). The committed `fixtures//expected/vocabulary.json` are its pretty form, one term and one scope per line, rebuilt byte for byte by the tests. + +**Words come from names, never from source.** A name is split by the universal word boundaries and its language's lexicon (the conventional names, file kinds, qualified kinds and interface marker [`profiles`](/docs/reference/cli/profiles/) prints). Each word is classed, never filtered, and weighted by the role of what carries it. + +**Rows are array indexes.** A term is its position in `terms`, a scope's parent its position in `scopes`. A scope carries its entity's id, opaque, so a city building or a navigator node finds its words. + +Field names are those of `VocabularyArtifact` in `packages/vocabulary/src/artifact.ts`. + +## Top level + +| Field | Type | Meaning | +|---|---|---| +| `kind` | string | always `codegraph.vocabulary/1` | +| `generatedBy` | string | always `@codegraph/vocabulary` | +| `view` | `ViewDescriptor` | `{name, filters}` | +| `corpus` | `{name, roots}` | | +| `langs` | string[] | the languages of the union | +| `weights` | object | the weight of each role and of `qualifier`, as counted: `module`, `type`, `operation`, `attribute`, `parameter`, `local`, `other`, `qualifier` | +| `classes` | string[] | `domain`, `technical`, `keyword`, `short`, in that order | +| `technical` | `{words, sha256}` | the shared technical list's size and a sha256 over its sorted words joined by `\n`; each language's own words apply on top | +| `terms` | `Term[]` | sorted by text in UTF-16 code-unit order, then class | +| `scopes` | `Scope[]` | the corpus first, then modules, then types, each level in id order | +| `diagnostics` | `{unplaced, withoutLexicon}` | kept entities no module contains (counted in the corpus alone), and languages core has no lexicon for (split by the universal rules alone); both sorted | + +## `terms[]` — `Term` + +One word in one class. A word two languages class differently (Go reserves `type`, Java does not) is two terms. + +| Field | Type | Meaning | +|---|---|---| +| `text` | string | lowercased | +| `class` | `"domain"` \| `"technical"` \| `"keyword"` \| `"short"` | `keyword`: the language reserves it; `short`: one letter or digit; `technical`: the shared list or the language's own; else `domain` | +| `modules`, `types` | number | how many module scopes, and how many type scopes, say it | +| `exclusive` | number | the one module scope that says it, when exactly one does | + +## `scopes[]` — `Scope` + +A type's words are those of everything that folds into it (`containingType`), a module's likewise, the corpus's every kept entity's. A nested type is its own scope. A stub never speaks. + +| Field | Type | Meaning | +|---|---|---| +| `level` | `"corpus"` \| `"module"` \| `"type"` | | +| `id` | string | the entity id; absent on the corpus | +| `name`, `kind` | string | the entity's name and kind; the corpus has its display name and no kind | +| `parent` | number | the nearest scope up the containment chain, else the corpus; absent on the corpus | +| `entities` | number | kept entities here; each is unnamed, conventional or counted in `roles` | +| `roles` | object | the entities that said words, by role | +| `skipped` | `{unnamed, conventional, numeric}` | names with no word: an unnamed entity, a name the language dictates (`toString`), and pieces of digits only, dropped | +| `distinct`, `distinctDomain` | number | terms in the table, and the `domain` ones among them: the cognitive-load proxy | +| `weight`, `occurrences` | number | sums over the table | +| `terms` | `[term, weight, occurrences, carriers][]` | by weight descending, then term; `carriers` counts distinct entities, so a name saying a word twice is two occurrences and one carrier | + +## Weights + +| Role | Weight | Carried by | +|---|---|---| +| `module`, `type` | 8 | an entity with `TModule` or `TType` | +| `attribute` | 4 | a `TStructural` no entity lists as a parameter or local | +| `operation` | 3 | `TInvocable` | +| `parameter`, `local` | 1 | a `TStructural` listed in some entity's `parameters` or `localVariables` | +| `other` | 1 | a named entity no role trait describes | +| `qualifier` | 1 | not a role: the words before the last separator of a qualified name (`com.acme` of `com.acme.order`), whatever carries it | + +Roles come from traits, never from the kind. `--weights` overrides them, and the artifact records the weights it was counted with. diff --git a/website/content/docs/reference/cli/_index.md b/website/content/docs/reference/cli/_index.md index b0a409f..3a573da 100644 --- a/website/content/docs/reference/cli/_index.md +++ b/website/content/docs/reference/cli/_index.md @@ -4,7 +4,7 @@ linkTitle: CLI weight: 1 --- -`codegraph` is one command with fifteen sub-commands. Every sub-command prints its own options with `codegraph --help`; the pages below are that output, with the conventions the commands share stated once here. +`codegraph` is one command with sixteen sub-commands. Every sub-command prints its own options with `codegraph --help`; the pages below are that output, with the conventions the commands share stated once here. ``` codegraph [options] @@ -24,7 +24,7 @@ global options: **`--json`** prints the same information as a machine-readable object on stdout. -**A sibling `model.db`** — the SQLite analysis store — is built on first use and reused afterwards. `--no-cache` reads the JSONL directly; it is accepted by `analyze`, `export`, `navigator`, `domain-facts` and `explain`. +**A sibling `model.db`** — the SQLite analysis store — is built on first use and reused afterwards. `--no-cache` reads the JSONL directly; it is accepted by `analyze`, `export`, `navigator`, `vocabulary`, `domain-facts` and `explain`. **Exit codes** distinguish a broken tool from a broken model: `0` success, `1` an internal bug, `2` a usage error, `3` findings. See [Exit codes](/docs/reference/exit-codes/). @@ -38,6 +38,7 @@ global options: | [`export`](/docs/reference/cli/export/) | Write the folded graph as DOT, JSON, CSV or PlantUML. | | [`city`](/docs/reference/cli/city/) | Write the code city: modules as districts, types as buildings. | | [`navigator`](/docs/reference/cli/navigator/) | Explore the model: a searchable tree with per-node dependency detail. | +| [`vocabulary`](/docs/reference/cli/vocabulary/) | Report the words the model's names are made of, per corpus, module or type; or drift against a glossary. | | [`domain-facts`](/docs/reference/cli/domain-facts/) | Write per-type domain dossiers: joined facts for domain extraction. | | [`explain`](/docs/reference/cli/explain/) | Explain every operation, type and module with an LLM, bottom-up, into a side-car. | | [`scm`](/docs/reference/cli/scm/) | Mine a repository's history into a deterministic `history.jsonl`. | diff --git a/website/content/docs/reference/cli/vocabulary.md b/website/content/docs/reference/cli/vocabulary.md new file mode 100644 index 0000000..ab956b4 --- /dev/null +++ b/website/content/docs/reference/cli/vocabulary.md @@ -0,0 +1,109 @@ +--- +title: "vocabulary" +weight: 16 +--- + +Report the words the model's names are made of, per corpus, module or type; or drift against a glossary. + +Every `TNamed` entity's name is split into words (`OrderLineRepository` → `order`, `line`, `repository`), each word classed `domain`, `technical`, `keyword` or `short` and weighted by the role of what carries it: a type's or a module's name weighs 8, an attribute's 4, an operation's 3, a parameter's or a local's 1. The words of a qualified name before its last separator (`com.acme` of `com.acme.order`) weigh 1 whatever carries them. Only names are read, never source. The splitting rules per language are the lexicons [`profiles`](/docs/reference/cli/profiles/) prints. + +## Synopsis + +``` +codegraph vocabulary [model.jsonl...] [--level ] [--scope ID] [--top N] + [--class ] [--weights KEY=N,…] + [--technical FILE] [--drift FILE] [--out FILE] [--json] + [--internal-only] [--declared-only] [--no-cache] +``` + +## Arguments + +| Argument | Meaning | Default | +|---|---|---| +| `` | One or more model.jsonl paths, loaded together as ONE union (decision 5). With no path, the model the extractor writes in this directory. | `-codegraph.jsonl` | + +## Options + +| Option | Meaning | Default | +|---|---|---| +| `--level ` | The corpus's words, or one block per module or type, ranked by distinct terms (the cognitive-load report). | `corpus` | +| `--scope ID` | Report one scope: a module or type by its entity id, or an operation, counted on demand. | — | +| `--top N` | Show the N heaviest words: 40 by default, 10 per block under `--level module\|type`. | — | +| `--class ` | Which words to show. | `domain` | +| `--weights KEY=N,…` | Override weights, e.g. `type=6,qualifier=0`. Keys: `module`, `type`, `operation`, `attribute`, `parameter`, `local`, `other`, `qualifier`. | — | +| `--technical FILE` | Replace the shared technical words with this file's, one word per line; each language's own still apply. | — | +| `--drift FILE` | Compare with a glossary, one business term per line: the terms no scope of `--level` says (exit 3), and the domain words no term contains. | — | +| `--out FILE` | Also write the [`vocabulary.json`](/docs/reference/artifacts/vocabulary-json/) artifact to this file. | — | +| `--json` | Print the same information as a machine-readable JSON object on stdout. | — | +| `--internal-only` | Drop stub (external) entities and every edge touching one. | — | +| `--declared-only` | Keep only `declared` facts; drop derived and dynamic-candidate edges. | — | +| `--no-cache` | Read the model.jsonl directly; never build or reuse a sibling model.db. | — | +| `-h, --help` | Show this help. | — | + +## Scopes + +A type's words are those of everything that folds into it, a module's likewise, the corpus's every kept entity's: the membership the city uses. A nested type is a scope of its own, so its members are not its outer type's. A stub never speaks, under any view: its name was chosen by another codebase. An operation's words are not stored; `--scope` counts them on demand, from the operation and everything written inside it short of a nested type. + +## Drift + +The glossary is a plain text file, one business term per line; blank lines and `#` comments are skipped. Each term is split by the same universal rules, so `Order Line` meets `order` and `line`. A term is *said* by a scope when every one of its words is, whatever class the code gives the word, so `--level module` asks for all of them in one module. Nothing is stemmed: `lines` is not `line`. The report lists the terms no scope says with their missing words, the terms it says and where, and the `domain` words the code says that no term contains. + +## Exit codes + +`0` ok · `1` internal error (a bug) · `2` usage error · `3` findings. + +`3` when the load was not clean, or with `--drift` when a business term is said by no scope of the level. An unknown `--scope` id, an unknown `--weights` key or a `--technical` line that is not one word is a usage error. + +## Example + +```console +$ codegraph vocabulary fixtures/java/expected/model.jsonl --top 5 +cache: fixtures/java/expected/model.db +codegraph vocabulary — words +models: fixtures/java/expected/model.jsonl +level: corpus +view: all (nothing filtered — stubs and inferred edges included) +weights: module 8 · type 8 · operation 3 · attribute 4 · parameter 1 · local 1 · other 1 · qualifier 1 +technical: 125 shared words (sha256 c16bd05be59a) + each language's own + +corpus src + 152 entities: 127 speak (module 3 · type 19 · operation 43 · attribute 21 · parameter 35 · local 6), 25 unnamed + 85 distinct terms, 75 domain · weight 503 · 146 occurrences + domain: 5 of 75 terms (--top 5), by weight descending, ties by term + + TERM CLASS WEIGHT OCC CARRIERS MODULES TYPES + order domain 40 12 12 3 4 + price domain 18 6 6 1 6 + basket domain 16 2 2 1 2 + channel domain 16 4 4 1 2 + discount domain 14 3 3 1 3 +``` + +`OCC` counts the word in names, `CARRIERS` the entities whose name says it, and `MODULES` and `TYPES` how many of those scopes say it in the whole corpus. + +```console +$ printf 'Order\nOrder Line\nCustomer\n' > glossary.txt +$ codegraph vocabulary fixtures/java/expected/model.jsonl --drift glossary.txt --level module --top 4 +… +glossary: glossary.txt, 3 business terms + +never said: 1 of 3 business terms, no module says all their words + Customer missing customer + +said: 2 of 3 business terms + Order 3 modules + Order Line 1 module: java:com.acme.order + +code only: 4 of 73 domain words no business term contains (--top 4), by weight descending, ties by term + + TERM WEIGHT OCC MODULES + price 18 6 1 + basket 16 2 1 + channel 16 4 1 + discount 14 3 1 +1 business term no module says — exiting 3 (findings). +``` + +## See also + +[vocabulary.json](/docs/reference/artifacts/vocabulary-json/) · [`profiles`](/docs/reference/cli/profiles/) · [CLI conventions](/docs/reference/cli/) diff --git a/website/content/docs/reference/exit-codes.md b/website/content/docs/reference/exit-codes.md index 324635c..7ca9be3 100644 --- a/website/content/docs/reference/exit-codes.md +++ b/website/content/docs/reference/exit-codes.md @@ -49,6 +49,7 @@ Valid options for 'codegraph city': --height METRIC, --height-scale Date: Mon, 26 Oct 2026 08:31:42 +0100 Subject: [PATCH 7/7] docs(plan): the vocabulary command as built MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §20.1's command paragraph now describes what was built: - the header's view, weights and technical list; - the 10-word blocks under --level; - an operation's scope on demand, and the usage errors that say what an id is; - the --weights and --technical rules, and --out beside the report; - the glossary's comments, merges and wordless lines; - drift's two halves and its exit 3; - the flags a report would not read, refused; - the JSON kind. §20.4 checks the drift and command item. Left in M19a: the coverage table on the real corpora. Co-Authored-By: Claude Opus 5.5 --- PLAN.md | 53 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/PLAN.md b/PLAN.md index fc4c45a..0b59268 100644 --- a/PLAN.md +++ b/PLAN.md @@ -5093,17 +5093,40 @@ scope per line, the snapshots' form. Size is measured on BroadleafCommerce **The command.** `codegraph vocabulary [--level corpus|module|type] [--scope ID] [--top N] [--class domain|technical|keyword|short|all] [--weights …] [--technical FILE] [--drift FILE] [--out vocabulary.json] -[--svg FILE] [--json] [--internal-only] [--declared-only]`. No flag: the -corpus root's top 40 `domain` terms as a table — term, class, weight, -occurrences, modules, types. `--level module`: one block per module, its top -N and its `distinct` column, sorted by `distinct` descending — the -cognitive-load report. `--drift FILE`: a plain text file, one business term -per line; each term is split with the universal rules so `Order Line` meets -`order` and `line`; a business term is SPOKEN in a scope when every one of -its words is, and the report lists the business terms no scope speaks and -the `domain` words the code speaks that no business term contains, by -`--level`. The Specy `.domain` file as input is deferred (§20.5): the plain -list first, because it exists in every company. +[--svg FILE] [--json] [--internal-only] [--declared-only] [--no-cache]` +(`--svg` arrives with M19b's layout), a shell over the package like +`analyze` over the analyzer. Every report's +header states the view, the weights and the technical list (its size, the +first 12 hex of its digest, and the file it came from). No flag: the corpus +root's top 40 `domain` terms as a table — term, class, weight, occurrences, +carriers, modules, types. `--level module`: one block per module, its top N +(10 by default) and its `distinct` column, sorted by `distinct` descending, +ties by id — the cognitive-load report. `--scope ID`: a stored module or +type, else an operation counted on demand (`operationScope`: the operation +and every kept entity written inside it, lambdas included, short of a type +or module, indexed by the artifact's terms; an artifact built under other +options is refused); an unknown id, a stub, or a member that is no operation +is a usage error that says what the id is. `--weights KEY=N,…` lays the keys +typed over the defaults (an unknown or repeated key, or a weight below 0, is +a usage error). `--technical FILE` replaces the shared list whole, one word +per line, `#` comments skipped; a line the splitter does not yield as one +word could never match a token and is refused by its number. `--out` writes +the artifact (compact) beside the report. `--drift FILE`: a plain text file, +one business term per line, `#` comments skipped, a term whose words repeat +an earlier one's merged, a line naming no word warned about by its number; +each term is split with the universal rules so `Order Line` meets `order` +and `line`; a business term is SPOKEN in a scope when every one of its words +is, whatever class the code gives it, and the report lists the business +terms no scope of `--level` speaks (with the words no scope says at all, or +"never together"), the ones it speaks and where, and the `domain` words the +code speaks that no business term contains, summed over the level's scopes. +Nothing is stemmed. A term spoken nowhere exits 3, as `analyze --report +cycles` does for a cycle: a CI job can gate on the glossary. `--class` with +`--drift`, and `--level` or `--drift` with `--scope`, are usage errors — a +flag the report would not read is refused, never dropped. `--json` is one +object, `codegraph.vocabularyReport/1`, `report: "words" | "drift"`. The +Specy `.domain` file as input is deferred (§20.5): the plain list first, +because it exists in every company. **Measured** (the table this milestone fills — the numbers below are what to collect, not results): @@ -5330,10 +5353,12 @@ never looks like a fact. the splitter's `qualifier` count and the `qualifier` weight. - [x] `aggregate.ts` on `containingType`/`containingModule`, `artifact.ts` (`codegraph.vocabulary/1`); the package now depends on the analyzer. -- [ ] `drift.ts` and `codegraph vocabulary` (§20.1): the report, `--level`, - `--scope`, `--class`, `--weights`, `--technical`, `--drift`, `--out`, +- [x] `drift.ts` and `codegraph vocabulary` (§20.1): the report, `--level`, + `--scope` (an operation's scope on demand), `--class`, `--weights`, + `--technical`, `--drift` (exit 3 on a term spoken nowhere), `--out`, `--json`, the view flags; usage errors for an unknown scope, role or - class. + class; `docs/cli.md`, the website's command and `vocabulary.json` + pages. - [x] `fixtures//expected/vocabulary.json` for the five fixtures and `fixtures/unicode`, byte-identical on every run (`CODEGRAPH_UPDATE_SNAPSHOTS=1` rewrites them); hand counts in each