From e2632e3563ea838c68cb09fa78b176874a095673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 13:35:33 +0200 Subject: [PATCH 1/4] feat(core): language lexicons on profiles Profile.lexicon is what the vocabulary splitter (M19, #6) must know about a language and cannot derive: the specification's reserved words (contextual keywords are identifiers, often domain words), the whole names the language or its runtime dictates, its plumbing words, the kinds whose name is a file path, and C#'s interface marker. Word boundaries are universal, so separators and sigils carry no data. The eleven shipped profiles carry one; TypeScript and JavaScript read one ECMASCRIPT_LEXICON. validateProfile gains four codes for entries that could never do what their list says. `codegraph profiles` prints the lexicon between the kinds and the notes, and --json carries it in a fixed key order. schemas/ is unchanged: a profile is not on the wire. Co-Authored-By: Claude Opus 5.5 --- METAMODEL.md | 26 +++++ docs/cli.md | 5 +- packages/cli/src/commands/profiles.ts | 47 +++++++- packages/cli/test/profiles.test.ts | 51 +++++++++ packages/core/src/profile.ts | 139 +++++++++++++++++++++++ packages/core/src/profiles/clojure.ts | 16 +++ packages/core/src/profiles/csharp.ts | 33 ++++++ packages/core/src/profiles/ecmascript.ts | 29 +++++ packages/core/src/profiles/elixir.ts | 29 +++++ packages/core/src/profiles/go.ts | 19 ++++ packages/core/src/profiles/java.ts | 24 ++++ packages/core/src/profiles/javascript.ts | 4 + packages/core/src/profiles/php.ts | 29 +++++ packages/core/src/profiles/python.ts | 29 +++++ packages/core/src/profiles/ruby.ts | 24 ++++ packages/core/src/profiles/rust.ts | 24 ++++ packages/core/src/profiles/typescript.ts | 4 + packages/core/test/profile.test.ts | 101 ++++++++++++++++ packages/core/test/profiles.test.ts | 87 +++++++++++++- 19 files changed, 713 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/profiles/ecmascript.ts diff --git a/METAMODEL.md b/METAMODEL.md index ed461c5..6e9952c 100644 --- a/METAMODEL.md +++ b/METAMODEL.md @@ -384,6 +384,7 @@ implemented* (robustness test of the schema). | `kinds` | `Record` | the licit trait compositions per entity kind | | `edges` | EdgeKind[] | edge kinds this language can emit | | `notes` | string[] | documented static-analysis blind spots (reflection, `Class.forName`, dynamic `require`, macros pre-expansion, …) | +| `lexicon` | `{keywords, conventional, technical?, fileKinds?, interfacePrefix?}` | the language's words, for the vocabulary (§5.3): reserved words, names the language dictates, plumbing words, kinds whose name is a file path, C#'s interface marker | **Validation** of an entity against its profile: 1. `kind` exists in the profile; @@ -421,6 +422,31 @@ keyed on; and the verdict is per profile, since two profiles may name the same kind under different rules. It is a **reader-side** optimisation and requires no encoding support — trait sets are deliberately not a wire-level concept. +### 5.3 The lexicon: a language's words + +The vocabulary (PLAN.md §20) splits every name into words — `OrderManagerRepository` +is `order`, `manager`, `repository` — and tags each one. Splitting is +**universal**: every character that is not a letter, a mark or a digit ends a +word, and so does a camel-case boundary, so sigils, separators and suffixes +(`@total`, `valid?`, `Order::Line`) need no data. What a language adds is the +lexicon, optional on a profile and present on every shipped one: + +| Key | Meaning | +|---|---| +| `keywords` | the specification's **reserved** words, as the language spells them. A contextual keyword (`record`, `var`, TypeScript's `type`) is a legal identifier, often a domain word, and is not listed | +| `conventional` | whole identifiers the language or its runtime dictates — overrides, protocol members, operators (`toString`, `__init__`, `<=>`). Their words say nothing about the domain, so such a name contributes none | +| `technical` | plumbing words of the language beyond the vocabulary's shared list, spelled as the splitter yields them (lowercase letters and digits) | +| `fileKinds` | kinds whose name is a file path (TypeScript's `module`, Ruby's `file`): the extension is not a word | +| `interfacePrefix` | `true` in C# alone: `IOrderService` names an interface `OrderService` | + +`validateProfile` rejects an entry that could never do what its list says: +listed twice (`lexicon-duplicate`); a keyword with no letter or a name holding +whitespace, a technical entry the splitter cannot yield (`lexicon-not-a-word`); +a conventional name or technical word a keyword already claims +(`lexicon-keyword-overlap`); a file kind the profile does not define +(`lexicon-unknown-kind`). The lexicon is not on the wire: `schemas/` is +unchanged by it. + --- ## 6. Stub diff --git a/docs/cli.md b/docs/cli.md index 9098bbe..ff288c7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -314,7 +314,10 @@ facts. ### `profiles` Prints the language profiles core ships — the per-kind required and optional -trait sets an extractor must respect. +trait sets an extractor must respect, the edge kinds, the lexicon the +vocabulary reads (reserved words, conventional names, technical words, file +kinds), and the notes. `--lang L` prints one profile in full; `--json` prints +the data as `{kind: "codegraph.profiles/1", profiles[]}`. ## The Java extractor diff --git a/packages/cli/src/commands/profiles.ts b/packages/cli/src/commands/profiles.ts index 0b3db75..acc4621 100644 --- a/packages/cli/src/commands/profiles.ts +++ b/packages/cli/src/commands/profiles.ts @@ -1,5 +1,5 @@ import { compareIds, toJsonString } from "@codegraph/analyzer"; -import { PROFILES, getProfile, type Profile } from "@codegraph/core"; +import { PROFILES, getProfile, type Lexicon, type Profile } from "@codegraph/core"; import type { ProfilesOptions } from "../args.js"; import { EXIT, UsageError, type ExitCode } from "../exit.js"; import { errLine, outLines, type IoSink } from "../io.js"; @@ -11,7 +11,8 @@ import { errLine, outLines, type IoSink } from "../io.js"; * - Read `PROFILES` / `getProfile(lang)` from `@codegraph/core`. Profiles are * DATA (CLAUDE.md invariant 8): print them, never synthesize one. * - No `--lang`: list every shipped profile. With `--lang`: print that - * profile's kinds (required/optional traits), edge kinds, spaces and notes. + * profile's kinds (required/optional traits), edge kinds, spaces, lexicon + * and notes. * - An unknown `--lang` is a USAGE error (exit 2) naming the languages that do * exist — no model was involved, so it cannot be a finding. * - `--json` prints the profile data on stdout; deterministic key and array @@ -135,7 +136,7 @@ function summaryLines(profiles: readonly Profile[]): readonly string[] { /** * The full spec for one language: every kind with its licit trait composition, * the licensed edge kinds, the declaration spaces where the profile declares - * any, and the notes in full. + * any, the lexicon, and the notes in full. * * Kinds are printed in sorted order so the same profile always renders the same * bytes and two languages can be diffed against each other; trait lists print @@ -168,6 +169,9 @@ function specLines(profile: Profile): readonly string[] { if (spaces !== undefined) lines.push(` space ${list([...spaces])}`); } + lines.push(""); + lines.push(...lexiconLines(profile.lexicon)); + lines.push(""); lines.push("notes:"); if (noteCount === 0) { @@ -179,11 +183,31 @@ function specLines(profile: Profile): readonly string[] { return lines; } +/** + * The words the vocabulary splitter reads (PLAN.md §20.1), every entry, in the + * order the profile declares it: a reader asking why `newOrder` gave no `new` + * looks for `new` here. Between the kinds and the notes, because the notes stay + * last — a reader of this text (the website's page) takes all that follows + * `notes:` as notes. The optional rows print only where the profile declares them. + */ +function lexiconLines(lexicon: Lexicon | undefined): readonly string[] { + if (lexicon === undefined) return ["lexicon:", " (none)"]; + const lines = [ + "lexicon:", + ` keywords ${list([...lexicon.keywords])}`, + ` conventional ${list([...lexicon.conventional])}`, + ]; + if (lexicon.technical !== undefined) lines.push(` technical ${list([...lexicon.technical])}`); + if (lexicon.fileKinds !== undefined) lines.push(` file kinds ${list([...lexicon.fileKinds])}`); + if (lexicon.interfacePrefix === true) lines.push(" interface prefix yes"); + return lines; +} + /** * The profile as JSON: the same data, with a canonical key and kind order so * that identical inputs produce byte-identical stdout (decision 6). Nothing is - * added, renamed or defaulted — `space` and `notes` stay absent where the - * profile omits them, because their absence is itself profile information. + * added, renamed or defaulted — `space`, `notes` and `lexicon` stay absent where + * the profile omits them, because their absence is itself profile information. */ function canonicalProfile(profile: Profile): Record { const kinds: Record = {}; @@ -208,6 +232,19 @@ function canonicalProfile(profile: Profile): Record { out["space"] = space; } if (profile.notes !== undefined) out["notes"] = profile.notes; + if (profile.lexicon !== undefined) out["lexicon"] = canonicalLexicon(profile.lexicon); + + return out; +} +/** The lexicon's keys in one fixed order; an absent key stays absent, never `[]` or `false`. */ +function canonicalLexicon(lexicon: Lexicon): Record { + const out: Record = { + keywords: lexicon.keywords, + conventional: lexicon.conventional, + }; + if (lexicon.technical !== undefined) out["technical"] = lexicon.technical; + if (lexicon.fileKinds !== undefined) out["fileKinds"] = lexicon.fileKinds; + if (lexicon.interfacePrefix !== undefined) out["interfacePrefix"] = lexicon.interfacePrefix; return out; } diff --git a/packages/cli/test/profiles.test.ts b/packages/cli/test/profiles.test.ts index 52b5784..ea9d081 100644 --- a/packages/cli/test/profiles.test.ts +++ b/packages/cli/test/profiles.test.ts @@ -100,6 +100,48 @@ describe("one language's full spec", () => { ); }); + /** + * The lexicon is what the vocabulary splitter reads (PLAN.md §20.1): someone + * asking why `newOrder` gave no `new` reads the keyword list here, so every + * entry must be printed, unwrapped, in the order the profile declares it. + */ + it.each(LANGS)("prints %s's lexicon in full, line for line", (lang) => { + const lexicon = profileOf(lang).lexicon; + if (lexicon === undefined) throw new Error(`${lang} ships no lexicon`); + const lines = invoke(["profiles", "--lang", lang]).io.stdoutLines(); + const start = lines.indexOf("lexicon:"); + const expected = [ + "lexicon:", + ` keywords ${lexicon.keywords.join(", ")}`, + ` conventional ${lexicon.conventional.length === 0 ? "(none)" : lexicon.conventional.join(", ")}`, + ...(lexicon.technical === undefined ? [] : [` technical ${lexicon.technical.join(", ")}`]), + ...(lexicon.fileKinds === undefined ? [] : [` file kinds ${lexicon.fileKinds.join(", ")}`]), + ...(lexicon.interfacePrefix === true ? [" interface prefix yes"] : []), + ]; + expect(start, "no lexicon section").toBeGreaterThan(0); + expect(lines.slice(start, start + expected.length)).toEqual(expected); + // The block ends there: a blank line, then the next section. + expect(lines[start + expected.length]).toBe(""); + }); + + // The notes stay last: they are the one section whose entries run to several + // sentences, and a reader of the text form (the website's page) takes + // everything after `notes:` as notes. + it("prints the lexicon after the kinds and before the notes", () => { + for (const lang of LANGS) { + const lines = invoke(["profiles", "--lang", lang]).io.stdoutLines(); + expect(lines.indexOf("lexicon:"), lang).toBeGreaterThan(lines.indexOf("kinds:")); + expect(lines.indexOf("lexicon:"), lang).toBeLessThan(lines.indexOf("notes:")); + } + }); + + it("prints the interface prefix and the file kinds only where a profile declares them", () => { + expect(invoke(["profiles", "--lang", "csharp"]).io.stdout()).toContain("\n interface prefix yes\n"); + expect(invoke(["profiles", "--lang", "java"]).io.stdout()).not.toContain("interface prefix"); + expect(invoke(["profiles", "--lang", "ts"]).io.stdout()).toContain("\n file kinds module\n"); + expect(invoke(["profiles", "--lang", "java"]).io.stdout()).not.toContain("file kinds"); + }); + it("prints the declaration spaces exactly where a profile declares them", () => { const ts = invoke(["profiles", "--lang", "ts"]).io.stdout(); expect(ts).toContain("space"); @@ -145,6 +187,15 @@ describe("--json prints the profile data as-is", () => { ); }); + it("writes the lexicon's keys in one fixed order, absent keys absent", () => { + const keysOf = (lang: string): readonly string[] => { + const [json] = payload(["profiles", "--lang", lang]) as readonly Profile[]; + return Object.keys(json?.lexicon ?? {}); + }; + expect(keysOf("csharp")).toEqual(["keywords", "conventional", "technical", "interfacePrefix"]); + expect(keysOf("ts")).toEqual(["keywords", "conventional", "technical", "fileKinds"]); + }); + it("says the same thing as the text form (decision 8)", () => { const text = invoke(["profiles", "--lang", "go"]).io.stdout(); const [json] = payload(["profiles", "--lang", "go"]) as readonly Profile[]; diff --git a/packages/core/src/profile.ts b/packages/core/src/profile.ts index ab8310a..5630654 100644 --- a/packages/core/src/profile.ts +++ b/packages/core/src/profile.ts @@ -30,8 +30,48 @@ export interface Profile { readonly space?: Readonly> | undefined; /** Documented static-analysis blind spots (reflection, macros, dynamic require…). */ readonly notes?: readonly string[] | undefined; + /** The language's words, for the vocabulary (PLAN.md §20.1); absent, the universal rules apply alone. */ + readonly lexicon?: Lexicon | undefined; } +/** + * What the vocabulary splitter must know about a language and cannot derive + * (METAMODEL.md §5, PLAN.md §20.1). Splitting itself is universal — every + * character that is not a letter, a mark or a digit ends a word, and so does a + * camel-case boundary — so sigils, separators and suffixes (`@total`, + * `valid?`, `Order::Line`, `` Repository`1 ``) need no data here. + */ +export interface Lexicon { + /** + * The reserved words of the language's specification, as it spells them: the + * identifiers its grammar forbids. A contextual keyword (`record`, `var`, + * TypeScript's `type`) is a legal identifier, often a domain word, and is not + * listed; nor is an entry with no letter (Java's `_`), which no word can equal. + */ + readonly keywords: readonly string[]; + /** + * Whole identifiers the language or its runtime dictates — an override, a + * protocol member, an operator: `toString`, `__init__`, `<=>`. Their words + * say nothing about the domain, so such a name contributes none. + */ + readonly conventional: readonly string[]; + /** Plumbing words of this language beyond the shared technical list, as the splitter yields them. */ + readonly technical?: readonly string[] | undefined; + /** Kinds whose name is a file path (`src/order.ts`): the extension is not a word. */ + readonly fileKinds?: readonly string[] | undefined; + /** C#'s convention: `IOrderService` names an interface `OrderService`; the `I` is a marker, not a word. */ + readonly interfacePrefix?: true | undefined; +} + +/** Zod mirror of {@link Lexicon}. */ +export const LexiconSchema = z.object({ + keywords: z.array(z.string()), + conventional: z.array(z.string()), + technical: z.array(z.string()).optional(), + fileKinds: z.array(z.string()).optional(), + interfacePrefix: z.literal(true).optional(), +}); + /** Zod mirror of {@link Profile}, for validating profile data loaded from JSON. */ export const ProfileSchema = z.object({ lang: z.string().min(1), @@ -45,6 +85,7 @@ export const ProfileSchema = z.object({ edges: z.array(EdgeKind), space: z.record(z.string(), z.array(Space)).optional(), notes: z.array(z.string()).optional(), + lexicon: LexiconSchema.optional(), }); /** The closed set of validation codes. Consumers match on these, not on messages. */ @@ -62,6 +103,10 @@ export const VALIDATION_CODES = [ "required-optional-overlap", "space-not-allowed", "unknown-space-kind", + "lexicon-duplicate", + "lexicon-not-a-word", + "lexicon-keyword-overlap", + "lexicon-unknown-kind", ] as const; export type ValidationCode = (typeof VALIDATION_CODES)[number]; @@ -413,5 +458,99 @@ export function validateProfile(profile: Profile): ValidationIssue[] { } } + if (profile.lexicon !== undefined) issues.push(...lexiconIssues(profile.lexicon, profile.kinds)); + + return issues; +} + +/** What the splitter yields: a lowercase run of letters, marks and digits. */ +const SPLITTER_TOKEN = /^[\p{L}\p{M}\p{N}]+$/u; +const LETTER = /\p{L}/u; +const WHITESPACE = /\s/u; + +/** + * An entry that could never do what its list says: listed twice, a keyword no + * word can equal, a technical word the splitter cannot yield, a name the + * keyword class would claim first, a file kind the profile never declares. + */ +function lexiconIssues(lexicon: Lexicon, kinds: Profile["kinds"]): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const lists: readonly (readonly [string, readonly string[] | undefined])[] = [ + ["keywords", lexicon.keywords], + ["conventional", lexicon.conventional], + ["technical", lexicon.technical], + ["fileKinds", lexicon.fileKinds], + ]; + for (const [name, entries] of lists) { + const seen = new Set(); + for (const entry of entries ?? []) { + if (seen.has(entry)) { + issues.push({ code: "lexicon-duplicate", path: `lexicon.${name}`, message: `${JSON.stringify(entry)} is listed twice` }); + } + seen.add(entry); + } + } + + for (const keyword of lexicon.keywords) { + if (!LETTER.test(keyword) || WHITESPACE.test(keyword)) { + issues.push({ + code: "lexicon-not-a-word", + path: "lexicon.keywords", + message: `${JSON.stringify(keyword)} is not a word: a keyword holds a letter and no whitespace`, + }); + } + } + for (const name of lexicon.conventional) { + if (name === "" || WHITESPACE.test(name)) { + issues.push({ + code: "lexicon-not-a-word", + path: "lexicon.conventional", + message: `${JSON.stringify(name)} is not an identifier: it is empty or holds whitespace`, + }); + } + } + for (const word of lexicon.technical ?? []) { + if (!SPLITTER_TOKEN.test(word) || word !== word.toLowerCase()) { + issues.push({ + code: "lexicon-not-a-word", + path: "lexicon.technical", + message: `${JSON.stringify(word)} can never match: the splitter yields lowercase runs of letters and digits`, + }); + } + } + + // Exact for a conventional NAME (a whole identifier); lowercased for a technical + // WORD, because classification lowercases a token and tries `keyword` first. + const keywords = new Set(lexicon.keywords); + const loweredKeywords = new Set(lexicon.keywords.map((keyword) => keyword.toLowerCase())); + for (const name of lexicon.conventional) { + if (keywords.has(name)) { + issues.push({ + code: "lexicon-keyword-overlap", + path: "lexicon.conventional", + message: `${name} is a keyword: a conventional name is one the language lets you write`, + }); + } + } + for (const word of lexicon.technical ?? []) { + if (loweredKeywords.has(word)) { + issues.push({ + code: "lexicon-keyword-overlap", + path: "lexicon.technical", + message: `${word} is a keyword: classification decides that first, so it is never technical`, + }); + } + } + + for (const kind of lexicon.fileKinds ?? []) { + if (!Object.hasOwn(kinds, kind)) { + issues.push({ + code: "lexicon-unknown-kind", + path: "lexicon.fileKinds", + message: `file kind "${kind}" is not a kind the profile defines`, + }); + } + } + return issues; } diff --git a/packages/core/src/profiles/clojure.ts b/packages/core/src/profiles/clojure.ts index 093ebe9..320a362 100644 --- a/packages/core/src/profiles/clojure.ts +++ b/packages/core/src/profiles/clojure.ts @@ -126,4 +126,20 @@ export const clojureProfile: Profile = { "Runtime resolution (`resolve`, `requiring-resolve`, `eval`, `(var-get (ns-resolve ...))`) and dynamically computed `require` are static blind spots: no `import` or `invocation` edge is emitted for them.", "`:refer :all` and `:refer` bring vars into scope without qualification; the `import` edge is namespace -> namespace, so the module layer stays exact even when a symbol's origin needs var resolution.", ], + + lexicon: { + // The special forms of clojure.org's reference (with `try`'s `catch` and `finally`) and the + // literals `nil`, `true`, `false`. The interop form `.` has no letter to match. + keywords: [ + "def", "if", "do", "let", "quote", "var", "fn", "loop", "recur", "throw", "try", "catch", + "finally", "monitor-enter", "monitor-exit", "new", "set!", "nil", "true", "false", + ], + // The `gen-class` prefix's entry point, constructor hooks and `Object` overrides. + conventional: [ + "-main", "-init", "-post-init", "-toString", "-equals", "-hashCode", + ], + technical: [ + "coll", "ctx", "kw", "ns", "opts", "seq", "sym", "xf", "xform", + ], + }, }; diff --git a/packages/core/src/profiles/csharp.ts b/packages/core/src/profiles/csharp.ts index 4521e7f..8b089b7 100644 --- a/packages/core/src/profiles/csharp.ts +++ b/packages/core/src/profiles/csharp.ts @@ -217,4 +217,37 @@ export const csharpProfile: Profile = { "Neither a `.sln` nor a `.csproj` is read: the extractor parses every `*.cs` under the source roots into ONE compilation (no MSBuild, no restore), so a legacy tree with no restorable project graph still extracts, and a missing package is a stub rather than a build failure. Consequences: one preprocessor configuration is parsed (no `--define` yet); source-generated and XAML/Razor code-behind partials are absent unless their output is under the roots; a name declared twice across projects that never see each other collides as one entity.", "The metrics pass (`codegraph-metrics`, PLAN.md §18): a property with accessors is joined to its accessor spaces — `{ get; set; }` is two, so it carries the pass's charged keys only (cognitive, exits, ABC). An implicit constructor (anchored on the type's name) and a record's positional members have no space and carry none of its keys. tree-sitter-c-sharp 0.23.5 misparses a C# 14 extension block: its members go unmeasured by the pass, and the enclosing class is joined by containment (charged keys only). tree-sitter does not evaluate `#if`: in a file with conditional blocks big-code-analysis measures EVERY branch, where Roslyn — and so this extractor's `sloc` — keeps the active one; on Humanizer such files are the 37 exact joins whose `sloc` and big-code-analysis's `ploc` disagree.", ], + + lexicon: { + // ECMA-334 §6.4.4: the 77 reserved keywords. Contextual keywords (`var`, `async`, `await`, + // `record`, `value`, `get`, `set`, the query words) are identifiers. + keywords: [ + "abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked", + "class", "const", "continue", "decimal", "default", "delegate", "do", "double", "else", + "enum", "event", "explicit", "extern", "false", "finally", "fixed", "float", "for", + "foreach", "goto", "if", "implicit", "in", "int", "interface", "internal", "is", "lock", + "long", "namespace", "new", "null", "object", "operator", "out", "override", "params", + "private", "protected", "public", "readonly", "ref", "return", "sbyte", "sealed", "short", + "sizeof", "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", + "try", "typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", "virtual", + "void", "volatile", "while", + ], + // `System.Object`'s overrides, the entry point, + // `IComparable`/`IDisposable`/`IEnumerable`/`ICloneable`, deconstruction, the indexer + // (`this[]`, as the extractor names it) and the operators' metadata names. + conventional: [ + "ToString", "Equals", "GetHashCode", "Finalize", "Main", "CompareTo", "Dispose", + "DisposeAsync", "GetEnumerator", "GetAsyncEnumerator", "MoveNext", "MoveNextAsync", "Clone", + "Deconstruct", "this[]", "op_Addition", "op_Subtraction", "op_Multiply", "op_Division", + "op_Modulus", "op_Equality", "op_Inequality", "op_LessThan", "op_GreaterThan", + "op_LessThanOrEqual", "op_GreaterThanOrEqual", "op_Implicit", "op_Explicit", + "op_UnaryNegation", "op_UnaryPlus", "op_LogicalNot", "op_OnesComplement", "op_Increment", + "op_Decrement", "op_True", "op_False", "op_BitwiseAnd", "op_BitwiseOr", "op_ExclusiveOr", + "op_LeftShift", "op_RightShift", "op_UnsignedRightShift", + ], + technical: [ + "args", "async", "attribute", "enumerable", "enumerator", "extension", "extensions", "linq", + ], + interfacePrefix: true, + }, }; diff --git a/packages/core/src/profiles/ecmascript.ts b/packages/core/src/profiles/ecmascript.ts new file mode 100644 index 0000000..f3005ab --- /dev/null +++ b/packages/core/src/profiles/ecmascript.ts @@ -0,0 +1,29 @@ +import type { Lexicon } from "../profile.js"; + +/** + * The words JavaScript and TypeScript share: TypeScript reserves exactly + * ECMAScript's words — `type`, `namespace`, `declare`, `readonly` and the rest + * of its own vocabulary are contextual, hence identifiers. One statement, read + * by both profiles, so the two cannot drift apart. + */ +export const ECMASCRIPT_LEXICON = { + // ECMA-262, Keywords and Reserved Words (ReservedWord), plus the words strict mode — every + // module — reserves. + keywords: [ + "await", "break", "case", "catch", "class", "const", "continue", "debugger", "default", + "delete", "do", "else", "enum", "export", "extends", "false", "finally", "for", "function", + "if", "import", "in", "instanceof", "new", "null", "return", "super", "switch", "this", + "throw", "true", "try", "typeof", "var", "void", "while", "with", "yield", "implements", + "interface", "let", "package", "private", "protected", "public", "static", + ], + // The constructor, `Object.prototype`'s overrides, `JSON.stringify`'s hook, and + // the well-known symbols' methods, spelled as the extractor names them. + conventional: [ + "constructor", "toString", "toLocaleString", "valueOf", "toJSON", "[Symbol.iterator]", + "[Symbol.asyncIterator]", "[Symbol.hasInstance]", "[Symbol.toPrimitive]", + "[Symbol.toStringTag]", "[Symbol.dispose]", "[Symbol.asyncDispose]", + ], + technical: [ + "args", "async", "callback", "cb", "fn", "json", "opts", "promise", "prototype", + ], +} as const satisfies Lexicon; diff --git a/packages/core/src/profiles/elixir.ts b/packages/core/src/profiles/elixir.ts index c59397f..dcb28ce 100644 --- a/packages/core/src/profiles/elixir.ts +++ b/packages/core/src/profiles/elixir.ts @@ -132,4 +132,33 @@ export const elixirProfile: Profile = { "Measures (TMetrics, METAMODEL.md §3.8): `sloc` on the file, every module and every invocable — lines of its own span that are neither blank nor comment-only, a comment being what the compiler's own parser reports as one: every non-blank line a heredoc, a sigil or a multi-line string spans is code, a `#` inside one included; `cyclomatic` on invocables: 1 + clause heads beyond the first + `case`/`cond`/`with`/`receive`/`try` clauses beyond the first + `if`/`unless` + `and`/`or`/`&&`/`||` + `rescue`/`catch` clauses. A nested `fn` contributes to its enclosing invocable: an anonymous function is not an entity.", "The metrics pass (`codegraph-metrics`, PLAN.md §18): big-code-analysis opens one space per `def` clause, so a function of several clauses is joined by containment and carries the charged keys only (cognitive, exits, ABC) — no `lloc`, Halstead or `mi`, which do not add up over clauses. `defprotocol` and `defimpl` bodies, `defguard` and `defdelegate` have no space and carry none of the pass's keys. The join's witness (`sloc` = big-code-analysis's `ploc`) cannot hold over a heredoc: big-code-analysis leaves a heredoc's text lines out of `ploc`, which METAMODEL's `sloc` counts (Phoenix's `mix_app_path`: `sloc` 19, `ploc` 13).", ], + + lexicon: { + // The Elixir syntax reference, Reserved words: the 15 words no variable, function or module + // may be named. + keywords: [ + "true", "false", "nil", "when", "and", "or", "not", "in", "fn", "do", "end", "catch", + "rescue", "after", "else", + ], + // The compiler's callbacks and reflection, the OTP behaviours' callbacks (`GenServer`, + // `Supervisor`), `String.Chars` and `Inspect`, and the module attributes the compiler reads, + // spelled as the extractor names attributes (`@moduledoc`). + conventional: [ + "__using__", "__before_compile__", "__after_compile__", "__after_verify__", + "__on_definition__", "__struct__", "__info__", "__deriving__", "__impl__", "__protocol__", + "start_link", "child_spec", "init", "handle_call", "handle_cast", "handle_info", + "handle_continue", "terminate", "code_change", "format_status", "to_string", "inspect", + "@moduledoc", "@doc", "@typedoc", "@spec", "@type", "@typep", "@opaque", "@callback", + "@macrocallback", "@optional_callbacks", "@impl", "@behaviour", "@derive", "@enforce_keys", + "@external_resource", "@compile", "@deprecated", "@dialyzer", "@on_load", "@vsn", + "@before_compile", "@after_compile", "@after_verify", "@on_definition", "@nifs", + "@fallback_to_any", "@file", + ], + technical: [ + "attrs", "behaviour", "callback", "changeset", "conn", "ecto", "ets", "gen", "live", "mix", + "opts", "otp", "phoenix", "pid", "plug", "repo", "schema", "server", "socket", "sup", + "supervisor", "web", + ], + fileKinds: ["file"], + }, }; diff --git a/packages/core/src/profiles/go.ts b/packages/core/src/profiles/go.ts index a381f71..4709355 100644 --- a/packages/core/src/profiles/go.ts +++ b/packages/core/src/profiles/go.ts @@ -109,4 +109,23 @@ export const goProfile: Profile = { "Invisible to static extraction: reflect, the plugin package, cgo, go:generate and go:linkname directives, and side effects of init() functions pulled in by blank imports (`import _ \"…\"`), which are still emitted as ordinary import edges.", "Build tags and GOOS/GOARCH constrained files mean a single extraction run sees one build configuration; entities excluded by the active tags are absent from the model.", ], + + lexicon: { + // The Go specification, Keywords: the 25 reserved words. Predeclared identifiers (`nil`, + // `true`, `int`, `len`, `new`…) may be redeclared, so they are not listed. + keywords: [ + "break", "case", "chan", "const", "continue", "default", "defer", "else", "fallthrough", + "for", "func", "go", "goto", "if", "import", "interface", "map", "package", "range", + "return", "select", "struct", "switch", "type", "var", + ], + // Package initialization, the entry point, and the methods the standard library's interfaces + // dictate (`fmt.Stringer`, `error` and `errors`, `sort.Interface`, `encoding`, `net/http`). + conventional: [ + "init", "main", "String", "GoString", "Error", "Unwrap", "Is", "As", "Len", "Less", "Swap", + "MarshalJSON", "UnmarshalJSON", "MarshalText", "UnmarshalText", "ServeHTTP", + ], + technical: [ + "cfg", "cmd", "ctx", "err", "errs", "internal", "mu", "mutex", "pkg", "srv", "svc", "wg", + ], + }, }; diff --git a/packages/core/src/profiles/java.ts b/packages/core/src/profiles/java.ts index bd47bca..403f516 100644 --- a/packages/core/src/profiles/java.ts +++ b/packages/core/src/profiles/java.ts @@ -183,4 +183,28 @@ export const javaProfile: Profile = { "Spoon materializes the implicit constructor of an anonymous class with synthetic parameters named `$anonymousN`. They are emitted (implicit members are, deliberately, so that `new Foo()` does not dangle), so the model contains a handful of parameter entities nobody wrote, carrying no anchor of their own — 5 of 15338 entities on commons-lang.", "The metrics pass (`codegraph-metrics`, PLAN.md §18) joins big-code-analysis spaces to entities on their anchors. Spoon starts a declaration AFTER its annotations and tree-sitter AT them; the pass bridges the two when nothing but annotation lines lies between (a LEADING join — most of a Spring corpus). It measures nothing on a lambda (big-code-analysis opens no space for one and charges its cognitive, exits and ABC to the enclosing method), on an implicit constructor (anchored on its class) or on an annotation element (no body): those carry none of the pass's keys.", ], + + lexicon: { + // JLS 21 §3.9 (reserved keywords) and §3.10.3/§3.10.8 (the literals `true`, `false`, `null`). + // `_` is a keyword too, with no letter to match. Contextual keywords (`record`, `var`, + // `yield`, `sealed`, `permits`, `when`, the module words) are identifiers. + keywords: [ + "abstract", "assert", "boolean", "break", "byte", "case", "catch", "char", "class", "const", + "continue", "default", "do", "double", "else", "enum", "extends", "final", "finally", + "float", "for", "goto", "if", "implements", "import", "instanceof", "int", "interface", + "long", "native", "new", "package", "private", "protected", "public", "return", "short", + "static", "strictfp", "super", "switch", "synchronized", "this", "throw", "throws", + "transient", "try", "void", "volatile", "while", "true", "false", "null", + ], + // `java.lang.Object`'s overrides, the entry point, and the + // `Comparable`/`Iterable`/`Serializable` contracts. + conventional: [ + "toString", "equals", "hashCode", "clone", "finalize", "main", "compareTo", "iterator", + "hasNext", "valueOf", "serialVersionUID", "readObject", "writeObject", "readObjectNoData", + "readResolve", "writeReplace", + ], + technical: [ + "annotation", "bean", "beans", "ejb", "jaxb", "jdbc", "jndi", "jpa", "pojo", "servlet", + ], + }, }; diff --git a/packages/core/src/profiles/javascript.ts b/packages/core/src/profiles/javascript.ts index 959614a..015227e 100644 --- a/packages/core/src/profiles/javascript.ts +++ b/packages/core/src/profiles/javascript.ts @@ -1,4 +1,5 @@ import type { Profile } from "../profile.js"; +import { ECMASCRIPT_LEXICON } from "./ecmascript.js"; /** * JavaScript — the untyped baseline. Two absences carry information: @@ -110,4 +111,7 @@ export const javascriptProfile: Profile = { "Getters/setters are emitted as `method` entities; reads and writes of the underlying value at call sites are `access` edges, so a property read that runs code is visible as access, not invocation.", "Class fields and object literal members share the `property` kind; object literals used as namespaces therefore appear as a `variable` with `property` children.", ], + + // The module is the file: its name is a path whose extension is not a word. + lexicon: { ...ECMASCRIPT_LEXICON, fileKinds: ["module"] }, }; diff --git a/packages/core/src/profiles/php.ts b/packages/core/src/profiles/php.ts index 081ee76..07abacd 100644 --- a/packages/core/src/profiles/php.ts +++ b/packages/core/src/profiles/php.ts @@ -123,4 +123,33 @@ export const phpProfile: Profile = { "Gradual typing: parameter, property and return type declarations are optional, so TTypedEntity.declaredType is often absent. Docblock types (@var, @param) are not authoritative and must not be promoted to declaredType.", "Namespaces are 1-N with files: TModule.definedIn lists every file declaring into the namespace. A file with no `namespace` statement declares into the global namespace.", ], + + lexicon: { + // The PHP manual, List of Keywords and its compile-time constants, plus the literals `true`, + // `false`, `null`; spelled lowercase, as PHP matches them case-insensitively. `enum` is + // context-sensitive; the type names reserved only as class names (`int`, `string`, `mixed`…) + // are not listed. + keywords: [ + "__halt_compiler", "abstract", "and", "array", "as", "break", "callable", "case", "catch", + "class", "clone", "const", "continue", "declare", "default", "die", "do", "echo", "else", + "elseif", "empty", "enddeclare", "endfor", "endforeach", "endif", "endswitch", "endwhile", + "eval", "exit", "extends", "final", "finally", "fn", "for", "foreach", "function", "global", + "goto", "if", "implements", "include", "include_once", "instanceof", "insteadof", + "interface", "isset", "list", "match", "namespace", "new", "or", "print", "private", + "protected", "public", "readonly", "require", "require_once", "return", "static", "switch", + "throw", "trait", "try", "unset", "use", "var", "while", "xor", "yield", "__CLASS__", + "__DIR__", "__FILE__", "__FUNCTION__", "__LINE__", "__METHOD__", "__NAMESPACE__", + "__PROPERTY__", "__TRAIT__", "true", "false", "null", + ], + // The magic methods. + conventional: [ + "__construct", "__destruct", "__call", "__callStatic", "__get", "__set", "__isset", + "__unset", "__sleep", "__wakeup", "__serialize", "__unserialize", "__toString", "__invoke", + "__set_state", "__clone", "__debugInfo", + ], + technical: [ + "arr", "closure", "facade", "middleware", "php", "str", + ], + fileKinds: ["codeFile"], + }, }; diff --git a/packages/core/src/profiles/python.ts b/packages/core/src/profiles/python.ts index fd26c88..1686077 100644 --- a/packages/core/src/profiles/python.ts +++ b/packages/core/src/profiles/python.ts @@ -105,4 +105,33 @@ export const pythonProfile: Profile = { "`from x import *` yields a module-level import edge only; the individual names it binds cannot be attributed.", "Conditional and `if TYPE_CHECKING:` imports are emitted like any other import — this profile declares no Space, so type-only dependencies are indistinguishable from runtime ones.", ], + + lexicon: { + // The Python 3.12 Language Reference §2.3.1: the 35 keywords. Soft keywords (`match`, `case`, + // `type`, `_`) are identifiers. + keywords: [ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", + "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", + "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", + "while", "with", "yield", + ], + // The receiver conventions and the data model's special names (Reference §3.3). + conventional: [ + "self", "cls", "__init__", "__new__", "__del__", "__repr__", "__str__", "__bytes__", + "__format__", "__hash__", "__bool__", "__eq__", "__ne__", "__lt__", "__le__", "__gt__", + "__ge__", "__len__", "__iter__", "__next__", "__aiter__", "__anext__", "__reversed__", + "__contains__", "__getitem__", "__setitem__", "__delitem__", "__missing__", "__getattr__", + "__getattribute__", "__setattr__", "__delattr__", "__dir__", "__get__", "__set__", + "__delete__", "__set_name__", "__call__", "__enter__", "__exit__", "__aenter__", "__aexit__", + "__await__", "__init_subclass__", "__class_getitem__", "__post_init__", "__add__", "__sub__", + "__mul__", "__matmul__", "__truediv__", "__floordiv__", "__mod__", "__pow__", "__neg__", + "__pos__", "__abs__", "__invert__", "__and__", "__or__", "__xor__", "__lshift__", + "__rshift__", "__int__", "__float__", "__index__", "__round__", "__copy__", "__deepcopy__", + "__reduce__", "__getstate__", "__setstate__", "__slots__", "__all__", "__name__", "__main__", + "__doc__", "__dict__", + ], + technical: [ + "args", "dict", "kwargs", "metaclass", "mixin", "py", "str", "tuple", + ], + }, }; diff --git a/packages/core/src/profiles/ruby.ts b/packages/core/src/profiles/ruby.ts index c2dfc18..727ccdc 100644 --- a/packages/core/src/profiles/ruby.ts +++ b/packages/core/src/profiles/ruby.ts @@ -113,4 +113,28 @@ export const rubyProfile: Profile = { "`--trace ` merges what `extractors/ruby/trace/codegraph_trace.rb` wrote from INSIDE the running program — the one thing the extractor never does is run corpus code, so the trace is a separate step, the user's, in a project that still boots. It names constants, files and lines, never a key: no entity is added and no edge of the source changes. A mixin the VM holds and no line writes is a `traitUsage` with provenance `generated`, anchored on the declaration of the type that has it. A call the program made at a site the baseline left unresolved is ONE `invocation` with provenance `dynamic-candidate`, `candidates` listing every method that site was seen to reach and `to` the first of them — what ran once, not what must run. A method the VM holds on a line that writes no `def` has no entity and gets none: it is counted.", "Audited (M18c) on rake 13.2.1, rubygems 3.6.2, Mastodon 4.3.0 and rails 8.0.2 — 100, 794, 2 541 and 3 334 files, every model `validate`-clean, rails in 4.7 s. Resolution rate 77.8 %, 54.4 %, 31.2 % and 57.0 % of the sites; Mastodon without `spec/` is 62.1 % (55 000 of its unbound calls are RSpec's DSL), rails' own `lib` and `app` 59.4 %. Four fifths of what stays unresolved in application code is `untyped-receiver` — the ceiling of any reader that does not run the code. ORACLES: Packwerk 3.3.1's reference extraction and ConstantResolver over Mastodon's autoload paths resolve 4 335 cross-file constant references in `app/` and `lib/`; the model has the same constant in the same file for 4 289 of them (98.9 %) — the 45 others are a `module Admin` WRAPPER, which Packwerk counts as a reference and the wrapper rule does not, and one an inherited constant Packwerk places by file name. The model names 346 more that Packwerk cannot: declarations outside the autoload paths. Run under its own tests in a container, rake's VM holds a method on the same line for 1 006 of the 1 012 methods the model declares in the 83 files it loaded (the six others: definitions under a platform or capability `if`, a `def` inside a method that never ran, methods of anonymous classes), and 200 methods the model has none for — 199 written by the metaprogramming loops of three files (`class_eval` of a string, `define_method` over a list), one the `else` branch of a platform test. Of the 2 829 calls observed during that run, 1 533 are edges the baseline wrote and 879 land on sites it had left untyped. The core-table misreading, measured on rubygems: 39 corpus bodies of 20 VM types (`Gem` alone 12), and 3 802 of 71 197 edges landing on the VM's stub of a type that corpus itself implements.", ], + + lexicon: { + // The Ruby documentation, Keywords: the 41 reserved words, as Ruby spells them. + keywords: [ + "__ENCODING__", "__LINE__", "__FILE__", "BEGIN", "END", "alias", "and", "begin", "break", + "case", "class", "def", "defined?", "do", "else", "elsif", "end", "ensure", "false", "for", + "if", "in", "module", "next", "nil", "not", "or", "redo", "rescue", "retry", "return", + "self", "super", "then", "true", "undef", "unless", "until", "when", "while", "yield", + ], + // The constructor and copy hooks, the conversion and comparison protocols, the operator + // methods, `Enumerable`'s `each`, `method_missing`, and the hooks a module or class receives. + conventional: [ + "initialize", "initialize_copy", "to_s", "to_str", "to_a", "to_ary", "to_h", "to_hash", + "to_proc", "inspect", "hash", "eql?", "==", "===", "=~", "!~", "!=", "<=>", "<", ">", "<=", + ">=", "+", "-", "*", "/", "%", "**", "<<", ">>", "&", "|", "^", "~", "!", "+@", "-@", "[]", + "[]=", "each", "coerce", "method_missing", "respond_to_missing?", "respond_to?", "included", + "extended", "inherited", "prepended", "method_added", "const_missing", + ], + technical: [ + "accessor", "attr", "concern", "concerns", "ext", "gem", "gemspec", "klass", "lambda", + "mixin", "proc", "rake", "rakefile", + ], + fileKinds: ["file"], + }, }; diff --git a/packages/core/src/profiles/rust.ts b/packages/core/src/profiles/rust.ts index 5c307f9..5c943ed 100644 --- a/packages/core/src/profiles/rust.ts +++ b/packages/core/src/profiles/rust.ts @@ -104,4 +104,28 @@ export const rustProfile: Profile = { "`static` items map to the `const` kind. Tuple-struct fields are named by their position (\"0\", \"1\", ...). Shadowed `let` bindings produce several localVariable entities disambiguated by (file, startLine).", "The `self` receiver is not emitted as a parameter; the receiver relation is the impl block's attachment.", ], + + lexicon: { + // The Rust Reference, Keywords: the strict and the reserved keywords. Weak keywords (`union`, + // `macro_rules`, `raw`, `safe`) are identifiers outside their one context. + keywords: [ + "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", + "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", + "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", + "type", "unsafe", "use", "where", "while", "abstract", "become", "box", "do", "final", "gen", + "macro", "override", "priv", "try", "typeof", "unsized", "virtual", "yield", + ], + // The entry point and the methods the standard library's traits name (`Display`/`Debug`, + // `Drop`, `Clone`, `PartialEq`/`Ord`, `Deref`, `IntoIterator`, `FromStr`, the conversion + // traits, `Default`, `IndexMut`, `Future`). + conventional: [ + "main", "fmt", "drop", "clone", "clone_from", "eq", "ne", "partial_cmp", "cmp", "deref", + "deref_mut", "into_iter", "from_str", "try_from", "try_into", "as_ref", "as_mut", + "borrow_mut", "index_mut", "default", "poll", + ], + technical: [ + "arc", "buf", "cell", "ctx", "err", "ffi", "iter", "mutex", "ok", "opt", "rc", "refcell", + "rs", "rx", "sys", "tx", "vec", + ], + }, }; diff --git a/packages/core/src/profiles/typescript.ts b/packages/core/src/profiles/typescript.ts index 4b986ba..9d3aad9 100644 --- a/packages/core/src/profiles/typescript.ts +++ b/packages/core/src/profiles/typescript.ts @@ -1,6 +1,7 @@ import type { TraitName } from "../names.js"; import type { Space } from "../primitives.js"; import type { Profile } from "../profile.js"; +import { ECMASCRIPT_LEXICON } from "./ecmascript.js"; /** * TypeScript profile (v2, PLAN.md §14.2 — the compiler-API extractor's @@ -246,4 +247,7 @@ export const typescriptProfile: Profile = { "A written value (TWithValue) is emitted for a `const` variable, a `readonly` property, an enum member and a parameter default whose initializer is CONSTANT-SHAPED — a literal, `-N`, an enum member, a type used as a value, arrays and operator expressions over those. TypeScript folds nothing at the declaration, so `4 * 25` rides `unevaluated` with its source text where Java would fold it; an enum member's value is the checker's computed constant. An initializer that is code — a call, a `new`, a function, an object literal — carries no value at all: absence is a claim. Such an initializer's calls and accesses are edges FROM the variable or property (the narrowest declared owner), which then carries TWithInvocations/TWithAccesses.", "The metrics pass (`codegraph-metrics`, PLAN.md §18): namespaces, enums, type aliases, interface members without a body and ambient declarations have no big-code-analysis space and carry none of the pass's keys. An exported class's decorators precede tree-sitter's class node; the pass joins across the decorator lines. In a `.tsx` file big-code-analysis's `ploc` can count a line or two fewer than this extractor's `sloc` (a JSX text token counts every line it spans here), so the join's witness differs there by that much.", ], + + // The module is the file: its name is a path whose extension is not a word. + lexicon: { ...ECMASCRIPT_LEXICON, fileKinds: ["module"] }, }; diff --git a/packages/core/test/profile.test.ts b/packages/core/test/profile.test.ts index 21d4b06..cf35586 100644 --- a/packages/core/test/profile.test.ts +++ b/packages/core/test/profile.test.ts @@ -6,6 +6,7 @@ import type { Model } from "../src/model.js"; import { SCHEMA_VERSION } from "../src/model.js"; import { type Profile, + ProfileSchema, validateEntity, validateModel, validateProfile, @@ -347,6 +348,106 @@ describe("validateProfile", () => { }); }); +/** + * The lexicon (PLAN.md §20.1) is profile data like the rest: what a language + * reserves, the names it dictates, its plumbing words. These rules catch an + * entry that could never do what its list says — a duplicate, a word the + * splitter cannot yield, a name the classification would decide otherwise. + */ +describe("validateProfile — the lexicon", () => { + const withLexicon = (lexicon: Profile["lexicon"]): Profile => ({ ...demo, lexicon }); + const base = { keywords: ["class", "new", "Self"], conventional: ["toString", "<=>"] }; + + it("accepts a well-formed lexicon, and a profile with none", () => { + expect(validateProfile(demo)).toEqual([]); + expect( + validateProfile( + withLexicon({ ...base, technical: ["impl", "utf8"], interfacePrefix: true, fileKinds: ["class"] }), + ), + ).toEqual([]); + }); + + it("rejects an entry listed twice, naming the list", () => { + const issues = validateProfile(withLexicon({ keywords: ["new", "new"], conventional: [] })); + expect(codes(issues)).toEqual(["lexicon-duplicate"]); + expect(issues[0]?.path).toBe("lexicon.keywords"); + for (const list of ["conventional", "technical", "fileKinds"] as const) { + const twice = validateProfile(withLexicon({ ...base, [list]: list === "fileKinds" ? ["class", "class"] : ["main", "main"] })); + expect(codes(twice), list).toEqual(["lexicon-duplicate"]); + expect(twice[0]?.path).toBe(`lexicon.${list}`); + } + }); + + it("rejects a keyword no word can equal: no letter, or whitespace inside", () => { + for (const keyword of ["_", ".", "", "yield from"]) { + const issues = validateProfile(withLexicon({ ...base, keywords: [keyword] })); + expect(codes(issues), JSON.stringify(keyword)).toEqual(["lexicon-not-a-word"]); + expect(issues[0]?.path).toBe("lexicon.keywords"); + } + }); + + it("rejects a conventional identifier that is empty or holds whitespace", () => { + for (const name of ["", "to s"]) { + expect(codes(validateProfile(withLexicon({ ...base, conventional: [name] }))), JSON.stringify(name)).toEqual([ + "lexicon-not-a-word", + ]); + } + }); + + it("rejects a technical entry the splitter could never yield", () => { + // The splitter yields lowercase runs of letters, marks and digits: these never match a token. + for (const word of ["Bean", "core_ext", "to-do", "", "a b"]) { + const issues = validateProfile(withLexicon({ ...base, technical: [word] })); + expect(codes(issues), JSON.stringify(word)).toEqual(["lexicon-not-a-word"]); + expect(issues[0]?.path).toBe("lexicon.technical"); + } + expect(validateProfile(withLexicon({ ...base, technical: ["résumé", "md5"] }))).toEqual([]); + }); + + it("rejects a conventional identifier that is also a keyword", () => { + const issues = validateProfile(withLexicon({ keywords: ["new"], conventional: ["new"] })); + expect(codes(issues)).toEqual(["lexicon-keyword-overlap"]); + expect(issues[0]?.path).toBe("lexicon.conventional"); + }); + + it("rejects a technical word a keyword already claims, whatever the keyword's case", () => { + // Classification lowercases and decides `keyword` first: `self` from `Self` is never technical. + const issues = validateProfile(withLexicon({ ...base, technical: ["self"] })); + expect(codes(issues)).toEqual(["lexicon-keyword-overlap"]); + expect(issues[0]?.path).toBe("lexicon.technical"); + }); + + it("rejects a file kind the profile does not declare", () => { + const issues = validateProfile(withLexicon({ ...base, fileKinds: ["module"] })); + expect(codes(issues)).toEqual(["lexicon-unknown-kind"]); + expect(issues[0]?.path).toBe("lexicon.fileKinds"); + }); +}); + +describe("ProfileSchema mirrors Profile", () => { + it("keeps every key of a profile with a full lexicon — nothing stripped, nothing added", () => { + const full: Profile = { + ...demo, + space: { class: ["type"] }, + notes: ["a blind spot"], + lexicon: { + keywords: ["class"], + conventional: ["toString"], + technical: ["impl"], + interfacePrefix: true, + fileKinds: ["class"], + }, + }; + expect(ProfileSchema.parse(full)).toEqual(full); + }); + + it("reads an absent interface prefix as absent, never as false", () => { + const lexicon = { keywords: ["class"], conventional: [] }; + expect(ProfileSchema.parse({ ...demo, lexicon }).lexicon).toEqual(lexicon); + expect(ProfileSchema.safeParse({ ...demo, lexicon: { ...lexicon, interfacePrefix: false } }).success).toBe(false); + }); +}); + /** * MM-4: profile validity is a function of `(kind, trait set)`, so the verdict * is computed once per distinct pair and shared. These tests exist for the diff --git a/packages/core/test/profiles.test.ts b/packages/core/test/profiles.test.ts index f17bec3..af8cf33 100644 --- a/packages/core/test/profiles.test.ts +++ b/packages/core/test/profiles.test.ts @@ -3,7 +3,7 @@ import { EDGE_KINDS, TRAIT_NAMES } from "../src/names.js"; import { SPACES } from "../src/primitives.js"; import { Entity } from "../src/entity.js"; import { parseModel, SCHEMA_VERSION } from "../src/model.js"; -import { validateEntity, validateModel, validateProfile, type Profile } from "../src/profile.js"; +import { ProfileSchema, validateEntity, validateModel, validateProfile, type Profile } from "../src/profile.js"; import { PROFILES, clojureProfile, @@ -758,3 +758,88 @@ describe("structural claims that must not silently regress", () => { }); }); }); + +/** + * The lexicon (PLAN.md §20.1): what the vocabulary splitter must know about a + * language and cannot derive. Every claim below is about the LANGUAGE — its + * specification's reserved words, the names its runtime dictates — so a later + * edit that "tidies" a list into something the language does not say fails here. + */ +describe("the lexicon of every shipped profile", () => { + const lexiconOf = (lang: string) => { + const lexicon = getProfile(lang)?.lexicon; + if (lexicon === undefined) throw new Error(`no lexicon for ${lang}`); + return lexicon; + }; + + it.each(profiles)("%s has a lexicon with its reserved words", (_lang, profile) => { + expect(profile.lexicon?.keywords.length ?? 0).toBeGreaterThan(0); + }); + + it.each(profiles)("%s parses through ProfileSchema unchanged", (_lang, profile) => { + expect(ProfileSchema.parse(profile)).toEqual(profile); + }); + + it("lists the conventional names the plan names, per language", () => { + expect(lexiconOf("java").conventional).toEqual(expect.arrayContaining(["toString", "equals", "hashCode", "main"])); + expect(lexiconOf("rb").conventional).toEqual(expect.arrayContaining(["initialize", "to_s", "inspect", "<=>"])); + expect(lexiconOf("ts").conventional).toEqual(expect.arrayContaining(["constructor"])); + expect(lexiconOf("ex").conventional).toEqual(expect.arrayContaining(["__using__", "start_link", "init", "handle_call"])); + expect(lexiconOf("csharp").conventional).toEqual( + expect.arrayContaining(["Equals", "GetHashCode", "ToString", "Dispose", "Main"]), + ); + expect(lexiconOf("python").conventional).toEqual(expect.arrayContaining(["__init__", "self", "cls"])); + }); + + // A contextual keyword is a legal identifier, and often a domain word: listing + // it would tag every `record`, `event` handler or `match` in the corpus as syntax. + it("lists reserved words only, never contextual ones", () => { + const contextual: Readonly> = { + java: ["record", "var", "yield", "sealed", "permits", "module", "when", "to"], + csharp: ["var", "async", "await", "record", "value", "get", "set", "where", "from", "select"], + ts: ["type", "namespace", "module", "declare", "readonly", "of", "get", "set", "async"], + js: ["of", "get", "set", "async", "from"], + python: ["match", "case", "type", "_"], + rust: ["union", "macro_rules", "default"], + php: ["enum"], + }; + for (const [lang, words] of Object.entries(contextual)) { + const keywords = new Set(lexiconOf(lang).keywords); + for (const word of words) expect(keywords.has(word), `${lang} lists ${word}`).toBe(false); + } + }); + + it("knows which languages reserve which words", () => { + // `type` is reserved by Go and Rust; to Java and TypeScript it is an identifier. + expect(lexiconOf("go").keywords).toContain("type"); + expect(lexiconOf("rust").keywords).toContain("type"); + expect(lexiconOf("java").keywords).not.toContain("type"); + expect(lexiconOf("ts").keywords).not.toContain("type"); + // Rust reserves both spellings of the receiver; Ruby spells its block ends. + expect(lexiconOf("rust").keywords).toEqual(expect.arrayContaining(["self", "Self"])); + expect(lexiconOf("rb").keywords).toEqual(expect.arrayContaining(["end", "begin", "BEGIN", "END"])); + expect(lexiconOf("ex").keywords).toEqual(expect.arrayContaining(["do", "end", "fn", "when"])); + }); + + it("strips the interface marker in C# alone", () => { + for (const [lang, profile] of profiles) { + expect(profile.lexicon?.interfacePrefix === true, lang).toBe(lang === "csharp"); + } + }); + + // A file kind's name is a path whose extension is not a word. Elixir's + // `module` is a dotted name (`AcmeOrder.Registry`), not a file: declaring it + // would drop `.Registry` as an extension. + it("declares exactly the kinds whose name is a file path", () => { + const fileKinds = Object.fromEntries( + profiles.flatMap(([lang, profile]) => (profile.lexicon?.fileKinds === undefined ? [] : [[lang, profile.lexicon.fileKinds]])), + ); + expect(fileKinds).toEqual({ + ex: ["file"], + js: ["module"], + php: ["codeFile"], + rb: ["file"], + ts: ["module"], + }); + }); +}); From 3b73151400d53b52ac6364bb3b8147577a9ac040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 13:35:33 +0200 Subject: [PATCH 2/4] docs(plan): M19a lexicon as built The lexicon drops the draft's separators, prefixes and suffixes, which could change no output once the splitter breaks on every non-word character, and turns the per-profile fileModules flag into per-kind fileKinds: Elixir's `module` is a dotted name, not a file. The split steps and principle 2 follow; the M19a profile item is ticked. Co-Authored-By: Claude Opus 5.5 --- PLAN.md | 77 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/PLAN.md b/PLAN.md index 2fadcad..d9dc667 100644 --- a/PLAN.md +++ b/PLAN.md @@ -4871,9 +4871,10 @@ Seven principles, locked up front: (constructor, lambda) contributes none and is counted. Coverage per extractor is a table in this section, filled by the milestone. 2. **The idiom is the profile's; the algorithm is one.** Keywords, - conventional member names, affixes and separators are DATA on the - profile (`lexicon`), specifiable without being implemented (invariant 8). - The splitter reads them and knows no language. + conventional names, technical words and file kinds are DATA on the + profile (`lexicon`), specifiable without being implemented (invariant 8); + word boundaries are universal and need none. The splitter reads the data + and knows no language. 3. **Classification, not filtering.** Every token is kept and tagged `domain | technical | keyword | short`; a consumer filters. Drift needs the technical words too, and "not shown" must stay distinguishable from @@ -4902,38 +4903,54 @@ the universal rules below — present on the eleven shipped: ```ts lexicon?: { - keywords: string[]; // reserved words, as the language spells them - conventional: string[]; // whole member names the language or its runtime dictates; + keywords: string[]; // the specification's RESERVED words, as the language spells them; + // a contextual keyword (record, var, TS's type) is an identifier + conventional: string[]; // whole identifiers the language or its runtime dictates; // skipped whole: toString/equals/hashCode/main (Java), // initialize/to_s/inspect/<=> (Ruby), constructor (TS), // __using__/start_link/init/handle_call (Elixir), // Equals/GetHashCode/ToString/Dispose/Main (C#) - technical?: string[]; // language-specific noise beyond the shared list: - // impl, bean (Java); concern (Ruby); live, web (Elixir) - idioms: { - separators: string[]; // "_", "-", ".", "::", "/", "$" - prefixes: string[]; // sigils stripped before splitting: "@@", "@", "$", "#", "_" - suffixes: string[]; // "?", "!", "=" (Ruby/Elixir), "`N" (C# generic arity) - interfacePrefix?: true; // C#: `IOrderService` → order, service (I + Upper + lower) - fileModules?: true; // the module is the file: strip the extension, split the path - }; + technical?: string[]; // language-specific plumbing beyond the shared list, as the + // splitter yields it: bean, jdbc (Java); concern (Ruby); live, web (Elixir) + fileKinds?: string[]; // kinds whose name is a file path: TS/JS `module`, Ruby and Elixir + // `file`, PHP `codeFile` — the extension is not a word + interfacePrefix?: true; // C#: `IOrderService` → order, service (I + Upper + lower) }; ``` -`ProfileSchema` mirrors it; `codegraph profiles --lang X` prints it in full -(the command's completeness rule); `schemas/` is unchanged — a profile is not -on the wire. Tests pin: every shipped lexicon has a non-empty keyword list, -its entries are distinct, and nothing in `conventional` is also a keyword. - -**The split** (`split.ts`): one function, `splitIdentifier(name, lexicon)`, -in six steps, each pinned by a table of hand cases per idiom: - -1. strip one leading prefix and one trailing suffix from the lexicon - (`@total` → `total`, `valid?` → `valid`, `save!`, `name=`, - `Repository\`1` → `Repository`), then the interface prefix when the flag - is set and the rest starts Upper + lower (`IOrderService`, not `IO`); -2. split on the lexicon's separators; for `fileModules`, drop the - extension first (`legacy/acme-report.ts` → `legacy`, `acme`, `report`); +The first draft carried `idioms.separators`, `prefixes` and `suffixes`; built +(M19a), they could change no output. The splitter must break on EVERY +character that is not a letter, a mark or a digit anyway — or `<=>`, +`[Symbol.iterator]` and `this[]` become tokens — and once it does, `@total`, +`valid?`, `Order::Line` and `` Repository`1 `` need no data. The draft's +`fileModules` flag was per PROFILE; file-ness is per KIND (Elixir's `file` is +a path, its `module` the dotted `AcmeOrder.Registry`, whose `.Registry` a +profile-wide flag would drop as an extension), so it became `fileKinds`. + +`ProfileSchema` mirrors it (a test parses every shipped profile through it +unchanged); `codegraph profiles --lang X` prints it in full, between the kinds +and the notes (the command's completeness rule); `schemas/` is unchanged — a +profile is not on the wire. `validateProfile` rejects an entry that could +never do what its list says, with four codes: `lexicon-duplicate`, +`lexicon-not-a-word` (a keyword with no letter, a name holding whitespace, a +technical entry the splitter cannot yield), `lexicon-keyword-overlap` (a +conventional name or a technical word a keyword claims first) and +`lexicon-unknown-kind`. Tests pin, per shipped profile: a non-empty keyword +list, no contextual keyword listed, `interfacePrefix` in C# alone, and exactly +the five file kinds above. TypeScript reserves exactly ECMAScript's words, so +the two profiles read one `ECMASCRIPT_LEXICON`. + +**The split** (`split.ts`): one function, `splitIdentifier(name, kind, +lexicon)`, in six steps, each pinned by a table of hand cases per idiom: + +1. for a kind in `fileKinds`, drop the extension of the last path segment + (`legacy/acme-report.ts` → `legacy/acme-report`); then the interface + prefix when the flag is set and the name starts `I` + Upper + lower + (`IOrderService`, not `IO`); +2. split on every character that is not a Unicode letter, mark or digit — + universal, no data: `@total` → `total`, `valid?` → `valid`, `save!`, + `name=`, `Order::Line`, `max_lines`, `acme-report`, `Repository\`1` → + `repository`, `1`; 3. camel-case boundaries with the acronym rule — lower→Upper, and an Upper run followed by Upper + lower splits before its last letter (`HTTPServer` → `http`, `server`; `getURLString` → `get`, `url`, @@ -5234,10 +5251,10 @@ never looks like a fact. **M19a — the lexicon, the split, the artifact** (closes the profile half of #19) -- [ ] `Profile.lexicon` in core: type, `ProfileSchema`, METAMODEL §5 row, +- [x] `Profile.lexicon` in core: type, `ProfileSchema`, METAMODEL §5 row, the eleven profiles filled (keywords from each language's specification; conventional names from its runtime), `codegraph - profiles --lang` printing it; the three lexicon tests; `schemas/` + profiles --lang` printing it; the lexicon's validation codes and tests; `schemas/` unchanged, proven by the drift check. - [ ] `packages/vocabulary/` (`@codegraph/vocabulary`, depends on core and analyzer only): `split.ts`, `classify.ts` with `DEFAULT_TECHNICAL`, From ced15d8593593b2e65bf5e7331faf66c978665e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 13:35:33 +0200 Subject: [PATCH 3/4] docs(website): render the profile lexicon profiles-page.mjs stops the kinds at `lexicon:` and renders each profile's reserved words, conventional names, technical words and file kinds as a list (Ruby's `|` operator rules out a table cell); the page is regenerated from the CLI. The metamodel page gains the lexicon row and section, and now says eleven shipped profiles, not nine. Co-Authored-By: Claude Opus 5.5 --- .../docs/reference/metamodel/profiles.md | 17 ++++- website/content/docs/reference/profiles.md | 74 ++++++++++++++++++- website/scripts/profiles-page.mjs | 20 ++++- 3 files changed, 106 insertions(+), 5 deletions(-) diff --git a/website/content/docs/reference/metamodel/profiles.md b/website/content/docs/reference/metamodel/profiles.md index a3175de..bc73be4 100644 --- a/website/content/docs/reference/metamodel/profiles.md +++ b/website/content/docs/reference/metamodel/profiles.md @@ -11,8 +11,9 @@ A profile is **data, not code**: the contract stating what a given language's ex | `kinds` | `Record` | the licit trait compositions per entity kind | | `edges` | EdgeKind[] | edge kinds this language can emit | | `notes` | string[] | documented static-analysis blind spots (reflection, `Class.forName`, dynamic `require`, macros pre-expansion, …) | +| `lexicon` | `{keywords, conventional, technical?, fileKinds?, interfacePrefix?}` | the language's words, for the vocabulary — see [The lexicon](#the-lexicon) | -The nine shipped profiles are rendered in full on [Language profiles](/docs/reference/profiles/). +The eleven shipped profiles are rendered in full on [Language profiles](/docs/reference/profiles/). ## Validation @@ -34,6 +35,20 @@ The rule above reads only the kind and the *set* of traits — never the entity Two bounds on that licence: a [stub](/docs/reference/metamodel/stubs/)'s exemption from the lower bound is an entity-level fact, so it is part of what the verdict is keyed on; and the verdict is per profile, since two profiles may name the same kind under different rules. It is a reader-side optimisation and requires no encoding support. +## The lexicon + +The vocabulary splits every name into words — `OrderManagerRepository` is `order`, `manager`, `repository` — and tags each one. Splitting is **universal**: every character that is not a letter, a mark or a digit ends a word, and so does a camel-case boundary, so sigils, separators and suffixes (`@total`, `valid?`, `Order::Line`) need no data. What a language adds is its lexicon, optional on a profile and present on every shipped one: + +| Key | Meaning | +|---|---| +| `keywords` | the specification's **reserved** words, as the language spells them. A contextual keyword (`record`, `var`, TypeScript's `type`) is a legal identifier, often a domain word, and is not listed | +| `conventional` | whole identifiers the language or its runtime dictates — overrides, protocol members, operators (`toString`, `__init__`, `<=>`). Their words say nothing about the domain, so such a name contributes none | +| `technical` | plumbing words of the language beyond the vocabulary's shared list, spelled as the splitter yields them | +| `fileKinds` | kinds whose name is a file path (TypeScript's `module`, Ruby's `file`): the extension is not a word | +| `interfacePrefix` | `true` in C# alone: `IOrderService` names an interface `OrderService` | + +Profile validation rejects an entry that could never do what its list says: one listed twice, a keyword with no letter, a technical entry the splitter cannot yield, a name a keyword already claims, a file kind the profile does not define. + ## Framework profiles The same rule one level up, consumed by the analyzer rather than the extractor. What an annotation *means* — `@Autowired` marks an injection point, `@Service` a stereotype, `@Qualifier` narrows candidates, `@Primary` wins a tie — is a declarative table mapping annotation identity (simple name plus declaring module) to a role. It lives in the analyzer: the extractor stays framework-blind, and the wiring a framework profile licenses is derived in memory, never serialized. diff --git a/website/content/docs/reference/profiles.md b/website/content/docs/reference/profiles.md index cf6c511..f453331 100644 --- a/website/content/docs/reference/profiles.md +++ b/website/content/docs/reference/profiles.md @@ -4,7 +4,7 @@ linkTitle: Language profiles weight: 4 --- -The profiles `@codegraph/core` ships: per entity kind, the traits an extractor must emit and the traits it may emit, plus the edge kinds the language can produce and the profile's documented blind spots. +The profiles `@codegraph/core` ships: per entity kind, the traits an extractor must emit and the traits it may emit, plus the edge kinds the language can produce, the words the vocabulary reads of it, and the profile's documented blind spots. {{< callout type="info" >}} Generated content. Every table on this page is a rendering of `codegraph profiles --json`; run it for the machine-readable form. @@ -52,6 +52,12 @@ A profile that is specified but has no shipped extractor is complete, not defici | `record` | `TNamed`, `TType`, `TChildOf` | `TWithImplements`, `TWithChildren`, `TSourceAnchor`, `TComment` | | `var` | `TNamed`, `TStructural` | `TSourceAnchor`, `TComment`, `TChildOf`, `TWithChildren`, `TAttachedTo`, `TTypedEntity`, `TInvocable`, `TWithParameters`, `TWithLocalVariables`, `TWithInvocations`, `TWithAccesses` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `def` · `if` · `do` · `let` · `quote` · `var` · `fn` · `loop` · `recur` · `throw` · `try` · `catch` · `finally` · `monitor-enter` · `monitor-exit` · `new` · `set!` · `nil` · `true` · `false` +- Conventional names: `-main` · `-init` · `-post-init` · `-toString` · `-equals` · `-hashCode` +- Technical words: `coll` · `ctx` · `kw` · `ns` · `opts` · `seq` · `sym` · `xf` · `xform` + **Notes** - Planned extractor: clj-kondo analysis output (`--config '{:output {:analysis true}}'`), which resolves var definitions, usages and namespace requires without loading the code. @@ -90,6 +96,13 @@ A profile that is specified but has no shipped extractor is complete, not defici | `record` | `TNamed`, `TType`, `TWithInheritances`, `TWithImplements`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TComment`, `TMetrics` | | `struct` | `TNamed`, `TType`, `TWithImplements`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TComment`, `TMetrics` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `abstract` · `as` · `base` · `bool` · `break` · `byte` · `case` · `catch` · `char` · `checked` · `class` · `const` · `continue` · `decimal` · `default` · `delegate` · `do` · `double` · `else` · `enum` · `event` · `explicit` · `extern` · `false` · `finally` · `fixed` · `float` · `for` · `foreach` · `goto` · `if` · `implicit` · `in` · `int` · `interface` · `internal` · `is` · `lock` · `long` · `namespace` · `new` · `null` · `object` · `operator` · `out` · `override` · `params` · `private` · `protected` · `public` · `readonly` · `ref` · `return` · `sbyte` · `sealed` · `short` · `sizeof` · `stackalloc` · `static` · `string` · `struct` · `switch` · `this` · `throw` · `true` · `try` · `typeof` · `uint` · `ulong` · `unchecked` · `unsafe` · `ushort` · `using` · `virtual` · `void` · `volatile` · `while` +- Conventional names: `ToString` · `Equals` · `GetHashCode` · `Finalize` · `Main` · `CompareTo` · `Dispose` · `DisposeAsync` · `GetEnumerator` · `GetAsyncEnumerator` · `MoveNext` · `MoveNextAsync` · `Clone` · `Deconstruct` · `this[]` · `op_Addition` · `op_Subtraction` · `op_Multiply` · `op_Division` · `op_Modulus` · `op_Equality` · `op_Inequality` · `op_LessThan` · `op_GreaterThan` · `op_LessThanOrEqual` · `op_GreaterThanOrEqual` · `op_Implicit` · `op_Explicit` · `op_UnaryNegation` · `op_UnaryPlus` · `op_LogicalNot` · `op_OnesComplement` · `op_Increment` · `op_Decrement` · `op_True` · `op_False` · `op_BitwiseAnd` · `op_BitwiseOr` · `op_ExclusiveOr` · `op_LeftShift` · `op_RightShift` · `op_UnsignedRightShift` +- Technical words: `args` · `async` · `attribute` · `enumerable` · `enumerator` · `extension` · `extensions` · `linq` +- Interface prefix: a leading `I` before Upper + lower is a marker, not a word + **Notes** - Partial classes (and partial methods) mean one entity id is declared across several files: the entity carries a single anchor for its primary declaration, while each edge carries sourceFile to say which declaration site produced it. Merging by id is correct here, not a collision. @@ -130,6 +143,13 @@ A profile that is specified but has no shipped extractor is complete, not defici | `parameter` | `TStructural`, `TChildOf` | `TNamed`, `TSourceAnchor`, `TWithValue` | | `protocol` | `TNamed`, `TType`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TComment`, `TMetrics` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `true` · `false` · `nil` · `when` · `and` · `or` · `not` · `in` · `fn` · `do` · `end` · `catch` · `rescue` · `after` · `else` +- Conventional names: `__using__` · `__before_compile__` · `__after_compile__` · `__after_verify__` · `__on_definition__` · `__struct__` · `__info__` · `__deriving__` · `__impl__` · `__protocol__` · `start_link` · `child_spec` · `init` · `handle_call` · `handle_cast` · `handle_info` · `handle_continue` · `terminate` · `code_change` · `format_status` · `to_string` · `inspect` · `@moduledoc` · `@doc` · `@typedoc` · `@spec` · `@type` · `@typep` · `@opaque` · `@callback` · `@macrocallback` · `@optional_callbacks` · `@impl` · `@behaviour` · `@derive` · `@enforce_keys` · `@external_resource` · `@compile` · `@deprecated` · `@dialyzer` · `@on_load` · `@vsn` · `@before_compile` · `@after_compile` · `@after_verify` · `@on_definition` · `@nifs` · `@fallback_to_any` · `@file` +- Technical words: `attrs` · `behaviour` · `callback` · `changeset` · `conn` · `ecto` · `ets` · `gen` · `live` · `mix` · `opts` · `otp` · `phoenix` · `pid` · `plug` · `repo` · `schema` · `server` · `socket` · `sup` · `supervisor` · `web` +- File kinds: `file` + **Notes** - The module is the file (the TypeScript rule): `TModule.definedIn` has exactly one entry, and every entity's module is the file it is written in. A `defmodule` is a `module` entity, child of its file, carrying TType — the module IS the type in Elixir (a struct is `%Mod{}`, a behaviour and a protocol are modules, `@spec f :: Mod.t()`). Nesting (`defmodule Inner` inside `defmodule Outer`) defines `Outer.Inner` as a second child of the file and aliases `Inner` inside `Outer`: it is an alias, not a containment. @@ -171,6 +191,12 @@ A profile that is specified but has no shipped extractor is complete, not defici | `struct` | `TNamed`, `TType`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TWithImplements`, `TComment` | | `typeAlias` | `TNamed`, `TType`, `TTypedEntity`, `TChildOf`, `TSourceAnchor` | `TComment` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `break` · `case` · `chan` · `const` · `continue` · `default` · `defer` · `else` · `fallthrough` · `for` · `func` · `go` · `goto` · `if` · `import` · `interface` · `map` · `package` · `range` · `return` · `select` · `struct` · `switch` · `type` · `var` +- Conventional names: `init` · `main` · `String` · `GoString` · `Error` · `Unwrap` · `Is` · `As` · `Len` · `Less` · `Swap` · `MarshalJSON` · `UnmarshalJSON` · `MarshalText` · `UnmarshalText` · `ServeHTTP` +- Technical words: `cfg` · `cmd` · `ctx` · `err` · `errs` · `internal` · `mu` · `mutex` · `pkg` · `srv` · `svc` · `wg` + **Notes** - Go has no inheritance: neither the TWithInheritances trait nor the inheritance edge kind belongs to this profile. Their absence is the profile's statement about the language, not a missing feature. @@ -202,6 +228,12 @@ A profile that is specified but has no shipped extractor is complete, not defici | `parameter` | `TNamed`, `TStructural`, `TTypedEntity`, `TChildOf` | `TSourceAnchor` | | `record` | `TNamed`, `TType`, `TWithImplements`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TComment`, `TMetrics` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `abstract` · `assert` · `boolean` · `break` · `byte` · `case` · `catch` · `char` · `class` · `const` · `continue` · `default` · `do` · `double` · `else` · `enum` · `extends` · `final` · `finally` · `float` · `for` · `goto` · `if` · `implements` · `import` · `instanceof` · `int` · `interface` · `long` · `native` · `new` · `package` · `private` · `protected` · `public` · `return` · `short` · `static` · `strictfp` · `super` · `switch` · `synchronized` · `this` · `throw` · `throws` · `transient` · `try` · `void` · `volatile` · `while` · `true` · `false` · `null` +- Conventional names: `toString` · `equals` · `hashCode` · `clone` · `finalize` · `main` · `compareTo` · `iterator` · `hasNext` · `valueOf` · `serialVersionUID` · `readObject` · `writeObject` · `readObjectNoData` · `readResolve` · `writeReplace` +- Technical words: `annotation` · `bean` · `beans` · `ejb` · `jaxb` · `jdbc` · `jndi` · `jpa` · `pojo` · `servlet` + **Notes** - Measures (TMetrics, METAMODEL.md §3.8): `sloc` on every type and invocable — lines of its own span that are neither blank nor comment-only, counted by a scanner that knows string and text-block literals, so a `"/*"` in the source does not swallow the rest of the file. `cyclomatic` on invocables only: 1 + if / for / foreach / while / do / non-default case label / catch / ternary / short-circuit && and || / switch pattern guard. Purely syntactic, hence immune to the noClasspath resolution ceiling. A nested lambda or anonymous class does NOT contribute to its enclosing method: it is its own invocable and carries its own count. A type's complexity is therefore not stored — it is the sum over its members, which the consumer computes. @@ -237,6 +269,13 @@ A profile that is specified but has no shipped extractor is complete, not defici | `property` | `TNamed`, `TStructural`, `TChildOf` | `TTypedEntity`, `TSourceAnchor`, `TComment` | | `variable` | `TNamed`, `TStructural` | `TSourceAnchor`, `TComment`, `TChildOf`, `TWithChildren`, `TTypedEntity` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `await` · `break` · `case` · `catch` · `class` · `const` · `continue` · `debugger` · `default` · `delete` · `do` · `else` · `enum` · `export` · `extends` · `false` · `finally` · `for` · `function` · `if` · `import` · `in` · `instanceof` · `new` · `null` · `return` · `super` · `switch` · `this` · `throw` · `true` · `try` · `typeof` · `var` · `void` · `while` · `with` · `yield` · `implements` · `interface` · `let` · `package` · `private` · `protected` · `public` · `static` +- Conventional names: `constructor` · `toString` · `toLocaleString` · `valueOf` · `toJSON` · `[Symbol.iterator]` · `[Symbol.asyncIterator]` · `[Symbol.hasInstance]` · `[Symbol.toPrimitive]` · `[Symbol.toStringTag]` · `[Symbol.dispose]` · `[Symbol.asyncDispose]` +- Technical words: `args` · `async` · `callback` · `cb` · `fn` · `json` · `opts` · `promise` · `prototype` +- File kinds: `module` + **Notes** - A module is a file: `TModule.definedIn` always has exactly one entry (1-1 cardinality, unlike a Java package). @@ -272,6 +311,13 @@ A profile that is specified but has no shipped extractor is complete, not defici | `property` | `TNamed`, `TStructural`, `TTypedEntity`, `TChildOf`, `TSourceAnchor` | `TComment` | | `trait` | `TNamed`, `TType`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TComment` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `__halt_compiler` · `abstract` · `and` · `array` · `as` · `break` · `callable` · `case` · `catch` · `class` · `clone` · `const` · `continue` · `declare` · `default` · `die` · `do` · `echo` · `else` · `elseif` · `empty` · `enddeclare` · `endfor` · `endforeach` · `endif` · `endswitch` · `endwhile` · `eval` · `exit` · `extends` · `final` · `finally` · `fn` · `for` · `foreach` · `function` · `global` · `goto` · `if` · `implements` · `include` · `include_once` · `instanceof` · `insteadof` · `interface` · `isset` · `list` · `match` · `namespace` · `new` · `or` · `print` · `private` · `protected` · `public` · `readonly` · `require` · `require_once` · `return` · `static` · `switch` · `throw` · `trait` · `try` · `unset` · `use` · `var` · `while` · `xor` · `yield` · `__CLASS__` · `__DIR__` · `__FILE__` · `__FUNCTION__` · `__LINE__` · `__METHOD__` · `__NAMESPACE__` · `__PROPERTY__` · `__TRAIT__` · `true` · `false` · `null` +- Conventional names: `__construct` · `__destruct` · `__call` · `__callStatic` · `__get` · `__set` · `__isset` · `__unset` · `__sleep` · `__wakeup` · `__serialize` · `__unserialize` · `__toString` · `__invoke` · `__set_state` · `__clone` · `__debugInfo` +- Technical words: `arr` · `closure` · `facade` · `middleware` · `php` · `str` +- File kinds: `codeFile` + **Notes** - fileInclude (include/require/_once) is the metamodel's only file-to-file dependency; its endpoints are `codeFile` entities. A file both declares entities and includes other files, so the same file appears as a codeFile node and as the `anchor.file` of the declarations it contains. @@ -301,6 +347,12 @@ A profile that is specified but has no shipped extractor is complete, not defici | `parameter` | `TNamed`, `TStructural`, `TTypedEntity`, `TChildOf` | `TSourceAnchor` | | `variable` | `TNamed`, `TStructural`, `TTypedEntity`, `TChildOf` | `TSourceAnchor`, `TComment` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `False` · `None` · `True` · `and` · `as` · `assert` · `async` · `await` · `break` · `class` · `continue` · `def` · `del` · `elif` · `else` · `except` · `finally` · `for` · `from` · `global` · `if` · `import` · `in` · `is` · `lambda` · `nonlocal` · `not` · `or` · `pass` · `raise` · `return` · `try` · `while` · `with` · `yield` +- Conventional names: `self` · `cls` · `__init__` · `__new__` · `__del__` · `__repr__` · `__str__` · `__bytes__` · `__format__` · `__hash__` · `__bool__` · `__eq__` · `__ne__` · `__lt__` · `__le__` · `__gt__` · `__ge__` · `__len__` · `__iter__` · `__next__` · `__aiter__` · `__anext__` · `__reversed__` · `__contains__` · `__getitem__` · `__setitem__` · `__delitem__` · `__missing__` · `__getattr__` · `__getattribute__` · `__setattr__` · `__delattr__` · `__dir__` · `__get__` · `__set__` · `__delete__` · `__set_name__` · `__call__` · `__enter__` · `__exit__` · `__aenter__` · `__aexit__` · `__await__` · `__init_subclass__` · `__class_getitem__` · `__post_init__` · `__add__` · `__sub__` · `__mul__` · `__matmul__` · `__truediv__` · `__floordiv__` · `__mod__` · `__pow__` · `__neg__` · `__pos__` · `__abs__` · `__invert__` · `__and__` · `__or__` · `__xor__` · `__lshift__` · `__rshift__` · `__int__` · `__float__` · `__index__` · `__round__` · `__copy__` · `__deepcopy__` · `__reduce__` · `__getstate__` · `__setstate__` · `__slots__` · `__all__` · `__name__` · `__main__` · `__doc__` · `__dict__` +- Technical words: `args` · `dict` · `kwargs` · `metaclass` · `mixin` · `py` · `str` · `tuple` + **Notes** - A module is a file: TModule.definedIn holds exactly one path (1-1 cardinality). A package is the module of its __init__.py. @@ -328,6 +380,13 @@ A profile that is specified but has no shipped extractor is complete, not defici | `module` | `TNamed`, `TType`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TAttachedTo`, `TComment`, `TMetrics`, `TWithInvocations`, `TWithAccesses` | | `parameter` | `TStructural`, `TChildOf` | `TNamed`, `TSourceAnchor`, `TWithValue` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `__ENCODING__` · `__LINE__` · `__FILE__` · `BEGIN` · `END` · `alias` · `and` · `begin` · `break` · `case` · `class` · `def` · `defined?` · `do` · `else` · `elsif` · `end` · `ensure` · `false` · `for` · `if` · `in` · `module` · `next` · `nil` · `not` · `or` · `redo` · `rescue` · `retry` · `return` · `self` · `super` · `then` · `true` · `undef` · `unless` · `until` · `when` · `while` · `yield` +- Conventional names: `initialize` · `initialize_copy` · `to_s` · `to_str` · `to_a` · `to_ary` · `to_h` · `to_hash` · `to_proc` · `inspect` · `hash` · `eql?` · `==` · `===` · `=~` · `!~` · `!=` · `<=>` · `<` · `>` · `<=` · `>=` · `+` · `-` · `*` · `/` · `%` · `**` · `<<` · `>>` · `&` · `|` · `^` · `~` · `!` · `+@` · `-@` · `[]` · `[]=` · `each` · `coerce` · `method_missing` · `respond_to_missing?` · `respond_to?` · `included` · `extended` · `inherited` · `prepended` · `method_added` · `const_missing` +- Technical words: `accessor` · `attr` · `concern` · `concerns` · `ext` · `gem` · `gemspec` · `klass` · `lambda` · `mixin` · `proc` · `rake` · `rakefile` +- File kinds: `file` + **Notes** - The module is the file (the TypeScript and Elixir rule): `TModule.definedIn` has exactly one entry, and every entity's module is the file it is written in. `class` and `module` are types below it, named and keyed by their fully qualified constant path (`Acme::Order`) — `::` is part of the name, never a separator of the key. A type written inside the body of a type that is an entity in the same file is its child; otherwise it is a child of the file. A value constant is keyed the same way (`Acme::Order::MAX_LINES`), a child of the type whose body assigns it. @@ -378,6 +437,12 @@ A profile that is specified but has no shipped extractor is complete, not defici | `trait` | `TNamed`, `TType`, `TWithChildren`, `TChildOf`, `TSourceAnchor` | `TComment` | | `typeAlias` | `TNamed`, `TType`, `TTypedEntity`, `TChildOf`, `TSourceAnchor` | `TComment` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `as` · `async` · `await` · `break` · `const` · `continue` · `crate` · `dyn` · `else` · `enum` · `extern` · `false` · `fn` · `for` · `if` · `impl` · `in` · `let` · `loop` · `match` · `mod` · `move` · `mut` · `pub` · `ref` · `return` · `self` · `Self` · `static` · `struct` · `super` · `trait` · `true` · `type` · `unsafe` · `use` · `where` · `while` · `abstract` · `become` · `box` · `do` · `final` · `gen` · `macro` · `override` · `priv` · `try` · `typeof` · `unsized` · `virtual` · `yield` +- Conventional names: `main` · `fmt` · `drop` · `clone` · `clone_from` · `eq` · `ne` · `partial_cmp` · `cmp` · `deref` · `deref_mut` · `into_iter` · `from_str` · `try_from` · `try_into` · `as_ref` · `as_mut` · `borrow_mut` · `index_mut` · `default` · `poll` +- Technical words: `arc` · `buf` · `cell` · `ctx` · `err` · `ffi` · `iter` · `mutex` · `ok` · `opt` · `rc` · `refcell` · `rs` · `rx` · `sys` · `tx` · `vec` + **Notes** - Rust has no inheritance: TWithInheritances and the `inheritance` edge kind are absent from this profile by design, not by omission. Supertrait bounds (`trait A: B`), generic bounds and where-clauses are `reference` edges. @@ -416,6 +481,13 @@ Declaration spaces: `abstractClass`, `arrowFunction`, `class`, `enum`, `function | `typeAlias` | `TNamed`, `TType`, `TChildOf`, `TSourceAnchor` | `TTypedEntity`, `TComment`, `TMetrics` | | `variable` | `TNamed`, `TStructural`, `TTypedEntity`, `TChildOf`, `TSourceAnchor` | `TComment`, `TWithChildren`, `TWithValue`, `TWithInvocations`, `TWithAccesses` | +**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language + +- Reserved words: `await` · `break` · `case` · `catch` · `class` · `const` · `continue` · `debugger` · `default` · `delete` · `do` · `else` · `enum` · `export` · `extends` · `false` · `finally` · `for` · `function` · `if` · `import` · `in` · `instanceof` · `new` · `null` · `return` · `super` · `switch` · `this` · `throw` · `true` · `try` · `typeof` · `var` · `void` · `while` · `with` · `yield` · `implements` · `interface` · `let` · `package` · `private` · `protected` · `public` · `static` +- Conventional names: `constructor` · `toString` · `toLocaleString` · `valueOf` · `toJSON` · `[Symbol.iterator]` · `[Symbol.asyncIterator]` · `[Symbol.hasInstance]` · `[Symbol.toPrimitive]` · `[Symbol.toStringTag]` · `[Symbol.dispose]` · `[Symbol.asyncDispose]` +- Technical words: `args` · `async` · `callback` · `cb` · `fn` · `json` · `opts` · `promise` · `prototype` +- File kinds: `module` + **Notes** - This is the only profile that populates `Entity.space` (METAMODEL.md §1.4). Type-space only (`space: ["type"]`): `interface`, `typeAlias`, a `const enum` (its members are inlined at emit, leaving no runtime entity), and a `namespace` that declares only types. Both spaces (`["type", "value"]`): `class`, `abstractClass`, a plain `enum`, and a `namespace` that declares at least one value. Value-space only (`["value"]`): `module`, `function`, `arrowFunction`, `constructor`, `variable`, `parameter`. A `method` or `property` is `["type"]` when written in an `interface` body — it exists only for the checker, and an access to it is a dependency on the interface's SHAPE — and `["value"]` when written in a class or an object literal. diff --git a/website/scripts/profiles-page.mjs b/website/scripts/profiles-page.mjs index 0d2a85f..a15fa3c 100644 --- a/website/scripts/profiles-page.mjs +++ b/website/scripts/profiles-page.mjs @@ -22,19 +22,25 @@ function parse(text) { const summary = /summary: (\d+) kinds, (\d+) edge kinds, (\d+) notes/.exec(lines.find((l) => l.startsWith("summary:"))); const edges = lines.find((l) => l.startsWith("edge kinds:")).slice("edge kinds:".length).split(",").map((s) => s.trim()); const kinds = []; - for (let i = lines.indexOf("kinds:") + 1; i < lines.length && !lines[i].startsWith("notes:"); i++) { + for (let i = lines.indexOf("kinds:") + 1; i < lines.length && lines[i] !== "lexicon:" && !lines[i].startsWith("notes:"); i++) { const kind = /^ (\S+)$/.exec(lines[i]); if (kind) { kinds.push({ name: kind[1], required: [], optional: [] }); continue; } const traits = /^ (required|optional)\s+(.*)$/.exec(lines[i]); if (traits) kinds.at(-1)[traits[1]] = traits[2].split(",").map((s) => s.trim()).filter(Boolean); } + // ` keywords a, b` … up to the blank line; absent rows stay absent. + const lexicon = {}; + for (let i = lines.indexOf("lexicon:") + 1; i > 0 && i < lines.length && lines[i] !== ""; i++) { + const row = /^ (keywords|conventional|technical|file kinds|interface prefix)\s+(.*)$/.exec(lines[i]); + if (row) lexicon[row[1]] = row[2] === "(none)" ? [] : row[2].split(", "); + } const notes = []; for (let n = lines.indexOf("notes:") + 1; n < lines.length; n++) { const l = lines[n]; if (l.startsWith(" - ")) notes.push(l.slice(4)); else if (l.trim() !== "" && notes.length) notes[notes.length - 1] += "\n" + l; } - return { counts: summary.slice(1).map(Number), edges, kinds, notes: notes.map((note) => FENCED.reduce((s, f) => s.replaceAll("`" + f + "`", "``" + f + "``"), note)) }; + return { counts: summary.slice(1).map(Number), edges, kinds, lexicon, notes: notes.map((note) => FENCED.reduce((s, f) => s.replaceAll("`" + f + "`", "``" + f + "``"), note)) }; } const profiles = Object.fromEntries(langs.map((lang) => [lang, parse(codegraph("profiles", "--lang", lang))])); @@ -46,7 +52,7 @@ linkTitle: Language profiles weight: 4 --- -The profiles \`@codegraph/core\` ships: per entity kind, the traits an extractor must emit and the traits it may emit, plus the edge kinds the language can produce and the profile's documented blind spots. +The profiles \`@codegraph/core\` ships: per entity kind, the traits an extractor must emit and the traits it may emit, plus the edge kinds the language can produce, the words the vocabulary reads of it, and the profile's documented blind spots. {{< callout type="info" >}} Generated content. Every table on this page is a rendering of \`codegraph profiles --json\`; run it for the machine-readable form. @@ -78,6 +84,14 @@ for (const lang of langs) { out.push("**Edge kinds:** " + p.edges.map((e) => "`" + e + "`").join(" · "), ""); out.push("| Kind | Required traits | Optional traits |", "|---|---|---|"); for (const k of p.kinds) out.push(`| \`${k.name}\` | ${tick(k.required)} | ${tick(k.optional)} |`); + const lx = p.lexicon; + const words = (xs) => (xs.length === 0 ? "(none)" : xs.map((x) => "`" + x + "`").join(" · ")); + out.push("", "**Lexicon** — what the [vocabulary](/docs/reference/metamodel/profiles/#the-lexicon) reads of this language", ""); + out.push("- Reserved words: " + words(lx["keywords"])); + out.push("- Conventional names: " + words(lx["conventional"])); + if (lx["technical"]) out.push("- Technical words: " + words(lx["technical"])); + if (lx["file kinds"]) out.push("- File kinds: " + words(lx["file kinds"])); + if (lx["interface prefix"]) out.push("- Interface prefix: a leading `I` before Upper + lower is a marker, not a word"); out.push("", "**Notes**", ""); for (const note of p.notes) out.push("- " + note); } From 51ed180366907e5f69fc43e5b1c141508229175c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 17:13:08 +0200 Subject: [PATCH 4/4] fix(website): count regex spaces in profiles page The page generator matched the CLI's indentation with runs of literal spaces, which eslint's no-regex-spaces rejects; the lexicon row added one more. Written as {2} and {4}, the script passes lint and writes the same page byte for byte. Co-Authored-By: Claude Opus 5.5 --- website/scripts/profiles-page.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/scripts/profiles-page.mjs b/website/scripts/profiles-page.mjs index a15fa3c..5b4047c 100644 --- a/website/scripts/profiles-page.mjs +++ b/website/scripts/profiles-page.mjs @@ -23,15 +23,15 @@ function parse(text) { const edges = lines.find((l) => l.startsWith("edge kinds:")).slice("edge kinds:".length).split(",").map((s) => s.trim()); const kinds = []; for (let i = lines.indexOf("kinds:") + 1; i < lines.length && lines[i] !== "lexicon:" && !lines[i].startsWith("notes:"); i++) { - const kind = /^ (\S+)$/.exec(lines[i]); + const kind = /^ {2}(\S+)$/.exec(lines[i]); if (kind) { kinds.push({ name: kind[1], required: [], optional: [] }); continue; } - const traits = /^ (required|optional)\s+(.*)$/.exec(lines[i]); + const traits = /^ {4}(required|optional)\s+(.*)$/.exec(lines[i]); if (traits) kinds.at(-1)[traits[1]] = traits[2].split(",").map((s) => s.trim()).filter(Boolean); } // ` keywords a, b` … up to the blank line; absent rows stay absent. const lexicon = {}; for (let i = lines.indexOf("lexicon:") + 1; i > 0 && i < lines.length && lines[i] !== ""; i++) { - const row = /^ (keywords|conventional|technical|file kinds|interface prefix)\s+(.*)$/.exec(lines[i]); + const row = /^ {2}(keywords|conventional|technical|file kinds|interface prefix)\s+(.*)$/.exec(lines[i]); if (row) lexicon[row[1]] = row[2] === "(none)" ? [] : row[2].split(", "); } const notes = [];