diff --git a/.gitattributes b/.gitattributes index 882f09c61e..6e1f657e1f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,13 @@ # Union misfiles fork entries once upstream has cut a release, so after every # upstream merge run `node scripts/changelog-reconcile.mjs` to rebuild the file. CHANGELOG.md merge=union + +# Third-party tree-sitter grammars that the native kernel compiles from source +# (codegraph-kernel/build.rs). Keep all of them out of GitHub's language +# statistics, and collapse only the generated parser.c files in pull-request +# diffs. The hand-written scanner.c files are where C gets patched or added, +# so they stay expanded for review. +# The /** is needed: an attributes pattern that names a directory does not +# reach the files inside it. +codegraph-kernel/grammars/** linguist-vendored +codegraph-kernel/grammars/**/parser.c linguist-generated diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e52b2c240..deb774534a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -457,7 +457,9 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - When several CodeGraph processes start at the same moment after one of them crashed, like the MCP server, a git hook's `codegraph sync` and `codegraph index`, only one of them now takes over the write lock the crashed process left behind. Before, two of them could both take it, the second one deleting the first one's lock as it did, and both went on to write to the index at once. A write lock held by another user's CodeGraph, or on Windows by one running as administrator, is no longer mistaken for a crashed one's and taken over either. Thanks @inth3shadows for the report and @vedantb7 for the first fix. (#2979) - A program that uses CodeGraph as a library and runs `indexAll()` again on a project it has already indexed now keeps every symbol findable by the words in its name, which the prompt hook and `codegraph_explore` use to match a question like "how does the order state machine work" to `OrderStateMachine`. Before, only the files changed since the last index kept their names' words, and a cancelled run lost them all; `codegraph index` was not affected. Thanks @rudycelekli. - The shared background server is no longer killed with "Main thread unresponsive" while it brings another project's index up to date, such as one a session names with `projectPath`: only its own project's index counted as progress, so a long update of any other looked like a hang, and the server was stopped and restarted again and again. Every index the server keeps up to date now counts, and so does the one a session updates on its own when the background server is unavailable. Thanks @mradwankhalil for the report and the first fix. (#2404) +- A long-running session, like the background server behind your agent, now notices a framework that arrives or goes while it runs, such as Express in a new `package.json`, or one app replaced by another with as many files. Files it indexes from then on are read with the frameworks the project has now; before, they were read with the ones it had when the session started, so a new Express app's routes were missing until a full `codegraph index`. Files indexed before the change are not read again, so `codegraph status` now says which frameworks changed since the last full index and recommends running one, including when only `package.json` changed. Thanks @nk33-dev for the report and the first fix, and @inth3shadows for the `package.json` case. (#2516, #2519, #3007) - A project a session queries through `projectPath` is now let go once it has gone its own 10 minutes without a query. Before, when its index was still being brought up to date as the session went on to query another project, it was held, locked, until that other project's 10 minutes were up, so its own session and `codegraph index` could wait nearly twice as long to take over. Thanks @bompus. (#2087) +- An MCP tool call whose `projectPath` names a file inside a project, such as the file an agent is looking at, now answers from that project. Before, it failed with "Path is not a directory", even though a path that doesn't exist at all worked. Sensitive folders such as `/etc` are still refused when the path names a file inside them. Thanks @inth3shadows for the report and @maxmilian for the fix. (#2982) - An index left incomplete by an indexing run that was stopped partway, by a crash, an out-of-memory kill or the server's own hang check, no longer gets the background server killed with "Main thread unresponsive" every time it starts: on a large project, restoring the database indexes the run had set aside took longer than the hang check allows, so each restart died at the same point and the index was never repaired. They are now restored in the background, and the project is served once they are back. Thanks @cbeaulieu-gt for the report and the first fix. (#1887) - A new agent session no longer waits up to about 12 minutes for its first answer when the background server is stuck: a server whose process still holds its connection open but never responds made the session wait out a long series of retries. The session now gives up within seconds and answers on its own, and switches back to the background server once that has been restarted. - An MCP session whose agent stops reading its answers, while keeping the session open, now shuts down the normal way the next time it answers: it stops its work, frees the project for the next session and exits cleanly, and other sessions sharing the background server carry on. Before, that answer crashed it with "Uncaught exception: Error: EPIPE". Thanks @iiijiashu for the report and the first fix. (#2991) @@ -465,6 +467,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - The bundled Node runtime moves to 24.22.1, Node's October security release, and every release now checks the Node it bundles against the checksums nodejs.org publishes. The new runtime also fixes a crash on Windows as a command like `codegraph sync` exits, after its work is done, printing `Assertion failed: !(handle->flags & UV_HANDLE_CLOSING)`, which made a script or CI run that had succeeded fail. Thanks @Ancienttwo for the report and the first fix, and @IdealBoy709 for the report on the security release. (#2617, #2780) - Building CodeGraph from source with bun, as in `bun run build`, no longer runs until it is stopped: bun ran one step of the build as the whole build again, so it kept starting itself over. Building with npm is unchanged. Thanks @bompus. - When CodeGraph can't open a project's database, it now says what went wrong: SQLite's error, the file's path, and what to check, such as another program holding the file open, a folder you can't write to, or a network share. Before, every such failure read "CodeGraph requires node:sqlite (Node.js 22.5+)", and an agent was told no project was loaded. That hint now appears only when Node really can't load its built-in SQLite, a case that used to end in an unhandled error instead. +- When a git worktree is answered from another checkout's index, `codegraph status` now suggests plain `codegraph init` in the worktree, without the `-i` flag that no longer does anything, and the warning your agent sees, with its results and in `codegraph_status`, now says you can run it there instead of telling the agent to build the index itself. Thanks @GaoSSR. - `codegraph upgrade` no longer wipes your agent instructions file, such as a global `AGENTS.md`, `CLAUDE.md` or `GEMINI.md`, when it refreshes CodeGraph's section in it: a stray `` line above that section, like one left behind when part of an older section was deleted by hand, made the refresh replace everything in between, often the whole file. CodeGraph now rewrites only its own section, in `codegraph install` and `codegraph uninstall` too, and keeps everything around it exactly as it was, Windows line endings included. Thanks @pzaj2 for the report. (#2506) - `codegraph install`, `upgrade` and `uninstall` now write through an instructions or config file that is a symbolic link, such as a `~/.codex/AGENTS.md` or `~/.claude.json` linked from your dotfiles, instead of replacing the link with a copy: the shared file gets CodeGraph's settings, uninstall takes them out again, and the link stays in place. A link CodeGraph can't write through, like one to a read-only file or into a folder that doesn't exist, is now reported by name and left alone, and an upgrade still refreshes your other agents. Thanks @jguerra6 for the report and @danusha2345 for the first fix. (#2721) - `codegraph install` and `upgrade` no longer break Codex's `config.toml` when CodeGraph's `[mcp_servers.codegraph]` entry is written differently from how CodeGraph writes it, such as indented, with spaces or quotes inside the brackets, or with Windows line endings: the entry is now updated in place, where before a second copy was added and Codex could load none of the file. A file already broken that way is repaired on the next install or upgrade, an example of the entry inside a multiline setting such as `developer_instructions` is no longer mistaken for it, uninstall removes the entry however it is written, and an entry you wrote with dotted keys or as an inline table is left as you wrote it. When your `mcp_servers` settings are one inline table, CodeGraph now says it can't add itself instead of breaking the file. Thanks @dencoseca for the report, and @danusha2345 and @rudycelekli for the first fixes. (#2562) @@ -475,6 +478,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Running `codegraph query`, `explore`, `node`, `callers`, `callees`, `impact`, `context`, `files` or `affected` while `codegraph index` rebuilds a project no longer makes the rebuild fail with "database is locked", and neither does the context hook Claude Code runs on its own when you ask about your code: like `codegraph status`, they now only read the index and say when a rebuild is running, and the hook stays quiet until the rebuild is done. They still answer from an index an older CodeGraph version built. Thanks @sealad886 for the report. (#2913) - Opening a project whose index needs a schema update no longer fails when several CodeGraph processes open it at the same moment, like agent sessions starting right after an upgrade, or the MCP server and a git hook: one of them updates the index while the others wait for it and then carry on. Before, the ones that lost the race applied the same update again, and failed with "UNIQUE constraint failed" or "database is locked". Thanks @inth3shadows for the report and @johnhldiver-beep for the first fix. (#2980) - A qualified name given to `codegraph callers`, `callees` or `impact`, or to `codegraph_node`, like `Format.Price`, now names only the symbol whose scope is spelled that way, so its answer no longer merges in `LegacyFormat.Price` or any other class whose name only ends with `Format`, nor a class that just sits in a file named `Format.cs`. In C#, `BasketService.AddItemToBasket` no longer also selects the `IBasketService` interface's method; ask for `IBasketService.AddItemToBasket` for that one. A file or folder still names the module in Rust, Python and Go, and a PHP namespace can be written with its own backslashes, like `Billing\Format::price`. No re-index needed. Thanks @Gobakabana for the report and @yang0228. (#2908) +- `codegraph callers` and the `codegraph_callers` tool now list each caller at the lines where it calls the symbol, every one of them, like `src/report.ts:9, 10` for a function that calls it twice, and a function that calls itself among its own callers, marked as calling itself. Before, a caller showed only the line its own declaration starts on, a second call in the same caller didn't show at all, and a recursive call was left out, though each is a line to edit when the symbol changes. `codegraph_node` lists its callers the same way, `codegraph callees` and `codegraph_callees` include a function that calls itself, and `--json` gives every caller its call-site lines. A caller with many calls shows the first few lines and how many more there are. Thanks @Gobakabana for the report and @johnhldiver-beep for the first fix. (#2909) - `codegraph affected --filter` now matches its glob against whole names: `tests/**/*.py` also finds the tests directly inside `tests/`, including a changed test file itself, `tests/*.py` no longer counts a changed `tests/x.py.bak` as a test, and `e2e/*.spec.ts` no longer matches a folder that only ends in `e2e`, like `xe2e/`. A pattern with a folder in it still matches that folder at any depth, so `e2e/*.spec.ts` keeps finding `apps/web/e2e/login.spec.ts`; start it with `/` to match from the project root only. Thanks @Jseca666. - `codegraph init` and `codegraph index` no longer report a file you deleted as one that "could not be read" while git still lists it, which it does until the deletion is staged: such a file is now skipped, and so is a symlink that points at nothing. A file that is there but can't be read is still reported. Thanks @xianzhi-xzw for the report and the first fix. (#2904) - A full index of a project that holds a video named `.ts`, like an MPEG-TS clip among test fixtures, no longer warns that the index is PARTIAL and missing a file, and `codegraph status` no longer reports the index as partial: the video is skipped on purpose, and it now counts as skipped. Thanks @Hetox for the original report. (#1910) @@ -795,12 +799,14 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In JavaScript and TypeScript, a value the file imports from your own project and passes along now links only to what it imports: a constant like `isCloudHosted`, a class like a `Star` model, a whole module (`import * as Slack from '../slack'`), or a stylesheet, image or GraphQL document (`import TCO21 from './themes/TCO21.scss'`) no longer links to a function elsewhere that only shares its name. Before, when the project had exactly one function of that name, such a value linked to it: outline's settings screens showed up among the callers of the server's `isCloudHosted()` check, and its stores among those of the `` icon, so `codegraph callers`, impact and `codegraph_explore` followed references that never happen. A component that wraps the function declared beside it, like `export default styled(Disclosure)`, and a page loaded lazily, like `lazy(() => import('../pages/Home'))`, still lead to that function. Re-index JavaScript and TypeScript projects after upgrading. - In JavaScript, an import of a TypeScript module written without an extension, like `import AudioModal from './audio_modal'` in a `.jsx` file next to `audio_modal.tsx`, now links to that module, as it already did from a TypeScript file. Before, a `.js` or `.jsx` file's imports of TypeScript modules were never followed, so in a codebase moving to TypeScript they were matched by name instead: an import under another name linked nothing, and a name that repeats could link to the wrong file, so mastodon's `` and `` tags rendered other components of those names than the ones each file imports. A JavaScript module beside a TypeScript one of the same name still comes first. Re-index JavaScript projects after upgrading. - `codegraph_explore`'s Flow now goes through the implementation your query names when an interface method has several that lead to the same place. Before, it took whichever one came first in the index: asking about prometheus's `Engine.execEvalStmt Queryable.Querier fanout.Querier NewMergeQuerier` traced the call through the TSDB's `DB.Querier` and left the `fanout.Querier` you named off the Flow. Any two routes of the same length are now settled this way, in favor of the one that passes through more of the symbols you named. +- `codegraph_explore`'s Flow now reaches every symbol your query names along a call path, even when a shorter route gets to one of them first through a function you didn't name. Before, for `entry deep leaf` where `entry` calls `deep` (through a helper) and `deep` calls `leaf`, but `entry` also reaches `leaf` through another helper, the Flow stopped at `deep` or skipped it, so a named step was missing and the agent went to read it; on Django, `SQLCompiler.as_sql pre_sql_setup get_select` left out `pre_sql_setup`. A Flow still goes through at most one function you didn't name between two you did, and never round a loop. Thanks @inth3shadows for the report and @johnhldiver-beep for the first fix. (#2970) - `codegraph_explore`'s "Dynamic-dispatch links among your symbols" list now shows every link on its own line. Before, the links from an interface method to its implementations all read alike, like `TrySet → TrySet`, so only one was listed, chosen by the order of the index rather than by your query: asking about gin's `tryToSetValue setter.TrySet multipartRequest.TrySet setByForm` showed `headerSource`'s implementation and hid `formSource`'s. Both ends of such a link are now named with their type, like `setter.TrySet → formSource.TrySet`. When there are more links than the list shows, the implementations your query names come first and those in tests or examples last, and a hop the Flow already shows is not repeated. - `codegraph_explore`, `codegraph_node`, `codegraph_callers`, `codegraph_callees`, `codegraph_impact` and the `codegraph callers`, `callees` and `impact` commands now find a symbol written with its package, the way Go code names it: `refresh.Discovery.refresh`, `storage.Queryable.Querier`, or `gin.Context.Next` in gin's root package. Before, such a name found nothing, so a question written that way lost the symbol and explore's Flow had nothing to connect. A name the search ranked below other results, like gin's `binding.Default`, is found as well, and a Python or Rust module followed by a class and method works the same way. - When a name has more definitions than one `codegraph_node` answer holds, every language and top-level folder it is defined in now shows up, in `codegraph_node` and `codegraph node`. Before, definitions filled the answer in file order, so forty in one folder could hide the only one in another language or a later folder behind a bare "+5 more". Now a language with no code in the answer gets one of its definitions in full, on top of the ones that fit, the list of the others starts with one from each folder, and the rest are counted by language and folder. Thanks @ferrine. - When `codegraph_explore` finds more dynamic-dispatch links than its list shows, the symbols you named now take turns, so each one's links make the list: asking about prometheus's `refresh.Discovery.refresh hetzner.refresher.refresh hcloudDiscovery.refresh` now lists hetzner's `robotDiscovery` beside the links into other discovery packages that used to fill the list. - `codegraph_explore` now shows a function you name even when it is too long for its share of the answer: its definition and the start of its body, with a note where the body is cut that says which lines were left out and which `codegraph_explore` query returns them. Before, a file made of one long function and a line that calls it could come back as just its last few lines, without the function, and asking for the file by path did the same. A trimmed answer also no longer suggests `codegraph_node`, which isn't offered by default. Thanks @Huy3ko for the report and @danusha2345 for the first fix. (#2486) - When a `codegraph_explore` query names symbols the index has never seen, the answer now says so first: it lists those names and points to `codegraph status`, and to how a folder skipped by default, like a root `build/` folder, is brought into the index. Before, the answer showed only what the query's other words matched, such as an unrelated `Main` and a C++ `Parser`, with nothing to say the names asked about were missing. Thanks @tippmar-nr for the report. (#2462) +- `codegraph_explore`, `codegraph explore` and `codegraph context` now find code named in Chinese, Japanese or Korean, like `寻路服务`, `ユーザー取得` or `로그인`, and explore's Flow now traces a call chain named that way, like `入口 末端`. Before, explore answered "No relevant code found" and context printed only its header, though `codegraph query` found the same name. A name written right against a word in another script, like `NavMeshAgent类型`, is read as two words, and a word with accented or non-Latin letters, like `état` or `получить`, is now searched whole instead of in broken pieces, so it can find different results than before. Queries in plain English letters are answered exactly as before. Thanks @andy-sg and @yimingll. - In C and C++, a struct, class, union or enum defined in a variable declaration, like `struct Foo { … } foo;`, is now indexed with its methods and enum values, next to the variable. Before, at most the variable was, so the type was missing from search and `codegraph_explore`, a class deriving from it had no base, and calls to its methods linked to nothing: in redis, the Sentinel's `sentinelState` and `redis-cli`'s `config` struct were missing. A struct, union or enum with no name of its own now takes the name of the variable it declares, so redis's `static struct { … } SPT;` is found as `SPT`, and one declared inside a function is named after its variable instead of ``. The comment above the declaration becomes the type's description. Re-index C and C++ projects after upgrading. - In TypeScript React code, a styled-components or emotion component declared with a type argument, like `const Wrapper = styled.div` followed by its CSS, or `styled(Base)`, is now indexed as a component, as one written without a type argument already was. Before, it was indexed as a plain constant, so `codegraph callers` and impact found none of the code that renders it, and its tags could link to another file's component of the same name instead: in outline/outline, `NudeButton` and `HStack` had no callers, and `` tags linked to a different `Text` than the one each file imports. Re-index TypeScript React projects after upgrading. diff --git a/README.md b/README.md index 3e3f958f65..68f44f5076 100644 --- a/README.md +++ b/README.md @@ -1078,12 +1078,15 @@ See [Get Started](#get-started) for the one-line install commands. ## Supported Agents The interactive installer auto-detects and configures each of these — wiring up -the MCP server (which delivers its own usage guidance, so no instructions file -is written): +the MCP server, which delivers its own usage guidance. For Claude Code, Codex +CLI, Gemini CLI and opencode it also writes a short marker-fenced CodeGraph +section into the agent's instructions file (`CLAUDE.md` / `AGENTS.md` / +`GEMINI.md`), so subagents and non-MCP agents learn the `codegraph explore` +command: - **Claude Code** - **Cursor** -- **Codex CLI** +- **Codex CLI** — the Codex IDE extension reads the same config: a global install writes `$CODEX_HOME/config.toml` (default `~/.codex/config.toml`), and a local install writes `.codex/config.toml`, which Codex uses only in a project you've marked as trusted - **opencode** — MCP entry is OpenCode 2's `mcp.servers.codegraph` with `codemode: false` (keeps `codegraph_explore` on the native tool list; `codegraph install` migrates the older `mcp.codegraph` shape) - **Hermes Agent** - **Gemini CLI** @@ -1185,7 +1188,7 @@ Framework routing is validated the same way, on a canonical app per framework: E **Two `codegraph serve --mcp` on one project fight over the index / auto-sync stops** — CodeGraph allows one live MCP *writer* per project (the shared background daemon, or a single direct-mode process). Extra clients should proxy to that daemon. If you set `CODEGRAPH_NO_DAEMON=1`, run only one `serve --mcp` for that project; a second instance exits with a clear writer-lock error (see `writer.pid` under `.codegraph/`; its persistent `writer.pid.mutation.lock` coordinates ownership changes and needs no stale-lock deletion). Prefer leaving the daemon enabled so multiple MCP hosts share one watcher. -**MCP tool calls fail with `Transport closed` while `codegraph status`/`sync` are healthy** — almost always WSL2 with the project on a Windows drive (a `/mnt/c` or `/mnt/d` path), where the local socket CodeGraph uses to share one background server across sessions is unreliable. CodeGraph now falls back to serving the session in-process instead of dropping the connection, and keeps trying to reach the shared server in the background: first after 5 seconds, backing off to every 5 minutes. `CODEGRAPH_DAEMON_RETRY_MS` and `CODEGRAPH_DAEMON_RETRY_MAX_MS` set those two delays, and `CODEGRAPH_DAEMON_RETRY_MS=0` stops retrying. If you still hit it, set `CODEGRAPH_NO_DAEMON=1` in your MCP server's environment to skip the shared server entirely (each session runs in its own process). Moving the project onto the Linux-native filesystem (e.g. under `~/` instead of `/mnt/`) restores the shared server. +**MCP tool calls fail with `Transport closed` while `codegraph status`/`sync` are healthy** — almost always WSL2 with the project on a Windows drive (a `/mnt/c` or `/mnt/d` path), where the local socket CodeGraph uses to share one background server across sessions is unreliable. CodeGraph works around this itself now: when the project's filesystem can't host that socket, the shared server puts it in the system temp directory instead, and when the shared server still can't be reached, or drops, the session is served in-process instead of losing the connection, and keeps trying to reach the shared server in the background: first after 5 seconds, backing off to every 5 minutes. `CODEGRAPH_DAEMON_RETRY_MS` and `CODEGRAPH_DAEMON_RETRY_MAX_MS` set those two delays, and `CODEGRAPH_DAEMON_RETRY_MS=0` stops retrying. If you still hit it, set `CODEGRAPH_NO_DAEMON=1` in your MCP server's environment to skip the shared server entirely (each session runs in its own process). Moving the project onto the Linux-native filesystem (e.g. under `~/` instead of `/mnt/`) restores the shared server. **Missing symbols** — The MCP server auto-syncs on save (wait a couple seconds). Run `codegraph sync` manually if needed. Check that the file's language is supported and isn't inside a `.gitignore`d or default-excluded directory (e.g. `node_modules`, `dist`). diff --git a/__tests__/add-lang-verify-extraction.test.ts b/__tests__/add-lang-verify-extraction.test.ts new file mode 100644 index 0000000000..2e3f38a3c7 --- /dev/null +++ b/__tests__/add-lang-verify-extraction.test.ts @@ -0,0 +1,97 @@ +/** + * scripts/add-lang/verify-extraction.mjs reads `codegraph status --json` from + * the codegraph that built an index. It used to call `codegraph` by bare name, + * which never works on Windows: npm installs codegraph there as a .cmd shim, + * and Node can neither find a .cmd by bare name nor run one without a shell. + * /add-lang's bench.sh then skipped its A/B as if extraction had failed. + * + * A stand-in for codegraph prints a healthy status and records the arguments + * it received, so each test can check what reached it. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +const SCRIPT = path.resolve('scripts/add-lang/verify-extraction.mjs'); +const roots: string[] = []; + +const FAKE_CODEGRAPH = ` +const fs = require('fs'); +fs.writeFileSync(process.env.CG_TEST_ARGV, JSON.stringify(process.argv.slice(2))); +console.log(JSON.stringify({ + initialized: true, backend: 'node-sqlite', fileCount: 2, nodeCount: 9, edgeCount: 7, + languages: ['lua'], nodesByKind: { file: 2, function: 5, variable: 2 }, +})); +`; + +function kit() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-verify-extraction-')); + roots.push(root); + const fake = path.join(root, 'fake-codegraph.cjs'); + fs.writeFileSync(fake, FAKE_CODEGRAPH); + // A folder name like npm's prefix under "Program Files (x86)". + const bin = path.join(root, 'bin (x86)'); + fs.mkdirSync(bin); + return { root, fake, bin, argv: path.join(root, 'argv.json') }; +} + +// The script's environment, with PATH limited to `dirs` plus what node needs, +// so no codegraph installed on the machine running the tests can answer. +function envWith(k: ReturnType, dirs: string[], extra: NodeJS.ProcessEnv = {}) { + const env: NodeJS.ProcessEnv = {}; + for (const [key, value] of Object.entries(process.env)) { + if (key.toUpperCase() !== 'PATH' && key !== 'CODEGRAPH_BIN') env[key] = value; + } + const system = process.platform === 'win32' ? [path.join(process.env.SystemRoot ?? 'C:\\Windows', 'System32')] : ['/usr/bin', '/bin']; + env.PATH = [...dirs, path.dirname(process.execPath), ...system].join(path.delimiter); + env.CG_TEST_ARGV = k.argv; + return { ...env, ...extra }; +} + +function verify(k: ReturnType, repo: string, env: NodeJS.ProcessEnv) { + const r = spawnSync(process.execPath, [SCRIPT, repo, 'lua'], { encoding: 'utf8', timeout: 60_000, env }); + const received = fs.existsSync(k.argv) ? JSON.parse(fs.readFileSync(k.argv, 'utf8')) : null; + return { status: r.status, output: `${r.stdout}${r.stderr}`, received }; +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('add-lang verify-extraction: which codegraph it asks', () => { + it('reads the status through CODEGRAPH_BIN, running a .js path with node', () => { + const k = kit(); + const repo = path.join(k.root, 'some repo'); + const r = verify(k, repo, envWith(k, [], { CODEGRAPH_BIN: k.fake })); + expect(r.output).toContain('RESULT: PASS'); + expect(r.status).toBe(0); + expect(r.received).toEqual(['status', repo, '--json']); + }); + + it.runIf(process.platform === 'win32')('finds the npm .cmd shim on PATH and passes an awkward repo path through cmd.exe intact', () => { + const k = kit(); + // The shape of npm's shim: cmd.exe runs it and %* hands the arguments on. + fs.writeFileSync(path.join(k.bin, 'codegraph.cmd'), '@ECHO off\r\nSETLOCAL\r\nSET "_prog=node"\r\n"%_prog%" "%~dp0..\\fake-codegraph.cjs" %*\r\n'); + // Spaces, & and parentheses, a defined %NAME%, a lone %, and a trailing + // backslash that must not escape the closing quote. + const repo = path.join(k.root, 'repo & co (x86) %CG_TEST_PCT% 50%') + '\\'; + const r = verify(k, repo, envWith(k, [k.bin], { CG_TEST_PCT: 'expanded' })); + expect(r.output).toContain('RESULT: PASS'); + expect(r.status).toBe(0); + expect(r.received).toEqual(['status', repo, '--json']); + }); + + it.runIf(process.platform !== 'win32')('runs the codegraph on PATH directly, as before', () => { + const k = kit(); + const shim = path.join(k.bin, 'codegraph'); + fs.writeFileSync(shim, `#!/bin/sh\nexec "${process.execPath}" "${k.fake}" "$@"\n`); + fs.chmodSync(shim, 0o755); + const repo = path.join(k.root, 'some repo'); + const r = verify(k, repo, envWith(k, [k.bin])); + expect(r.output).toContain('RESULT: PASS'); + expect(r.status).toBe(0); + expect(r.received).toEqual(['status', repo, '--json']); + }); +}); diff --git a/__tests__/callers-call-sites.test.ts b/__tests__/callers-call-sites.test.ts new file mode 100644 index 0000000000..5fbb9b8c6d --- /dev/null +++ b/__tests__/callers-call-sites.test.ts @@ -0,0 +1,203 @@ +/** + * `codegraph callers` lists every place a caller calls the symbol, and a + * symbol that calls itself among its callers (#2909). + * + * Callers kept one row per caller (#1086), with the caller's first edge only, + * and left the symbol itself out, so a method called on two lines of `build` + * showed one, and a recursive call none at all, though the index stores them + * and each is a line to edit when the symbol changes. The rows stay one per + * caller; the sites go inside the row: `src/report.ts:9, 10`. The CLI, the + * codegraph_callers / codegraph_callees tools and codegraph_node's trail read + * them from one place, `getCallSites`, while `getCallers` / `getCallees`, which + * ranking and the flow and impact walks use, read as they did. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { ToolHandler } from '../src/mcp/tools'; +import type { Node } from '../src/types'; + +const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); +const BUSY = 30; + +let root = ''; +let cg: CodeGraph; +let handler: ToolHandler; + +const files: Record = { + // The issue's example, with the recursion written `this.walk()`. + 'src/report.ts': [ + 'export class Logger {', + ' log(message: string): void {}', + '}', + '', + 'export class Report {', + ' constructor(private readonly logger: Logger) {}', + '', + ' build(): void {', + " this.logger.log('start');", + " this.logger.log('end');", + ' }', + '}', + '', + 'export class TreeNode {', + ' children: TreeNode[] = [];', + '', + ' walk(): void {', + ' this.walk();', + ' }', + '}', + '', + ].join('\n'), + 'src/once.ts': [ + "import { Logger } from './report';", + '', + 'export function once(logger: Logger): void {', + " logger.log('only');", + '}', + '', + ].join('\n'), + 'src/math.ts': [ + 'export function factorial(n: number): number {', + ' return n <= 1 ? 1 : n * factorial(n - 1);', + '}', + '', + 'export function tally(): number {', + ' return factorial(3) + factorial(4);', + '}', + '', + ].join('\n'), + // A caller with more call sites than any list shows. + 'src/busy.ts': [ + "import { Logger } from './report';", + '', + 'export function busy(logger: Logger): void {', + ...Array.from({ length: BUSY }, (_, i) => ` logger.log('${i}');`), + '}', + '', + ].join('\n'), +}; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-call-sites-')); + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + cg = await CodeGraph.init(root, { index: true }); + handler = new ToolHandler(cg); +}); + +afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +function named(name: string, file: string): Node { + const node = cg.getNodesInFile(file).find((n) => n.name === name); + expect(node, `${file}:${name}`).toBeDefined(); + return node!; +} + +function cli(...args: string[]): string { + const result = spawnSync(process.execPath, [BIN, ...args, '-p', root], { + encoding: 'utf-8', + env: { ...process.env, CODEGRAPH_NO_DAEMON: '1', NO_COLOR: '1', DO_NOT_TRACK: '1', CODEGRAPH_TELEMETRY: '0' }, + timeout: 60_000, + }); + expect(result.status, result.stderr).toBe(0); + return result.stdout; +} + +async function tool(name: string, args: Record): Promise { + const res = await handler.execute(name, args); + return res.content?.[0]?.text ?? ''; +} + +describe('call sites of a symbol (#2909)', () => { + it('keeps one row per caller with every site in it', () => { + const rows = cg.getCallSites(named('log', 'src/report.ts').id, 'callers'); + const build = rows.filter((row) => row.node.name === 'build'); + expect(build).toHaveLength(1); + expect(build[0]!.edges.map((e) => e.line)).toEqual([9, 10]); + expect(build[0]!.recursive).toBe(false); + expect(rows.find((row) => row.node.name === 'once')!.edges.map((e) => e.line)).toEqual([4]); + }); + + it('lists a symbol that calls itself as its own caller and callee', () => { + for (const [name, file, line] of [['walk', 'src/report.ts', 18], ['factorial', 'src/math.ts', 2]] as const) { + const node = named(name, file); + for (const direction of ['callers', 'callees'] as const) { + const self = cg.getCallSites(node.id, direction).filter((row) => row.node.id === node.id); + expect(self, `${name} ${direction}`).toHaveLength(1); + expect(self[0]!.recursive).toBe(true); + expect(self[0]!.edges.map((e) => e.line)).toEqual([line]); + } + } + }); + + it('leaves getCallers and getCallees as they were: the first edge, never the symbol itself', () => { + for (const node of cg.getNodesByKind('method').concat(cg.getNodesByKind('function'))) { + for (const direction of ['callers', 'callees'] as const) { + const before = (direction === 'callers' ? cg.getCallers(node.id) : cg.getCallees(node.id)) + .map((c) => [c.node.id, c.edge.line, c.edge.kind]); + const now = cg.getCallSites(node.id, direction) + .filter((row) => !row.recursive) + .map((row) => [row.node.id, row.edges[0]!.line, row.edges[0]!.kind]); + expect(now, `${node.name} ${direction}`).toEqual(before); + } + } + }); +}); + +describe('codegraph callers / callees', () => { + it('prints a caller at the lines it calls from, and a recursive one as calling itself', () => { + const text = cli('callers', 'Logger.log'); + expect(text).toMatch(/method\s+build\n\s+src\/report\.ts:9, 10\n/); + expect(text).toMatch(/function\s+once\n\s+src\/once\.ts:4\n/); + expect(text.match(/\bbuild\b/g)).toHaveLength(1); + const walk = cli('callers', 'TreeNode.walk'); + expect(walk).toMatch(/method\s+walk \(calls itself\)\n\s+src\/report\.ts:18\n/); + // Callees keep the callee's own line: where a call leads. + expect(cli('callees', 'tally')).toMatch(/function\s+factorial\n\s+src\/math\.ts:1\n/); + }); + + it('caps one caller at ten sites in text, and lists them all in JSON', () => { + expect(cli('callers', 'Logger.log')).toMatch(/function\s+busy\n\s+src\/busy\.ts:4, 5, 6, 7, 8, 9, 10, 11, 12, 13, \+20 more\n/); + const out = JSON.parse(cli('callers', 'Logger.log', '--json')); + const busy = out.definitions[0].callers.find((row: { name: string }) => row.name === 'busy'); + expect(busy.lines).toEqual(Array.from({ length: BUSY }, (_, i) => 4 + i)); + expect(busy.recursive).toBe(false); + const build = out.callers.find((row: { name: string }) => row.name === 'build'); + expect(build).toMatchObject({ lines: [9, 10], recursive: false }); + const buildId = out.definitions[0].callers.find((row: { name: string }) => row.name === 'build').id; + expect(out.definitions[0].edges.filter((e: { source: string }) => e.source === buildId).map((e: { line: number }) => e.line)) + .toEqual([9, 10]); + const walk = JSON.parse(cli('callers', 'TreeNode.walk', '--json')); + expect(walk.callers).toEqual([expect.objectContaining({ name: 'walk', lines: [18], recursive: true })]); + }); +}); + +describe('codegraph_callers / codegraph_callees and the codegraph_node trail', () => { + it('lists every site of a caller, capped, and a recursive call', async () => { + const callers = await tool('codegraph_callers', { symbol: 'Logger.log' }); + expect(callers).toContain('- build (method) - src/report.ts:9, 10'); + expect(callers).toContain('- once (function) - src/once.ts:4'); + expect(callers).toContain('- busy (function) - src/busy.ts:4, 5, 6, 7, 8, 9, 10, 11, 12, 13, +20 more'); + expect(await tool('codegraph_callers', { symbol: 'factorial' })).toContain('- factorial (function) - src/math.ts:2 — calls itself'); + expect(await tool('codegraph_callees', { symbol: 'factorial' })).toContain('- factorial (function) - src/math.ts:1 — calls itself'); + }); + + it('shows callers in the trail at their sites, three at most, and the recursion once', async () => { + const log = await tool('codegraph_node', { symbol: 'Logger.log' }); + expect(log).toMatch(/\*\*Called by ←\*\* .*build \(src\/report\.ts:9, 10\)/); + expect(log).toContain('busy (src/busy.ts:4, 5, 6, +27 more)'); + const factorial = await tool('codegraph_node', { symbol: 'factorial' }); + expect(factorial).toMatch(/\*\*Called by ←\*\* .*factorial \(src\/math\.ts:2, calls itself\)/); + expect(factorial).toContain('tally (src/math.ts:6)'); + expect(factorial).not.toMatch(/\*\*Calls →\*\*.*factorial/); + }); +}); diff --git a/__tests__/cjk-query-tokens.test.ts b/__tests__/cjk-query-tokens.test.ts new file mode 100644 index 0000000000..031c2987e7 --- /dev/null +++ b/__tests__/cjk-query-tokens.test.ts @@ -0,0 +1,341 @@ +/** + * Chinese, Japanese and Korean names in explore and context queries (#516, #1262). + * + * The index already stores a name in any script as one full-text token, and + * `codegraph query 로그인` found it, but every query tokenizer between a + * question and the index knew only ASCII: `extractSearchTerms` split on + * `[^a-zA-Z0-9]`, context's symbol extraction matched ASCII patterns only, and + * explore's named-symbol and Flow tokens had to start with `[A-Za-z_$]` and run + * three characters. So `codegraph_explore 寻路服务` said "No relevant code + * found", `codegraph context 로그인` printed only its header, and `入口 末端` + * had no Flow. Words in any script are now taken whole; Han, kana and Hangul + * count from two characters and are cut from a neighbouring script + * (`NavMeshAgent类型`). An ASCII query reads exactly as it did: the reference + * copies of the old tokenizers at the bottom of this file pin that. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { ToolHandler, exploreQueryTokens } from '../src/mcp/tools'; +import { flowTokens } from '../src/graph/named-symbol-flow'; +import { extractSymbolsFromQuery } from '../src/context'; +import { LOW_CONFIDENCE_MARKER } from '../src/context/markers'; +import { + extractSearchTerms, splitAtCjkSwitches, isQueryNameToken, STOP_WORDS, getStemVariants, +} from '../src/search/query-utils'; + +describe('query words in any script (#516, #1262)', () => { + it('keeps Han, kana and Hangul words whole, from two characters', () => { + expect(extractSearchTerms('寻路服务')).toEqual(['寻路服务']); + expect(extractSearchTerms('入口 末端')).toEqual(['入口', '末端']); + expect(extractSearchTerms('로그인 인증확인')).toEqual(['로그인', '인증확인']); + // The prolonged sound mark ー belongs to the katakana around it. + expect(extractSearchTerms('ユーザー取得')).toEqual(['ユーザー取得']); + expect(extractSearchTerms('类 的')).toEqual([]); + }); + + it('cuts a name where Han, kana or Hangul meets another script', () => { + expect(splitAtCjkSwitches('NavMeshAgent类型')).toEqual(['NavMeshAgent', '类型']); + expect(splitAtCjkSwitches('用户ID')).toEqual(['用户', 'ID']); + expect(splitAtCjkSwitches('寻路服务.计算路径')).toEqual(['寻路服务.计算路径']); + expect(splitAtCjkSwitches('get_用户')).toEqual(['get_用户']); + const terms = extractSearchTerms('NavMeshAgent类型'); + expect(terms).toEqual(expect.arrayContaining(['navmeshagent', 'nav', 'mesh', 'agent', '类型'])); + expect(terms).not.toContain('agent类型'); + }); + + it('keeps other scripts whole at three characters', () => { + // main: `état` became `tat`, and `größe` nothing at all. + expect(extractSearchTerms('état machine', { stems: false })).toEqual(['état', 'machine']); + expect(extractSearchTerms('größe', { stems: false })).toEqual(['größe']); + // Russian `на` (on) stays out, like English `is`. + expect(extractSearchTerms('получить данные на сервере', { stems: false })) + .toEqual(['получить', 'данные', 'сервере']); + }); + + it('gives context the names in other scripts, whole', () => { + expect(extractSymbolsFromQuery('寻路服务 计算路径')).toEqual(['寻路服务', '计算路径']); + expect(extractSymbolsFromQuery('로그인')).toEqual(['로그인']); + expect(extractSymbolsFromQuery('état machine')).toEqual(['machine', 'état']); + expect(extractSymbolsFromQuery('NavMeshAgent类型')).toEqual(['NavMeshAgent', 'NavMeshAgent类型', '类型']); + expect(extractSymbolsFromQuery('get用户信息')).toEqual(['get用户信息', '用户信息']); + }); + + it('reads names in any script as explore and Flow tokens', () => { + for (const tokens of [exploreQueryTokens, flowTokens]) { + expect(tokens('入口 末端')).toEqual(['入口', '末端']); + expect(tokens('寻路服务.计算路径 로그인')).toEqual(['寻路服务.计算路径', '로그인']); + expect(tokens('NavMeshAgent类型 类')).toEqual(['NavMeshAgent', '类型']); + expect(tokens('auth.ts état на')).toEqual(['auth', 'état']); + } + expect(isQueryNameToken('入口')).toBe(true); + expect(isQueryNameToken('на')).toBe(false); + }); +}); + +describe('an ASCII query reads exactly as before (#516, #1262)', () => { + const words = [ + 'how', 'does', 'the', 'auth', 'flow', 'reach', 'the', 'database', 'scrapeLoop', 'run', 'UserService', + 'getCallGraph', 'build_report', 'MAX_RETRIES', 'REST', 'HTTP2', 'app.isPackaged', 'Session.request', + 'pkg::run', 'src/auth/session.ts', 'Program.cs', 'caching', 'eviction', 'handlers', 'x', 'is', 'to', + 'refreshTokenService', 'tokio', 'poll', 'Harness::poll', 'fn/3', 'mod:fn', 'v2', 'IOStream', 'XMLHttpRequest', + '$state', 'items$', '_private', '__init__', 'a1b2c3', 'TODO:', '(foo)', '[bar]', 'baz,', 'qux;', '"quoted"', + "it's", 'e2e/*.spec.ts', 'stage_apply::run', 'C++', 'node.js', '--flag', 'snake_Case_Mixed', 'ABCDef', + ]; + const corpus: string[] = [ + ...words, + 'how does the order state machine work', + 'DataRequest task validate', + 'entryPoint middleStep tailStep', + 'Engine.execEvalStmt Queryable.Querier fanout.Querier NewMergeQuerier', + 'why does getUserName return null when the cache is cold?', + 'trace triggerRender -> triggerUpdate in App.tsx', + 'what calls build_report and MAX_RETRIES (see utils/report.py:120)', + ]; + // Every 3-word window of the word list, so words meet each other's punctuation. + for (let i = 0; i + 3 <= words.length; i++) corpus.push(words.slice(i, i + 3).join(' ')); + + it('gives the same search terms', () => { + for (const q of corpus) { + expect(extractSearchTerms(q), q).toEqual(oldExtractSearchTerms(q)); + expect(extractSearchTerms(q, { stems: false }), q).toEqual(oldExtractSearchTerms(q, { stems: false })); + } + }); + + it('gives the same context symbols', () => { + for (const q of corpus) expect(extractSymbolsFromQuery(q), q).toEqual(oldExtractSymbolsFromQuery(q)); + }); + + it('gives the same explore and Flow tokens', () => { + for (const q of corpus) { + expect(exploreQueryTokens(q), q).toEqual(oldNameTokens(q).slice(0, 16)); + expect(flowTokens(q), q).toEqual(oldNameTokens(q).slice(0, 16)); + } + }); +}); + +describe('explore and context on Korean, Chinese and Japanese names (#516, #1262)', () => { + let root: string; + let cg: CodeGraph; + let handler: ToolHandler; + + const write = (rel: string, body: string) => { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), body); + }; + const explore = async (query: string): Promise => { + const res = await handler.execute('codegraph_explore', { query }); + expect(res.isError).toBeFalsy(); + return res.content?.[0]?.text ?? ''; + }; + const context = async (query: string): Promise => + String(await cg.buildContext(query, { format: 'markdown' })); + + beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cjk-query-')); + // #516's Korean fixture. + write('src/auth.ts', [ + 'export function 로그인(사용자명: string): boolean {', + ' return 인증확인(사용자명);', + '}', + '', + 'export function 인증확인(사용자명: string): boolean {', + ' return 사용자명.length > 0;', + '}', + '', + ].join('\n')); + // #1262's Chinese fixture. + write('src/nav.ts', [ + 'export class 寻路服务 {', + ' 计算路径(起点: number, 终点: number): number[] {', + ' return [起点, 终点];', + ' }', + '}', + '', + 'export class NavMeshAgent {', + ' move(): void {', + ' new 寻路服务().计算路径(1, 2);', + ' }', + '}', + '', + ].join('\n')); + // A Chinese call chain across three files, for the Flow. + write('src/flow/a.ts', 'import { 中间 } from "./b";\nexport function 入口(): number {\n return 中间() + 1;\n}\n'); + write('src/flow/b.ts', 'import { 末端 } from "./c";\nexport function 中间(): number {\n return 末端() * 2;\n}\n'); + write('src/flow/c.ts', 'export function 末端(): number {\n return 42;\n}\n'); + // Japanese: kanji, katakana and the prolonged sound mark. + write('src/order.ts', [ + 'export function 注文処理(注文番号: string): boolean {', + ' return 在庫確認(注文番号);', + '}', + '', + 'export function 在庫確認(番号: string): boolean {', + ' return 番号.length > 0;', + '}', + '', + 'export class ユーザー管理 {', + ' ユーザー取得(名前: string): string {', + ' return 名前;', + ' }', + '}', + '', + ].join('\n')); + write('src/other.ts', 'export function unrelatedHelper(): number {\n return 42;\n}\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + handler = new ToolHandler(cg); + }, 60_000); + + afterAll(() => { + handler?.closeAll(); + cg?.destroy(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('explores a Korean, a Chinese and a Japanese name', async () => { + // main: "No relevant code found" for all three. + const korean = await explore('로그인'); + expect(korean).not.toContain('No relevant code found'); + expect(korean).toContain('export function 로그인(사용자명: string): boolean {'); + const chinese = await explore('寻路服务'); + expect(chinese).not.toContain('No relevant code found'); + expect(chinese).toContain('export class 寻路服务 {'); + const japanese = await explore('ユーザー取得'); + expect(japanese).not.toContain('No relevant code found'); + expect(japanese).toContain('ユーザー取得(名前: string): string {'); + }); + + it('leads a Chinese call chain with its Flow', async () => { + // main: "No relevant code found"; the ASCII twin of this query leads with the Flow. + const text = await explore('入口 末端'); + expect(text.startsWith('**Flow (call path among the symbols you queried)**')).toBe(true); + expect(text).toMatch(/1\. 入口 \(src\/flow\/a\.ts:2\)[\s\S]*2\. 中间 \(src\/flow\/b\.ts:2\)[\s\S]*3\. 末端 \(src\/flow\/c\.ts:1\)/); + }); + + it('finds the English name beside a Chinese word without calling it missing', async () => { + const text = await explore('NavMeshAgent类型'); + expect(text).toContain('export class NavMeshAgent {'); + expect(text).not.toContain('Not in the index'); + }); + + it('builds context for names in each script', async () => { + // main: the header and the query, nothing else. + expect(await context('로그인')).toContain('- **로그인** (function) - src/auth.ts:1'); + const japanese = await context('注文処理 在庫確認'); + expect(japanese).toContain('- **注文処理** (function) - src/order.ts:1'); + expect(japanese).toContain('- **在庫確認** (function) - src/order.ts:5'); + // Names the user wrote out are not "common words". + expect(japanese).not.toContain(LOW_CONFIDENCE_MARKER); + expect(await context('寻路服务 计算路径')).not.toContain(LOW_CONFIDENCE_MARKER); + }); +}); + +// --------------------------------------------------------------------------- +// The ASCII-only tokenizers as they were before #516 / #1262, kept verbatim as +// the reference an ASCII query must still match. +// --------------------------------------------------------------------------- + +function oldExtractSearchTerms(query: string, options?: { stems?: boolean }): string[] { + const includeStems = options?.stems !== false; + const tokens = new Set(); + const compoundPattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:[A-Z][a-z]+)+|[A-Z][a-z]+(?:[A-Z][a-z]*)+)\b/g; + let match; + while ((match = compoundPattern.exec(query)) !== null) { + if (match[1] && match[1].length >= 3) tokens.add(match[1].toLowerCase()); + } + const snakePattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:_[a-zA-Z0-9]+)+)\b/g; + while ((match = snakePattern.exec(query)) !== null) { + if (match[1] && match[1].length >= 3) tokens.add(match[1].toLowerCase()); + } + const camelSplit = query + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2'); + const normalised = camelSplit.replace(/[_.]+/g, ' '); + const words = normalised.split(/[^a-zA-Z0-9]+/).filter(Boolean); + for (const word of words) { + const lower = word.toLowerCase(); + if (lower.length < 3) continue; + if (STOP_WORDS.has(lower)) continue; + tokens.add(lower); + } + if (includeStems) { + const stems = new Set(); + for (const token of tokens) { + for (const variant of getStemVariants(token)) { + if (!tokens.has(variant) && !STOP_WORDS.has(variant)) stems.add(variant); + } + } + for (const stem of stems) tokens.add(stem); + } + return [...tokens]; +} + +function oldExtractSymbolsFromQuery(query: string): string[] { + const symbols = new Set(); + const camelCasePattern = /\b([A-Z][a-z]+(?:[A-Z][a-z]*)*|[a-z]+(?:[A-Z][a-z]*)+)\b/g; + let match; + while ((match = camelCasePattern.exec(query)) !== null) { + if (match[1] && match[1].length >= 2) symbols.add(match[1]); + } + const snakeCasePattern = /\b([a-z][a-z0-9]*(?:_[a-z0-9]+)+)\b/gi; + while ((match = snakeCasePattern.exec(query)) !== null) { + if (match[1] && match[1].length >= 3) symbols.add(match[1]); + } + const screamingPattern = /\b([A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)\b/g; + while ((match = screamingPattern.exec(query)) !== null) { + if (match[1]) symbols.add(match[1]); + } + const acronymPattern = /\b([A-Z]{2,})\b/g; + while ((match = acronymPattern.exec(query)) !== null) { + if (match[1]) symbols.add(match[1]); + } + const dotPattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:\.[a-zA-Z][a-zA-Z0-9]*)+)\b/g; + while ((match = dotPattern.exec(query)) !== null) { + if (match[1]) { + symbols.add(match[1]); + for (const part of match[1].split('.')) if (part.length >= 2) symbols.add(part); + } + } + const lowercasePattern = /\b([a-z][a-z0-9]{2,})\b/g; + while ((match = lowercasePattern.exec(query)) !== null) { + if (match[1]) symbols.add(match[1]); + } + const commonWords = new Set([ + 'the', 'and', 'for', 'with', 'from', 'this', 'that', 'have', 'been', + 'will', 'would', 'could', 'should', 'does', 'done', 'make', 'made', + 'use', 'used', 'using', 'work', 'works', 'find', 'found', 'show', + 'call', 'called', 'calling', 'get', 'set', 'add', 'all', 'any', + 'how', 'what', 'when', 'where', 'which', 'who', 'why', + 'not', 'but', 'are', 'was', 'were', 'has', 'had', 'its', + 'can', 'did', 'may', 'also', 'into', 'than', 'then', 'them', + 'each', 'other', 'some', 'such', 'only', 'same', 'about', + 'after', 'before', 'between', 'through', 'during', 'without', + 'again', 'further', 'once', 'here', 'there', 'both', 'just', + 'more', 'most', 'very', 'being', 'having', 'doing', + 'system', 'need', 'needs', 'want', 'wants', 'like', 'look', + 'change', 'changes', 'changed', 'changing', + 'layer', 'handle', 'handles', 'handling', 'incoming', 'outgoing', + 'data', 'flow', 'flows', 'level', 'levels', 'request', 'requests', + 'response', 'responses', 'implement', 'implements', 'implementation', + 'interface', 'interfaces', 'class', 'classes', 'method', 'methods', + 'trigger', 'triggers', 'affected', 'affect', 'affects', + 'else', 'code', 'failing', 'failed', 'silently', 'decide', 'decides', + 'return', 'returns', 'returned', 'take', 'takes', 'taken', + 'check', 'checks', 'checked', 'create', 'creates', 'created', + 'read', 'reads', 'write', 'writes', 'written', + 'start', 'starts', 'stop', 'stops', 'run', 'runs', 'running', + ]); + return Array.from(symbols).filter((s) => !commonWords.has(s.toLowerCase())); +} + +/** explore's and the Flow's token filter before #1262, uncapped. */ +function oldNameTokens(query: string): string[] { + const FILE_EXT = /\.(?:java|kt|kts|ts|tsx|js|jsx|mjs|cjs|cs|py|go|rb|php|swift|rs|cpp|cc|cxx|c|h|hpp|scala|lua|dart|vue|svelte|astro|erl|hrl)$/i; + return [...new Set( + query.split(/[\s,()[\]]+/) + .map((t) => t.replace(FILE_EXT, '').trim()) + .filter((t) => t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)), + )]; +} diff --git a/__tests__/explore-exact-target.test.ts b/__tests__/explore-exact-target.test.ts index d54d36798f..b1d11958f3 100644 --- a/__tests__/explore-exact-target.test.ts +++ b/__tests__/explore-exact-target.test.ts @@ -107,7 +107,7 @@ class SQLCompiler: def get_select(self, with_col_aliases=False): # GET_SELECT_BODY -${filler('sel', 34)} +${filler('sel', 60)} return [] def get_qualify_sql(self): @@ -235,10 +235,14 @@ class IExact: return result.content?.[0]?.text ?? ''; }; - it('fixture sanity: the flow runs as_sql → get_qualify_sql → get_select and compiler.py goes focused', async () => { + it('fixture sanity: the flow runs as_sql → pre_sql_setup → setup_query → get_select and compiler.py goes focused', async () => { const text = await explore('SQLCompiler.as_sql pre_sql_setup get_select'); expect(text).toContain('**Flow (call path among the symbols you queried)'); - expect(text).toMatch(/1\. as_sql \(compiler\.py:\d+\)[\s\S]*2\. get_qualify_sql[\s\S]*3\. get_select/); + // Through every symbol the query names (#2970), as on django itself: the + // shorter route through the unnamed `get_qualify_sql` skipped `pre_sql_setup`. + // The named and spine bodies outgrow compiler.py's share, as django's do + // (18,390 chars against 8,895), which is what makes it a per-symbol view. + expect(text).toMatch(/1\. as_sql \(compiler\.py:\d+\)[\s\S]*2\. pre_sql_setup[\s\S]*3\. setup_query[\s\S]*4\. get_select/); expect(sectionFor(text, COMPILER), 'the family file renders as a per-symbol view').toContain('· focused'); }); diff --git a/__tests__/flow-named-longer-route.test.ts b/__tests__/flow-named-longer-route.test.ts new file mode 100644 index 0000000000..0bc3f24bfb --- /dev/null +++ b/__tests__/flow-named-longer-route.test.ts @@ -0,0 +1,97 @@ +/** + * Explore's Flow keeps a named symbol that a shorter route reaches first (#2970). + * + * The named walk kept one route per node, replaced only by a better one of the + * same length. In the issue's + * + * entry → mid1 → deep → leaf + * entry → mid2 → leaf + * + * `leaf` was reached first through the unnamed `mid2`, so the route through the + * named `deep` was turned away for being longer, and `entry deep leaf` got the + * Flow `entry → mid1 → deep`, without `leaf`. A node now keeps every route no + * other route to it beats on depth, named symbols and bridge streak together. + * + * What stays: the one-unnamed-bridge budget, the deepest-sink choice, and a + * route never going round a cycle, which a walk that kept routes by named + * count alone would do: `ping → pong → ping → pong → done` counts both twice. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; +import { resolveNamedSymbolFlow } from '../src/graph/named-symbol-flow'; + +const FILES: Record = { + // The issue's example. + 'src/a.ts': `export function entry() { mid1(); mid2(); } +export function mid1() { deep(); } +export function mid2() { leaf(); } +export function deep() { leaf(); } +export function leaf() { return 1; } +`, + // A named step past two unnamed ones on the way: still out of reach. + 'src/budget.ts': `export function origin() { relay(); hop(); } +function relay() { linker(); } +function linker() { dest(); } +function hop() { dest(); } +export function dest() { return 1; } +`, + // Two named functions calling each other on the way to a third. + 'src/cycle.ts': `export function ping(n: number): number { return n > 0 ? pong(n - 1) : done(); } +export function pong(n: number): number { return ping(n); } +export function done(): number { return 0; } +`, +}; + +let root = ''; +let cg: CodeGraph; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-flow-longer-route-')); + for (const [rel, content] of Object.entries(FILES)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + cg = await CodeGraph.init(root, { index: true }); +}); + +afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +/** The lead chain, as names. */ +const lead = (query: string): string[] => (resolveNamedSymbolFlow(cg, query).chains[0]?.steps ?? []).map((s) => s.node.name); + +describe('a named symbol a shorter route reaches first (#2970)', () => { + it('stays on the Flow, through the named step', () => { + expect(lead('entry deep leaf')).toEqual(['entry', 'mid1', 'deep', 'leaf']); + }); + + it("is in codegraph_explore's Flow section", async () => { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query: 'entry deep leaf' }); + const text = res.content?.[0]?.text ?? ''; + const flow = text.slice(text.indexOf('**Flow (call path among the symbols you queried)**')); + expect(flow).toMatch( + /^\*\*Flow[^\n]*\n\n1\. entry \(src\/a\.ts:1\)\n\s+↓ calls\n2\. mid1 \(src\/a\.ts:2\)\n\s+↓ calls\n3\. deep \(src\/a\.ts:4\)\n\s+↓ calls\n4\. leaf \(src\/a\.ts:5\)\n\n/, + ); + }); + + it('still reads one unnamed step, and no further', () => { + expect(lead('origin dest')).toEqual(['origin', 'hop', 'dest']); + expect(lead('entry leaf')).toEqual(['entry', 'mid2', 'leaf']); + }); + + it('never goes round a cycle', () => { + // As on main: the deepest route that passes each node once. + expect(lead('ping pong done')).toEqual(['pong', 'ping', 'done']); + const chains = resolveNamedSymbolFlow(cg, 'ping pong done', { maxChains: 3 }).chains; + for (const chain of chains) { + const ids = chain.steps.map((s) => s.node.id); + expect(new Set(ids).size).toBe(ids.length); + } + }); +}); diff --git a/__tests__/framework-hint-resolver-only.test.ts b/__tests__/framework-hint-resolver-only.test.ts new file mode 100644 index 0000000000..cb7203df38 --- /dev/null +++ b/__tests__/framework-hint-resolver-only.test.ts @@ -0,0 +1,103 @@ +/** + * `codegraph status` recommends a re-index when a framework arrives or leaves + * through `package.json` alone. + * + * #2516 records the frameworks a full index extracted with, and a sync that + * detects a different set says the files indexed before it lack the change. A + * sync detects again only when it extracts a file, though: add `@sveltejs/kit` + * (or `express`) to `package.json` with no source file edited and nothing is + * recorded, so status stayed silent while `goto('/login')` in an unchanged + * `+layout.svelte` (or an unchanged Express app's routes) stayed unlinked. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; + +const KIT = JSON.stringify({ name: 'app', devDependencies: { '@sveltejs/kit': '^2.0.0', svelte: '^4.0.0' } }); +const NO_KIT = JSON.stringify({ name: 'app', devDependencies: { svelte: '^4.0.0' } }); + +describe('framework changes since the last full index, from package.json alone', () => { + let root: string; + let cg: CodeGraph | undefined; + const write = (rel: string, content: string) => { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + }; + const navigates = (): number => { + const nodes = cg!.getNodesInFile('src/routes/+layout.svelte'); + return cg!.getOutgoingEdgesFrom(nodes.map((n) => n.id)).filter((e) => e.kind === 'navigates').length; + }; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fw-hint-')); + write('src/routes/+layout.svelte', '\n'); + write('src/routes/login/+page.svelte', '

login

\n'); + }); + + afterEach(() => { + cg?.destroy(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('names the SvelteKit router when @sveltejs/kit is added after the index', async () => { + write('package.json', NO_KIT); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + + write('package.json', KIT); + cg.close(); + cg = CodeGraph.openSync(root); + await cg.sync(); + // The unchanged layout's goto is still unlinked: only a re-index links it... + expect(navigates()).toBe(0); + // ...which is what status must now say. + expect(cg.getFrameworkChangesSinceIndex()?.added).toContain('sveltekit-router'); + + cg.close(); + cg = await CodeGraph.recreate(root); + await cg.indexAll(); + expect(navigates()).toBe(1); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + }); + + it('names it as removed when the dependency goes', async () => { + write('package.json', KIT); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(navigates()).toBe(1); + + write('package.json', NO_KIT); + cg.close(); + cg = CodeGraph.openSync(root); + expect(cg.getFrameworkChangesSinceIndex()?.removed).toContain('sveltekit-router'); + }); + + it('names Express when it is added with no source file changed', async () => { + write('package.json', JSON.stringify({ name: 'app', dependencies: {} })); + write('src/server.js', "const express = require('express');\nconst app = express();\napp.get('/hello', (req, res) => res.send('hi'));\n"); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + + write('package.json', JSON.stringify({ name: 'app', dependencies: { express: '^4.0.0' } })); + cg.close(); + cg = CodeGraph.openSync(root); + await cg.sync(); + expect(cg.getFrameworkChangesSinceIndex()?.added).toContain('express'); + }); + + it('says nothing when the project is as it was indexed', async () => { + write('package.json', KIT); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + cg.close(); + cg = CodeGraph.openSync(root); + await cg.sync(); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + }); +}); diff --git a/__tests__/mcp-nested-repo-projectpath.test.ts b/__tests__/mcp-nested-repo-projectpath.test.ts index ba5c825584..b0fa858f92 100644 --- a/__tests__/mcp-nested-repo-projectpath.test.ts +++ b/__tests__/mcp-nested-repo-projectpath.test.ts @@ -155,6 +155,19 @@ describe('projectPath inside a nested git repository the ancestor index excludes expect(text(res)).toContain('parent-only.ts'); }); + it('gives the same #155 notice when projectPath names a file in that worktree (#2982)', async () => { + const wt = path.join(parent, 'wt'); + git(parent, 'worktree', 'add', '-q', '-b', 'feature', wt); + handler = new ToolHandler(cg); + const res = await handler.execute('codegraph_search', { + query: 'parentOnly', + projectPath: path.join(wt, 'src', 'sub.ts'), + }); + expect(res.isError).toBeUndefined(); + expect(text(res)).toContain('different git worktree'); + expect(text(res)).toContain('parent-only.ts'); + }); + it('notices the nested repository joining the index without a restart', async () => { handler = new ToolHandler(cg); const before = await handler.execute('codegraph_search', { query: 'nestedOnly', projectPath: nested }); diff --git a/__tests__/mcp-projectpath-file.test.ts b/__tests__/mcp-projectpath-file.test.ts new file mode 100644 index 0000000000..63b9b4ab79 --- /dev/null +++ b/__tests__/mcp-projectpath-file.test.ts @@ -0,0 +1,94 @@ +/** + * A `projectPath` that names a file inside an indexed project (#2982). + * + * Passing the file it is looking at is an easy agent mistake. It used to be + * refused with `isError: true` ("Path is not a directory") while a path that + * doesn't exist at all walked up to the project's `.codegraph/` fine. A file + * now resolves through its directory like any other path inside the project, + * and the sensitive-folder refusal still applies to a file inside one. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { ToolHandler, __setLoadCodeGraphForTests } from '../src/mcp/tools'; + +describe('projectPath naming a file (#2982)', { timeout: 30_000 }, () => { + let project: string; + let handler: ToolHandler | undefined; + + beforeAll(async () => { + project = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-2982-'))); + fs.mkdirSync(path.join(project, 'src')); + fs.writeFileSync(path.join(project, 'src', 'a.ts'), 'export function fileTarget() { return 1; }\n'); + const cg = await CodeGraph.init(project, { config: { include: ['**/*.ts'], exclude: [] } }); + await cg.indexAll(); + cg.close(); + __setLoadCodeGraphForTests(CodeGraph); + }); + + afterAll(async () => { + __setLoadCodeGraphForTests(null); + if (handler) await handler.closeAll(); + fs.rmSync(project, { recursive: true, force: true }); + }); + + it('resolves a file inside the project to that project', async () => { + handler = new ToolHandler(null); + const res = await handler.execute('codegraph_search', { + query: 'fileTarget', + projectPath: path.join(project, 'src', 'a.ts'), + }); + + expect(res.isError).toBeUndefined(); + expect(res.content[0]!.text).toMatch(/fileTarget/); + }); + + it.runIf(process.platform !== 'win32')('still refuses a file inside a sensitive folder', async () => { + const res = await new ToolHandler(null).execute('codegraph_search', { + query: 'anything', + projectPath: '/etc/hosts', + }); + + expect(res.isError).toBe(true); + expect(res.content[0]!.text).toMatch(/sensitive/); + }); + + // A file never reaches more than the folder holding it: whatever the folder's + // answer is (the project's results, a sensitive-folder refusal, the + // not-indexed guidance), the file gets that same answer. + it('answers a file exactly as it answers the folder holding it', async () => { + const ask = async (projectPath: string) => { + const h = new ToolHandler(null); + try { + const res = await h.execute('codegraph_search', { query: 'fileTarget', projectPath }); + return { isError: res.isError, text: res.content[0]!.text }; + } finally { + await h.closeAll(); + } + }; + const ssh = path.join(os.homedir(), '.ssh'); + const outside = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-2982-outside-'))); + try { + fs.mkdirSync(ssh, { recursive: true }); + fs.writeFileSync(path.join(ssh, 'known_hosts'), 'example.com ssh-ed25519 AAAA\n'); + fs.writeFileSync(path.join(outside, 'notes.txt'), 'hi\n'); + const files = [ + path.join(project, 'src', 'a.ts'), + path.join(ssh, 'known_hosts'), + path.join(outside, 'notes.txt'), + ...(process.platform === 'win32' && fs.existsSync('C:\\Windows\\win.ini') ? ['C:\\Windows\\win.ini'] : []), + ]; + for (const file of files) { + expect({ file, ...(await ask(file)) }).toEqual({ file, ...(await ask(path.dirname(file))) }); + } + const refused = await ask(path.join(ssh, 'known_hosts')); + expect(refused.isError).toBe(true); + expect(refused.text).toMatch(/sensitive/); + } finally { + fs.rmSync(ssh, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/__tests__/sync-framework-detection.test.ts b/__tests__/sync-framework-detection.test.ts new file mode 100644 index 0000000000..9be6d58ab4 --- /dev/null +++ b/__tests__/sync-framework-detection.test.ts @@ -0,0 +1,252 @@ +/** + * A long-lived graph keeps framework detection current across syncs (#2516, + * #2519). + * + * The MCP server's watcher holds one CodeGraph for the whole session. Its + * framework detection used to be cached from the first time it ran, so a + * framework that arrived mid-session, like Express in a new `package.json`, + * was missing from every file synced after it until `codegraph index`. A sync + * now detects again when what detection read changed, without walking the + * project on a watcher's scoped sync, and says when files indexed earlier + * were read without the change. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as cp from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { detectFrameworksWithSkips } from '../src/resolution/frameworks'; +import { clearDeclaredDependencies, declaredDependencies } from '../src/resolution/frameworks/package-deps'; +import type { ResolutionContext } from '../src/resolution/types'; + +// Pass-through wrappers, so a test can count framework detections and see the git commands run. +vi.mock('child_process', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, execFileSync: vi.fn(actual.execFileSync) }; +}); +vi.mock('../src/resolution/frameworks', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, detectFrameworksWithSkips: vi.fn(actual.detectFrameworksWithSkips) }; +}); + +const EXPRESS = JSON.stringify({ dependencies: { express: '^4.18.0' } }); +const ROUTES = "const app = express();\napp.get('/hello', function hello(req, res) {});\n"; + +describe('framework detection in a long-lived graph (#2516)', () => { + let root: string; + let cg: CodeGraph | undefined; + const write = (rel: string, body: string) => { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), body); + }; + const routes = (rel: string) => cg!.getNodesInFile(rel).filter((n) => n.kind === 'route').map((n) => n.name); + + beforeEach(() => { + root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-frameworks-'))); + }); + + afterEach(() => { + cg?.close(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it.each(['full', 'scoped'] as const)('reads a file added with a new framework using it (%s sync)', async (mode) => { + write('src/app.ts', 'export function boot() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + // An ordinary edit first, so this graph has detected (no frameworks) and cached it. + write('src/app.ts', 'export function boot() { return 2; }\n'); + await cg.sync(mode === 'scoped' ? { paths: ['src/app.ts'] } : {}); + expect(cg.getDetectedFrameworks()).not.toContain('express'); + + write('package.json', EXPRESS); + write('src/routes.js', ROUTES); + await cg.sync(mode === 'scoped' ? { paths: ['src/routes.js'] } : {}); + expect(cg.getDetectedFrameworks()).toContain('express'); + expect(routes('src/routes.js')).toEqual(['GET /hello']); + }); + + it('detects again when an edit gives a file a framework import the skip table had passed over', async () => { + write('src/a.js', 'export const a = 1;\n'); + write('src/b.js', 'export const b = 1;\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + cg.close(); + // A fresh process: the first sync detects with the skips the index run recorded, so a.js is not read. + cg = CodeGraph.openSync(root); + write('src/b.js', 'export const b = 2;\n'); + await cg.sync({ paths: ['src/b.js'] }); + + // The resolver detects on its own after each sync; the routes show what extraction detected. + write('src/a.js', "import { Hono } from 'hono';\nconst app = new Hono();\napp.get('/ping', () => 'pong');\n"); + await cg.sync({ paths: ['src/a.js'] }); + expect(routes('src/a.js')).toEqual(['GET /ping']); + }); + + it('detects again when a file is added, with no manifest change', async () => { + write('src/app.ts', 'export function boot() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getDetectedFrameworks()).not.toContain('svelte'); + // Svelte is detected from a .svelte file alone. + write('src/Widget.svelte', '\n\n'); + await cg.sync({ paths: ['src/Widget.svelte'] }); + expect(cg.getDetectedFrameworks()).toContain('svelte'); + }); + + it('detects again when a source file detection read is edited', async () => { + write('src/app.ts', 'export function boot() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getDetectedFrameworks()).not.toContain('react-native-bridge'); + // The bridge is detected from a TurboModule spec in the first files' source. + write('src/app.ts', "export const Native = TurboModuleRegistry.getEnforcing('Native');\n"); + await cg.sync({ paths: ['src/app.ts'] }); + expect(cg.getDetectedFrameworks()).toContain('react-native-bridge'); + }); + + it('notices a manifest edit at the next sync, and says what the files indexed before it lack', async () => { + write('src/server.js', ROUTES); + write('src/util.js', 'function util() { return 1; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(routes('src/server.js')).toEqual([]); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + + // No source file changes with the manifest; the next edit anywhere syncs it. + write('package.json', EXPRESS); + write('src/util.js', 'function util() { return 2; }\n'); + await cg.sync({ paths: ['src/util.js'] }); + expect(cg.getDetectedFrameworks()).toContain('express'); + // server.js was indexed without Express and is unchanged: status recommends a re-index. + expect(routes('src/server.js')).toEqual([]); + expect(cg.getFrameworkChangesSinceIndex()).toEqual({ added: ['express'], removed: [] }); + + // Going back to what the index was built with clears it. + write('package.json', JSON.stringify({ dependencies: {} })); + write('src/util.js', 'function util() { return 3; }\n'); + await cg.sync({ paths: ['src/util.js'] }); + expect(cg.getDetectedFrameworks()).not.toContain('express'); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + + // A full re-index reads every file with the new frameworks, and the note goes. + write('package.json', EXPRESS); + cg.close(); + cg = await CodeGraph.recreate(root); + await cg.indexAll(); + expect(routes('src/server.js')).toEqual(['GET /hello']); + expect(cg.getFrameworkChangesSinceIndex()).toBeNull(); + }); +}); + +describe('a scoped sync in a fresh process keeps off the full scan (#2516)', () => { + let root: string; + let cg: CodeGraph | undefined; + const git = (...args: string[]) => cp.execFileSync('git', args, { cwd: root, stdio: 'pipe' }); + // The full scan's untracked listing. (The scope matcher's embedded-repo + // discovery lists with `--directory`, once per process, by design.) + const scans = () => vi.mocked(cp.execFileSync).mock.calls + .filter(([cmd, args]) => cmd === 'git' && Array.isArray(args) && args.join(' ') === 'ls-files -z -o --exclude-standard'); + + beforeEach(() => { + root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-noscan-'))); + }); + + afterEach(() => { + cg?.close(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('detects from the index, once, and not again while nothing it read changed', async () => { + git('init', '-q'); + git('config', 'user.email', 'test@example.invalid'); + git('config', 'user.name', 'Test'); + fs.writeFileSync(path.join(root, '.gitignore'), '.codegraph/\n'); + fs.writeFileSync(path.join(root, 'package.json'), EXPRESS); + fs.mkdirSync(path.join(root, 'src')); + fs.writeFileSync(path.join(root, 'src', 'routes.js'), ROUTES); + fs.writeFileSync(path.join(root, 'src', 'tool.py'), 'def run():\n return 1\n'); + git('add', '-A'); + git('commit', '-qm', 'init'); + const built = CodeGraph.initSync(root); + await built.indexAll(); + built.close(); + + // The watcher's process: open, then sync one edited file at a time. + cg = CodeGraph.openSync(root); + vi.mocked(cp.execFileSync).mockClear(); + vi.mocked(detectFrameworksWithSkips).mockClear(); + // The fork's http-routing detector reads every JS/TS source file, so an edit to one + // is detection input and detects again; the first sync edits routes.js to show that. + // The edits after it are to a Python file no detector reads. + fs.writeFileSync(path.join(root, 'src', 'routes.js'), ROUTES + '// edit 1\n'); + await cg.sync({ paths: ['src/routes.js'] }); + for (const n of [2, 3]) { + fs.writeFileSync(path.join(root, 'src', 'tool.py'), `def run():\n return ${n}\n`); + await cg.sync({ paths: ['src/tool.py'] }); + } + expect(scans()).toEqual([]); + // The fork's sync detects with the skip table; the resolver's own pass after each sync passes no `stale`. + const syncDetections = vi.mocked(detectFrameworksWithSkips).mock.calls.filter(([, , options]) => options?.stale); + expect(syncDetections).toHaveLength(1); + expect(cg.getNodesInFile('src/routes.js').filter((n) => n.kind === 'route').map((n) => n.name)).toEqual(['GET /hello']); + }); +}); + +describe('declared dependencies in a long-lived graph (#2519)', () => { + let root: string; + let cg: CodeGraph | undefined; + const write = (rel: string, body: string) => { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), body); + }; + + beforeEach(() => { + root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-deps-'))); + }); + + afterEach(() => { + cg?.close(); + cg = undefined; + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('sees one app swapped for another with as many files', async () => { + write('web/package.json', JSON.stringify({ dependencies: { react: '^18.0.0' } })); + write('web/a.js', 'export function a() { return 1; }\n'); + write('web/b.js', 'export function b() { return 2; }\n'); + cg = CodeGraph.initSync(root); + await cg.indexAll(); + expect(cg.getDetectedFrameworks()).toContain('react'); + + fs.rmSync(path.join(root, 'web'), { recursive: true, force: true }); + write('server/package.json', EXPRESS); + write('server/a.js', ROUTES); + write('server/b.js', 'export function b() { return 2; }\n'); + await cg.sync(); + expect(cg.getDetectedFrameworks()).toContain('express'); + expect(cg.getDetectedFrameworks()).not.toContain('react'); + cg.reinitializeResolver(); + expect(cg.getDetectedFrameworks()).toContain('express'); + expect(cg.getDetectedFrameworks()).not.toContain('react'); + }); + + it('reads the manifests again once told to forget them', () => { + write('package.json', JSON.stringify({ dependencies: { react: '^18.0.0' } })); + const files = ['a.js']; + const context = { + getAllFiles: () => files, + fileExists: (rel: string) => fs.existsSync(path.join(root, rel)), + readFile: (rel: string) => (fs.existsSync(path.join(root, rel)) ? fs.readFileSync(path.join(root, rel), 'utf8') : null), + } as unknown as ResolutionContext; + expect([...declaredDependencies(context)]).toEqual(['react']); + write('package.json', EXPRESS); + // Same file count: the count alone keeps the old answer. + expect([...declaredDependencies(context)]).toEqual(['react']); + clearDeclaredDependencies(context); + expect([...declaredDependencies(context)]).toEqual(['express']); + }); +}); diff --git a/__tests__/worktree-detection.test.ts b/__tests__/worktree-detection.test.ts index a7eada3334..3e405b9719 100644 --- a/__tests__/worktree-detection.test.ts +++ b/__tests__/worktree-detection.test.ts @@ -21,6 +21,7 @@ import { detectUnindexedWorktree, worktreeMismatchWarning, unindexedWorktreeNotice, + worktreeMismatchNotice, gitWorktreeRoot, gitWorktreeRoots, gitCommonDir, @@ -108,10 +109,29 @@ describe('detectWorktreeIndexMismatch (issue #155)', () => { const msg = worktreeMismatchWarning(detectWorktreeIndexMismatch(worktree, mainRepo)!); expect(msg).toContain(real(worktree)); expect(msg).toContain(real(mainRepo)); - expect(msg).toContain('codegraph init'); + expect(msg).toContain('Run "codegraph init" in this worktree'); + expect(msg).not.toContain('init -i'); + }); + + it('words the fix as the user\'s option when an agent reads it', () => { + const m = detectWorktreeIndexMismatch(worktree, mainRepo)!; + expectAgentAdvice(worktreeMismatchWarning(m, 'agent')); + expectAgentAdvice(worktreeMismatchNotice(m)); }); }); +/** + * What an agent is told about a borrowed index. Indexing is the user's + * decision (the server instructions tell the agent not to run `codegraph init` + * itself), so the text says the user can build a worktree-local index, never + * tells the agent to, and doesn't name the `-i` flag, which does nothing now. + */ +function expectAgentAdvice(text: string): void { + expect(text).toContain('The user can run "codegraph init" in this worktree'); + expect(text).not.toContain('Run "codegraph init'); + expect(text).not.toContain('init -i'); +} + /** * The inverse case (issue #1236): a worktree placed OUTSIDE the main checkout. * @@ -272,7 +292,7 @@ describe('worktree mismatch surfaces on hot read tools (issue #155)', () => { expect(res.isError).toBeFalsy(); expect(text).toContain('different git worktree'); expect(text).toContain(real(worktree)); - expect(text).toContain('codegraph init'); + expectAgentAdvice(text); }); it('does NOT prefix when the default project is the main checkout itself', async () => { @@ -287,6 +307,7 @@ describe('worktree mismatch surfaces on hot read tools (issue #155)', () => { const text = res.content[0].text; expect(text).toContain('different git working tree'); expect(text).toContain(real(worktree)); + expectAgentAdvice(text); }); it('caches detection — a later tool call needs no further git spawn', async () => { @@ -389,7 +410,7 @@ describe('worktree mismatch verdict re-resolves when the index root changes (iss * super-repo descends into its submodules and gitlinked clones, so a query run * from inside one resolves up to the parent index — which genuinely contains * that nested repo's symbols. The warning's premise is false there, and its - * "run codegraph init -i" advice would fragment the unified index. (#1031, #1033) + * "codegraph init" advice would fragment the unified index. (#1031, #1033) */ describe('detectWorktreeIndexMismatch — nested repos covered by the parent index (#1031, #1033)', () => { let parent: string; // super-repo that owns the .codegraph index diff --git a/docs/SEARCH_QUALITY_LOOP.md b/docs/SEARCH_QUALITY_LOOP.md index 97d57dedf4..3c70027a9b 100644 --- a/docs/SEARCH_QUALITY_LOOP.md +++ b/docs/SEARCH_QUALITY_LOOP.md @@ -11,10 +11,10 @@ A language is NOT verified until an LLM can reliably use CodeGraph's MCP tools t ```bash npm run build rm -rf /.codegraph -node dist/bin/codegraph.js init -iv +node dist/bin/codegraph.js init -v ``` -The `-iv` flag gives verbose output showing extraction progress, node/edge counts, and timing. +`init` builds the index; the `-v` flag gives verbose output showing extraction progress, node/edge counts, and timing. ### 2. Quick sanity check @@ -470,7 +470,7 @@ test().catch(console.error); ```bash npm run build rm -rf /.codegraph -node dist/bin/codegraph.js init -iv +node dist/bin/codegraph.js init -v # Re-run the failing tests from above ``` diff --git a/docs/design/callback-edge-synthesis.md b/docs/design/callback-edge-synthesis.md index 0e8a018808..ce11105f1b 100644 --- a/docs/design/callback-edge-synthesis.md +++ b/docs/design/callback-edge-synthesis.md @@ -37,7 +37,7 @@ We synthesize `dispatcher → callback` edges that static parsing misses. It wor ```bash npm run build rm -rf /tmp/codegraph-corpus/excalidraw/.codegraph -( cd /tmp/codegraph-corpus/excalidraw && codegraph init -i ) +( cd /tmp/codegraph-corpus/excalidraw && codegraph init ) # synthesized edges (provenance='heuristic', metadata.synthesizedBy in {callback,event-emitter,…,http-client,queue-job,event-bus}): sqlite3 /tmp/codegraph-corpus/excalidraw/.codegraph/codegraph.db \ "select s.name||' → '||t.name||' '||coalesce(e.metadata,'') from edges e \ diff --git a/docs/design/dynamic-dispatch-coverage-playbook.md b/docs/design/dynamic-dispatch-coverage-playbook.md index 61965e352b..0ccc629e0d 100644 --- a/docs/design/dynamic-dispatch-coverage-playbook.md +++ b/docs/design/dynamic-dispatch-coverage-playbook.md @@ -150,7 +150,7 @@ question and a real repo (add to `.claude/skills/agent-eval/corpus.json`). Examp ### Step 2 — Measure the hole (deterministic, no agent) ```bash -rm -rf /.codegraph && ( cd && codegraph init -i ) +rm -rf /.codegraph && ( cd && codegraph init ) node scripts/agent-eval/probe-explore.mjs " " # does the Flow section connect them? node scripts/agent-eval/probe-node.mjs # trail: is the next hop missing? ``` @@ -228,7 +228,7 @@ point **locates the hole**. Confirm it's dynamic by reading the break symbol's b | `sqlite3 /.codegraph/codegraph.db` | direct edge/node inspection (provenance, metadata, counts) | Probe scripts use the built `dist/` — run `npm run build` first. Reindex after any -extraction or resolution change (`rm -rf /.codegraph && codegraph init -i`) — the +extraction or resolution change (`rm -rf /.codegraph && codegraph init`) — the synthesizer/resolvers run at index time. Test fixtures: keep a tiny per-pattern fixture (see `/tmp/cb-fixture/bus.js`; **move into `__tests__/`** when shipping). diff --git a/scripts/add-lang/bench.sh b/scripts/add-lang/bench.sh index b0fb30ee88..5110ca0c81 100755 --- a/scripts/add-lang/bench.sh +++ b/scripts/add-lang/bench.sh @@ -10,6 +10,10 @@ # # Usage: bench.sh "" [headless|tmux|all] # Env: CORPUS corpus dir (default ~/codegraph-corpora, shared with agent-eval) +# CODEGRAPH_BIN index and verify with this codegraph instead of the one on +# PATH: a binary, or a build's dist/bin/codegraph.js (run with +# node). Exported, so verify-extraction.mjs uses it too. The A/B's +# MCP server is run-all.sh's own CG_BIN (default: codegraph on PATH). set -uo pipefail LANG_TOKEN="${1:?usage: bench.sh \"\" [mode]}" @@ -23,10 +27,25 @@ AGENT_EVAL="$(cd "$HARNESS/../agent-eval" && pwd)" CORPUS="${CORPUS:-$HOME/codegraph-corpora}" REPO="$CORPUS/$NAME" -command -v codegraph >/dev/null || { echo "no codegraph on PATH (build + ./scripts/local-install.sh first)"; exit 1; } +if [ -n "${CODEGRAPH_BIN:-}" ]; then + # A path is made absolute, since indexing runs from inside the repo. + if [ -e "$CODEGRAPH_BIN" ]; then + CODEGRAPH_BIN="$(cd "$(dirname "$CODEGRAPH_BIN")" && pwd)/$(basename "$CODEGRAPH_BIN")" + else + command -v "$CODEGRAPH_BIN" >/dev/null || { echo "CODEGRAPH_BIN not found: $CODEGRAPH_BIN"; exit 1; } + fi + export CODEGRAPH_BIN + # Stands in for the PATH codegraph in this script's own calls. + case "$CODEGRAPH_BIN" in + *.js|*.mjs|*.cjs) codegraph() { node "$CODEGRAPH_BIN" "$@"; } ;; + *) codegraph() { "$CODEGRAPH_BIN" "$@"; } ;; + esac +else + command -v codegraph >/dev/null || { echo "no codegraph on PATH (build + ./scripts/local-install.sh first)"; exit 1; } +fi echo "==================== add-lang bench: $NAME ($LANG_TOKEN) ====================" -echo "codegraph: $(command -v codegraph) -> $(codegraph --version 2>/dev/null || echo '?')" +echo "codegraph: ${CODEGRAPH_BIN:-$(command -v codegraph)} -> $(codegraph --version 2>/dev/null || echo '?')" # 1. Ensure the repo (shallow clone, reuse if present). mkdir -p "$CORPUS" @@ -40,7 +59,7 @@ fi # 2. Wipe + index with the binary under test. echo "→ wiping .codegraph and indexing" rm -rf "$REPO/.codegraph" -( cd "$REPO" && codegraph init -i ) || { echo "indexing failed"; exit 1; } +( cd "$REPO" && codegraph init ) || { echo "indexing failed"; exit 1; } # 3. Verify extraction (cheap guard before the paid A/B). echo "→ verifying extraction" diff --git a/scripts/add-lang/verify-extraction.mjs b/scripts/add-lang/verify-extraction.mjs index ac70f59c92..febf0a8a64 100755 --- a/scripts/add-lang/verify-extraction.mjs +++ b/scripts/add-lang/verify-extraction.mjs @@ -4,12 +4,16 @@ // can drive a write-extractor -> build -> re-check loop. // // Usage: node scripts/add-lang/verify-extraction.mjs -// Set CG_BIN to the executable or JavaScript CLI that built the index. -// Without CG_BIN, uses `codegraph` from PATH. +// Reads `codegraph status --json` using whatever codegraph is on PATH, +// so it reflects the binary that built the index. Set CODEGRAPH_BIN (or CG_BIN) +// to read it with another one: a binary, or a build's dist/bin/codegraph.js, +// which runs with this node. bench.sh exports CODEGRAPH_BIN when it is given one. // // Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run. import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import path from 'node:path'; const [repo, lang] = process.argv.slice(2); if (!repo || !lang) { @@ -17,13 +21,52 @@ if (!repo || !lang) { process.exit(2); } +// On Windows, npm installs codegraph as a .cmd shim (`npm link` too), and +// execFileSync can neither find a .cmd by bare name nor run one without a +// shell. So look where Windows would: the first PATH folder holding codegraph +// under one of PATHEXT's runnable extensions. Only a .cmd or .bat needs the +// shell; an .exe or .com is found by the plain call below on its own. +function findWindowsShim() { + const runnable = ['.com', '.exe', '.bat', '.cmd']; + const exts = (process.env.PATHEXT || runnable.join(';')).toLowerCase().split(';').filter((e) => runnable.includes(e)); + for (const dir of (process.env.PATH || '').split(path.delimiter)) { + if (!dir) continue; + for (const ext of exts) { + const file = path.join(dir, `codegraph${ext}`); + if (existsSync(file)) return ext === '.bat' || ext === '.cmd' ? file : null; + } + } + return null; +} + +// One token of a command line for cmd.exe, which hands it on to node.exe +// through the shim's %*. Double quotes keep it one argument for the C runtime +// that splits node.exe's command line, so a backslash run before the closing +// quote is doubled (or it would escape that quote). Then every cmd +// metacharacter, the quotes included, gets a caret, so cmd neither splits the +// token nor expands a %NAME% inside it. A Windows path can't contain a double +// quote, so a token with one is refused rather than mangled. +function quoteForCmd(token) { + if (/["\r\n]/.test(token)) throw new Error(`cannot pass ${JSON.stringify(token)} through cmd.exe`); + return `"${token.replace(/(\\+)$/, '$1$1')}"`.replace(/[()[\]%!^"`<>&|;, *?]/g, '^$&'); +} + +function runCodegraph(args) { + const options = { encoding: 'utf8' }; + const bin = process.env.CODEGRAPH_BIN || process.env.CG_BIN || (process.platform === 'win32' ? findWindowsShim() : null); + if (!bin) return execFileSync('codegraph', args, options); + if (/\.[cm]?js$/i.test(bin)) return execFileSync(process.execPath, [bin, ...args], options); + if (process.platform === 'win32' && /\.(bat|cmd)$/i.test(bin)) { + // A single command string with no args array: Node hands it to cmd.exe + // unchanged, so the quoting above is all there is. + return execFileSync([bin, ...args].map(quoteForCmd).join(' '), { ...options, shell: true }); + } + return execFileSync(bin, args, options); +} + let status; try { - const binary = process.env.CG_BIN || 'codegraph'; - const args = ['status', repo, '--json']; - const javascript = /\.[cm]?js$/i.test(binary); - const out = execFileSync(javascript ? process.execPath : binary, - javascript ? [binary, ...args] : args, { encoding: 'utf8' }); + const out = runCodegraph(['status', repo, '--json']); status = JSON.parse(out); } catch (e) { console.error(`[verify] could not read codegraph status for ${repo}: ${e.message}`); diff --git a/scripts/agent-eval/audit.sh b/scripts/agent-eval/audit.sh index e3d026a7cd..685e904855 100755 --- a/scripts/agent-eval/audit.sh +++ b/scripts/agent-eval/audit.sh @@ -54,6 +54,8 @@ fi # 3. Wipe + re-index with THIS version (the index must be built by the same # binary that serves it — different versions extract differently). +# Keep -i: may predate 0.9.8, where `init` built the index only +# with it. Newer versions index by default and still accept the flag. echo "→ [3/4] wiping .codegraph and re-indexing with $ACTUAL" rm -rf "$REPO/.codegraph" ( cd "$REPO" && "$CG_BIN" init -i ) diff --git a/site/src/content/docs/getting-started/installation.md b/site/src/content/docs/getting-started/installation.md index 4f9b90986f..2ef050e677 100644 --- a/site/src/content/docs/getting-started/installation.md +++ b/site/src/content/docs/getting-started/installation.md @@ -11,10 +11,10 @@ npx @colbymchenry/codegraph The installer will: -- Ask which agent(s) to configure — auto-detecting installed ones from **Claude Code**, **Cursor**, **Codex CLI**, **opencode**, **Hermes Agent**, **Gemini CLI**, **Antigravity IDE**, and **Kiro**. +- Ask which agent(s) to configure — auto-detecting installed ones from **Claude Code**, **Cursor**, **Codex CLI**, **opencode**, **Hermes Agent**, **Gemini CLI**, **Antigravity IDE**, **Kiro**, and **GitHub Copilot** (VS Code, Copilot CLI, JetBrains IDEs). - Prompt to install `codegraph` on your `PATH` (so agents can launch the MCP server). - Ask whether configs apply to all your projects or just this one. -- Write each chosen agent's MCP server config, plus a small marker-fenced CodeGraph section in the agent's instructions file (`CLAUDE.md` / `AGENTS.md` / `GEMINI.md`). Cursor and Kiro get the MCP config only. Removed cleanly by `codegraph uninstall`. +- Write each chosen agent's MCP server config. Claude Code, Codex CLI, Gemini CLI and opencode also get a small marker-fenced CodeGraph section in their instructions file (`CLAUDE.md` / `AGENTS.md` / `GEMINI.md`); the others get the MCP config only. Removed cleanly by `codegraph uninstall`. - Set up auto-allow permissions when Claude Code is one of the targets. The installer **wires up your agents only — it does not index your code.** After it finishes, build each project's graph yourself with `codegraph init` (step 3 below). @@ -38,7 +38,7 @@ codegraph install --print-config codex # print snippet, no file wr ## 2. Restart your agent -Restart your agent (Claude Code / Cursor / Codex CLI / opencode / Hermes Agent / Gemini CLI / Antigravity IDE / Kiro) for the MCP server to load. +Restart your agent (Claude Code / Cursor / Codex CLI / opencode / Hermes Agent / Gemini CLI / Antigravity IDE / Kiro / VS Code, the Copilot CLI, or your JetBrains IDE for GitHub Copilot) for the MCP server to load. ## 3. Initialize projects diff --git a/site/src/content/docs/getting-started/introduction.md b/site/src/content/docs/getting-started/introduction.md index 13d5ce8375..045c440f5b 100644 --- a/site/src/content/docs/getting-started/introduction.md +++ b/site/src/content/docs/getting-started/introduction.md @@ -5,7 +5,7 @@ description: What CodeGraph is, and why it makes AI coding agents faster and mor CodeGraph is a **local-first code-intelligence tool**. It parses your codebase with [tree-sitter](https://tree-sitter.github.io/), stores every symbol, edge, and file in a local SQLite database, and exposes the result as a queryable **knowledge graph** — over the [Model Context Protocol (MCP)](/codegraph/reference/mcp-server/), a CLI, and a TypeScript library. -It exists to make AI coding agents — Claude Code, Cursor, Codex CLI, opencode, Hermes Agent, Gemini CLI, Antigravity IDE, and Kiro — **answer structural questions without scanning files**. Instead of fanning out across `grep`, `glob`, and `Read` to reconstruct how code fits together, an agent queries a pre-built index and gets the answer in a handful of calls. +It exists to make AI coding agents — Claude Code, Cursor, Codex CLI, opencode, Hermes Agent, Gemini CLI, Antigravity IDE, Kiro, and GitHub Copilot — **answer structural questions without scanning files**. Instead of fanning out across `grep`, `glob`, and `Read` to reconstruct how code fits together, an agent queries a pre-built index and gets the answer in a handful of calls. ## Why it matters diff --git a/site/src/content/docs/getting-started/quickstart.md b/site/src/content/docs/getting-started/quickstart.md index e1ea543f0d..f7d94074ed 100644 --- a/site/src/content/docs/getting-started/quickstart.md +++ b/site/src/content/docs/getting-started/quickstart.md @@ -25,7 +25,7 @@ Already have Node? `npm i -g @colbymchenry/codegraph` works on any version. Code codegraph install ``` -Auto-detects and configures Claude Code, Cursor, Codex CLI, opencode, Hermes Agent, Gemini CLI, Antigravity IDE, and Kiro — wiring the CodeGraph MCP server into each. This step connects your agents only; it does **not** index any code. (Shortcut: `npx @colbymchenry/codegraph` downloads and runs the installer in one go.) +Auto-detects and configures Claude Code, Cursor, Codex CLI, opencode, Hermes Agent, Gemini CLI, Antigravity IDE, Kiro, and GitHub Copilot (VS Code, Copilot CLI, JetBrains IDEs) — wiring the CodeGraph MCP server into each. This step connects your agents only; it does **not** index any code. (Shortcut: `npx @colbymchenry/codegraph` downloads and runs the installer in one go.) ## 3. Initialize each project diff --git a/site/src/content/docs/guides/indexing.md b/site/src/content/docs/guides/indexing.md index a72083f97f..46ca286a7c 100644 --- a/site/src/content/docs/guides/indexing.md +++ b/site/src/content/docs/guides/indexing.md @@ -26,7 +26,7 @@ When a file named by an unresolved import appears later, incremental sync retrie ## Stay fresh automatically -**You don't need to run `codegraph sync` by hand during an agent session.** When your agent (Claude Code, Cursor, Codex, opencode, Hermes, Gemini, Antigravity, Kiro) launches `codegraph serve --mcp`, three layers cooperate to keep the index in step with your code — and to never give the agent a quiet wrong answer in the small window between an edit and the next sync. +**You don't need to run `codegraph sync` by hand during an agent session.** When your agent (Claude Code, Cursor, Codex, opencode, Hermes, Gemini, Antigravity, Kiro, GitHub Copilot) launches `codegraph serve --mcp`, three layers cooperate to keep the index in step with your code — and to never give the agent a quiet wrong answer in the small window between an edit and the next sync. ### 1. File watcher with debounced auto-sync (always on) diff --git a/site/src/content/docs/reference/integrations.md b/site/src/content/docs/reference/integrations.md index ffc4da8c3e..c4ee89cadc 100644 --- a/site/src/content/docs/reference/integrations.md +++ b/site/src/content/docs/reference/integrations.md @@ -9,13 +9,13 @@ The interactive installer auto-detects and configures each supported agent — w - **Claude Code** - **Cursor** -- **Codex CLI** +- **Codex CLI** — the Codex IDE extension reads the same config: a global install writes `$CODEX_HOME/config.toml` (default `~/.codex/config.toml`), and a local install writes `.codex/config.toml`, which Codex uses only in a project you've marked as trusted - **opencode** - **Hermes Agent** - **Gemini CLI** - **Antigravity IDE** - **Kiro** -- **GitHub Copilot** (VS Code, Copilot CLI, JetBrains IDEs) +- **GitHub Copilot** — Copilot Chat in VS Code (`copilot-vscode`), the Copilot CLI (`copilot-cli`), and the Copilot plugin in JetBrains IDEs (`copilot-jetbrains`) - **Devin** (CLI and Desktop) Run `npx @colbymchenry/codegraph` and pick your agent(s); see [Installation](/codegraph/getting-started/installation/) for the non-interactive flags. diff --git a/site/src/pages/index.astro b/site/src/pages/index.astro index a469c00b4f..be97b13542 100644 --- a/site/src/pages/index.astro +++ b/site/src/pages/index.astro @@ -94,7 +94,7 @@ const install = 'npx @colbymchenry/codegraph';

MCP server

Expose the graph to Claude Code, Cursor, Codex, opencode, Hermes, - Gemini, Antigravity, and Kiro over MCP — agents answer in a handful of calls. + Gemini, Antigravity, Kiro, and GitHub Copilot over MCP — agents answer in a handful of calls.

diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index f44e8f4adb..e6796a999f 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -88,6 +88,7 @@ import { BROWSER_ENV, DEFAULT_UI_PORT } from '../ui-server/constants'; import type { UiServerHandle } from '../ui-server'; import { lookupSymbolNodes, describeSymbolNode, groupDefinitions } from '../graph/symbol-lookup'; import { isDocumentationNode } from '../graph/traversal'; +import { callSiteLines, formatCallSiteLines } from '../graph/call-sites'; import { goPackageReaderFor } from '../graph/go-package'; import type { Node, Edge } from '../types'; import { isTestPath } from '../search/query-utils'; @@ -411,6 +412,9 @@ function cliNode(node: Node) { return { name: node.name, kind: node.kind, filePath: node.filePath, startLine: node.startLine }; } +/** Call-site lines `codegraph callers` prints for one caller; `--json` lists them all. */ +const CLI_CALL_SITES_SHOWN = 10; + /** Attribute a group's edges to every overload of this definition. */ function cliDefinition(group: Node[]) { const head = group[0]!; @@ -1198,6 +1202,10 @@ program const buildInfo = cg.getIndexBuildInfo(); const reindexRecommended = cg.isIndexStale(); + // Frameworks that came or went since the last full index (#2516), as a + // sync recorded them or as detected now: files indexed before then were + // read without the change. + const frameworkChanges = cg.getFrameworkChangesSinceIndex(); const indexState = cg.getIndexState(); // Zero on a healthy index; non-zero at rest means a resolution pass was // interrupted, so some files' call edges are missing (#1187). @@ -1236,7 +1244,9 @@ program builtWithVersion: buildInfo.version, builtWithExtractionVersion: buildInfo.extractionVersion, currentExtractionVersion: EXTRACTION_VERSION, - reindexRecommended, + reindexRecommended: reindexRecommended || frameworkChanges !== null, + // { added, removed } frameworks since the last full index, or null. + frameworksChangedSinceIndex: frameworkChanges, // 'complete' | 'partial' (files silently dropped) | 'indexing' // (a run was killed mid-index — the index is truncated) | // 'failed' | null (predates the marker). @@ -1359,6 +1369,15 @@ program info('Run "codegraph index" (full rebuild) or "codegraph sync"'); console.log(); } + if (frameworkChanges) { + const changed = [ + ...(frameworkChanges.added.length > 0 ? [`added ${frameworkChanges.added.join(', ')}`] : []), + ...(frameworkChanges.removed.length > 0 ? [`removed ${frameworkChanges.removed.join(', ')}`] : []), + ].join('; '); + warn(`Frameworks changed since the last full index (${changed}); files indexed before then were read without the change.`); + info('Run "codegraph index" to re-index every file with them'); + console.log(); + } cg.destroy(); } catch (err) { @@ -2471,29 +2490,47 @@ for (const direction of ['callers', 'callees'] as const) { const note = filteredOut ? `no definition of "${symbol}" matches file "${options.file}" — showing all definitions instead.` : undefined; + // Every site of each neighbour (#2909): a caller that calls the symbol on + // two lines has both, and a symbol that calls itself is a row of its own. const collected = groups.map((group) => { const nodes = new Map(); const edges = new Map(); + const recursive = new Set(); for (const target of group) { - const connections = direction === 'callers' ? cg.getCallers(target.id) : cg.getCallees(target.id); - for (const { node, edge } of connections) { - nodes.set(node.id, node); - edges.set(`${edge.source}->${edge.target}:${edge.kind}`, edge); + for (const row of cg.getCallSites(target.id, direction)) { + nodes.set(row.node.id, row.node); + if (row.recursive) recursive.add(row.node.id); + for (const edge of row.edges) { + edges.set(`${edge.source}|${edge.target}|${edge.kind}|${edge.line ?? -1}|${edge.column ?? -1}`, edge); + } } } - return { group, nodes: [...nodes.values()], edges: [...edges.values()] }; + return { group, nodes: [...nodes.values()], edges: [...edges.values()], recursive }; }); - const relationships = (node: Node, edges: Edge[]) => [...new Set(edges - .filter((edge) => (direction === 'callers' ? edge.source : edge.target) === node.id) - .map((edge) => edge.kind))]; + const ownEdges = (node: Node, edges: Edge[]) => + edges.filter((edge) => (direction === 'callers' ? edge.source : edge.target) === node.id); + const relationships = (node: Node, edges: Edge[]) => [...new Set(ownEdges(node, edges).map((edge) => edge.kind))]; + // A caller is listed at the lines its calls sit on, the lines to edit; a + // callee at its own declaration, where a call leads. + const location = (node: Node, edges: Edge[]) => { + const lines = direction === 'callers' ? callSiteLines(ownEdges(node, edges)) : []; + if (lines.length > 0) return `:${formatCallSiteLines(lines, CLI_CALL_SITES_SHOWN)}`; + return node.startLine ? `:${node.startLine}` : ''; + }; + const row = (node: Node, edges: Edge[], recursive: boolean) => ({ + ...cliNode(node), + relationships: relationships(node, edges), + lines: callSiteLines(ownEdges(node, edges)), + recursive, + }); if (options.json) { - const definitions = collected.map(({ group, nodes, edges }) => { + const definitions = collected.map(({ group, nodes, edges, recursive }) => { const limited = nodes.slice(0, limit); const shown = new Set(limited.map((node) => node.id)); return { ...cliDefinition(group), - [direction]: limited.map((node) => ({ id: node.id, ...cliNode(node), relationships: relationships(node, edges) })), + [direction]: limited.map((node) => ({ id: node.id, ...row(node, edges, recursive.has(node.id)) })), edges: edges.filter((edge) => shown.has(direction === 'callers' ? edge.source : edge.target)), total: nodes.length, limit, @@ -2505,6 +2542,7 @@ for (const direction of ['callers', 'callees'] as const) { for (const node of nodes) union.set(node.id, node); } const total = union.size; + const allEdges = collected.flatMap((entry) => entry.edges); console.log(JSON.stringify({ symbol, targets: groups.flat().map((node) => cliDefinition([node]).definition), @@ -2514,8 +2552,8 @@ for (const direction of ['callers', 'callees'] as const) { filteredOut, note, definitions, - [direction]: [...union.values()].slice(0, limit).map((node) => ({ ...cliNode(node), - relationships: relationships(node, collected.flatMap((entry) => entry.edges)) })), + [direction]: [...union.values()].slice(0, limit).map((node) => + row(node, allEdges, collected.some((entry) => entry.recursive.has(node.id)))), total, limit, truncated: total > limit, @@ -2525,7 +2563,7 @@ for (const direction of ['callers', 'callees'] as const) { if (ambiguous) { console.log(chalk.bold(`\n${title} of "${symbol}" — ${groups.length} distinct definitions (narrow with --file):`)); } - for (const { group, nodes, edges } of collected) { + for (const { group, nodes, edges, recursive } of collected) { const limited = nodes.slice(0, limit); const total = nodes.length; const truncated = total > limit; @@ -2541,11 +2579,11 @@ for (const direction of ['callers', 'callees'] as const) { else info(`No ${direction} found for "${symbol}"`); } for (const node of limited) { - const loc = node.startLine ? `:${node.startLine}` : ''; const kinds = relationships(node, edges).filter((kind) => kind !== 'calls'); const relation = kinds.length ? ` [${kinds.join(', ')}]` : ''; - console.log(chalk.cyan(node.kind.padEnd(12)) + chalk.white(node.name) + chalk.dim(relation)); - console.log(chalk.dim(` ${node.filePath}${loc}`)); + const itself = recursive.has(node.id) ? ' (calls itself)' : ''; + console.log(chalk.cyan(node.kind.padEnd(12)) + chalk.white(node.name) + chalk.dim(itself + relation)); + console.log(chalk.dim(` ${node.filePath}${location(node, edges)}`)); console.log(); } if (truncated) console.log(chalk.dim(`Showing ${limited.length} of ${total}; pass --limit to widen.`)); diff --git a/src/codegraph.ts b/src/codegraph.ts index e399e7402e..f4c7a34063 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -49,6 +49,7 @@ import type { } from './extraction'; import type { ReferenceResolver, ResolutionResult } from './resolution'; import { GraphTraverser, GraphQueryManager } from './graph'; +import type { CallSiteRow } from './graph'; import { findNamedCopybooks, type NamedCopybook } from './graph/cobol-copybooks'; import { ContextBuilder, createContextBuilder } from './context'; import { Mutex, FileLock, escapesRoot } from './utils'; @@ -150,6 +151,32 @@ export interface IndexOptions { /** project_metadata key for the Python package-root digest (see sync). */ const PYTHON_ROOTS_METADATA_KEY = 'python_root_fingerprint'; const WORKSPACE_METADATA_KEY = 'js_workspace_fingerprint'; +/** Metadata key: the frameworks (sorted JSON names) the last full index extracted every file with. */ +const FRAMEWORKS_AT_INDEX_KEY = 'frameworks_at_index'; +/** + * Metadata key: `{ added, removed }` — the frameworks a sync detected since the + * last full index, against {@link FRAMEWORKS_AT_INDEX_KEY}; '' when none. + */ +const FRAMEWORKS_SINCE_INDEX_KEY = 'frameworks_since_index'; +/** + * Metadata key: the frameworks (sorted JSON names) the resolver had at the last + * full index. A sync detects again only when it extracts a file, so one that + * finds just `package.json` changed records nothing in + * {@link FRAMEWORKS_SINCE_INDEX_KEY}; the resolver detects on every open, so + * its set is compared against this one as it is now. + */ +const RESOLVER_FRAMEWORKS_AT_INDEX_KEY = 'resolver_frameworks_at_index'; + +/** A JSON list of names from metadata, or null when absent or unreadable. */ +function parseNameList(value: string | null): string[] | null { + if (!value) return null; + try { + const parsed: unknown = JSON.parse(value); + return Array.isArray(parsed) && parsed.every((n) => typeof n === 'string') ? parsed : null; + } catch { + return null; + } +} export class CodeGraph { private db: DatabaseConnection; @@ -855,6 +882,7 @@ export class CodeGraph { // extraction stamp (the bulk would still be stale). See extraction-version.ts. if (result.success) this.stampPythonRootFingerprint(); if (result.success) this.stampWorkspaceFingerprint(); + if (result.success) this.stampIndexedFrameworks(); if (result.success && result.filesIndexed > 0) { try { @@ -984,6 +1012,83 @@ export class CodeGraph { } catch { /* metadata is advisory */ } } + /** Record the frameworks a full index extracted every file with; none has come or gone since. */ + private stampIndexedFrameworks(): void { + const names = this.orchestrator.getDetectedFrameworkNames(); + if (names === null) return; + try { + this.queries.setMetadata(FRAMEWORKS_AT_INDEX_KEY, JSON.stringify([...names].sort())); + this.queries.setMetadata(FRAMEWORKS_SINCE_INDEX_KEY, ''); + this.queries.setMetadata(RESOLVER_FRAMEWORKS_AT_INDEX_KEY, JSON.stringify([...this.resolver.getDetectedFrameworks()].sort())); + } catch { /* metadata is advisory */ } + } + + /** + * A sync detected frameworks again because what detection read had changed + * (#2516): a framework arrived mid-session, like Express in a new + * `package.json`, or one left. Files the sync extracted were read with the + * new set already. The resolver detects again when its set differs, so + * resolution and the passes after it use the new one too. Files indexed + * before the change were extracted without it, so the change against the + * last full index is recorded for `codegraph status`, which recommends a + * re-index: re-extracting every file a framework can touch would mean + * re-parsing most of the project inside a watcher's sync. + */ + private followDetectedFrameworks(detected: string[]): void { + const now = [...detected].sort(); + const resolverNow = [...this.resolver.getDetectedFrameworks()].sort(); + if (now.join('\0') !== resolverNow.join('\0')) this.resolver.initialize(); + try { + const atIndex = parseNameList(this.queries.getMetadata(FRAMEWORKS_AT_INDEX_KEY)); + if (!atIndex) return; // indexed before this was recorded: nothing to compare with + const added = now.filter((name) => !atIndex.includes(name)); + const removed = atIndex.filter((name) => !now.includes(name)); + const value = added.length > 0 || removed.length > 0 ? JSON.stringify({ added, removed }) : ''; + if ((this.queries.getMetadata(FRAMEWORKS_SINCE_INDEX_KEY) ?? '') !== value) { + this.queries.setMetadata(FRAMEWORKS_SINCE_INDEX_KEY, value); + } + } catch { /* metadata is advisory */ } + } + + /** + * The frameworks that came or went since the last full index (#2516): files + * indexed before the change were read without the frameworks added, and with + * the ones removed, so a full re-index brings them in line. Null when nothing + * changed; an index built before a half of this was recorded gives nothing + * for that half. + * + * The first half is what a sync detected against the frameworks the index + * extracted every file with. + * + * A sync that only finds a manifest changed (`express` or `@sveltejs/kit` + * added to `package.json`, no source file edited) extracts nothing, so it + * never detects again and records nothing: the files indexed before it still + * lack the framework. The resolver's set as detected on open, against the + * one the last full index had, covers that. + */ + getFrameworkChangesSinceIndex(): { added: string[]; removed: string[] } | null { + const names = (list: unknown): string[] => (Array.isArray(list) ? list.filter((n): n is string => typeof n === 'string') : []); + const added = new Set(); + const removed = new Set(); + try { + const value = this.queries.getMetadata(FRAMEWORKS_SINCE_INDEX_KEY); + if (value) { + const parsed = JSON.parse(value) as { added?: unknown; removed?: unknown }; + for (const name of names(parsed.added)) added.add(name); + for (const name of names(parsed.removed)) removed.add(name); + } + } catch { /* metadata is advisory */ } + try { + const atIndex = parseNameList(this.queries.getMetadata(RESOLVER_FRAMEWORKS_AT_INDEX_KEY)); + if (atIndex) { + const now = this.resolver.getDetectedFrameworks(); + for (const name of now) if (!atIndex.includes(name)) added.add(name); + for (const name of atIndex) if (!now.includes(name)) removed.add(name); + } + } catch { /* metadata is advisory */ } + return added.size > 0 || removed.size > 0 ? { added: [...added].sort(), removed: [...removed].sort() } : null; + } + async sync(options: IndexOptions = {}): Promise { if (options.requireComplete && process.env.CODEGRAPH_SYNC_RESYNTHESIS === '0') { throw new Error('Complete reconciliation requires synthesized-edge refresh to be enabled.'); @@ -1102,6 +1207,10 @@ export class CodeGraph { } }); + // Frameworks detected again because what detection read changed: the + // passes below use the new set (#2516). + if (result.detectedFrameworks) this.followDetectedFrameworks(result.detectedFrameworks); + // Fold the store phase's WAL BEFORE the post-store reads below // (resolution reads on the main thread) — same rationale as // indexAll's fold between store and resolution. @@ -2956,6 +3065,18 @@ export class CodeGraph { return this.traverser.getCallees(nodeId, maxDepth); } + /** + * Direct callers or callees of a function/method, each with every site that + * joins them, the symbol itself included when it calls itself + * + * @param nodeId - ID of the function/method node + * @param direction - `'callers'` or `'callees'` + * @returns One row per neighbour, in the order getCallers / getCallees list them + */ + getCallSites(nodeId: string, direction: 'callers' | 'callees'): CallSiteRow[] { + return this.traverser.getCallSites(nodeId, direction); + } + /** * Calculate the impact radius of a node * diff --git a/src/context/index.ts b/src/context/index.ts index d93899fbdf..d143f18ff8 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -25,7 +25,10 @@ import { GraphTraverser } from '../graph'; import { formatContextAsMarkdown, formatContextAsJson } from './formatter'; import { logDebug } from '../errors'; import { validatePathWithinRoot, isConfigLeafNode } from '../utils'; -import { isTestFile, extractSearchTerms, scorePathRelevance, getStemVariants, isDistinctiveIdentifier } from '../search/query-utils'; +import { + isTestFile, extractSearchTerms, scorePathRelevance, getStemVariants, isDistinctiveIdentifier, + blankNonAsciiWords, splitAtCjkSwitches, meetsQueryWordFloor, +} from '../search/query-utils'; import { LOW_CONFIDENCE_MARKER } from './markers'; import { findNamedCopybooks, isCopybookInclude } from '../graph/cobol-copybooks'; import { describeSynthesizedHop } from '../graph/synthesized-hop'; @@ -61,17 +64,21 @@ export const MAX_QUERY_TERMS = 128; * - SCREAMING_SNAKE: MAX_RETRIES * - dot.notation: app.isPackaged (extracts both sides) * - Single words that look like identifiers (no spaces, not common English words) + * - Names in other scripts, whole: 로그인, 寻路服务, état (#516, #1262) * * @param query - Natural language query * @returns Array of potential symbol names */ -function extractSymbolsFromQuery(query: string): string[] { +export function extractSymbolsFromQuery(query: string): string[] { const symbols = new Set(); + // The ASCII patterns below would read `état` as `tat`; they see such words + // blanked out, and the Unicode pass at the end takes them whole. + const asciiView = blankNonAsciiWords(query); // Extract CamelCase identifiers (2+ chars, starts with letter) const camelCasePattern = /\b([A-Z][a-z]+(?:[A-Z][a-z]*)*|[a-z]+(?:[A-Z][a-z]*)+)\b/g; let match; - while ((match = camelCasePattern.exec(query)) !== null) { + while ((match = camelCasePattern.exec(asciiView)) !== null) { if (match[1] && match[1].length >= 2) { symbols.add(match[1]); } @@ -79,7 +86,7 @@ function extractSymbolsFromQuery(query: string): string[] { // Extract snake_case identifiers const snakeCasePattern = /\b([a-z][a-z0-9]*(?:_[a-z0-9]+)+)\b/gi; - while ((match = snakeCasePattern.exec(query)) !== null) { + while ((match = snakeCasePattern.exec(asciiView)) !== null) { if (match[1] && match[1].length >= 3) { symbols.add(match[1]); } @@ -87,7 +94,7 @@ function extractSymbolsFromQuery(query: string): string[] { // Extract SCREAMING_SNAKE_CASE const screamingPattern = /\b([A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)\b/g; - while ((match = screamingPattern.exec(query)) !== null) { + while ((match = screamingPattern.exec(asciiView)) !== null) { if (match[1]) { symbols.add(match[1]); } @@ -95,7 +102,7 @@ function extractSymbolsFromQuery(query: string): string[] { // Extract ALL_CAPS acronyms (2+ chars, e.g., REST, HTTP, LRU, API) const acronymPattern = /\b([A-Z]{2,})\b/g; - while ((match = acronymPattern.exec(query)) !== null) { + while ((match = acronymPattern.exec(asciiView)) !== null) { if (match[1]) { symbols.add(match[1]); } @@ -103,7 +110,7 @@ function extractSymbolsFromQuery(query: string): string[] { // Extract dot.notation and split into parts (e.g., "app.isPackaged" -> ["app", "isPackaged"]) const dotPattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:\.[a-zA-Z][a-zA-Z0-9]*)+)\b/g; - while ((match = dotPattern.exec(query)) !== null) { + while ((match = dotPattern.exec(asciiView)) !== null) { if (match[1]) { // Add both the full path and individual parts symbols.add(match[1]); @@ -119,12 +126,26 @@ function extractSymbolsFromQuery(query: string): string[] { // Extract plain lowercase identifiers (3+ chars, not already matched) // Catches symbol names like "undo", "redo", "history", "render", "parse" const lowercasePattern = /\b([a-z][a-z0-9]{2,})\b/g; - while ((match = lowercasePattern.exec(query)) !== null) { + while ((match = lowercasePattern.exec(asciiView)) !== null) { if (match[1]) { symbols.add(match[1]); } } + // Names with a letter outside ASCII, whole runs only (#2005 keeps partial + // Chinese words out of search): `로그인`, `寻路服务`, `état`. A name that + // switches between Han, kana or Hangul and another script, or joins words + // with `_`, is tried as written (`get用户信息`) and by its words, so + // `NavMeshAgent类型` also names `类型`; its ASCII side is the patterns' above. + for (const run of query.match(/[\p{L}\p{N}\p{M}_$]+/gu) ?? []) { + if (!/[^\x00-\x7F]/.test(run)) continue; + const words = run.split(/[_$]+/).flatMap(splitAtCjkSwitches); + if (words.length > 1 && meetsQueryWordFloor(run)) symbols.add(run); + for (const word of words) { + if (/[^\x00-\x7F]/.test(word) && meetsQueryWordFloor(word)) symbols.add(word); + } + } + // Filter out common English words that aren't likely symbol names const commonWords = new Set([ 'the', 'and', 'for', 'with', 'from', 'this', 'that', 'have', 'been', diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 08cdd4d873..e8050ae644 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -169,6 +169,12 @@ export interface SyncResult { definitionDelta?: string[]; /** Files whose exported-name set changed (adds, drops, renames, re-exports, default moves). */ exportDelta?: string[]; + /** + * The frameworks extraction detected, when this sync had to detect them + * again because what detection read had changed (#2516). Absent when the + * cached detection still held. + */ + detectedFrameworks?: string[]; } /** @@ -193,6 +199,27 @@ function readSourceOrStamp(fullPath: string): string | null { return isMpegTsBytes(fullPath, bytes) ? null : bytes.toString('utf8'); } +/** + * What framework detection found at a path, to tell later whether it changed: + * a file's size and modification time, a folder's sub-folders, or '-' when + * nothing is there. + */ +function diskSignature(fullPath: string, isFolder = false): string { + try { + if (isFolder) { + return fs.readdirSync(fullPath, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + .join('/'); + } + const stats = fs.statSync(fullPath, { throwIfNoEntry: false }); + return stats ? `${stats.size}:${stats.mtimeMs}` : '-'; + } catch { + return '-'; + } +} + /** Whether these bytes, read from `filePath`, are a `.ts` MPEG transport stream (#1910). */ function isMpegTsBytes(filePath: string, bytes: Buffer): boolean { if (!hasMpegTsExtension(filePath) || !isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))) return false; @@ -1990,6 +2017,23 @@ export class ExtractionOrchestrator { private detectedFrameworkNames: string[] | null = null; private conventionInvalidatedFiles = new Set(); private pendingLiquidThemeRoots: Record | null = null; + /** + * What the last detection read, so a sync can tell when it went stale + * (#2516): the indexed source files it read or probed, which a sync's own + * change lists cover, and every other path with what was there then + * ({@link diskSignature}): the manifests, like `package.json`, that no sync + * reports. Null until detection runs. + */ + private detectionInputs: { sourceFiles: Set; diskPaths: Map } | null = null; + /** Files were added or removed since the last detection: the file list it read is out of date. */ + private detectionFileListChanged = false; + /** + * What detection read, by project-relative path, kept between detections: a + * source file until a sync changes it, any other path while its signature + * holds. Detecting again then reads only what changed, not the couple of + * hundred files some detectors sample (react-native-bridge reads up to 200). + */ + private detectionReads = new Map(); /** * Scope matcher for SCOPED syncs, memoized on the mtimes of the two root * files it is derived from (`codegraph.json`, `.gitignore`). See @@ -2046,11 +2090,32 @@ export class ExtractionOrchestrator { * detection. Graph-query methods (getNodesByName etc.) return empty because * the DB hasn't been populated yet, but detect() only uses readFile, * fileExists, and getAllFiles, so that's fine. + * + * With `inputs`, every path a detector reads or probes is recorded there, + * and answered from {@link detectionReads} when it was read before. */ private frameworkSourceContext: ResolutionContext | null = null; - private buildDetectionContext(files: string[]): ResolutionContext { + private buildDetectionContext(files: string[], inputs?: NonNullable): ResolutionContext { const rootDir = this.rootDir; + const listed = inputs ? new Set(files) : null; + const reads = inputs ? this.detectionReads : null; + // What is at a path, once per path until a sync says it changed. + const probe = (relativePath: string): { full: string; signature: string; content?: string | null } | null => { + const rel = normalizePath(relativePath); + let known = reads?.get(rel); + if (!known) { + const full = validatePathWithinRoot(rootDir, relativePath); + if (!full) return null; + known = { full, signature: diskSignature(full) }; + reads?.set(rel, known); + } + if (inputs) { + if (listed!.has(rel)) inputs.sourceFiles.add(rel); + else if (!inputs.diskPaths.has(rel)) inputs.diskPaths.set(rel, known.signature); + } + return known; + }; return { getNodesInFile: () => [], getNodesByName: () => [], @@ -2061,24 +2126,22 @@ export class ExtractionOrchestrator { getAllFiles: () => files, getProjectRoot: () => rootDir, fileExists: (relativePath: string) => { - const full = validatePathWithinRoot(rootDir, relativePath); - if (!full) return false; - try { - return fs.existsSync(full); - } catch { - return false; - } + const known = probe(relativePath); + return known !== null && known.signature !== '-'; }, readFile: (relativePath: string) => { - const full = validatePathWithinRoot(rootDir, relativePath); - if (!full) return null; - try { - // Framework detectors scan source by name; a file over the size - // limit was never indexed and must not be decoded here either (#1910). - return readBoundedSourceSync(full).bytes?.toString('utf8') ?? null; - } catch { - return null; + const known = probe(relativePath); + if (!known) return null; + if (known.content === undefined) { + try { + // Framework detectors scan source by name; a file over the size + // limit was never indexed and must not be decoded here either (#1910). + known.content = readBoundedSourceSync(known.full).bytes?.toString('utf8') ?? null; + } catch { + known.content = null; + } } + return known.content; }, // Monorepo support — needed by framework detect()s that probe // subpackage manifests (e.g. fabric-view looking at @@ -2089,6 +2152,11 @@ export class ExtractionOrchestrator { relativePath === '.' || relativePath === '' ? rootDir : path.join(rootDir, relativePath); + // A folder appearing or going changes what a monorepo probe sees. + if (inputs) { + const key = normalizePath(path.relative(rootDir, target)) + '/'; + if (!inputs.diskPaths.has(key)) inputs.diskPaths.set(key, diskSignature(target, true)); + } try { return readDirEntries(target) .filter((entry) => entry.isDirectory()) @@ -2103,20 +2171,33 @@ export class ExtractionOrchestrator { /** * Detect frameworks on demand using the current scanned files (or a fresh * scan if none are provided). Cached on the orchestrator so repeat calls - * inside a single run don't re-scan. A sync passes the files whose content - * it is about to re-index as `stale`; every other file's indexed hash is its - * current content, so detectors may reuse the skips recorded against it. + * inside a single run don't re-scan; a sync drops the cache when what it + * read has changed ({@link refreshFrameworkDetection}). A sync passes the + * files whose content it is about to re-index as `stale`; every other file's + * indexed hash is its current content, so detectors may reuse the skips + * recorded against it. */ private ensureDetectedFrameworks(files?: string[], stale?: ReadonlySet): string[] { if (this.detectedFrameworkNames !== null) return this.detectedFrameworkNames; const fileList = files ?? scanDirectory(this.rootDir); - const context = this.buildDetectionContext(fileList); + const inputs = { sourceFiles: new Set(), diskPaths: new Map() }; + const context = this.buildDetectionContext(fileList, inputs); + this.detectionInputs = inputs; + this.detectionFileListChanged = false; this.detectedFrameworkNames = ( - stale ? detectFrameworksWithSkips(context, this.queries, { stale }) : detectFrameworks(context) + stale + // A file a skip let the detector pass over still counts as read: editing it must detect again. + ? detectFrameworksWithSkips(context, this.queries, { stale, onSkipped: (file) => inputs.sourceFiles.add(file) }) + : detectFrameworks(context) ).map((r) => r.name); // Route extractors resolve imports across the whole project, not just this run's files. - this.frameworkSourceContext = this.buildDetectionContext([...new Set([...this.queries.getAllFilePaths(), ...fileList])]); - this.gateFrameworksByApp(); + const projectFiles = [...new Set([...this.queries.getAllFilePaths(), ...fileList])]; + this.frameworkSourceContext = this.buildDetectionContext(projectFiles); + this.gateFrameworksByApp(this.buildDetectionContext(projectFiles, inputs)); + // Keep only what this detection read. + for (const rel of this.detectionReads.keys()) { + if (!inputs.sourceFiles.has(rel) && !inputs.diskPaths.has(rel)) this.detectionReads.delete(rel); + } return this.detectedFrameworkNames; } @@ -2128,10 +2209,11 @@ export class ExtractionOrchestrator { * unsampled package is not gated and its extractor runs in every package. * The sample comes from every indexed file, not only this run's, so a * scoped sync that re-detects from one changed file gates exactly as the - * full index did. + * full index did. `context` is the one detection recorded its reads through, + * so a manifest edit brings a new detection about. */ - private gateFrameworksByApp(): void { - const declared = declaredDependencies(this.frameworkSourceContext!); + private gateFrameworksByApp(context: ResolutionContext): void { + const declared = declaredDependencies(context); this.gatedFrameworks = new Map(); for (const name of this.detectedFrameworkNames ?? []) { const deps = getFrameworkResolver(name)?.appDependencies; @@ -2190,8 +2272,12 @@ export class ExtractionOrchestrator { private dependenciesDeclaredIn(dir: string): Set | null { if (this.manifestDependencies.has(dir)) return this.manifestDependencies.get(dir)!; let declared: Set | null = null; + const manifest = path.join(this.rootDir, dir, 'package.json'); + // Kept until the next detection, so an edit of this manifest must bring one about. + const rel = dir ? `${dir}/package.json` : 'package.json'; + if (this.detectionInputs && !this.detectionInputs.diskPaths.has(rel)) this.detectionInputs.diskPaths.set(rel, diskSignature(manifest)); try { - const pkg = JSON.parse(fs.readFileSync(path.join(this.rootDir, dir, 'package.json'), 'utf8')) as Record; + const pkg = JSON.parse(fs.readFileSync(manifest, 'utf8')) as Record; declared = new Set(); for (const field of ['dependencies', 'devDependencies', 'peerDependencies']) { const group = pkg[field]; @@ -2202,6 +2288,43 @@ export class ExtractionOrchestrator { return declared; } + /** The frameworks extraction last detected, or null before any detection. */ + getDetectedFrameworkNames(): string[] | null { + return this.detectedFrameworkNames; + } + + /** + * Bring framework detection up to date before a sync extracts `filesToIndex` + * (#2516). Detection is cached for the life of this orchestrator, so in a + * long-lived process (the MCP server's watcher) a framework that arrives + * mid-session, like Express in a new `package.json`, would otherwise be + * missing from every file synced after it. It runs again only when what it + * read changed: files were added or removed since, a source file it read is + * among those changed, or a manifest it read or probed changed on disk. + * Otherwise this costs a set lookup per changed file and a stat per manifest, + * and when it runs, it reads again only what changed ({@link detectionReads}). + * + * `scannedFiles` is a full reconcile's scan. A scoped sync never walks the + * project: the file list is the index's, which already holds this sync's + * removals, plus the files it adds. Returns the frameworks detected when + * detection ran. + */ + private refreshFrameworkDetection(scannedFiles: string[] | null, filesToIndex: string[], addedFiles: string[], removedFiles: string[]): string[] | undefined { + for (const file of filesToIndex) this.detectionReads.delete(file); + for (const file of removedFiles) this.detectionReads.delete(file); + const inputs = this.detectionInputs; + let stale = this.detectedFrameworkNames === null || inputs === null || this.detectionFileListChanged || + filesToIndex.some((f) => inputs.sourceFiles.has(f)); + for (const [rel, signature] of inputs?.diskPaths ?? []) { + if (diskSignature(path.join(this.rootDir, rel), rel.endsWith('/')) === signature) continue; + stale = true; + this.detectionReads.delete(rel); + } + if (!stale) return undefined; + this.detectedFrameworkNames = null; + return this.ensureDetectedFrameworks(scannedFiles ?? [...this.queries.getAllFilePaths(), ...addedFiles], new Set(filesToIndex)); + } + /** * Index all files in the project */ @@ -2285,8 +2408,10 @@ export class ExtractionOrchestrator { // Names are passed to each parse call so framework-specific extractors // (route nodes, middleware, etc.) run after the tree-sitter pass. // Framework detection is reset each run so adding e.g. requirements.txt - // between runs is picked up without restarting the process. + // between runs is picked up without restarting the process. It reads + // everything afresh: no sync has told this process what changed on disk. this.detectedFrameworkNames = null; + this.detectionReads.clear(); const tFw = Date.now(); const frameworkNames = this.ensureDetectedFrameworks(files); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] framework-detect: ${Date.now() - tFw}ms`); @@ -3820,13 +3945,20 @@ export class ExtractionOrchestrator { } // File conventions can change without editing any page modules. + let detectedFrameworks: string[] | undefined; if (filesToIndex.length > 0 || filesRemoved > 0) { - const previous = this.detectedFrameworkNames ?? []; const hadFileRoutes = this.detectedFrameworkNames === null ? this.queries.getNodesByKind('route').some((n) => n.id.startsWith(`route:react-router:${n.filePath}:file:`)) - : previous.includes('react-router-files'); - this.detectedFrameworkNames = null; - const detected = this.ensureDetectedFrameworks(currentFiles, new Set(filesToIndex)); + : this.detectedFrameworkNames.includes('react-router-files'); + // The frameworks every file below is extracted with, current (#2516). + if (addedFilePaths.length > 0 || removedFilePaths.length > 0) this.detectionFileListChanged = true; + const tDetect = Date.now(); + detectedFrameworks = this.refreshFrameworkDetection( + scopedPaths && scopedPaths.length > 0 ? null : currentFiles, filesToIndex, addedFilePaths, removedFilePaths); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) { + console.error(`[phase-timing] sync-framework-detect: ${Date.now() - tDetect}ms (${detectedFrameworks ? 'detected again' : 'unchanged'})`); + } + const detected = this.detectedFrameworkNames ?? []; for (const [framework, matches] of [ ['solid-start', isSolidStartRoute], ['vike', isVikePage], ['qwik-city', isQwikCityRoute], ['waku', isWakuRouteFile], ['analog', isAnalogPage], @@ -3841,12 +3973,6 @@ export class ExtractionOrchestrator { filesModified++; } } - if (scopedPaths?.length) { - this.detectedFrameworkNames = [...new Set([ - ...previous.filter((name) => name !== 'react-router-files'), ...detected, - ])]; - this.gateFrameworksByApp(); - } if (hadFileRoutes !== detected.includes('react-router-files')) { const scope = this.scopedSyncMatcher(); for (const filePath of new Set([...this.queries.getAllFilePaths(), ...currentFiles])) { @@ -3962,6 +4088,7 @@ export class ExtractionOrchestrator { ...(failedFilePaths.length > 0 ? { failedFilePaths } : {}), definitionDelta: definitionDelta.length > 0 ? definitionDelta : undefined, ...(exportDelta.length > 0 ? { exportDelta } : {}), + ...(detectedFrameworks ? { detectedFrameworks } : {}), }; } diff --git a/src/graph/call-sites.ts b/src/graph/call-sites.ts new file mode 100644 index 0000000000..9b30a93267 --- /dev/null +++ b/src/graph/call-sites.ts @@ -0,0 +1,22 @@ +/** + * How a caller's call sites read, wherever callers are listed: `codegraph + * callers`, codegraph_callers and the trail of codegraph_node (#2909). Each + * site is a line to edit when the symbol changes, so the list says them all, + * up to a cap that keeps one busy caller from crowding out the rest. + */ +import type { Edge } from '../types'; + +/** The lines `edges` sit on, ascending, once each. An edge with no line adds none. */ +export function callSiteLines(edges: readonly Edge[]): number[] { + const lines = new Set(); + for (const edge of edges) { + if (typeof edge.line === 'number' && edge.line > 0) lines.add(edge.line); + } + return [...lines].sort((a, b) => a - b); +} + +/** `9, 10, 14`, or past `max` lines `9, 10, 14, +7 more`. */ +export function formatCallSiteLines(lines: readonly number[], max: number): string { + const shown = lines.slice(0, max).join(', '); + return lines.length > max ? `${shown}, +${lines.length - max} more` : shown; +} diff --git a/src/graph/index.ts b/src/graph/index.ts index 97f6d10cec..65ed8a78d6 100644 --- a/src/graph/index.ts +++ b/src/graph/index.ts @@ -5,6 +5,8 @@ */ export { GraphTraverser } from './traversal'; +export type { CallSiteRow } from './traversal'; +export { callSiteLines, formatCallSiteLines } from './call-sites'; export { GraphQueryManager } from './queries'; export { buildTypeHierarchy, diff --git a/src/graph/named-symbol-flow.ts b/src/graph/named-symbol-flow.ts index a77474acfa..43fbcf0e87 100644 --- a/src/graph/named-symbol-flow.ts +++ b/src/graph/named-symbol-flow.ts @@ -34,7 +34,7 @@ import type CodeGraph from '../index'; import type { Node, Edge } from '../types'; -import { isTestFile } from '../search/query-utils'; +import { isTestFile, splitAtCjkSwitches, isQueryNameToken } from '../search/query-utils'; import { qualifiedSymbolMatches } from './symbol-lookup'; import { goPackageReaderFor } from './go-package'; @@ -267,14 +267,19 @@ function isPreciseToken(token: string): boolean { return /[._$]|::|\//.test(token) || /[a-z][A-Z]/.test(token) || /^[A-Z]/.test(token); } -/** The symbol-shaped tokens of a query, deduped and capped. */ +/** + * The symbol-shaped tokens of a query, deduped and capped: names in any + * script, cut where Han, kana or Hangul meets another script, at least two + * characters in those scripts (`入口 末端`) and three elsewhere (#1262). + */ export function flowTokens(query: string): string[] { return [ ...new Set( query .split(/[\s,()[\]]+/) .map((t) => t.replace(FILE_EXT, '').trim()) - .filter((t) => t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) + .flatMap(splitAtCjkSwitches) + .filter(isQueryNameToken) ), ].slice(0, MAX_TOKENS); } @@ -394,6 +399,7 @@ const DIRECTED_VISIT_CAP = 12_000; /** How the `named` walk reached a node, and what the route there carries. */ interface WalkStep { + /** The route this one extends: its key in the walk's parent map. */ prev: string | null; edge: Edge | null; node: Node; @@ -405,6 +411,17 @@ interface WalkStep { named: number; } +/** A node's callees, read once per search however many routes reach it. */ +type CalleesOf = (id: string) => ReturnType; + +/** Whether the route ending at `key` passes `nodeId`: a walk back of at most `maxHops` steps. */ +function onRoute(parent: ReadonlyMap, key: string, nodeId: string): boolean { + for (let cur: string | null = key; cur !== null; cur = parent.get(cur)!.prev) { + if (parent.get(cur)!.node.id === nodeId) return true; + } + return false; +} + /** * Breadth-first over `calls` edges — synthesized ones included, which is what * carries a flow across a callback, a re-render or a JSX child. @@ -413,62 +430,91 @@ interface WalkStep { * at most `maxBridge` unnamed symbols may sit between two of them. That cap is * what bounds the frontier, so {@link NAMED_VISIT_CAP} is generous. * - * Two routes of the same length to one node are settled by the agent's naming, - * not by the order the index lists callees in: the route through more named - * symbols wins. An interface method calls every implementation, so - * prometheus's `Queryable.Querier` reaches `NewMergeQuerier` through - * `fanout.Querier` and the TSDB's `DB.Querier` alike; keeping whichever was - * listed first took the Flow through `DB.Querier` when the query named - * `fanout.Querier`. A rival route comes from another node at the same depth, - * and every node at one depth is expanded before any node at the next, so the - * choice is final before anything is reached through it. Ending on fewer - * unnamed hops ranks first, so a choice never spends bridge budget that a later - * hop needs (with the default budget of one, only a route node's streak can - * differ). + * A node keeps every route to it that no other route to it beats: one that got + * there no later, through as many named symbols, with no more unnamed hops to + * end on. One route per node lost a named symbol (#2970): in `entry → mid2 → + * leaf` beside `entry → mid1 → deep → leaf`, `leaf` was reached first through + * the unnamed `mid2`, so the route through the named `deep` was turned away for + * being longer, and a query naming all three got a Flow without `leaf`. A + * longer route that passes fewer named symbols is still turned away, so it + * can't win the deepest-sink choice over a better one. * - * Returns the parent map, so a caller can reconstruct any reached node's path. + * At one depth the same rule settles which route the agent's naming prefers, + * not the order the index lists callees in: an interface method calls every + * implementation, so prometheus's `Queryable.Querier` reaches `NewMergeQuerier` + * through `fanout.Querier` and the TSDB's `DB.Querier` alike, and the route + * through the named `fanout.Querier` beats the other. Every node at one depth is + * expanded before any node at the next, so a route beaten at its own depth is + * dropped before anything is reached through it. Ending on fewer unnamed hops + * is no worse, so no route spends bridge budget a later hop needs. + * + * Returns the parent map keyed by route, so a caller can reconstruct the path + * of any route that reached a named symbol (`reached`). */ function walkCalls( - cg: CodeGraph, + calleesOf: CalleesOf, seed: Node, named: ReadonlySet, maxHops: number, maxBridge: number ): { parent: Map; reached: string[] } { const parent = new Map(); - parent.set(seed.id, { - prev: null, - edge: null, - node: seed, - depth: 0, - streak: 0, - named: named.has(seed.id) ? 1 : 0, - }); - const queue: string[] = [seed.id]; + /** The routes each reached node keeps, by key; its size is the nodes reached. */ + const routes = new Map(); + const beaten = new Set(); + const queue: string[] = []; + /** Where each route waits in the queue. */ + const slot = new Map(); + const beats = (a: WalkStep, b: WalkStep) => a.depth <= b.depth && a.named >= b.named && a.streak <= b.streak; + const keep = (step: WalkStep): void => { + const kept = routes.get(step.node.id) ?? []; + if (kept.some((other) => beats(parent.get(other)!, step))) return; + const key = String(parent.size); + parent.set(key, step); + // A route beaten here is one of this depth, still waiting in the queue: the + // better route takes its place there, so the walk visits nodes in the order + // it reached them, whichever route won. + let place = -1; + const live: string[] = []; + for (const other of kept) { + if (!beats(step, parent.get(other)!)) { + live.push(other); + continue; + } + beaten.add(other); + const at = slot.get(other)!; + if (place < 0 || at < place) place = at; + } + live.push(key); + routes.set(step.node.id, live); + if (place < 0) { + place = queue.length; + queue.push(key); + } else { + queue[place] = key; + } + slot.set(key, place); + }; + keep({ prev: null, edge: null, node: seed, depth: 0, streak: 0, named: named.has(seed.id) ? 1 : 0 }); const reached: string[] = []; - for (let head = 0; head < queue.length && parent.size < NAMED_VISIT_CAP; head++) { - const id = queue[head]!; - const at = parent.get(id)!; - if (id !== seed.id && named.has(id)) reached.push(id); + for (let head = 0; head < queue.length && routes.size < NAMED_VISIT_CAP; head++) { + const key = queue[head]!; + if (beaten.has(key)) continue; + const at = parent.get(key)!; + const id = at.node.id; + if (id !== seed.id && named.has(id)) reached.push(key); if (at.depth >= maxHops - 1) continue; - for (const c of cg.getCallees(id)) { + for (const c of calleesOf(id)) { if (!FLOW_EDGE_KINDS.has(c.edge.kind)) continue; const isNamed = named.has(c.node.id); // A route node is a connector, not a symbol the reader would have named: // crossing one costs no bridge budget. const streak = isNamed ? 0 : c.node.kind === 'route' ? at.streak : at.streak + 1; if (streak > maxBridge) continue; - const depth = at.depth + 1; - const namedOnRoute = at.named + (isNamed ? 1 : 0); - const prior = parent.get(c.node.id); - if (prior) { - // Reached already: only a better route of the same length replaces it. - if (prior.depth !== depth) continue; - if (streak > prior.streak || (streak === prior.streak && namedOnRoute <= prior.named)) continue; - } else { - queue.push(c.node.id); - } - parent.set(c.node.id, { prev: id, edge: c.edge, node: c.node, depth, streak, named: namedOnRoute }); + // A route never passes a node twice: going round a cycle counts its named + // symbols again and would out-rank the flow it repeats. + if (onRoute(parent, key, c.node.id)) continue; + keep({ prev: key, edge: c.edge, node: c.node, depth: at.depth + 1, streak, named: at.named + (isNamed ? 1 : 0) }); } } return { parent, reached }; @@ -652,7 +698,7 @@ function dropOverrideHandoffs(cg: CodeGraph, tokenNodes: ReadonlyMap, + parent: ReadonlyMap, target: string ): FlowStep[] { const steps: FlowStep[] = []; @@ -702,14 +748,24 @@ export function resolveNamedSymbolFlow( if (steps) found.push(steps); } } else { + const callees = new Map>(); + const calleesOf: CalleesOf = (id) => { + let out = callees.get(id); + if (!out) { + out = cg.getCallees(id); + callees.set(id, out); + } + return out; + }; for (const seed of [...flow.named.values()].slice(0, MAX_SEEDS)) { - const { parent, reached } = walkCalls(cg, seed, namedIds, maxHops, maxBridge); + const { parent, reached } = walkCalls(calleesOf, seed, namedIds, maxHops, maxBridge); // Explore's rule: the DEEPEST named sink this seed can reach, measured // once override hand-offs are dropped so a hand-off can't win on length. + // `reached` holds route keys: a node can be reached by several routes. let deepest: FlowStep[] | null = null; - for (const id of reached) { - const steps = dropOverrideHandoffs(cg, flow.tokenNodes, chainTo(parent, id)); - if (!deepest || steps.length > deepest.length || (steps.length === deepest.length && parent.get(id)!.named > deepest.filter(step => namedIds.has(step.node.id)).length)) deepest = steps; + for (const key of reached) { + const steps = dropOverrideHandoffs(cg, flow.tokenNodes, chainTo(parent, key)); + if (!deepest || steps.length > deepest.length || (steps.length === deepest.length && parent.get(key)!.named > deepest.filter(step => namedIds.has(step.node.id)).length)) deepest = steps; } if (deepest) found.push(deepest); } diff --git a/src/graph/traversal.ts b/src/graph/traversal.ts index 672d9ef1a8..98c5aea49c 100644 --- a/src/graph/traversal.ts +++ b/src/graph/traversal.ts @@ -28,6 +28,34 @@ interface TraversalStep { depth: number; } +/** + * The edges that make one symbol another's caller. `instantiates` counts: + * constructing a class (`Foo(...)` / `new Foo()`) is calling its constructor, + * so the instantiation site is a caller of the class (#774). + */ +const CALL_EDGE_KINDS: EdgeKind[] = ['calls', 'references', 'imports', 'instantiates', 'navigates']; + +/** One edge's site: two edges are the same only on source, target, kind and position. */ +function edgeSite(e: Edge): string { + return `${e.source}|${e.target}|${e.kind}|${e.line ?? -1}|${e.column ?? -1}`; +} + +/** + * A direct caller (or callee) of a symbol with every edge between them, from + * {@link GraphTraverser.getCallSites}. + */ +export interface CallSiteRow { + /** The caller, or for callees the callee. */ + node: Node; + /** + * Every edge between the two, once per site, in index order. The first is + * the edge {@link GraphTraverser.getCallers} reports for this neighbour. + */ + edges: Edge[]; + /** The symbol calls itself: this row's node is the symbol asked about. */ + recursive: boolean; +} + /** * Graph traverser for BFS and DFS traversal */ @@ -76,8 +104,6 @@ export class GraphTraverser { // Edge-identity dedup so a `direction:'both'` scan — which encounters A→B // from both endpoints — records each edge once. const seenEdges = new Set(); - const edgeKey = (e: Edge) => - `${e.source}|${e.target}|${e.kind}|${e.line ?? -1}|${e.column ?? -1}`; const queue: TraversalStep[] = [{ node: startNode, edge: null, depth: 0 }]; if (opts.includeStart) { @@ -138,7 +164,7 @@ export class GraphTraverser { // adjacency scan (rather than once per dequeue) is what preserves // parallel edges to the same target (#1090); `nextNode` is guaranteed // to be in `nodes` at this point (just added, or already in-set). - const ek = edgeKey(adjEdge); + const ek = edgeSite(adjEdge); if (!seenEdges.has(ek)) { seenEdges.add(ek); edges.push(adjEdge); @@ -340,7 +366,7 @@ export class GraphTraverser { // caller of the class. Without it, `callers ` surfaced only the // importing file (via `imports`) and missed every construction site — // the opposite of "what breaks if I change this class?" (#774). - const incomingEdges = this.queries.getIncomingEdges(nodeId, ['calls', 'references', 'imports', 'instantiates', 'navigates']); + const incomingEdges = this.queries.getIncomingEdges(nodeId, CALL_EDGE_KINDS); if (incomingEdges.length === 0) return; // Batch-fetch all caller nodes in one round-trip instead of one @@ -395,7 +421,7 @@ export class GraphTraverser { // (`Foo(...)` / `new Foo()`) has that class as a callee, so callers and // callees stay inverses of each other and `trace` can cross the // instantiation boundary (function → class → its methods) (#774). - const outgoingEdges = this.queries.getOutgoingEdges(nodeId, ['calls', 'references', 'imports', 'instantiates', 'navigates']); + const outgoingEdges = this.queries.getOutgoingEdges(nodeId, CALL_EDGE_KINDS); if (outgoingEdges.length === 0) return; // Batch-fetch callee nodes (was N+1 — see getCallersRecursive note). @@ -413,6 +439,42 @@ export class GraphTraverser { } } + /** + * The direct callers (or callees) of a symbol, each with every edge that + * joins them: one row per neighbour, in the order {@link getCallers} and + * {@link getCallees} list them. Those keep a neighbour's first edge only and + * leave out the symbol itself; here a caller that calls it on two lines has + * both sites, and a symbol that calls itself is a row of its own, flagged + * `recursive`. Each of those sites is a line to edit when the symbol + * changes, so `codegraph callers`, codegraph_callers and codegraph_node's + * trail list them (#2909). The walks that rank and trace keep their own + * one-row-per-neighbour reading. + */ + getCallSites(nodeId: string, direction: 'callers' | 'callees'): CallSiteRow[] { + const edges = direction === 'callers' + ? this.queries.getIncomingEdges(nodeId, CALL_EDGE_KINDS) + : this.queries.getOutgoingEdges(nodeId, CALL_EDGE_KINDS); + if (edges.length === 0) return []; + const neighbour = (e: Edge) => (direction === 'callers' ? e.source : e.target); + const nodes = this.queries.getNodesByIds(edges.map(neighbour)); + const rows = new Map(); + const sites = new Set(); + for (const edge of edges) { + const node = nodes.get(neighbour(edge)); + if (!node) continue; + const site = edgeSite(edge); + if (sites.has(site)) continue; + sites.add(site); + let row = rows.get(node.id); + if (!row) { + row = { node, edges: [], recursive: node.id === nodeId }; + rows.set(node.id, row); + } + row.edges.push(edge); + } + return [...rows.values()]; + } + /** * Get the call graph for a function (both callers and callees) * diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 67025426f9..f0bd5150fa 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -38,7 +38,7 @@ import { import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind, GraphStats } from '../types'; import type { NodeSpan } from '../db/queries'; -import { isDistinctiveIdentifier, isTestFile, isTestPath, normalizeNameToken, STOP_WORDS } from '../search/query-utils'; +import { isDistinctiveIdentifier, isQueryNameToken, isTestFile, isTestPath, normalizeNameToken, splitAtCjkSwitches, STOP_WORDS } from '../search/query-utils'; import { groupDefinitions, isQualifiedSymbol, lastQualifierPart, matchesSymbol, qualifiedSymbolMatches } from '../graph/symbol-lookup'; import { goPackageReaderFor } from '../graph/go-package'; import { @@ -64,6 +64,8 @@ import { countImplementers } from '../graph/type-hierarchy'; import { isNameGuess } from '../graph/edge-trust'; import { isDocumentationNode } from '../graph/traversal'; import { describeSynthesizedHop } from '../graph/synthesized-hop'; +import { callSiteLines, formatCallSiteLines } from '../graph/call-sites'; +import type { CallSiteRow } from '../graph/traversal'; import { isCopybookInclude, MAX_COPYBOOK_INCLUDES, type NamedCopybook } from '../graph/cobol-copybooks'; import { findAllSymbols, @@ -153,7 +155,7 @@ function wslSharedIndexGuidance(err: WslSharedIndexError): string { export { PathRefusalError } from '../errors'; import { PathRefusalError } from '../errors'; import { indexedHashInput } from '../file-limits'; -import { resolve as resolvePath, relative as relativePath } from 'path'; +import { resolve as resolvePath, relative as relativePath, dirname } from 'path'; /** Maximum output length to prevent context bloat (characters) */ const MAX_OUTPUT_LENGTH = 15000; @@ -1351,15 +1353,18 @@ export function elidedBodyNote(filePath: string, name: string, from: number, to: /** * The symbol-shaped tokens of an explore query, the way its named-symbol * seeding reads them: split on spaces, commas and brackets, a source-file - * extension dropped (`Program.cs` names `Program`), at least 3 characters, - * plain or qualified identifiers only, deduped, at most 16. + * extension dropped (`Program.cs` names `Program`), cut where Han, kana or + * Hangul meets another script (`NavMeshAgent类型`), plain or qualified names + * in any script of at least 3 characters, or 2 in Han, kana or Hangul + * (`入口`, `로그인`), deduped, at most 16. */ export function exploreQueryTokens(query: string): string[] { const FILE_EXT = /\.(?:java|kt|kts|ts|tsx|js|jsx|mjs|cjs|cs|py|go|rb|php|swift|rs|cpp|cc|cxx|c|h|hpp|scala|lua|dart|vue|svelte|astro|erl|hrl)$/i; return [...new Set( query.split(/[\s,()[\]]+/) .map((t) => t.replace(FILE_EXT, '').trim()) - .filter((t) => t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) + .flatMap(splitAtCjkSwitches) + .filter(isQueryNameToken) )].slice(0, 16); } @@ -2357,6 +2362,9 @@ export function getStaticTools(): ToolDefinition[] { */ const DEFAULT_MCP_TOOLS = new Set(['explore', 'sessions']); +/** Call-site lines codegraph_callers shows per caller (#2909), so one busy caller stays one line. */ +const MCP_CALL_SITES_SHOWN = 10; + /** realpath when the path exists, the path itself otherwise — never throws. */ function canonicalPath(p: string): string { try { @@ -3352,7 +3360,16 @@ export class ToolHandler { // serve it. Resolved on the main thread so the watcher lives here even // when dispatch is off-loaded to a worker. if (typeof pathCheck === 'string') { - await this.awaitProjectGate(pathCheck); + // A file inside a project (an agent passing the file it is looking at) + // stands for its directory, like any other path inside it (#2982). + // Rewritten here so the gate, the worker, the worktree notice and + // getCodeGraph all see the same directory. + let projectPath = pathCheck; + try { + if (statSync(pathCheck).isFile()) projectPath = dirname(pathCheck); + } catch { /* a missing path walks up to its .codegraph/ as before */ } + if (projectPath !== pathCheck) args = { ...args, projectPath }; + await this.awaitProjectGate(projectPath); } // The `path` and `pattern` properties used by codegraph_files are // also path-shaped — apply the same cap. @@ -3673,27 +3690,35 @@ export class ToolHandler { ? `\n\n> **Note:** no definition of "${symbol}" matches file "${fileFilter}" — showing all definitions instead.` : ''; + // Every site of each caller (#2909): one row per caller, at the lines its + // calls sit on, and the symbol itself when it calls itself. const collect = (defNodes: Node[]) => { - const seen = new Set(); const callers: Node[] = []; const labels = new Map(); + const sites = new Map(); + const recursive = new Set(); for (const node of defNodes) { - for (const c of cg.getCallers(node.id)) { - if (!seen.has(c.node.id)) { - seen.add(c.node.id); - callers.push(c.node); - answerPaths.add(c.node.filePath); - const label = this.edgeLabel(c.edge); - if (label) labels.set(c.node.id, label); + for (const c of cg.getCallSites(node.id, 'callers')) { + const known = sites.get(c.node.id); + if (known) { + known.push(...c.edges); + continue; } + sites.set(c.node.id, [...c.edges]); + if (c.recursive) recursive.add(c.node.id); + callers.push(c.node); + answerPaths.add(c.node.filePath); + const label = this.edgeLabel(c.edges[0]!); + if (label) labels.set(c.node.id, label); } } - return { callers, labels }; + const locations = new Map(callers.map((n) => [n.id, this.callSiteLocation(n, sites.get(n.id) ?? [])])); + return { callers, labels, locations, recursive }; }; // Single definition (or same-file overloads): the familiar flat list. if (groups.length === 1) { - const { callers, labels } = collect(groups[0]!); + const { callers, labels, locations, recursive } = collect(groups[0]!); if (callers.length === 0) { return this.answerResult(cg, `No callers found for "${symbol}"${allMatches.note}${filterNote}`, answerPaths); } @@ -3705,7 +3730,7 @@ export class ToolHandler { const cut = callers.length > limit ? `\n\n> Showing ${limit} of ${callers.length} callers; pass \`limit\` (up to 100) to widen.` : ''; - const formatted = this.formatNodeList(callers.slice(0, limit), `Callers of ${symbol}`, labels) + cut + note + filterNote; + const formatted = this.formatNodeList(callers.slice(0, limit), `Callers of ${symbol}`, labels, locations, recursive) + cut + note + filterNote; return this.answerResult(cg, this.truncateOutput(formatted), answerPaths); } @@ -3716,16 +3741,14 @@ export class ToolHandler { `**Callers of ${symbol} — ${groups.length} distinct definitions (narrow with \`file\`)**`, ]; for (const group of groups) { - const { callers, labels } = collect(group); + const { callers, labels, locations, recursive } = collect(group); lines.push('', this.definitionHeading(group)); if (callers.length === 0) { lines.push('- (no callers)'); continue; } for (const node of callers.slice(0, limit)) { - const location = node.startLine ? `:${node.startLine}` : ''; - const label = labels.get(node.id); - lines.push(`- ${node.name} (${node.kind}) - ${node.filePath}${location}${label ? ` — via ${label}` : ''}`); + lines.push(this.formatNodeListEntry(node, labels, locations, recursive)); } if (callers.length > limit) { lines.push(`- … +${callers.length - limit} more (pass \`limit\` to widen)`); @@ -3756,26 +3779,30 @@ export class ToolHandler { ? `\n\n> **Note:** no definition of "${symbol}" matches file "${fileFilter}" — showing all definitions instead.` : ''; + // Callees are listed at their own declarations, where a call leads; a + // symbol that calls itself is one of them (#2909). const collect = (defNodes: Node[]) => { const seen = new Set(); const callees: Node[] = []; const labels = new Map(); + const recursive = new Set(); for (const node of defNodes) { - for (const c of cg.getCallees(node.id)) { + for (const c of cg.getCallSites(node.id, 'callees')) { if (!seen.has(c.node.id)) { seen.add(c.node.id); + if (c.recursive) recursive.add(c.node.id); callees.push(c.node); answerPaths.add(c.node.filePath); - const label = this.edgeLabel(c.edge); + const label = this.edgeLabel(c.edges[0]!); if (label) labels.set(c.node.id, label); } } } - return { callees, labels }; + return { callees, labels, recursive }; }; if (groups.length === 1) { - const { callees, labels } = collect(groups[0]!); + const { callees, labels, recursive } = collect(groups[0]!); if (callees.length === 0) { return this.answerResult(cg, `No callees found for "${symbol}"${allMatches.note}${filterNote}`, answerPaths); } @@ -3787,7 +3814,7 @@ export class ToolHandler { const cut = callees.length > limit ? `\n\n> Showing ${limit} of ${callees.length} callees; pass \`limit\` (up to 100) to widen.` : ''; - const formatted = this.formatNodeList(callees.slice(0, limit), `Callees of ${symbol}`, labels) + cut + note + filterNote; + const formatted = this.formatNodeList(callees.slice(0, limit), `Callees of ${symbol}`, labels, undefined, recursive) + cut + note + filterNote; return this.answerResult(cg, this.truncateOutput(formatted), answerPaths); } @@ -3796,16 +3823,14 @@ export class ToolHandler { `**Callees of ${symbol} — ${groups.length} distinct definitions (narrow with \`file\`)**`, ]; for (const group of groups) { - const { callees, labels } = collect(group); + const { callees, labels, recursive } = collect(group); lines.push('', this.definitionHeading(group)); if (callees.length === 0) { lines.push('- (no callees)'); continue; } for (const node of callees.slice(0, limit)) { - const location = node.startLine ? `:${node.startLine}` : ''; - const label = labels.get(node.id); - lines.push(`- ${node.name} (${node.kind}) - ${node.filePath}${location}${label ? ` — via ${label}` : ''}`); + lines.push(this.formatNodeListEntry(node, labels, undefined, recursive)); } if (callees.length > limit) { lines.push(`- … +${callees.length - limit} more (pass \`limit\` to widen)`); @@ -9690,33 +9715,35 @@ export class ToolHandler { */ private formatTrail(cg: CodeGraph, node: Node): string { const TRAIL_CAP = 12; - const fmt = (e: { node: Node; edge: Edge }) => { - const base = `${e.node.name} (${e.node.filePath}:${e.node.startLine})`; - const synth = this.synthEdgeNote(e.edge); + /** Call-site lines shown per caller, so one busy caller can't crowd the trail. */ + const TRAIL_CALL_SITES = 3; + const fmt = (row: CallSiteRow, where: string) => { + const base = `${row.node.name} (${where})`; + const synth = this.synthEdgeNote(row.edges[0]!); if (synth) return `${base} [${synth}]`; - return isNameGuess(e.edge) ? `${base} [name match, unverified]` : base; - }; - const collect = (edges: Array<{ node: Node; edge: Edge }>): Array<{ node: Node; edge: Edge }> => { - const seen = new Set([node.id]); - const out: Array<{ node: Node; edge: Edge }> = []; - for (const e of edges) { - if (seen.has(e.node.id)) continue; - seen.add(e.node.id); - out.push(e); - } - return out; + return isNameGuess(row.edges[0]!) ? `${base} [name match, unverified]` : base; }; - const callees = collect(cg.getCallees(node.id)); - const callers = collect(cg.getCallers(node.id)); + // A callee reads at its declaration, where following it leads. A caller + // reads at the lines its calls sit on, every one a line to edit when this + // symbol changes (#2909), and a symbol that calls itself is among its own + // callers, once. + const callees = cg.getCallSites(node.id, 'callees').filter((row) => !row.recursive); + const callers = cg.getCallSites(node.id, 'callers'); const dups = this.nearDuplicateNote(cg, node); if (callees.length === 0 && callers.length === 0 && !dups) return ''; + const callee = (row: CallSiteRow) => fmt(row, `${row.node.filePath}:${row.node.startLine}`); + const caller = (row: CallSiteRow) => { + const sites = callSiteLines(row.edges); + const at = sites.length > 0 ? formatCallSiteLines(sites, TRAIL_CALL_SITES) : String(row.node.startLine); + return fmt(row, `${row.node.filePath}:${at}${row.recursive ? ', calls itself' : ''}`); + }; const lines: string[] = ['', '**Trail — codegraph_node any of these to follow it (no Read needed)**']; if (dups) lines.push(`**Near-duplicates (update together) ≈** ${dups}`); if (callees.length > 0) { - lines.push(`**Calls →** ${callees.slice(0, TRAIL_CAP).map(fmt).join(', ')}${callees.length > TRAIL_CAP ? `, +${callees.length - TRAIL_CAP} more` : ''}`); + lines.push(`**Calls →** ${callees.slice(0, TRAIL_CAP).map(callee).join(', ')}${callees.length > TRAIL_CAP ? `, +${callees.length - TRAIL_CAP} more` : ''}`); } if (callers.length > 0) { - lines.push(`**Called by ←** ${callers.slice(0, TRAIL_CAP).map(fmt).join(', ')}${callers.length > TRAIL_CAP ? `, +${callers.length - TRAIL_CAP} more` : ''}`); + lines.push(`**Called by ←** ${callers.slice(0, TRAIL_CAP).map(caller).join(', ')}${callers.length > TRAIL_CAP ? `, +${callers.length - TRAIL_CAP} more` : ''}`); } return lines.join('\n'); } @@ -9743,7 +9770,9 @@ export class ToolHandler { // (e.g. the server resolved up from a nested worktree to the main checkout). // Queries then reflect that tree's branch, not the worktree being edited. // status shows the verbose, multi-line form; the read tools get the compact - // one-liner via withWorktreeNotice. Both share the cached detection. + // one-liner via withWorktreeNotice. Both share the cached detection, and + // both are worded for the agent: the user can index the worktree, the + // agent doesn't. const mismatch = this.worktreeMismatchFor(args.projectPath as string | undefined); const lines: string[] = [ @@ -9752,7 +9781,7 @@ export class ToolHandler { '', ]; if (mismatch) { - lines.push(`> ⚠ ${worktreeMismatchWarning(mismatch).replace(/\n/g, '\n> ')}`, ''); + lines.push(`> ⚠ ${worktreeMismatchWarning(mismatch, 'agent').replace(/\n/g, '\n> ')}`, ''); } lines.push( `**Files indexed:** ${stats.fileCount}`, @@ -10155,20 +10184,45 @@ export class ToolHandler { return lines.join('\n'); } - private formatNodeList(nodes: Node[], title: string, labels?: Map): string { + private formatNodeList( + nodes: Node[], + title: string, + labels?: Map, + locations?: Map, + recursive?: Set, + ): string { const lines: string[] = [`**${title} (${nodes.length} found)**`, '']; + for (const node of nodes) lines.push(this.formatNodeListEntry(node, labels, locations, recursive)); + return lines.join('\n'); + } - for (const node of nodes) { - const location = node.startLine ? `:${node.startLine}` : ''; - // Compact: just name, kind, location — plus the relationship when it - // isn't a plain call (callback registration, instantiation, …). - const label = labels?.get(node.id); - lines.push( - `- ${node.name} (${node.kind}) - ${node.filePath}${location}${label ? ` — via ${label}` : ''}` - ); - } + /** + * One callers/callees entry. Compact: name, kind, location (the caller's + * call sites when `locations` has them, else the declaration's line), the + * relationship when it isn't a plain call (callback registration, + * instantiation, …), and whether the symbol calls itself. + */ + private formatNodeListEntry( + node: Node, + labels?: Map, + locations?: Map, + recursive?: Set, + ): string { + const location = locations?.get(node.id) ?? (node.startLine ? `:${node.startLine}` : ''); + const label = labels?.get(node.id); + const itself = recursive?.has(node.id) ? ' — calls itself' : ''; + return `- ${node.name} (${node.kind}) - ${node.filePath}${location}${label ? ` — via ${label}` : ''}${itself}`; + } - return lines.join('\n'); + /** + * `:9, 10` — where a caller's calls sit, the lines to edit when the symbol + * changes (#2909), capped at {@link MCP_CALL_SITES_SHOWN}; the caller's own + * line when its edges carry none. + */ + private callSiteLocation(node: Node, edges: readonly Edge[]): string { + const lines = callSiteLines(edges); + if (lines.length > 0) return `:${formatCallSiteLines(lines, MCP_CALL_SITES_SHOWN)}`; + return node.startLine ? `:${node.startLine}` : ''; } /** diff --git a/src/resolution/frameworks/index.ts b/src/resolution/frameworks/index.ts index 36cba92cc3..801d1c8701 100644 --- a/src/resolution/frameworks/index.ts +++ b/src/resolution/frameworks/index.ts @@ -161,12 +161,13 @@ interface SkipStore { * skip is valid only while the file's indexed hash is its current content, so * a caller that runs before this run's edits are stored passes them as * `stale`: those files are read again. `persist: false` reads the skips - * without writing any (a read-only connection). + * without writing any (a read-only connection). `onSkipped` hears of each + * file a skip let a detector pass over unread. */ export function detectFrameworksWithSkips( context: ResolutionContext, store: SkipStore, - options: { stale?: ReadonlySet; persist?: boolean } = {} + options: { stale?: ReadonlySet; persist?: boolean; onSkipped?: (file: string) => void } = {} ): FrameworkResolver[] { let known = new Map>(); try { @@ -179,7 +180,7 @@ export function detectFrameworksWithSkips( } catch { known = new Map(); // No skip table yet: detect from the files alone. } - const skips = new SynthSkips(known); + const skips = new SynthSkips(known, options.onSkipped); const detected = detectFrameworks({ ...context, synthSkips: skips }); const rows = skips.take(); if (options.persist !== false && rows.length > 0) { diff --git a/src/resolution/frameworks/package-deps.ts b/src/resolution/frameworks/package-deps.ts index 907c679431..e1a5c4bc40 100644 --- a/src/resolution/frameworks/package-deps.ts +++ b/src/resolution/frameworks/package-deps.ts @@ -31,9 +31,19 @@ const SCRIPT_FILE = /\.(?:[cm]?[jt]sx?|vue|svelte|astro)$/; * proshop-shaped repo indexed its React Router routes (extraction is not * gated on detection) and then resolved none of its navigation. Re-reading * when the file count changes costs one manifest sweep per index. + * + * The count alone misses a manifest edit and a swap of one app for another + * of the same size (#2519), so a long-lived context also forgets its answer + * with {@link clearDeclaredDependencies}, which the resolver calls with its + * other per-context memos, before it detects frameworks again. */ const cache = new WeakMap }>(); +/** Forget what `context`'s manifests declared, so the next question reads them again. */ +export function clearDeclaredDependencies(context: ResolutionContext): void { + cache.delete(context); +} + /** Every dependency name declared in the manifests read (see the file comment), de-duplicated. */ export function declaredDependencies(context: ResolutionContext): Set { const files = context.getAllFiles(); diff --git a/src/resolution/index.ts b/src/resolution/index.ts index 0f04849530..aa581c7a6e 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -36,6 +36,7 @@ import { GO_TYPE_KINDS, isGoBareName, isGoUnknownQualified, goTypePositionTarget import { ResolverPool, adaptiveEngageEnabled, minRefsForBulkLoad, minRefsForPool, shouldEngageAdaptively } from './resolver-pool'; import { resolveAliasBinding } from './alias-binding'; import { detectFrameworksWithSkips, getResolvingFrameworks } from './frameworks'; +import { clearDeclaredDependencies } from './frameworks/package-deps'; import { collectSynthesisInputs, synthesizeCallbackEdges } from './callback-synthesizer'; import { createYielder, type MaybeYield } from './cooperative-yield'; import { MAX_SOURCE_FILE_SIZE_BYTES } from '../file-limits'; @@ -230,6 +231,12 @@ export class ReferenceResolver { */ /** `persist: false` on a read-only connection: reuse detection skips without writing them. */ initialize(persist = true): void { + // A full index resolves by the package.json files on disk now, as the + // fingerprint it stamps says, even in a process that read them earlier. + this.workspacePackages = undefined; + // Detection reads through the caches (file contents, existence probes, + // declared dependencies): clear them first, or a process that read the + // manifests earlier detects by what they said then (#2519). this.clearCaches(); this.frameworksByLanguage.clear(); this.frameworks = detectFrameworksWithSkips(this.context, this.queries, { persist }); @@ -468,6 +475,7 @@ export class ReferenceResolver { clearSwiftTypeVisibility(this.context); clearTypeParameterMemos(this.context); clearRustScopeMemos(this.context); + clearDeclaredDependencies(this.context); } } diff --git a/src/resolution/synth-skips.ts b/src/resolution/synth-skips.ts index 63c15ad1d4..1c9b59391e 100644 --- a/src/resolution/synth-skips.ts +++ b/src/resolution/synth-skips.ts @@ -50,11 +50,17 @@ export class SynthSkips { private readonly recorded: Array<[string, string, string]> = []; private readonly hashes = new Map(); - constructor(private readonly known: Map>) {} + /** `onSkipped` hears of each file a skip answered for, so a caller can tell what the run depended on. */ + constructor( + private readonly known: Map>, + private readonly onSkipped?: (file: string) => void, + ) {} /** True when `pass` skipped `file` at its current indexed content. */ has(pass: string, file: string): boolean { - return this.known.get(pass)?.has(file) ?? false; + const skipped = this.known.get(pass)?.has(file) ?? false; + if (skipped) this.onSkipped?.(file); + return skipped; } /** Record that `pass` skipped `file` after reading exactly `content`. */ diff --git a/src/search/query-utils.ts b/src/search/query-utils.ts index 35457ec5e5..cff8c5d9fe 100644 --- a/src/search/query-utils.ts +++ b/src/search/query-utils.ts @@ -141,6 +141,90 @@ export function getStemVariants(term: string): string[] { return [...variants].filter(v => v.length >= 3 && v !== t); } +/** + * A Han, Hiragana, Katakana or Hangul character: scripts that write a word in + * one or two characters and put no space between words. By script extension, + * so the marks they share, like Japanese's prolonged sound mark `ー` and the + * iteration mark `々`, count with the word they sit in. + */ +const CJK_CHAR = /[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]/u; +const LETTER_OR_DIGIT = /[\p{L}\p{N}]/u; +const COMBINING_MARK = /\p{M}/u; +const NON_ASCII = /[^\x00-\x7F]/; + +/** Does `text` hold a Han, kana or Hangul character? */ +export function hasCjk(text: string): boolean { + return CJK_CHAR.test(text); +} + +/** + * `text` cut where a Han, kana or Hangul letter meets a letter or digit of + * another script: `NavMeshAgent类型` → `NavMeshAgent`, `类型` (#1262). Chinese + * and Japanese put no space between a name and the words around it, so the + * script switch is the only boundary such a query has; read as one word, it + * searched for `agent类型`, which nothing is called. Any other character, like + * `.`, `::` or `_`, ends a run without a cut, so `寻路服务.计算路径` stays whole. + */ +export function splitAtCjkSwitches(text: string): string[] { + if (!hasCjk(text)) return [text]; + const parts: string[] = []; + let start = 0; + let offset = 0; + let prevCjk: boolean | null = null; + for (const ch of text) { + if (LETTER_OR_DIGIT.test(ch)) { + const cjk = CJK_CHAR.test(ch); + if (prevCjk !== null && cjk !== prevCjk) { + parts.push(text.slice(start, offset)); + start = offset; + } + prevCjk = cjk; + } else if (!COMBINING_MARK.test(ch)) { + prevCjk = null; + } + offset += ch.length; + } + parts.push(text.slice(start)); + return parts.filter((part) => part.length > 0); +} + +/** + * Is `word` long enough to search for? Two characters in Han, kana or Hangul, + * where one or two make a word (`入口`, `寻路`, `로그`); three in any other + * script, so short words like English `is` or Russian `на` and `из` stay out. + */ +export function meetsQueryWordFloor(word: string): boolean { + return hasCjk(word) ? [...word].length >= 2 : word.length >= 3; +} + +/** + * Is `token` written like a name, in any script: `Session.request`, `pkg::run`, + * `로그인`, `寻路服务`, `寻路服务.计算路径`, `état`? Meets the word floor, so two + * characters do in Han, kana or Hangul and three elsewhere (#516, #1262). An + * ASCII token reads exactly as the ASCII-only test this replaces read it. + */ +export function isQueryNameToken(token: string): boolean { + return meetsQueryWordFloor(token) + && /^[\p{L}\p{Nl}_$][\p{L}\p{N}\p{M}_$]*(?:(?:::|\.)[\p{L}\p{N}\p{M}_$]+)*$/u.test(token); +} + +/** + * `text` with every word that has a non-ASCII letter but no Han, kana or + * Hangul blanked to spaces, for the patterns that only know ASCII letters: + * their `\b` reads `é` as a gap, so `état` gave `tat` and `ÉtatMachine` gave + * `tatMachine`. Such words are taken whole instead. Han, kana and Hangul stay, + * as no ASCII pattern matches inside them and their edge rightly ends the + * ASCII name beside them (`NavMeshAgent类型`). + */ +export function blankNonAsciiWords(text: string): string { + if (!NON_ASCII.test(text)) return text; + return text.replace(/[\p{L}\p{N}\p{M}]+/gu, (run) => + splitAtCjkSwitches(run) + .map((part) => (NON_ASCII.test(part) && !hasCjk(part) ? ' '.repeat(part.length) : part)) + .join(''), + ); +} + /** * Extract meaningful search terms from a natural language query. * Splits camelCase, PascalCase, snake_case, SCREAMING_SNAKE, and dot.notation @@ -152,16 +236,23 @@ export function getStemVariants(term: string): string[] { * * Also generates stem variants (e.g., "caching"→"cache", "eviction"→"evict") * so FTS prefix matching can find related code symbols. + * + * Words in any script are kept whole, as FTS stores a name: `로그인`, `寻路服务`, + * `état` (#516, #1262). A word in Han, kana or Hangul counts from two + * characters, and a run that switches between those and another script is cut + * at the switch (see {@link splitAtCjkSwitches}). An ASCII query's terms are + * exactly what they were. */ export function extractSearchTerms(query: string, options?: { stems?: boolean }): string[] { const includeStems = options?.stems !== false; const tokens = new Set(); + const asciiView = blankNonAsciiWords(query); // First, extract and preserve compound identifiers before splitting // CamelCase: scrapeLoop, UserService, getCallGraph const compoundPattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:[A-Z][a-z]+)+|[A-Z][a-z]+(?:[A-Z][a-z]*)+)\b/g; let match; - while ((match = compoundPattern.exec(query)) !== null) { + while ((match = compoundPattern.exec(asciiView)) !== null) { if (match[1] && match[1].length >= 3) { tokens.add(match[1].toLowerCase()); // preserve full compound: "scrapeloop" } @@ -169,7 +260,7 @@ export function extractSearchTerms(query: string, options?: { stems?: boolean }) // snake_case: scrape_loop, user_service const snakePattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:_[a-zA-Z0-9]+)+)\b/g; - while ((match = snakePattern.exec(query)) !== null) { + while ((match = snakePattern.exec(asciiView)) !== null) { if (match[1] && match[1].length >= 3) { tokens.add(match[1].toLowerCase()); } @@ -183,12 +274,13 @@ export function extractSearchTerms(query: string, options?: { stems?: boolean }) // Replace underscores and dots with spaces (snake_case, dot.notation) const normalised = camelSplit.replace(/[_.]+/g, ' '); - // Split on any non-alphanumeric character - const words = normalised.split(/[^a-zA-Z0-9]+/).filter(Boolean); + // Split on anything but a letter, digit or combining mark, in any script, + // then where Han, kana or Hangul meets another script. + const words = normalised.split(/[^\p{L}\p{N}\p{M}]+/u).filter(Boolean).flatMap(splitAtCjkSwitches); for (const word of words) { const lower = word.toLowerCase(); - if (lower.length < 3) continue; + if (!meetsQueryWordFloor(lower)) continue; if (STOP_WORDS.has(lower)) continue; tokens.add(lower); } @@ -531,9 +623,15 @@ export function kindBonus(kind: Node['kind']): number { * `FLAT`. A leading-capital-only word ("Screen", "Zustand") is also treated as * a plain word — sentence-start capitalization and proper nouns aren't reliable * identifier signals. + * + * A name in Han, kana or Hangul (`寻路服务`, `ユーザー取得`, `로그인`) has no case + * or underscore to show it is one, and Chinese and Japanese prose has no + * spaces to set a word apart, so such a token, set apart, counts as a name the + * user chose (#516, #1262). */ export function isDistinctiveIdentifier(token: string): boolean { if (!token) return false; + if (hasCjk(token)) return true; // snake_case / SCREAMING_SNAKE, or an embedded digit → a deliberate identifier. if (/[_0-9]/.test(token)) return true; // An uppercase letter anywhere AFTER the first char → a camelCase/PascalCase diff --git a/src/sync/worktree.ts b/src/sync/worktree.ts index 67357fed6e..6bceabcf89 100644 --- a/src/sync/worktree.ts +++ b/src/sync/worktree.ts @@ -208,8 +208,8 @@ export function detectWorktreeIndexMismatch( // and gitlinked clones, so a query run from inside one resolves up to the // parent index — whose graph *does* contain that nested repo's files. The // warning's premise ("results are a different branch; symbols changed only - // here are missing") is false there, and its "run codegraph init -i" advice - // would needlessly fragment the unified workspace index. A genuine borrowed + // here are missing") is false there, and its "codegraph init" advice would + // needlessly fragment the unified workspace index. A genuine borrowed // worktree and the index root are the SAME repository (they share a git // common dir); a submodule/embedded clone is a DIFFERENT repository and does // not — so suppress only when the two clearly differ. (#1031, #1033) @@ -280,15 +280,32 @@ function nativeRealpath(p: string): string { } } -/** One-line-per-fact warning describing a detected mismatch. */ -export function worktreeMismatchWarning(m: WorktreeIndexMismatch): string { +/** + * The way out of a borrowed index: one built in the worktree itself. A person + * at the CLI is told to build it. An agent is told the user can: indexing is + * the user's decision, and the server instructions tell the agent not to run + * `codegraph init` itself, so a tool result must not tell it to. + */ +function worktreeIndexAdvice(audience: 'user' | 'agent'): string { + return audience === 'agent' + ? 'The user can run "codegraph init" in this worktree for a worktree-local index.' + : 'Run "codegraph init" in this worktree for a worktree-local index.'; +} + +/** + * One-line-per-fact warning describing a detected mismatch: `codegraph status` + * prints it for the user, and `codegraph_status` shows the agent's wording. + */ +export function worktreeMismatchWarning( + m: WorktreeIndexMismatch, + audience: 'user' | 'agent' = 'user', +): string { return ( `This CodeGraph index belongs to a different git working tree.\n` + ` Running in: ${m.worktreeRoot}\n` + ` Index from: ${m.indexRoot}\n` + `Results reflect that tree's code (often a different branch), not this worktree — ` + - `symbols changed only here are missing. Run "codegraph init -i" in this worktree ` + - `for a worktree-local index.` + `symbols changed only here are missing. ${worktreeIndexAdvice(audience)}` ); } @@ -296,13 +313,14 @@ export function worktreeMismatchWarning(m: WorktreeIndexMismatch): string { * Compact, single-line variant for prefixing a tool's result. Read tools * return their answer inline, so the heads-up has to ride on the same payload * the agent is already reading — a multi-line block would bury the result. + * Its reader is an agent, through MCP, the prompt hook or the `codegraph + * explore` and `codegraph node` CLI. */ export function worktreeMismatchNotice(m: WorktreeIndexMismatch): string { return ( `⚠ CodeGraph results below come from a different git worktree (${m.indexRoot}), ` + `not where you're working (${m.worktreeRoot}) — they may reflect another branch, ` + - `and symbols changed only here are missing. Run "codegraph init -i" here for a ` + - `worktree-local index.` + `and symbols changed only here are missing. ${worktreeIndexAdvice('agent')}` ); }