diff --git a/.gitignore b/.gitignore index 6679462..5f51c65 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,9 @@ __pycache__/ # Claude specific exclusions .claude/ -# Miscellaneous git templates and cmds +# Miscellaneous git/ +new_feature_research/ # Never commit secrets (tokens may match GLEAN_* or glean_tok_* patterns) .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 84525d0..324081a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,35 @@ For what Glean Code is and how to run it, see the [README](README.md). ### Added +- **Flow mapper (`/flow`)** — a local capture layer that records the investigations you run, + enriches their citations with real document text, and finds the connections between them. + Chat turns group by the `chatId` the API returns, so threading is exact rather than inferred. + Linking runs in three tiers — shared ticket identifiers, shared phrases anchored on document + titles, and cross-session links through a shared or linked document — and every link stores + the evidence for it. `/flow timeline` renders a self-contained HTML timeline with no external + references. Data lives in `~/.gleancode/flow.db` (`0600`), partitioned by instance, mode, and + `act_as` so fictional corpus rows can never link to real tenant content. Capture defaults to + **mock only**; recording live data is opt-in via `flow_capture`, because a local cache has no + permission model. Full guide: [docs/FLOW_MAPPER.md](docs/FLOW_MAPPER.md). +- **`/flow show` draws a rail.** Sessions are nodes on a vertical spine in the order they + happened; a connection branches off it on a yellow `├──◆`, stacking the two documents that + bridge around a `↓` with the evidence that earned the link. Each cited document is tagged with + its datasource in a consistent colour. Colour is decoration only — piped or under `NO_COLOR` + the glyphs still carry the structure — and no line exceeds the terminal width at any size. + `--docs ` sets how many documents are listed per session before the rest are counted. +- **`/flow show` orders by what tells you something.** Connections are ranked by kind before + score, because a `shared-citation` link scores `1.00` and says only "you ran this twice" + while the `linked-document` link that found something scores lower — re-running one + investigation used to bury the discovery under a wall of `1.00`s. Documents a thread kept + returning to lead; the rest hold citation order, since sorting by rank interleaves the turns + (every turn's citations restart at rank 0). `--links ` caps connections per session. +- **Per-datasource colours** — `ui.DATASOURCE_COLOURS` and `ui.datasource_colour()`, so one + source looks the same wherever it appears. Unknown datasources fall back to grey rather than + being assigned a colour, which would let an unfamiliar source impersonate a familiar one. +- **`width` on `ui.rule()`** — so a block that caps its own columns can draw rules that match. +- **Three more MCP tools** — `get_flow`, `get_flow_summary`, and `get_flow_collapsed` expose the + captured graph to an agent, with the same `[MOCK MODE]` banner and partition rules. +- **`flow_capture` config key** — `mock` (default), `on`, or `off`. - **`/mcp`** — inspect, configure, and run the bundled MCP server without leaving the REPL. `/mcp status` reports the installed `mcp` version, whether it can actually run the server, and any running instance's pid, URL, uptime, and mode. `/mcp config [client]` prints the @@ -43,8 +72,16 @@ For what Glean Code is and how to run it, see the [README](README.md). - **CI workflow renamed** from `tests.yml` to `release.yml`, and it now publishes the built zipapp as a downloadable workflow artifact. +### Changed + +- **`session_links` records both ends of a link** (`to_doc`), and `get_flow_summary` returns the + structured parts — document titles, the shared evidence, and each end's session id — instead + of only a sentence built for a human. Schema version 2; `connect()` migrates an existing + database by adding the column, since `CREATE TABLE IF NOT EXISTS` never reaches one. + ### Fixed +- **Mock `/getdocuments` returned no document content.** Real `/getdocuments` returns a body; the mock returned metadata only, which left anything downstream of a citation with nothing to read. It now returns the corpus document's text. - **`pip install "mcp[cli]"` broke fresh installs.** The MCP SDK's 2.0.0 release renamed `FastMCP` to `MCPServer` and removed the `mcp.server.fastmcp` module `glean_mcp.py` imports, so an unpinned install resolved to 2.x and failed on import. Install instructions now pin diff --git a/README.md b/README.md index fcb926a..88e6ede 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ A local, terminal-first client for the Glean Client REST API. Inspired by Claude - [Tokens and auth](#tokens-and-auth) - [Config keys](#config-keys) - [MCP server](#mcp-server) +- [Flow mapper](#flow-mapper) — map what you have investigated - [Project layout](#project-layout) - [Running tests](#running-tests) - [Documentation](#documentation) — the full reference set @@ -57,6 +58,7 @@ A local, terminal-first client for the Glean Client REST API. Inspired by Claude - **Offline by default** — a real mock corpus of interlinked documents across five faux datasources, so every command is explorable without credentials. See [docs/MOCK_CORPUS.md](docs/MOCK_CORPUS.md) - **Browser SSO or API token** — `/auth login` runs OAuth 2.1 + PKCE against your instance, or paste a Glean-issued token. Secure refs keep real secrets in environment variables, never on disk - **MCP server** (`glean_mcp.py`) for Claude Code, Claude Desktop, and Cursor +- **Flow mapper** — `/flow` records the investigations you run, enriches their citations with real text, and finds connections between conversations that never shared context. `/flow show` draws them as a rail with each connection branching off it and the evidence that earned it; `/flow timeline` renders the same graph as a self-contained HTML page. Local SQLite, opt-in for live data. See [docs/FLOW_MAPPER.md](docs/FLOW_MAPPER.md) - Terminal niceties: `/help ` for every command, tab completion that cycles matches, a powerline-style status bar, and `/scaffold` to generate stdlib-only starter projects ## Getting started @@ -146,29 +148,11 @@ A native VS Code extension that brings the full Glean Code REPL — slash comman ![Glean Code VS Code extension preview](assets/vscode_extension_glean-code-cli.png) -### Flow Mapper - -`/flow` records the investigations you run — every `/chat` and `/search`, and the documents they -cited — then finds the connections between them. Not only the obvious "both mentioned INC-1183", -but the indirect case: two conversations sharing no vocabulary at all, connected because a -document cited by one refers to the other's subject in passing. - -Below, a checkout incident and a customer renewal link on `incident, checkout`. Neither -conversation mentions the other. The QBR simply refers to "the checkout incident" in prose — -no ticket number, nothing to join on. - -![Flow Mapper preview — /flow show drawing two linked investigations](assets/flow_mapper_preview.png) - -Sessions run down a rail in the order you worked; each connection branches off it carrying the -evidence that earned it, so every link can be read rather than taken on trust. Capture is local -SQLite, defaults to recording mock traffic only, and the whole thing works offline against the -built-in corpus — no token needed to try it. - ## Commands at a glance | Area | Commands | | --- | --- | -| Shell | `/help` `/status` `/doctor` `/auth` `/login` `/logout` `/open` `/ask` `/config` `/mode` `/mcp` `/history` `/clear` `/exit` | +| Shell | `/help` `/status` `/doctor` `/auth` `/login` `/logout` `/open` `/ask` `/config` `/mode` `/mcp` `/flow` `/history` `/clear` `/exit` | | Chat and search | `/chat` `/search` `/autocomplete` `/recommendations` `/feedback` `/datasources.list` | | Indexing — read & debug | `/datasources.status` `/datasources.config` `/documents.status` `/documents.count` `/users.count` `/documents.access` `/debug.document` `/debug.documents` `/debug.user` `/indexing.rotate-token` | | Indexing — single write | `/index.document` `/index.permissions` `/index.user` `/index.group` `/index.membership` and their `/index.delete-*` partners | @@ -266,6 +250,20 @@ want a server that isn't owned by a client. Setup for all three clients, the tool table, and the mock-mode rationale: **[docs/MCP.md](docs/MCP.md)**. +## Flow mapper + +`/flow` records the investigations you run — every `/chat` and `/search`, and the documents they cited — then finds the connections between them. Not only the obvious "both mentioned INC-1183", but the indirect case: two conversations sharing no vocabulary at all, connected because a document cited by one refers to the other's subject in passing. + +Below, a checkout incident and a customer renewal link on `incident, checkout`. Neither conversation mentions the other. The QBR simply refers to "the checkout incident" in prose — no ticket number, nothing to join on. + +![Flow mapper — /flow show drawing two linked investigations](assets/flow_mapper_preview.png) + +Sessions run down a rail in the order you worked; each connection branches off it carrying the evidence that earned it, so every link can be read rather than taken on trust. `/flow timeline` renders the same graph as a self-contained HTML page. + +Capture is a local SQLite database at `~/.gleancode/flow.db`, partitioned so mock content can never link to real tenant content. It defaults to recording **mock traffic only** — a local cache has no permission model, so recording live data is opt-in via `flow_capture`. + +The whole feature works offline against the built-in corpus, with no token. Full guide: **[docs/FLOW_MAPPER.md](docs/FLOW_MAPPER.md)**. + ## Project layout ```text @@ -280,6 +278,7 @@ glean-code-cli/ config.py config file load and save help_docs.py per-command documentation mcp_control.py /mcp — MCP server diagnostics and process control + flow.py /flow — capture, enrich, link, and render investigations mock_corpus.py the fake corpus every mock endpoint reads from _indexing_walk.py --path file walking for indexing commands completion.py readline tab completion @@ -288,7 +287,7 @@ glean-code-cli/ auth_commands.py /auth command handlers auth/ OAuth 2.1 + PKCE: oauth, pkce, callback_server, token_store, manager - tests/ 17 test modules, stdlib unittest only + tests/ 18 test modules, stdlib unittest only docs/ full reference set — see below ``` @@ -313,7 +312,7 @@ files, and they outrank the `Glean Code.app` launcher in `Cmd+Space`: export PYTHONPYCACHEPREFIX="$HOME/.cache/python" ``` -776 tests covering the client and every mock response, commands and dispatch, config, UI, auth, completion, help docs, the mock corpus, indexing-walk, scaffold, the installer, and the MCP server. Development notes: [docs/TESTING.md](docs/TESTING.md). +834 tests covering the client and every mock response, commands and dispatch, config, UI, auth, completion, help docs, the mock corpus, indexing-walk, scaffold, the installer, the MCP server, and the flow mapper. Development notes: [docs/TESTING.md](docs/TESTING.md). ## Documentation @@ -328,6 +327,7 @@ export PYTHONPYCACHEPREFIX="$HOME/.cache/python" | [docs/SSO_OAUTH.md](docs/SSO_OAUTH.md) | Browser SSO via OAuth 2.1 + PKCE | | [docs/SECURE_TOKENS.md](docs/SECURE_TOKENS.md) | Secure refs, masking matrix, mock-mode fallback | | [docs/MCP.md](docs/MCP.md) | MCP server setup for Claude Code, Claude Desktop, Cursor | +| [docs/FLOW_MAPPER.md](docs/FLOW_MAPPER.md) | `/flow` — capturing investigations, linking them, and the retention questions | | [docs/REST_PATHS.md](docs/REST_PATHS.md) | Every REST path this client targets, and how to retarget them | | [docs/TESTING.md](docs/TESTING.md) | Test-suite development notes | | [SUPPORT.md](SUPPORT.md) | Best-effort support expectations, triage order, how to file a good bug report | diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 19437a5..a451775 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -238,6 +238,87 @@ Inspect, configure, and run the bundled MCP server without leaving the REPL. --- +#### /flow + +Map what you have investigated: captured chats, the documents they cited, and the connections between them. Full guide: [docs/FLOW_MAPPER.md](FLOW_MAPPER.md). + +```text +/flow [--docs ] [--links ] [--limit ] [--min-score ] [--output ] [--print] [--all] [--older-than ] +``` + +| Subcommand | Description | +| --- | --- | +| `status` | Capture setting, database path and size, and what has been recorded. The default when no subcommand is given. | +| `enrich` | Fetch document text for captured citations, via `/getdocuments` then `/summarize`. Required before linking. | +| `link` | Find document-to-document and cross-session links. | +| `show` | Draw the captured investigations as a vertical rail, with connections branching off it. | +| `timeline` | Write a self-contained HTML timeline and open it. | +| `purge` | Delete captured data, after confirming. | + +| Flag | Description | +| --- | --- | +| `--docs` | `show`: documents listed per session before the rest are counted. Default `6`. | +| `--links` | `show`: connections drawn per session before the rest are counted. Default `3`. | +| `--limit` | `enrich`: documents to fetch in one run. Default `50`. | +| `--min-score` | `link`: phrase-link threshold, 0–1. Default `0.45`. Higher is stricter. | +| `--output` | `timeline`: where to write the HTML. Default a temp file. | +| `--print` | `timeline`: write without opening a browser. | +| `--all` | `purge`: every instance and mode, not just the current one. | +| `--older-than` | `purge`: only sessions older than this many days. | + +```text +/flow status +/flow enrich +/flow link --min-score 0.6 +/flow show +/flow show --docs 3 +/flow timeline --output ~/flow.html +/flow purge --older-than 30 +``` + +**Output** — `show` draws each session as a node on a vertical rail, with its questions, sources, and any connection branching off in yellow: + +```text +── flow: acme-be.glean.com · mock ──────────────────────────────────────────── + + ●─ 1 what happened in the checkout incident? + │ Tue 18 Aug 2026, 22:05 · 4 turns · 6 sources + │ ↳ who owned the fix? + │ + │ ▪ confluence Postmortem: Checkout Latency Incident (INC-1183) + │ ▪ slack War room thread: checkout 5xx spike + │ ▪ jira INC-1183 — Elevated 5xx on checkout API + │ … 3 more documents + │ + ├──◆ linked-document 0.60 → 2 ─────────────────────────────────────────── + │ Postmortem: Checkout Latency Incident (INC-1183) + │ ↓ shares: incident, checkout + │ Customer QBR — Northwind Retail + │ + ●─ 2 what are the risks going into the Northwind renewal? + │ Tue 18 Aug 2026, 22:05 · 2 turns · 3 sources + │ + │ ▪ gdrive Customer QBR — Northwind Retail + │ ▪ jira SUP-882 — Northwind: search results missing Confluence… + │ ▪ slack Northwind attachment issue — need connector eyes + │ +────────────────────────────────────────────────────────────────────────────── +``` + +Documents a thread kept returning to lead the list; the rest hold the order they were cited in. Connections are ordered by how much they tell you — a `linked-document` link found something, while a `shared-citation` link between two runs of the same question is trivially certain and says little — so the discovery is never buried under a wall of `1.00` scores. + +Each document is tagged with its datasource in a consistent colour, so a source is recognisable before you read its name. A connection names the session it reaches (`→ 2`), stacks the two documents that bridge around a `↓`, and prints the shared evidence — `shares: incident, checkout` — so every link can be read rather than taken on trust. Colour is decoration only: piped, redirected, or under `NO_COLOR`, the glyphs still carry the structure. + +`status` prints a table; `timeline` reports the file written; `purge` confirms before deleting. + +**Capture is opt-in for live data.** The `flow_capture` config key defaults to `mock`, so real tenant content is never recorded until you set it to `on`. A local database has no permission model — see [the retention section](FLOW_MAPPER.md#privacy-retention-and-the-parts-to-think-about) before enabling it against a tenant. + +**Mock mode** — the built-in corpus is what this feature was tuned against: seven identifier clusters plus a QBR that references an incident in prose with no ticket number, which is the link worth finding. + +**Endpoint** — `(local — ~/.gleancode/flow.db; enrich calls /getdocuments or /summarize)` + +--- + #### /ask Translate a natural-language request into a sequence of Glean Code slash commands using Glean Assistant as the planner. Read [docs/NATURAL_LANGUAGE.md](NATURAL_LANGUAGE.md) for the full design. diff --git a/docs/FLOW_MAPPER.md b/docs/FLOW_MAPPER.md new file mode 100644 index 0000000..90d5466 --- /dev/null +++ b/docs/FLOW_MAPPER.md @@ -0,0 +1,330 @@ +# Flow mapper + +`/flow` records the investigations you run through Glean Code, enriches the documents they +cite with real text, and finds the connections between them — including between conversations +that never shared any context. + +Everything is local: one SQLite file at `~/.gleancode/flow.db`, no dependencies beyond the +standard library, no network beyond the calls you were making anyway. + +## Contents + +- [The five-minute tour](#the-five-minute-tour) +- [What gets captured, and when](#what-gets-captured-and-when) +- [The commands](#the-commands) +- [How linking works](#how-linking-works) +- [The timeline](#the-timeline) +- [MCP tools](#mcp-tools) +- [Mock mode and live mode](#mock-mode-and-live-mode) +- [Privacy, retention, and the parts to think about](#privacy-retention-and-the-parts-to-think-about) +- [Troubleshooting](#troubleshooting) + +## The five-minute tour + +Run this in mock mode and you'll see the whole feature work against the built-in corpus, with +no credentials and no network. + +```text +/mode mock +/config set instance acme-be.glean.com +/config set flow_capture on + +/chat "what happened in the checkout incident?" +/chat "who owned the fix?" +/chat "what are the risks going into the Northwind renewal?" --new + +/flow enrich +/flow link +/flow show +``` + +The last command draws the two investigations and the connection between them: + +```text +── flow: acme-be.glean.com · mock ──────────────────────────────────────────── + + ●─ 1 what happened in the checkout incident? + │ Tue 18 Aug 2026, 22:05 · 4 turns · 6 sources + │ ↳ who owned the fix? + │ + │ ▪ confluence Postmortem: Checkout Latency Incident (INC-1183) + │ ▪ slack War room thread: checkout 5xx spike + │ ▪ jira INC-1183 — Elevated 5xx on checkout API + │ … 3 more documents + │ + ├──◆ linked-document 0.60 → 2 ─────────────────────────────────────────── + │ Postmortem: Checkout Latency Incident (INC-1183) + │ ↓ shares: incident, checkout + │ Customer QBR — Northwind Retail + │ + ●─ 2 what are the risks going into the Northwind renewal? + │ Tue 18 Aug 2026, 22:05 · 2 turns · 3 sources + │ + │ ▪ gdrive Customer QBR — Northwind Retail + │ ▪ jira SUP-882 — Northwind: search results missing Confluence… + │ ▪ slack Northwind attachment issue — need connector eyes + │ +────────────────────────────────────────────────────────────────────────────── +``` + +Nothing in those two conversations shares a word. They connect because the Northwind QBR +mentions *"the checkout incident in June"* in passing — no ticket number, no shared vocabulary, +just a document referring to the other investigation's subject. + +> **`--new` goes after the message.** `/chat --new "…"` makes the parser read your message as +> the flag's value. Write `/chat "…" --new`. Without it, `/chat` continues the current thread — +> which is usually what you want, and is exactly why the two questions above stay in one session +> while the third starts its own. + +Then look at it: + +```text +/flow timeline +``` + +## What gets captured, and when + +Capture is governed by one config key: + +| `flow_capture` | Behaviour | +| --- | --- | +| `mock` | **Default.** Records only mock-mode traffic. Real tenant content never touches the database. | +| `on` | Records both modes. | +| `off` | Records nothing. | + +```text +/config set flow_capture on +``` + +Two endpoints are recorded: + +- **`/chat`** — your question, the answer, and every cited document. Turns are grouped by the + `chatId` the API returns, so a conversation is one session with no guessing involved. +- **`/search`** — the query and its results. A search runs within the proximity window (10 + minutes) of a session's last activity attaches to that session, because a search you run + mid-investigation is part of it. + +Capture happens inside `GleanClient._post`, the single funnel every Client API call passes +through, so it covers the REPL and the MCP server alike. **A capture failure can never break a +command** — errors there are swallowed by design. + +Every row is tagged with the instance, the mode, and the `act_as` value in force. Nothing is +ever queried or linked across those boundaries. + +## The commands + +```text +/flow +``` + +### `/flow status` + +Capture setting, database path and size, and what's been recorded. + +```text + capture on → recording in mock mode + database ~/.gleancode/flow.db (64 KB) + scope acme-be.glean.com · mock + sessions 2 + turns 6 + documents 9 (9 enriched) + document links 14 + session links 1 +``` + +### `/flow enrich` + +Citations arrive as metadata — title, URL, datasource — with no text. Enrichment fetches the +content, trying `/getdocuments` first and falling back to `/summarize`. **Linking needs this; +run it before `/flow link`.** + +`--limit ` caps how many documents are fetched in one go (default 50). Documents that return +nothing stay in the graph as metadata rather than disappearing. + +### `/flow link` + +Finds document-to-document and session-to-session links. Re-runs cleanly — links are rebuilt, +not appended. + +`--min-score <0-1>` sets the phrase-link threshold (default `0.45`). Raise it if you see links +you disagree with; identifier links are unaffected because they're exact. + +### `/flow show` + +The terminal view. Sessions run down a vertical rail in the order they happened; connections +branch off it. + +`--docs ` sets how many documents are listed per session before the rest are counted +(default `6`). `--links ` does the same for connections (default `3`). + +**What leads the list** + +Documents a thread kept returning to come first; everything else holds the order it was cited +in. Sorting by the citation rank instead would interleave the turns, because every turn's +citations restart at rank 0 — a tangential follow-up's top hit would land level with the +document the thread is actually about. + +Connections are ordered by how much they tell you, not by score. A `shared-citation` link is +trivially certain — two sessions cited the same document — and scores `1.00`; a +`linked-document` link is the one that found something and typically scores lower. Ordering by +score would bury the discovery, which matters as soon as you run the same investigation twice: +each re-run adds a `1.00` link to every earlier identical session. + +**Reading the rail** + +| | | +| --- | --- | +| `●─ 1` | A session. The number is its position on the rail, and it's what a connection points at. | +| `│` | The rail itself — everything indented off it belongs to the session above. | +| `↳` | A follow-up question in the same thread. Four are shown, then a count. | +| `▪ jira` | A cited document, tagged with its datasource. Each source keeps the same colour everywhere. | +| `├──◆` | A connection leaving this session, drawn in yellow so it reads as a departure from the rail. | +| `→ 2` | Which session the connection reaches. It is not always the next one down. | +| `↓` | The direction of the link, with the shared evidence beside it. | + +The bridge is the part worth reading closely. It names the document in *this* session, the +document in the *other* one, and what they share: + +```text + ├──◆ linked-document 0.60 → 2 ─────────────────────────────────────────── + │ Postmortem: Checkout Latency Incident (INC-1183) + │ ↓ shares: incident, checkout + │ Customer QBR — Northwind Retail +``` + +A `shared-citation` link collapses to one line, because both ends are the same document: + +```text + ├──◆ shared-citation 1.00 → 3 ──────────────────────────────────────────── + │ both cited: Postmortem: Checkout Latency Incident (INC-1183) +``` + +**Colour is decoration, never the message.** Piped, redirected, or with `NO_COLOR` set, every +line still carries its glyph, so the rail and the branch survive `\| less` and `> flow.txt`. +The block also fits whatever width it's given, capped at 100 columns so long lines stay +readable on a wide screen, and truncates titles with `…` rather than wrapping them. + +### `/flow timeline` + +Writes a self-contained HTML file and opens it. `--output ` chooses where; `--print` +writes without opening, which is what you want over SSH. + +### `/flow purge` + +Deletes captured data, after confirming. Scoped to the current instance and mode by default. + +```text +/flow purge # this instance and mode +/flow purge --older-than 30 # …and only sessions older than 30 days +/flow purge --all # everything in the database +``` + +## How linking works + +Three tiers. Each stores its evidence, so every link can be explained rather than asserted — a +graph you can't interrogate is worse than no graph. + +**1. Identifier — exact, score 1.0.** A shared ticket key or repo reference: `INC-1183`, +`SUP-882`, `acme/payments#1841`. No false positives; a shared identifier is a real relationship. + +**2. Phrase — scored, this is the one that finds the interesting links.** Two signals combine: + +- a **shared bigram** — the same two-word phrase in both documents +- a **title anchor** — a word from one document's *title* appearing in the other's text + +The title anchor is what makes this precise, and it's worth understanding why. Rarity alone +doesn't work: in a small set, a word used once anywhere scores higher than a meaningful word +used four times, so ranking by rarity surfaces accidents like "across" and "percent". A title is +a curated human label, so prose echoing one is a genuine reference. That's what connects a QBR +saying *"the checkout incident in June"* to a postmortem titled *"Checkout Latency Incident"* — +the two share no bigram at all, but `checkout` and `incident` both sit in the postmortem's title. + +**3. Session links.** Two sessions connect when they cite the same document +(`shared-citation`), or when a document cited by one links to a document cited by the other +(`linked-document`). The second is the indirect case — the one where neither conversation knew +about the other. + +## The timeline + +`/flow timeline` writes a single HTML file with no external references: no CDN, no framework, +no fonts, nothing that phones home. It follows your system light/dark preference. + +- Sessions run down a timeline with real dates. +- Multi-turn threads collapse into one box with a turn count; click to expand. +- Repeated questions fold into a single row with a `×n` badge. +- Questions appear in the session header rather than inside the document graph, so the graph + stays about content. +- Connections render between sessions with their evidence line visible. +- Mock-captured data carries a banner saying so. + +## MCP tools + +Three tools expose the graph to an agent, alongside the existing four: + +| Tool | Returns | +| --- | --- | +| `get_flow` | The full graph as JSON — sessions, turns, citations, and every link. Optional `session_id` narrows it. | +| `get_flow_summary` | The narrative: what was investigated, what connected, and why, in prose | +| `get_flow_collapsed` | The compact view — threads folded into counted nodes | + +They read the same partition rules as the REPL, so an agent sees one instance and one mode. When +the server runs with `GLEAN_MOCK=1`, every response carries the `[MOCK MODE]` banner. + +Setup is unchanged — see [docs/MCP.md](MCP.md). + +## Mock mode and live mode + +**Mock mode is where this feature is at its best**, and not as a consolation. The corpus is +seventy interlinked documents with known relationships: seven identifier clusters, and the +QBR-to-postmortem reference that has no identifier at all. That makes link quality *testable* — +the test suite asserts the checkout connection is found and that its evidence is specific rather +than generic — which is otherwise very hard to pin down. + +**Live mode works, and is off by default.** `flow_capture` must be set to `on` explicitly. The +mechanics are identical; what changes is what's in the file. + +## Privacy, retention, and the parts to think about + +Read this before turning capture on against a real tenant. + +**A local cache has no permission model.** Glean filters what you can see at query time. Once a +document's text is in `flow.db`, that filtering is gone. The copy survives you losing access to +the document, the token being revoked, and the document being deleted or restricted at source. + +What the implementation does about it: + +- The database is created `0600`, matching `config.json` and `auth.json`. +- Capture defaults to mock, so real content is never recorded by accident. +- Rows are partitioned by instance, mode, and `act_as`. An impersonated view is someone else's + view of the tenant; linking across those would build connections no single person is entitled + to see, so it can't happen. +- `/flow purge --older-than ` exists for retention. + +What it does **not** do, and you should decide about: + +- There is no automatic expiry. Retention is a command you run, not a policy that enforces itself. +- The file is not encrypted. It's as exposed as anything else in your home directory — which is + fine until it's copied, backed up, or synced to a cloud drive. +- Nothing re-checks permissions. There is no mechanism to notice that a captured document is no + longer one you can see. + +If that's more than you want, `flow_capture` stays on `mock` and the feature remains a genuinely +useful offline tool. + +## Troubleshooting + +**`/flow link` finds nothing.** Run `/flow enrich` first — linking needs document text, and +citations arrive without it. `/flow status` shows how many documents are still unenriched. + +**Links look wrong.** Raise the threshold: `/flow link --min-score 0.65`. Identifier links are +exact and unaffected. Every link carries its evidence in `/flow show`, so you can see what +triggered it. + +**Two conversations that should be separate are one session.** `/chat` continues the current +thread by default. Use `/chat "…" --new`, with the flag *after* the message. + +**Nothing is being captured.** Check `/flow status`: capture defaults to `mock`, so live traffic +is ignored until you set `/config set flow_capture on`. + +**The timeline won't open.** Use `--print` and open the file yourself; a headless or remote shell +has no browser to hand off to. diff --git a/docs/MCP.md b/docs/MCP.md index 642b76c..319a907 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -90,6 +90,9 @@ by `/login` in the REPL). You can also pass them as environment variables: | `chat` | Chat with the Glean Assistant; pass `chat_id` to continue a thread | | `list_agents` | List available agents; optional `query` filter | | `run_agent` | Run an agent by id and return its output | +| `get_flow` | The captured investigation graph — sessions, citations, links. See [docs/FLOW_MAPPER.md](FLOW_MAPPER.md) | +| `get_flow_summary` | What was investigated and what connected to what, in prose | +| `get_flow_collapsed` | The compact view — threads folded into counted nodes | ## Running the MCP server on mock data @@ -149,7 +152,8 @@ inside `glean`: server running pid 73343, up 4m endpoint http://127.0.0.1:8791/mcp would serve mock [MOCK MODE banner active] - tools search, chat, list_agents, run_agent + tools search, chat, list_agents, run_agent, get_flow, + get_flow_summary, get_flow_collapsed log ~/.gleancode/mcp.log ──────────────────────────────────────────────────────────────────── ``` diff --git a/docs/TESTING.md b/docs/TESTING.md index a889ee0..3a3fadd 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2,7 +2,7 @@ Notes on the test suite added during development of glean-code-cli. See [Running tests](../README.md#running-tests) for the user-facing instructions on how to run the tests. -All 776 tests pass. Here's what was added across the development passes: +All 834 tests pass. Here's what was added across the development passes: `tests/test_commands_extended.py` (155 new tests) — covers all previously untested commands: @@ -73,3 +73,23 @@ All 776 tests pass. Here's what was added across the development passes: - Command dispatch — bare `/mcp` shows status, unknown subcommands and clients error, `--url` without a server errors, a non-numeric `--port` errors Every test redirects the state and log paths at a temp directory, so `~/.gleancode/` is never touched. + +`tests/test_flow.py` (58 new tests) — covers the flow mapper: + +- Capture gating — the `flow_capture` key defaults to `mock`, so live tenant content is never recorded by accident; `on` records both modes and `off` records nothing +- Capture — chat turns, citations, and search snippets land in SQLite; a shared `chatId` is one session and a new one starts another; every row is tagged with instance and mode; a capture failure cannot break the API call +- Enrichment — document text is fetched, and comes from `/getdocuments` now that the mock returns a body +- Linking — identifier links are exact and score 1.0; two sessions with no shared wording connect through a document that mentions the other's subject in passing; link evidence is asserted to be specific rather than generic; links never cross an instance or mode partition +- Title anchoring — the scorer prefers a word from the other document's title over a rarer word that appears nowhere meaningful, which is what stops "across" and "percent" being offered as evidence +- Queries and purge — summary, collapsed, and single-session views are JSON-serialisable; purge is scoped to a partition or clears everything +- Rendering — the timeline is self-contained (no external references), well-formed HTML, badges mock data, renders connections with their evidence, and handles an empty database +- Command dispatch — bare `/flow` shows status, unknown subcommands error, and bad `--min-score` / `--limit` / `--docs` values are rejected +- Schema migration — a database written before `session_links` carried `to_doc` gains the column on open, keeps its existing rows, and can be reopened repeatedly without the migration running twice +- `/flow show` rendering — sessions hang off a vertical rail, documents are labelled with their datasource, and a connection branches off with both document titles, the shared evidence, and the session number it reaches +- Colour is decoration — with colour disabled the output contains no escape sequences and every structural glyph (`●─`, `│`, `├──◆`, `↓`, `▪`) is still present, so the shape survives being piped +- Width — no line exceeds the terminal width at 40, 58, 84, or 120 columns, with colour on and off. Measured on *visible* width, since ANSI escapes have zero display width and `len()` would pass a broken layout +- Ordering — informative links (`linked-document`) come before trivial ones (`shared-citation`) regardless of score, verified against a database built by running the same investigation twice; documents a thread returned to lead the list, and singly-cited ones keep citation order rather than being interleaved by rank +- Overflow — `--links` caps connections per session and counts the remainder, and both `--docs` and `--links` reject non-integers +- Datasource colours — known sources each get a distinct colour, lookup ignores case and padding, and an unknown source falls back to grey rather than borrowing a familiar source's colour + +The module patches out the mock client's simulated 0.25s network latency; without that these 58 tests take 31 seconds instead of 2. diff --git a/glean_code/client.py b/glean_code/client.py index e7f296c..5356d41 100644 --- a/glean_code/client.py +++ b/glean_code/client.py @@ -40,6 +40,7 @@ except Exception: # pragma: no cover urllib = None # type: ignore +from . import flow as _flow from . import mock_corpus from .config import Config @@ -66,13 +67,26 @@ def _headers(self) -> Dict[str, str]: h["X-Glean-ActAs"] = self.config.act_as return h + def _capture(self, path: str, body: Dict[str, Any], resp: Dict[str, Any]) -> None: + """Hand the exchange to the flow mapper, if capture is on for this mode. + + Every Client API call funnels through _post, so this one hook covers + the REPL and the MCP server alike. flow.record never raises: a capture + bug must not fail a user's command. + """ + setting = getattr(self.config, "flow_capture", _flow.DEFAULT_CAPTURE) + if _flow.capture_enabled(setting, self.config.effective_mode): + _flow.record(self.config, path, body, resp) + def _post(self, path: str, body: Dict[str, Any]) -> Dict[str, Any]: if self.config.effective_mode == "mock": mock_corpus.use_path(self.config.mock_corpus_path) try: - return _mock_response(path, body) + resp = _mock_response(path, body) except mock_corpus.CorpusError as e: raise GleanError(str(e)) from None + self._capture(path, body, resp) + return resp base = self.config.effective_base_url if not base: @@ -85,7 +99,9 @@ def _post(self, path: str, body: Dict[str, Any]) -> Dict[str, Any]: raw = resp.read().decode("utf-8") if not raw: return {} - return json.loads(raw) + parsed = json.loads(raw) + self._capture(path, body, parsed) + return parsed except urllib.error.HTTPError as e: body_text = e.read().decode("utf-8", "replace") raise GleanError(f"HTTP {e.code} from {path}: {body_text}") from None @@ -712,7 +728,13 @@ def _mock_response(path: str, body: Dict[str, Any]) -> Dict[str, Any]: for s in specs: hit = mock_corpus.find(s) if hit: - docs.append(mock_corpus.as_document(hit)) + doc = mock_corpus.as_document(hit) + # Real /getdocuments returns document text; the mock used to + # return metadata only, which left anything downstream of a + # citation (the flow mapper especially) with nothing to read. + doc["body"] = {"mimeType": "text/plain", + "textContent": mock_corpus.expand(hit["body"])} + docs.append(doc) continue # Unknown id/url: echo it back so callers still get what they asked for. docs.append({"id": s.get("id") or s.get("url"), diff --git a/glean_code/commands.py b/glean_code/commands.py index 36a7530..6d97be1 100644 --- a/glean_code/commands.py +++ b/glean_code/commands.py @@ -21,6 +21,7 @@ from . import ui from . import _indexing_walk as _walk +from . import flow as _flow from . import mcp_control as _mcp from .client import GleanClient, GleanError from .config import Config, SECURE_REFS, is_secure_ref, resolve_secure @@ -2448,6 +2449,275 @@ def cmd_scaffold(s: Session, pos, flags): ui.print_info(f"Run with: python3 {out_path}") +# -------------------- flow mapper -------------------- + + +def _flow_scope(s: Session): + """Every query is scoped to one (instance, mode) partition. + + Fictional corpus rows and real tenant rows live in the same file and must + never be linked to each other, so nothing here is ever queried globally. + """ + return (s.config.instance or "local").strip(), s.config.effective_mode + + +def _flow_status(s: Session) -> None: + st = _flow.stats() + instance, mode = _flow_scope(s) + setting = getattr(s.config, "flow_capture", _flow.DEFAULT_CAPTURE) + on = _flow.capture_enabled(setting, mode) + capture = ui.style(f"{setting} → {'recording' if on else 'not recording'} in {mode} mode", + ui.C.GREEN if on else ui.C.GREY) + rows = [ + ("capture", capture), + ("database", f"{st['path']} ({st['size'] / 1024:.0f} KB)"), + ("scope", f"{instance} · {mode}"), + ("sessions", str(st["sessions"])), + ("turns", str(st["turns"])), + ("documents", f"{st['documents']} ({st['enriched']} enriched)"), + ("document links", str(st["doc_links"])), + ("session links", str(st["session_links"])), + ] + print(ui.rule("flow")) + print(ui.kv_table(rows)) + print(ui.rule()) + if st["documents"] and st["enriched"] < st["documents"]: + ui.print_info(f"{st['documents'] - st['enriched']} documents have no text yet — " + "run /flow enrich to fetch it.") + + +def _flow_enrich(s: Session, flags) -> None: + try: + limit = int(flags.get("limit") or 50) + except (TypeError, ValueError): + ui.print_err("--limit must be an integer.") + return + enriched, attempted = _flow.enrich(s.client, s.config, limit=limit) + if not attempted: + ui.print_info("Every captured document already has text.") + return + ui.print_ok(f"Enriched {enriched} of {attempted} documents.") + if enriched < attempted: + ui.print_info("The rest returned no content — they stay in the graph as metadata.") + + +def _flow_link(s: Session, flags) -> None: + instance, mode = _flow_scope(s) + try: + min_score = float(flags.get("min-score") or flags.get("min_score") or 0.45) + except (TypeError, ValueError): + ui.print_err("--min-score must be a number.") + return + docs = _flow.link_documents(instance=instance, mode=mode, min_score=min_score) + sess = _flow.link_sessions(instance=instance, mode=mode) + ui.print_ok(f"{docs} document links, {sess} session links.") + if sess: + ui.print_info("See them with /flow show, or /flow timeline for the rendered view.") + + +def _flow_fit(text: str, width: int, collapse: bool = True) -> str: + """One line, no wider than width. Long titles lose their tail, not the shape. + + Questions and titles arrive with newlines and runs of spaces in them, so + they are collapsed first. Lines this module composed itself are already + spaced deliberately — pass collapse=False to leave that spacing alone. + """ + text = " ".join((text or "").split()) if collapse else (text or "") + if width <= 1 or len(text) <= width: + return text + return text[: max(1, width - 1)].rstrip() + "…" + + +def _flow_bridge(c, order, width: int) -> None: + """One connection, branching off the rail. + + The two documents stack around a downward arrow carrying the shared + evidence, so which way the link runs and why are both visible without + reading prose. Drawn in yellow — the rail stays grey, so the branch reads + as a departure from the spine rather than part of it. + """ + accent = lambda t: ui.style(t, ui.C.YELLOW) # noqa: E731 + dim = lambda t: ui.style(t, ui.C.GREY) # noqa: E731 + + target = order.get(c["_to_session"], c["_to_session"]) + label = f"{c['kind']} {float(c['score']):.2f} → {target}" + # " " + "├──◆" + " " + label + " " + bar = "─" * max(0, width - 8 - len(label)) + print(f" {accent('├──◆')} {accent(label)} {dim(bar)}") + + body_w = width - 9 + if c["kind"] == "shared-citation": + print(f" {accent('│')} {dim('both cited:')} " + f"{_flow_fit(c['to_title'] or c['to_doc'] or '?', body_w - 12)}") + else: + print(f" {accent('│')} {_flow_fit(c['from_title'] or c['from_doc'] or '?', body_w)}") + shares = c.get("shares") or c.get("via_kind") or "related content" + print(f" {accent('│')} {accent('↓')} {dim('shares: ' + _flow_fit(shares, body_w - 12))}") + print(f" {accent('│')} {_flow_fit(c['to_title'] or c['to_doc'] or '?', body_w)}") + + +def _flow_show(s: Session, flags) -> None: + instance, mode = _flow_scope(s) + summary = _flow.get_flow_summary(instance=instance, mode=mode) + if not summary["sessions"]: + ui.print_info("Nothing captured for this instance and mode yet.") + return + try: + max_docs = int(flags.get("docs") or 6) + except (TypeError, ValueError): + ui.print_err("--docs must be an integer.") + return + try: + max_links = int(flags.get("links") or 3) + except (TypeError, ValueError): + ui.print_err("--links must be an integer.") + return + + width = max(40, min(ui.term_width(), 100)) + rail = lambda g: ui.style(g, ui.C.GREY) # noqa: E731 + dim = lambda t: ui.style(t, ui.C.GREY) # noqa: E731 + + sessions = summary["sessions"] + order = {sess["session_id"]: i + 1 for i, sess in enumerate(sessions)} + + # Hang each connection off the session it leaves from. A link may reach a + # session outside this partition, or arrive from one — keep it either way + # and point the arrow at whichever end we can actually name. + outgoing = {} + for c in summary["connections"]: + a, b = c["a_session"], c["b_session"] + if a in order: + outgoing.setdefault(a, []).append(dict(c, _to_session=b)) + elif b in order: + flipped = dict(c, _to_session=a, + from_title=c["to_title"], to_title=c["from_title"], + from_doc=c["to_doc"], to_doc=c["from_doc"]) + outgoing.setdefault(b, []).append(flipped) + + ds_w = min(12, max(6, max((len(d.get("datasource") or "") for sess in sessions + for d in sess["documents"]), default=6))) + + print(ui.rule(f"flow: {instance} · {mode}", width=width)) + print() + for sess in sessions: + n = order[sess["session_id"]] + head = sess["questions"][0] if sess["questions"] else f"session {sess['session_id']}" + print(f" {ui.style('●', ui.C.BLUE)}{rail('─')} " + f"{ui.style(str(n), ui.C.BLUE, ui.C.BOLD)} " + f"{ui.style(_flow_fit(head, width - 8), ui.C.WHITE, ui.C.BOLD)}") + + docs = sess["documents"] + turns = sess.get("turn_count") or len(sess["questions"]) + meta = (f"{_flow.when(sess['started_at'])} · " + f"{turns} turn{'s' if turns != 1 else ''} · " + f"{len(docs)} source{'s' if len(docs) != 1 else ''}") + print(f" {rail('│')} {dim(_flow_fit(meta, width - 8, collapse=False))}") + for q in sess["questions"][1:4]: + print(f" {rail('│')} {dim('↳ ' + _flow_fit(q, width - 12))}") + more_q = len(sess["questions"]) - 4 + if more_q > 0: + print(f" {rail('│')} {dim(f'↳ +{more_q} more')}") + + if docs: + print(f" {rail('│')}") + for d in docs[:max_docs]: + name = (d.get("datasource") or "—")[:ds_w] + colour = ui.datasource_colour(d.get("datasource")) + title = _flow_fit(d.get("title") or d["doc_id"], width - ds_w - 12) + print(f" {rail('│')} {ui.style('▪', colour)} " + f"{ui.style(name.ljust(ds_w), colour)} {title}") + if len(docs) > max_docs: + rest = len(docs) - max_docs + print(f" {rail('│')} " + f"{dim(f'… {rest} more document' + ('s' if rest != 1 else ''))}") + + print(f" {rail('│')}") + links = outgoing.get(sess["session_id"], []) + for c in links[:max_links]: + _flow_bridge(c, order, width) + print(f" {rail('│')}") + if len(links) > max_links: + hidden = len(links) - max_links + plural = "s" if hidden != 1 else "" + print(f" {rail('│')} " + f"{dim(f'… {hidden} weaker connection{plural} not shown')}") + print(f" {rail('│')}") + + if not summary["connections"]: + print(dim(" no connections yet — run /flow enrich, then /flow link")) + print() + print(ui.rule(width=width)) + + +def _flow_timeline(s: Session, flags) -> None: + import tempfile + instance, mode = _flow_scope(s) + out = flags.get("output") or flags.get("o") + if not out: + out = Path(tempfile.gettempdir()) / "glean-flow.html" + try: + path = _flow.write_timeline(Path(out), instance=instance, mode=mode) + except OSError as e: + ui.print_err(f"Could not write the timeline: {e}") + return + ui.print_ok(f"Wrote {path}") + if flags.get("print") or flags.get("no-open") or flags.get("no_open"): + return + try: + webbrowser.open(path.as_uri(), new=2) + ui.print_info("Opened in your browser.") + except Exception as e: # noqa: BLE001 - headless is normal over SSH + ui.print_info(f"Open it manually: {path} ({e})") + + +def _flow_purge(s: Session, pos, flags) -> None: + instance, mode = _flow_scope(s) + everything = bool(flags.get("all")) + days = flags.get("older-than") or flags.get("older_than") + try: + days = float(days) if days is not None else None + except (TypeError, ValueError): + ui.print_err("--older-than must be a number of days.") + return + + what = ("everything in the database" if everything + else f"captured data for {instance} · {mode}" + + (f" older than {days:g} days" if days else "")) + try: + confirm = input(ui.style(f"Delete {what}? [y/N]: ", ui.C.YELLOW)).strip().lower() + except (EOFError, KeyboardInterrupt): + print() + ui.print_info("Cancelled.") + return + if confirm not in ("y", "yes"): + ui.print_info("Cancelled.") + return + + removed = _flow.purge(instance=None if everything else instance, + mode=None if everything else mode, + older_than_days=days) + ui.print_ok(f"Removed {removed} session{'s' if removed != 1 else ''}.") + + +@register("flow") +def cmd_flow(s: Session, pos, flags): + sub = (pos[0] if pos else "status").lower() + if sub == "status": + _flow_status(s) + elif sub == "enrich": + _flow_enrich(s, flags) + elif sub == "link": + _flow_link(s, flags) + elif sub == "show": + _flow_show(s, flags) + elif sub == "timeline": + _flow_timeline(s, flags) + elif sub == "purge": + _flow_purge(s, pos[1:], flags) + else: + ui.print_err("Usage: /flow ") + + # -------------------- natural-language planner -------------------- _PLANNER_SYSTEM_PROMPT = ( diff --git a/glean_code/completion.py b/glean_code/completion.py index f133e50..0651dcf 100644 --- a/glean_code/completion.py +++ b/glean_code/completion.py @@ -33,6 +33,8 @@ _MCP_SUBCMDS = ["status", "config", "start", "stop"] _MCP_CLIENTS = ["claude-code", "claude-desktop", "cursor"] +_FLOW_SUBCMDS = ["status", "enrich", "link", "show", "timeline", "purge"] + class _Completer: def __init__(self) -> None: @@ -120,6 +122,12 @@ def _complete(self, text: str) -> List[str]: partial = "" if ends_with_space else tokens[-1] return [c for c in _MCP_CLIENTS if c.startswith(partial)] + # ── /flow subcommands ──────────────────────────────────────────── + if cmd == "flow": + if len(tokens) == 1 or (len(tokens) == 2 and not ends_with_space): + partial = "" if ends_with_space else tokens[-1] + return [s for s in _FLOW_SUBCMDS if s.startswith(partial)] + # ── /help ────────────────────────────────────────── if cmd == "help": if ends_with_space or len(tokens) == 2: diff --git a/glean_code/config.py b/glean_code/config.py index 5eb9c90..fcd83c7 100644 --- a/glean_code/config.py +++ b/glean_code/config.py @@ -46,6 +46,7 @@ class Config: theme: str = "glean" # glean | mono | neon default_page_size: int = 10 mock_corpus_path: Optional[str] = None # JSON file backing mock mode; falls back to the built-in corpus + flow_capture: str = "mock" # mock | on | off — see docs/FLOW_MAPPER.md window_title: str = "full" # full | plain (no hostname) | off history: list = field(default_factory=list) diff --git a/glean_code/flow.py b/glean_code/flow.py new file mode 100644 index 0000000..beada7d --- /dev/null +++ b/glean_code/flow.py @@ -0,0 +1,948 @@ +"""Flow mapper — local capture, enrichment, and link discovery. + +Every Client API call the CLI makes passes through one funnel +(`GleanClient._post`), so a single hook there records what you asked and what +came back. Chat turns, their citations, and search results land in a local +SQLite database; documents are enriched with real text; and a linker connects +related material — both document-to-document and across chat sessions that +never shared any context. + +The interesting case is the indirect one. Two unrelated investigations — a +checkout incident and a customer renewal — connect because a QBR document +mentions the incident in passing, with no ticket number to join on. + +Design notes worth keeping in view: + + * Rows are tagged with instance, mode, and act_as. Fictional mock content + must never link to a real tenant's documents, and one person's + impersonated view must never link to another's. + * Capture defaults to mock only. In live mode this file becomes a copy of + company content with the permission model stripped off, so turning it on + there is a deliberate act with a retention policy attached. + * sqlite3 is in the standard library, so the zero-dependency rule holds. +""" +from __future__ import annotations + +import json +import os +import re +import sqlite3 +import time +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +from .config import CONFIG_DIR + +DB_PATH = CONFIG_DIR / "flow.db" + +# Capture modes for the `flow_capture` config key. +CAPTURE_MODES = ("mock", "on", "off") +DEFAULT_CAPTURE = "mock" + +SCHEMA_VERSION = 2 + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT +); +CREATE TABLE IF NOT EXISTS sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chat_id TEXT, + instance TEXT NOT NULL, + mode TEXT NOT NULL, + act_as TEXT, + started_at REAL NOT NULL, + ended_at REAL +); +CREATE TABLE IF NOT EXISTS turns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + role TEXT NOT NULL, + text TEXT NOT NULL, + endpoint TEXT NOT NULL, + ts REAL NOT NULL +); +CREATE TABLE IF NOT EXISTS documents ( + doc_id TEXT NOT NULL, + instance TEXT NOT NULL, + mode TEXT NOT NULL, + title TEXT, + url TEXT, + datasource TEXT, + author TEXT, + updated_at REAL, + content TEXT, + content_source TEXT, + fetched_at REAL, + PRIMARY KEY (doc_id, instance, mode) +); +CREATE TABLE IF NOT EXISTS citations ( + turn_id INTEGER NOT NULL REFERENCES turns(id) ON DELETE CASCADE, + doc_id TEXT NOT NULL, + rank INTEGER NOT NULL, + PRIMARY KEY (turn_id, doc_id) +); +CREATE TABLE IF NOT EXISTS doc_links ( + a_doc TEXT NOT NULL, + b_doc TEXT NOT NULL, + instance TEXT NOT NULL, + mode TEXT NOT NULL, + kind TEXT NOT NULL, + score REAL NOT NULL, + evidence TEXT, + PRIMARY KEY (a_doc, b_doc, instance, mode, kind) +); +CREATE TABLE IF NOT EXISTS session_links ( + a_session INTEGER NOT NULL, + b_session INTEGER NOT NULL, + kind TEXT NOT NULL, + score REAL NOT NULL, + via_doc TEXT, + to_doc TEXT, + evidence TEXT, + PRIMARY KEY (a_session, b_session, kind, via_doc) +); +CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id); +CREATE INDEX IF NOT EXISTS idx_citations_doc ON citations(doc_id); +""" + + +class FlowError(Exception): + """Anything the user needs to be told about, phrased for a terminal.""" + + +# ---------------------------------------------------------------- connection + + +def connect(path: Optional[Path] = None) -> sqlite3.Connection: + """Open (creating if needed) the flow database with 0600 permissions.""" + target = Path(path) if path else DB_PATH + target.parent.mkdir(parents=True, exist_ok=True) + fresh = not target.exists() + conn = sqlite3.connect(str(target)) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.executescript(_SCHEMA) + _migrate(conn) + conn.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)", + (str(SCHEMA_VERSION),)) + conn.commit() + if fresh: + try: + os.chmod(target, 0o600) + except OSError: + pass + return conn + + +def _migrate(conn: sqlite3.Connection) -> None: + """Add columns to tables that already exist. + + `CREATE TABLE IF NOT EXISTS` is a no-op on an existing table, so a new + column never reaches a database written by an earlier version. Each entry + is additive and idempotent; nothing here drops or rewrites data. + """ + added = { + "session_links": {"to_doc": "TEXT"}, # v2: the far end of the bridge + } + for table, columns in added.items(): + try: + have = {r["name"] for r in conn.execute(f"PRAGMA table_info({table})")} + except sqlite3.Error: + continue + for name, decl in columns.items(): + if name not in have: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {decl}") + + +def db_size(path: Optional[Path] = None) -> int: + target = Path(path) if path else DB_PATH + try: + return target.stat().st_size + except OSError: + return 0 + + +# ---------------------------------------------------------------- capture + + +def capture_enabled(capture_setting: str, mode: str) -> bool: + """Should this call be recorded? + + 'mock' — the default — records only fictional corpus traffic, so the + database never accumulates real tenant content by accident. + """ + if capture_setting == "off": + return False + if capture_setting == "on": + return True + return mode == "mock" + + +def _partition(config) -> Tuple[str, str, Optional[str]]: + instance = (getattr(config, "instance", None) or "local").strip() + return instance, config.effective_mode, getattr(config, "act_as", None) + + +def _session_for(conn, chat_id, instance, mode, act_as, ts) -> int: + """Find the session for a chat id, or open one. Threading is exact. + + /chat echoes chatId back, so turns of one conversation share a key and + need no time-proximity guessing. Non-chat traffic attaches to the most + recent session in the same partition within the proximity window. + """ + if chat_id: + row = conn.execute( + "SELECT id FROM sessions WHERE chat_id = ? AND instance = ? AND mode = ?" + " AND IFNULL(act_as,'') = IFNULL(?,'')", + (chat_id, instance, mode, act_as), + ).fetchone() + if row: + return int(row["id"]) + cur = conn.execute( + "INSERT INTO sessions (chat_id, instance, mode, act_as, started_at)" + " VALUES (?, ?, ?, ?, ?)", + (chat_id, instance, mode, act_as, ts), + ) + return int(cur.lastrowid) + + +def _recent_session(conn, instance, mode, act_as, ts, window: float) -> Optional[int]: + row = conn.execute( + "SELECT s.id, MAX(t.ts) AS last_ts FROM sessions s JOIN turns t ON t.session_id = s.id" + " WHERE s.instance = ? AND s.mode = ? AND IFNULL(s.act_as,'') = IFNULL(?,'')" + " GROUP BY s.id ORDER BY last_ts DESC LIMIT 1", + (instance, mode, act_as), + ).fetchone() + if row and row["last_ts"] is not None and (ts - float(row["last_ts"])) <= window: + return int(row["id"]) + return None + + +def _upsert_document(conn, doc: Dict[str, Any], instance: str, mode: str) -> None: + md = doc.get("metadata") or {} + author = md.get("author") + if isinstance(author, dict): + author = author.get("name") or author.get("email") + conn.execute( + "INSERT INTO documents (doc_id, instance, mode, title, url, datasource," + " author, updated_at, content, content_source, fetched_at)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL)" + " ON CONFLICT(doc_id, instance, mode) DO UPDATE SET" + " title = COALESCE(excluded.title, title)," + " url = COALESCE(excluded.url, url)," + " datasource = COALESCE(excluded.datasource, datasource)," + " author = COALESCE(excluded.author, author)," + " updated_at = COALESCE(excluded.updated_at, updated_at)", + (doc.get("id") or doc.get("url"), instance, mode, doc.get("title"), + doc.get("url"), doc.get("datasource"), author, md.get("updateTime")), + ) + + +def record(config, path: str, body: Dict[str, Any], response: Dict[str, Any], + db: Optional[Path] = None, proximity_window: float = 600.0) -> None: + """Record one API exchange. Never raises — capture must not break a call.""" + try: + _record(config, path, body, response, db, proximity_window) + except Exception: # noqa: BLE001 - a capture bug must not fail the user's command + pass + + +def _record(config, path, body, response, db, proximity_window) -> None: + if path not in ("/chat", "/search"): + return + instance, mode, act_as = _partition(config) + ts = time.time() + conn = connect(db) + try: + if path == "/chat": + chat_id = response.get("chatId") or body.get("chatId") + session_id = _session_for(conn, chat_id, instance, mode, act_as, ts) + question = "" + msgs = body.get("messages") or [] + if msgs: + frags = msgs[-1].get("fragments") or [] + question = "".join(f.get("text", "") for f in frags) + if question: + conn.execute( + "INSERT INTO turns (session_id, role, text, endpoint, ts)" + " VALUES (?, 'user', ?, ?, ?)", + (session_id, question, path, ts), + ) + for msg in response.get("messages") or []: + text = "".join(f.get("text", "") for f in (msg.get("fragments") or [])) + cur = conn.execute( + "INSERT INTO turns (session_id, role, text, endpoint, ts)" + " VALUES (?, 'assistant', ?, ?, ?)", + (session_id, text, path, ts), + ) + turn_id = int(cur.lastrowid) + for rank, cite in enumerate(msg.get("citations") or []): + doc = cite.get("sourceDocument") or {} + doc_id = doc.get("id") or doc.get("url") + if not doc_id: + continue + _upsert_document(conn, doc, instance, mode) + conn.execute( + "INSERT OR IGNORE INTO citations (turn_id, doc_id, rank)" + " VALUES (?, ?, ?)", (turn_id, doc_id, rank), + ) + else: # /search + session_id = _recent_session(conn, instance, mode, act_as, ts, proximity_window) + if session_id is None: + session_id = _session_for(conn, None, instance, mode, act_as, ts) + query = body.get("query") or "" + cur = conn.execute( + "INSERT INTO turns (session_id, role, text, endpoint, ts)" + " VALUES (?, 'search', ?, ?, ?)", (session_id, query, path, ts), + ) + turn_id = int(cur.lastrowid) + for rank, res in enumerate(response.get("results") or []): + doc_id = res.get("id") or res.get("url") + if not doc_id: + continue + _upsert_document(conn, res, instance, mode) + snips = res.get("snippets") or [] + if snips: + conn.execute( + "UPDATE documents SET content = COALESCE(content, ?)," + " content_source = COALESCE(content_source, 'search-snippet')," + " fetched_at = COALESCE(fetched_at, ?)" + " WHERE doc_id = ? AND instance = ? AND mode = ?", + (snips[0].get("text", ""), ts, doc_id, instance, mode), + ) + conn.execute( + "INSERT OR IGNORE INTO citations (turn_id, doc_id, rank)" + " VALUES (?, ?, ?)", (turn_id, doc_id, rank), + ) + conn.commit() + finally: + conn.close() + + +# ---------------------------------------------------------------- enrichment + + +def documents_needing_content(conn, instance: str, mode: str, + limit: int = 50) -> List[sqlite3.Row]: + return conn.execute( + "SELECT * FROM documents WHERE instance = ? AND mode = ?" + " AND (content IS NULL OR content = '')" + " ORDER BY fetched_at IS NOT NULL, doc_id LIMIT ?", + (instance, mode, limit), + ).fetchall() + + +def enrich(client, config, db: Optional[Path] = None, + limit: int = 50) -> Tuple[int, int]: + """Fill in document text for captured citations. + + Tries /getdocuments first (real content when the API returns it) and falls + back to /summarize, which yields a usable extract in both modes. Returns + (enriched, attempted). + """ + instance, mode, _ = _partition(config) + conn = connect(db) + enriched = 0 + try: + rows = documents_needing_content(conn, instance, mode, limit) + for row in rows: + text, source = _fetch_content(client, row) + if not text: + continue + conn.execute( + "UPDATE documents SET content = ?, content_source = ?, fetched_at = ?" + " WHERE doc_id = ? AND instance = ? AND mode = ?", + (text, source, time.time(), row["doc_id"], instance, mode), + ) + enriched += 1 + conn.commit() + return enriched, len(rows) + finally: + conn.close() + + +def _fetch_content(client, row) -> Tuple[Optional[str], Optional[str]]: + spec: Dict[str, Any] = {} + if row["url"]: + spec["url"] = row["url"] + elif row["doc_id"]: + spec["id"] = row["doc_id"] + if not spec: + return None, None + + try: + resp = client.get_documents( + ids=[spec["id"]] if "id" in spec else None, + urls=[spec["url"]] if "url" in spec else None, + ) + for doc in resp.get("documents") or []: + body = doc.get("body") + if isinstance(body, dict): + body = body.get("textContent") or body.get("text") + text = body or doc.get("content") or doc.get("text") + if text: + return str(text), "getdocuments" + except Exception: # noqa: BLE001 - fall through to summarize + pass + + try: + resp = client.summarize(**spec) + summary = resp.get("summary") + if summary: + return str(summary), "summarize" + except Exception: # noqa: BLE001 - nothing else to try + pass + return None, None + + +# ---------------------------------------------------------------- linking + +# Ticket keys and repo references are the highest-precision signal available. +_ID_RE = re.compile(r"\b(?:INC|PLAT|PLAN|SEC|SUP|REV|PAY)-\d+\b|\b[\w.-]+/[\w.-]+#\d+\b") + +_STOPWORDS = frozenset(""" +a an and are as at be by do does for from has have how i in is it its of on or our +the their there this to us was we what when where which who why will with you your +be been being if then than that these those they them he she his her not no yes but +can could should would may might must about into over under after before during more +most some any each every other another such only just also very much many few both +""".split()) + +_WORD_RE = re.compile(r"[a-z0-9][a-z0-9'-]*") + + +def _tokens(text: str) -> List[str]: + return [w for w in _WORD_RE.findall((text or "").lower()) + if w not in _STOPWORDS and len(w) > 2] + + +def _bigrams(tokens: Sequence[str]) -> List[str]: + return [f"{a} {b}" for a, b in zip(tokens, tokens[1:])] + + +def build_idf(docs: Sequence[sqlite3.Row]) -> Dict[str, float]: + """Rarity weight per phrase. A phrase in most documents carries no signal.""" + n = max(1, len(docs)) + seen: Counter = Counter() + for d in docs: + text = f"{d['title'] or ''} {d['content'] or ''}" + toks = _tokens(text) + for phrase in set(toks) | set(_bigrams(toks)): + seen[phrase] += 1 + # A plain ratio is enough here and keeps the maths readable. + return {p: 1.0 - (c / n) for p, c in seen.items()} + + +def _identifiers(row) -> set: + return set(_ID_RE.findall(f"{row['title'] or ''} {row['content'] or ''}")) + + +def _doc_parts(row) -> Tuple[set, set, set]: + """(title words, all words, bigrams) for one document.""" + title = row["title"] or "" + all_toks = _tokens(f"{title} {row['content'] or ''}") + return set(_tokens(title)), set(all_toks), set(_bigrams(all_toks)) + + +def link_documents(db: Optional[Path] = None, instance: str = "", mode: str = "", + min_score: float = 0.45, max_phrases: int = 3) -> int: + """Discover document-to-document links. Returns the number written. + + Three tiers, each storing its evidence so a link can be explained rather + than asserted: + + identifier — a shared ticket key or repo reference. Exact. + phrase — a shared bigram, or a word from one document's *title* + appearing in the other's text. The title anchor is what + makes this precise: rarity alone promotes accidents, since + in a small corpus a word used once anywhere scores higher + than a meaningful word used four times. A title is a + curated label, so prose echoing one is a real reference — + this is what connects a QBR saying "the checkout incident + in June" to a postmortem titled "Checkout Latency Incident" + with no ticket number shared between them. + """ + conn = connect(db) + written = 0 + try: + docs = conn.execute( + "SELECT * FROM documents WHERE instance = ? AND mode = ?" + " AND content IS NOT NULL AND content != ''", + (instance, mode), + ).fetchall() + if len(docs) < 2: + return 0 + + idf = build_idf(docs) + ids = {d["doc_id"]: _identifiers(d) for d in docs} + parts = {d["doc_id"]: _doc_parts(d) for d in docs} + + conn.execute("DELETE FROM doc_links WHERE instance = ? AND mode = ?", + (instance, mode)) + for i, a in enumerate(docs): + for b in docs[i + 1:]: + a_id, b_id = a["doc_id"], b["doc_id"] + + shared_ids = ids[a_id] & ids[b_id] + if shared_ids: + written += _write_link(conn, a_id, b_id, instance, mode, + "identifier", 1.0, + ", ".join(sorted(shared_ids))) + continue + + a_title, a_words, a_grams = parts[a_id] + b_title, b_words, b_grams = parts[b_id] + scored: Dict[str, float] = {} + for g in a_grams & b_grams: # a shared phrase + scored[g] = idf.get(g, 0.0) + for w in (a_words & b_title) | (b_words & a_title): + scored[w] = max(scored.get(w, 0.0), idf.get(w, 0.0) * 0.9) + if not scored: + continue + + ranked = sorted(scored, key=lambda p: -scored[p])[:max_phrases] + score = sum(scored[p] for p in ranked) / len(ranked) + if score >= min_score: + written += _write_link(conn, a_id, b_id, instance, mode, + "phrase", round(score, 3), + ", ".join(ranked)) + conn.commit() + return written + finally: + conn.close() + + +def _write_link(conn, a, b, instance, mode, kind, score, evidence) -> int: + lo, hi = sorted((a, b)) + conn.execute( + "INSERT OR REPLACE INTO doc_links (a_doc, b_doc, instance, mode, kind, score, evidence)" + " VALUES (?, ?, ?, ?, ?, ?, ?)", (lo, hi, instance, mode, kind, score, evidence), + ) + return 1 + + +def link_sessions(db: Optional[Path] = None, instance: str = "", mode: str = "") -> int: + """Connect sessions that never shared context. + + 'shared-citation' — both sessions cited the same document. + 'linked-document' — a document cited by one session links to a document + cited by the other. This is the indirect case: two + investigations connected only because a document + mentions the other's subject in passing. + """ + conn = connect(db) + written = 0 + try: + rows = conn.execute( + "SELECT s.id AS sid, c.doc_id AS doc_id FROM sessions s" + " JOIN turns t ON t.session_id = s.id JOIN citations c ON c.turn_id = t.id" + " WHERE s.instance = ? AND s.mode = ?", (instance, mode), + ).fetchall() + by_session: Dict[int, set] = defaultdict(set) + for r in rows: + by_session[int(r["sid"])].add(r["doc_id"]) + + links = conn.execute( + "SELECT a_doc, b_doc, kind, score, evidence FROM doc_links" + " WHERE instance = ? AND mode = ?", (instance, mode), + ).fetchall() + link_map: Dict[str, List[sqlite3.Row]] = defaultdict(list) + for l in links: + link_map[l["a_doc"]].append(l) + link_map[l["b_doc"]].append(l) + + conn.execute("DELETE FROM session_links") + sids = sorted(by_session) + for i, a in enumerate(sids): + for b in sids[i + 1:]: + shared = by_session[a] & by_session[b] + if shared: + doc = sorted(shared)[0] + conn.execute( + "INSERT OR REPLACE INTO session_links" + " (a_session, b_session, kind, score, via_doc, to_doc, evidence)" + " VALUES (?, ?, 'shared-citation', 1.0, ?, ?, ?)", + (a, b, doc, doc, f"both sessions cited {doc}"), + ) + written += 1 + continue + best = None + for doc_a in by_session[a]: + for l in link_map.get(doc_a, []): + other = l["b_doc"] if l["a_doc"] == doc_a else l["a_doc"] + if other in by_session[b]: + cand = (float(l["score"]), doc_a, other, l["kind"], l["evidence"]) + if best is None or cand[0] > best[0]: + best = cand + if best: + score, doc_a, other, kind, evidence = best + conn.execute( + "INSERT OR REPLACE INTO session_links" + " (a_session, b_session, kind, score, via_doc, to_doc, evidence)" + " VALUES (?, ?, 'linked-document', ?, ?, ?, ?)", + (a, b, score, doc_a, other, + f"{doc_a} -> {other} ({kind}: {evidence})"), + ) + written += 1 + conn.commit() + return written + finally: + conn.close() + + +# ---------------------------------------------------------------- queries + + +def stats(db: Optional[Path] = None) -> Dict[str, Any]: + conn = connect(db) + try: + def one(sql: str) -> int: + return int(conn.execute(sql).fetchone()[0]) + partitions = conn.execute( + "SELECT instance, mode, COUNT(*) AS n FROM sessions" + " GROUP BY instance, mode ORDER BY n DESC" + ).fetchall() + return { + "sessions": one("SELECT COUNT(*) FROM sessions"), + "turns": one("SELECT COUNT(*) FROM turns"), + "documents": one("SELECT COUNT(*) FROM documents"), + "enriched": one("SELECT COUNT(*) FROM documents WHERE content IS NOT NULL AND content != ''"), + "citations": one("SELECT COUNT(*) FROM citations"), + "doc_links": one("SELECT COUNT(*) FROM doc_links"), + "session_links": one("SELECT COUNT(*) FROM session_links"), + "partitions": [dict(r) for r in partitions], + "path": str(Path(db) if db else DB_PATH), + "size": db_size(db), + } + finally: + conn.close() + + +def get_flow(db: Optional[Path] = None, session_id: Optional[int] = None, + instance: str = "", mode: str = "") -> Dict[str, Any]: + """The full graph: sessions, their turns and citations, and every link.""" + conn = connect(db) + try: + where = "WHERE instance = ? AND mode = ?" + args: List[Any] = [instance, mode] + if session_id is not None: + where += " AND id = ?" + args.append(session_id) + sessions = [] + for s in conn.execute(f"SELECT * FROM sessions {where} ORDER BY started_at", args): + turns = [] + for t in conn.execute( + "SELECT * FROM turns WHERE session_id = ? ORDER BY ts, id", (s["id"],) + ): + cites = conn.execute( + "SELECT c.doc_id, c.rank, d.title, d.url, d.datasource" + " FROM citations c LEFT JOIN documents d" + " ON d.doc_id = c.doc_id AND d.instance = ? AND d.mode = ?" + " WHERE c.turn_id = ? ORDER BY c.rank", (instance, mode, t["id"]) + ).fetchall() + turns.append({**dict(t), "citations": [dict(c) for c in cites]}) + sessions.append({**dict(s), "turns": turns}) + + sids = {s["id"] for s in sessions} + slinks = [dict(r) for r in conn.execute("SELECT * FROM session_links") + if r["a_session"] in sids or r["b_session"] in sids] + dlinks = [dict(r) for r in conn.execute( + "SELECT * FROM doc_links WHERE instance = ? AND mode = ? ORDER BY score DESC", + (instance, mode))] + return {"sessions": sessions, "session_links": slinks, + "document_links": dlinks, "instance": instance, "mode": mode} + finally: + conn.close() + + +# Ordering for display. A shared citation is trivially certain and says little +# — two sessions cited the same document. A linked-document link is the one +# that found something, and it must not sit underneath a wall of 1.0 scores. +_LINK_INTEREST = {"linked-document": 0, "shared-citation": 1} + + +def get_flow_summary(db: Optional[Path] = None, instance: str = "", + mode: str = "") -> Dict[str, Any]: + """The compressed narrative: what was investigated and what connected.""" + flow = get_flow(db, None, instance, mode) + sessions = [] + doc_titles: Dict[str, str] = {} + for s in flow["sessions"]: + questions = [t["text"] for t in s["turns"] if t["role"] in ("user", "search") and t["text"]] + docs: Dict[str, Dict[str, Any]] = {} + seq = 0 + for t in s["turns"]: + for c in t["citations"]: + entry = docs.setdefault(c["doc_id"], {"doc_id": c["doc_id"], + "title": c.get("title"), + "datasource": c.get("datasource"), + "rank": c.get("rank"), + "cited": 0, + "seq": seq}) + entry["cited"] += 1 + seq += 1 + if c.get("rank") is not None: + best = entry.get("rank") + entry["rank"] = c["rank"] if best is None else min(best, c["rank"]) + if c.get("title"): + doc_titles[c["doc_id"]] = c["title"] + sessions.append({ + "session_id": s["id"], + "chat_id": s["chat_id"], + "started_at": s["started_at"], + "turn_count": len(s["turns"]), + "questions": questions, + # Documents a thread kept coming back to lead; everything else + # holds the order it was cited in. + # + # Sorting by rank instead would interleave the turns, because every + # turn's citations restart at rank 0 — a tangential question's top + # hit would land level with the document the thread is actually + # about. `seq` keeps each turn's results together and in its own + # relevance order, which is what a reader is expecting to see. + "documents": sorted(docs.values(), + key=lambda d: (-d.get("cited", 0), d.get("seq", 0))), + }) + + # The evidence a session link carries is a sentence built for a human. The + # renderer wants the parts, so read them back off the document link itself. + by_pair = {} + for l in flow["document_links"]: + by_pair[(l["a_doc"], l["b_doc"])] = l + by_pair[(l["b_doc"], l["a_doc"])] = l + + connections = [] + titles = {s["session_id"]: (s["questions"][0] if s["questions"] else f"session {s['session_id']}") + for s in sessions} + for l in flow["session_links"]: + from_doc = l.get("via_doc") + to_doc = l.get("to_doc") or from_doc + dl = by_pair.get((from_doc, to_doc)) if from_doc and to_doc else None + connections.append({ + "a": titles.get(l["a_session"], l["a_session"]), + "b": titles.get(l["b_session"], l["b_session"]), + "a_session": l["a_session"], + "b_session": l["b_session"], + "kind": l["kind"], "score": l["score"], "why": l["evidence"], + "from_doc": from_doc, + "to_doc": to_doc, + "from_title": doc_titles.get(from_doc or "", from_doc), + "to_title": doc_titles.get(to_doc or "", to_doc), + "shares": (dl["evidence"] if dl else None), + "via_kind": (dl["kind"] if dl else None), + }) + connections.sort(key=lambda c: (_LINK_INTEREST.get(c["kind"], 99), + -float(c["score"] or 0))) + return {"instance": instance, "mode": mode, + "sessions": sessions, "connections": connections} + + +def get_flow_collapsed(db: Optional[Path] = None, instance: str = "", + mode: str = "") -> Dict[str, Any]: + """Repeated questions and multi-turn threads folded into counted nodes.""" + flow = get_flow(db, None, instance, mode) + threads = [] + for s in flow["sessions"]: + questions = Counter(t["text"] for t in s["turns"] + if t["role"] in ("user", "search") and t["text"]) + docs = {c["doc_id"]: c.get("title") for t in s["turns"] for c in t["citations"]} + threads.append({ + "session_id": s["id"], + "chat_id": s["chat_id"], + "turn_count": len(s["turns"]), + "started_at": s["started_at"], + "questions": [{"text": q, "count": n} for q, n in questions.most_common()], + "document_count": len(docs), + "documents": [{"doc_id": k, "title": v} for k, v in docs.items()], + }) + return {"instance": instance, "mode": mode, "threads": threads, + "connections": flow["session_links"]} + + +def purge(db: Optional[Path] = None, instance: Optional[str] = None, + mode: Optional[str] = None, older_than_days: Optional[float] = None) -> int: + """Delete captured data. Returns the number of sessions removed.""" + conn = connect(db) + try: + clauses, args = [], [] + if instance: + clauses.append("instance = ?"); args.append(instance) + if mode: + clauses.append("mode = ?"); args.append(mode) + if older_than_days: + clauses.append("started_at < ?") + args.append(time.time() - older_than_days * 86400) + where = (" WHERE " + " AND ".join(clauses)) if clauses else "" + removed = int(conn.execute(f"SELECT COUNT(*) FROM sessions{where}", args).fetchone()[0]) + conn.execute(f"DELETE FROM sessions{where}", args) + if not clauses: + for table in ("turns", "citations", "documents", "doc_links", "session_links"): + conn.execute(f"DELETE FROM {table}") + else: + conn.execute( + "DELETE FROM documents WHERE 1=1" + + (" AND instance = ?" if instance else "") + + (" AND mode = ?" if mode else ""), + [a for a, c in zip(args, clauses) if c.startswith(("instance", "mode"))], + ) + conn.execute("DELETE FROM turns WHERE session_id NOT IN (SELECT id FROM sessions)") + conn.execute("DELETE FROM citations WHERE turn_id NOT IN (SELECT id FROM turns)") + conn.execute("DELETE FROM session_links WHERE a_session NOT IN (SELECT id FROM sessions)" + " OR b_session NOT IN (SELECT id FROM sessions)") + conn.commit() + conn.execute("VACUUM") + return removed + finally: + conn.close() + + +# ---------------------------------------------------------------- rendering + +_CSS = """ +:root { --bg:#fbfaf7; --fg:#1c1c1a; --muted:#6b6a66; --line:#e2e0da; + --card:#ffffff; --accent:#343ced; --warn:#b35309; --chip:#f1efe9; } +@media (prefers-color-scheme: dark) { + :root { --bg:#16171a; --fg:#e8e6e1; --muted:#9a9892; --line:#2c2e33; + --card:#1d1f23; --accent:#8f97ff; --warn:#e0a35c; --chip:#25272c; } +} +* { box-sizing:border-box; } +body { margin:0; padding:2rem 1.25rem 4rem; background:var(--bg); color:var(--fg); + font:15px/1.55 ui-sans-serif,-apple-system,"Segoe UI",Roboto,sans-serif; } +.wrap { max-width:820px; margin:0 auto; } +h1 { font-size:1.35rem; margin:0 0 .25rem; letter-spacing:-.01em; } +.sub { color:var(--muted); font-size:.85rem; margin-bottom:1.5rem; } +.banner { background:var(--warn); color:#fff; padding:.5rem .75rem; border-radius:6px; + font-size:.8rem; margin-bottom:1.25rem; } +.tl { position:relative; padding-left:1.5rem; } +.tl:before { content:""; position:absolute; left:.32rem; top:.4rem; bottom:.4rem; + width:2px; background:var(--line); } +.node { position:relative; margin-bottom:1.1rem; } +.node:before { content:""; position:absolute; left:-1.32rem; top:1.15rem; width:9px; height:9px; + border-radius:50%; background:var(--accent); } +.card { background:var(--card); border:1px solid var(--line); border-radius:9px; padding:.85rem 1rem; } +.q { font-weight:600; } +.meta { color:var(--muted); font-size:.78rem; margin-top:.15rem; } +details { margin-top:.6rem; } +summary { cursor:pointer; color:var(--accent); font-size:.82rem; } +summary::marker { color:var(--muted); } +.turn { border-left:2px solid var(--line); margin:.5rem 0 0 .2rem; padding:.1rem 0 .1rem .7rem; } +.role { color:var(--muted); font-size:.72rem; text-transform:uppercase; letter-spacing:.04em; } +.chips { margin-top:.6rem; display:flex; flex-wrap:wrap; gap:.35rem; } +.chip { background:var(--chip); border:1px solid var(--line); border-radius:999px; + padding:.15rem .6rem; font-size:.76rem; text-decoration:none; color:var(--fg); } +.ds { color:var(--muted); } +.link { border-left:3px solid var(--accent); background:var(--card); border-radius:0 8px 8px 0; + padding:.6rem .85rem; margin:0 0 1.1rem 0; font-size:.85rem; } +.why { color:var(--muted); font-size:.78rem; margin-top:.2rem; font-family:ui-monospace,monospace; } +.count { background:var(--chip); border-radius:999px; padding:0 .4rem; font-size:.72rem; + color:var(--muted); } +.empty { color:var(--muted); font-style:italic; } +""" + + +def _esc(text: Any) -> str: + s = "" if text is None else str(text) + return (s.replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """)) + + +def when(ts: Optional[float]) -> str: + """A timestamp as a person reads it. Shared by the HTML and the terminal.""" + if not ts: + return "unknown time" + return time.strftime("%a %d %b %Y, %H:%M", time.localtime(float(ts))) + + +def render_timeline(db: Optional[Path] = None, instance: str = "", + mode: str = "") -> str: + """A self-contained HTML timeline. No CDN, no framework, no network.""" + collapsed = get_flow_collapsed(db, instance, mode) + flow_data = get_flow(db, None, instance, mode) + turns_by_session = {s["id"]: s["turns"] for s in flow_data["sessions"]} + titles = {t["session_id"]: (t["questions"][0]["text"] if t["questions"] + else f"session {t['session_id']}") + for t in collapsed["threads"]} + + links_by_a: Dict[int, List[Dict[str, Any]]] = defaultdict(list) + for l in collapsed["connections"]: + links_by_a[l["a_session"]].append(l) + + parts: List[str] = [] + if mode == "mock": + parts.append('') + + if not collapsed["threads"]: + parts.append('

Nothing captured yet. Run some /chat or ' + '/search commands with capture on.

') + + parts.append('
') + for thread in collapsed["threads"]: + sid = thread["session_id"] + parts.append('
') + parts.append(f'
{_esc(titles.get(sid, ""))}
') + chat_id = f' · {_esc(thread["chat_id"])}' if thread["chat_id"] else "" + parts.append(f'
{_esc(when(thread["started_at"]))}' + f'{chat_id} · {thread["turn_count"]} turns · ' + f'{thread["document_count"]} documents
') + + # Repeated questions fold into one row with a count. + extra = [q for q in thread["questions"][1:]] + if extra: + parts.append(f'
{len(extra)} more question' + f'{"s" if len(extra) != 1 else ""} in this thread') + for q in extra: + badge = f' ×{q["count"]}' if q["count"] > 1 else "" + parts.append(f'
asked
' + f'{_esc(q["text"])}{badge}
') + parts.append('
') + + turns = turns_by_session.get(sid, []) + if turns: + parts.append(f'
Show all {len(turns)} messages') + for t in turns: + parts.append(f'
{_esc(t["role"])}
' + f'{_esc(t["text"])}
') + parts.append('
') + + if thread["documents"]: + parts.append('
') + for d in thread["documents"]: + label = _esc(d["title"] or d["doc_id"]) + parts.append(f'{label}') + parts.append('
') + parts.append('
') + + for l in links_by_a.get(sid, []): + other = titles.get(l["b_session"], f'session {l["b_session"]}') + parts.append( + f'
Connects to {_esc(other)}' + f' {_esc(l["kind"])} {l["score"]}' + f'
{_esc(l["evidence"])}
') + parts.append('
') + + body = "\n".join(parts) + scope = f'{_esc(instance)} · {_esc(mode)} mode' + return f""" + + +Glean Code — flow +
+

Flow

+
{scope} · {len(collapsed["threads"])} sessions · +{len(collapsed["connections"])} connections · rendered {_esc(when(time.time()))}
+{body} +
+""" + + +def write_timeline(path: Path, db: Optional[Path] = None, instance: str = "", + mode: str = "") -> Path: + html = render_timeline(db, instance, mode) + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(html, encoding="utf-8") + return target diff --git a/glean_code/help_docs.py b/glean_code/help_docs.py index 8583636..4dfc435 100644 --- a/glean_code/help_docs.py +++ b/glean_code/help_docs.py @@ -153,6 +153,37 @@ ], "endpoint": "(local — spawns glean_mcp.py)", }, + "flow": { + "summary": "Map what you have investigated: captured chats, their sources, and what connects.", + "usage": "/flow [--docs ] [--links ] [--limit ] " + "[--min-score ] [--output ] [--print] [--all] [--older-than ]", + "params": [ + ("status", "Capture setting, database size, and what has been recorded."), + ("enrich", "Fetch document text for captured citations. Needed before linking."), + ("link", "Find document and cross-session links. Run after enrich."), + ("show", "Print sessions, their sources, and the connections between them."), + ("timeline", "Write a self-contained HTML timeline and open it."), + ("purge", "Delete captured data. Confirms first."), + ("--docs", "show: documents listed per session before the rest are counted. Default 6."), + ("--links", "show: connections drawn per session before the rest are counted. Default 3."), + ("--limit", "enrich: how many documents to fetch. Default 50."), + ("--min-score", "link: phrase-link threshold, 0-1. Default 0.45. Higher is stricter."), + ("--output", "timeline: where to write the HTML. Default a temp file."), + ("--print", "timeline: write the file without opening a browser."), + ("--all", "purge: every instance and mode, not just the current one."), + ("--older-than", "purge: only sessions older than this many days."), + ], + "examples": [ + "/flow status", + "/flow enrich", + "/flow link --min-score 0.6", + "/flow show", + "/flow show --docs 3", + "/flow timeline --output ~/flow.html", + "/flow purge --older-than 30", + ], + "endpoint": "(local — ~/.gleancode/flow.db; enrich calls /getdocuments or /summarize)", + }, "doctor": { "summary": "Run a health check on your Glean Code setup. " "Inspects config, URL shape, DNS, TCP and runs a tiny auth probe.", @@ -1023,7 +1054,7 @@ COMMAND_GROUPS: List[Tuple[str, List[str]]] = [ - ("Shell", ["help", "status", "doctor", "login", "auth", "logout", "open", "ask", "config", "mode", "mcp", "history", "clear", "exit"]), + ("Shell", ["help", "status", "doctor", "login", "auth", "logout", "open", "ask", "config", "mode", "mcp", "flow", "history", "clear", "exit"]), ("Chat & Search", ["chat", "search", "datasources.list", "datasources.status", "autocomplete", "recommendations", "feedback"]), ("Agents & Tools", ["agents.list", "agents.run", "tools.list", "tools.call"]), ("Docs & People", ["docs.get", "docs.permissions", "entities.list", "people.get"]), diff --git a/glean_code/mcp_control.py b/glean_code/mcp_control.py index a7bc64b..a8988bc 100644 --- a/glean_code/mcp_control.py +++ b/glean_code/mcp_control.py @@ -265,4 +265,5 @@ def client_config(url: Optional[str] = None, def tool_names() -> List[str]: """The tools the server exposes. Kept in sync with glean_mcp.py by test.""" - return ["search", "chat", "list_agents", "run_agent"] + return ["search", "chat", "list_agents", "run_agent", + "get_flow", "get_flow_summary", "get_flow_collapsed"] diff --git a/glean_code/ui.py b/glean_code/ui.py index 394cf7f..2e7783b 100644 --- a/glean_code/ui.py +++ b/glean_code/ui.py @@ -61,6 +61,29 @@ class C: WHITE = "\033[38;5;255m" +# Per-datasource colour. A source should be recognisable before you read its +# name, and the same source should look the same everywhere it appears. +# Anything not listed falls back to grey rather than picking a colour at +# random, so an unfamiliar datasource never impersonates a familiar one. +DATASOURCE_COLOURS = { + "gdrive": C.BLUE, + "confluence": C.CYAN, + "jira": C.PURPLE, + "slack": C.GREEN, + "github": C.GREY, + "sharepoint": C.TEAL, + "notion": C.WHITE, + "salesforce": C.BLUE, + "zendesk": C.TEAL, + "gmail": C.RED, + "outlook": C.RED, +} + + +def datasource_colour(name: Optional[str]) -> str: + return DATASOURCE_COLOURS.get((name or "").strip().lower(), C.GREY) + + def supports_colour() -> bool: if os.environ.get("NO_COLOR"): return False @@ -229,8 +252,9 @@ def box(title: str, body: str, colour: str = C.BLUE) -> str: return "\n".join(lines) -def rule(label: str = "", colour: str = C.GREY) -> str: - w = term_width() +def rule(label: str = "", colour: str = C.GREY, width: Optional[int] = None) -> str: + """A horizontal rule. Pass width to match a block that caps its own columns.""" + w = width if width is not None else term_width() if label: bar = "─" * max(0, w - len(label) - 4) return style(f"── {label} {bar}", colour) diff --git a/glean_mcp.py b/glean_mcp.py index 34929f8..0202eb0 100644 --- a/glean_mcp.py +++ b/glean_mcp.py @@ -116,6 +116,7 @@ def _installed_mcp_version() -> Optional[str]: from glean_code.config import Config from glean_code.client import GleanClient, GleanError +from glean_code import flow as _flow MOCK_ENV_VAR = "GLEAN_MOCK" @@ -308,6 +309,105 @@ def run_agent(agent_id: str, input: str) -> str: return _label(result) +# ── flow mapper ─────────────────────────────────────────────────────────────── + + +def _flow_scope() -> tuple: + """Flow data is partitioned; never read across instance or mode.""" + return (_cfg.instance or "local").strip(), _cfg.effective_mode + + +@mcp.tool() +def get_flow(session_id: Optional[int] = None) -> str: + """Get the captured investigation graph: sessions, questions, cited documents, and links. + + Call this when the user asks what they looked into previously, whether two + topics are related, or where a document came up before. Returns every + session in the current instance and mode, each with its turns and + citations, plus document-to-document and session-to-session links with the + evidence for each. + + Pass session_id to narrow to one investigation. + + When the server is started with GLEAN_MOCK=1 this returns fictional demo + data from a built-in corpus, prefixed with a [MOCK MODE] banner. + """ + instance, mode = _flow_scope() + try: + data = _flow.get_flow(session_id=session_id, instance=instance, mode=mode) + except Exception as e: # noqa: BLE001 + return f"Error reading the flow database: {e}" + if not data["sessions"]: + return _label(f"No captured sessions for {instance} in {mode} mode.") + return _label(json.dumps(data, indent=2, default=str)) + + +@mcp.tool() +def get_flow_summary() -> str: + """Summarise what was investigated and which investigations connect to each other. + + Call this instead of get_flow when the user wants the narrative rather than + the raw graph — "what have I been looking at", "is this related to anything + I've seen". Returns one entry per session with its questions and documents, + then the connections between sessions with a plain-language reason for each. + + Connections are the useful part: two investigations that never shared + context can be linked because a document cited by one mentions the other's + subject in passing. + + When the server is started with GLEAN_MOCK=1 this returns fictional demo + data from a built-in corpus, prefixed with a [MOCK MODE] banner. + """ + instance, mode = _flow_scope() + try: + data = _flow.get_flow_summary(instance=instance, mode=mode) + except Exception as e: # noqa: BLE001 + return f"Error reading the flow database: {e}" + if not data["sessions"]: + return _label(f"No captured sessions for {instance} in {mode} mode.") + + lines = [f"{len(data['sessions'])} session(s) in {instance} ({mode} mode):", ""] + for s in data["sessions"]: + head = s["questions"][0] if s["questions"] else f"session {s['session_id']}" + lines.append(f"[{s['session_id']}] {head}") + for q in s["questions"][1:]: + lines.append(f" also asked: {q}") + for d in s["documents"]: + lines.append(f" cited: {d['title'] or d['doc_id']}") + lines.append("") + if data["connections"]: + lines.append("Connections between sessions:") + for c in data["connections"]: + lines.append(f" {c['a']}") + lines.append(f" <-> {c['b']}") + lines.append(f" {c['kind']} (score {c['score']}): {c['why']}") + else: + lines.append("No connections found between sessions.") + return _label("\n".join(lines)) + + +@mcp.tool() +def get_flow_collapsed() -> str: + """Get the compact view: threads folded into single nodes with turn and question counts. + + Call this when the full graph would be too much — a long history, or a + first look before drilling in with get_flow. Repeated questions collapse + into one entry with a count, and each thread reports how many turns and + documents it holds rather than listing them all. + + When the server is started with GLEAN_MOCK=1 this returns fictional demo + data from a built-in corpus, prefixed with a [MOCK MODE] banner. + """ + instance, mode = _flow_scope() + try: + data = _flow.get_flow_collapsed(instance=instance, mode=mode) + except Exception as e: # noqa: BLE001 + return f"Error reading the flow database: {e}" + if not data["threads"]: + return _label(f"No captured sessions for {instance} in {mode} mode.") + return _label(json.dumps(data, indent=2, default=str)) + + # ── entry point ─────────────────────────────────────────────────────────────── def _parse_args(argv: Optional[list] = None) -> "argparse.Namespace": diff --git a/tests/test_flow.py b/tests/test_flow.py new file mode 100644 index 0000000..2c67624 --- /dev/null +++ b/tests/test_flow.py @@ -0,0 +1,680 @@ +"""Tests for the flow mapper — capture, enrichment, linking, and rendering. + +Every test points the database at a temporary file, so ~/.gleancode/flow.db is +never touched. Nothing here reaches the network: all traffic is mock mode. +""" +import html.parser +import io +import json +import re +import sys +import tempfile +import time +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from glean_code import flow, ui +from glean_code.client import GleanClient +from glean_code.commands import HANDLERS, Session +from glean_code.config import Config + +INSTANCE = "acme-be.glean.com" + + +def setUpModule(): + """The mock client sleeps 0.25s per call to feel like a network hop. + + That realism is worth having in the REPL and worth nothing here, where it + turns a 1-second file into a 30-second one. + """ + global _no_sleep + _no_sleep = mock.patch("glean_code.client.time.sleep", lambda *_: None) + _no_sleep.start() + + +def tearDownModule(): + _no_sleep.stop() + + +class _Db: + """A temp database, plus a client whose capture writes into it.""" + + def __enter__(self): + self.tmp = tempfile.TemporaryDirectory() + self.path = Path(self.tmp.name) / "flow.db" + real = flow.record + self._patch = mock.patch.object( + flow, "record", + lambda cfg, p, b, r, db=None, proximity_window=600.0: + real(cfg, p, b, r, db=self.path, proximity_window=proximity_window), + ) + self._patch.start() + return self + + def __exit__(self, *exc): + self._patch.stop() + self.tmp.cleanup() + return False + + def client(self, **kw): + cfg = Config(mode="mock", instance=INSTANCE, flow_capture="on", **kw) + return GleanClient(cfg), cfg + + +class TestCaptureGate(unittest.TestCase): + """Capture defaults to mock only — live content is opt-in, deliberately.""" + + def test_default_records_mock_only(self): + self.assertTrue(flow.capture_enabled("mock", "mock")) + self.assertFalse(flow.capture_enabled("mock", "live")) + + def test_on_records_both(self): + self.assertTrue(flow.capture_enabled("on", "live")) + self.assertTrue(flow.capture_enabled("on", "mock")) + + def test_off_records_nothing(self): + self.assertFalse(flow.capture_enabled("off", "mock")) + self.assertFalse(flow.capture_enabled("off", "live")) + + def test_default_is_mock(self): + self.assertEqual(flow.DEFAULT_CAPTURE, "mock") + self.assertEqual(Config().flow_capture, "mock") + + +class TestCapture(unittest.TestCase): + def test_chat_records_turns_and_citations(self): + with _Db() as db: + client, _ = db.client() + client.chat("what happened in the checkout incident?") + st = flow.stats(db.path) + self.assertEqual(st["sessions"], 1) + self.assertGreaterEqual(st["turns"], 2) # question + answer + self.assertGreater(st["documents"], 0) + self.assertGreater(st["citations"], 0) + + def test_same_chat_id_is_one_session(self): + """Threading is exact: /chat echoes chatId, so no time-guessing.""" + with _Db() as db: + client, _ = db.client() + first = client.chat("what happened in the checkout incident?") + client.chat("who owned the fix?", chat_id=first["chatId"]) + self.assertEqual(flow.stats(db.path)["sessions"], 1) + + def test_new_chat_id_is_a_new_session(self): + with _Db() as db: + client, _ = db.client() + client.chat("checkout incident") + client.chat("northwind renewal") # no chat_id -> new thread + self.assertEqual(flow.stats(db.path)["sessions"], 2) + + def test_search_is_captured_with_snippets(self): + with _Db() as db: + client, cfg = db.client() + client.search("checkout incident", page_size=3) + conn = flow.connect(db.path) + rows = conn.execute("SELECT * FROM turns WHERE role = 'search'").fetchall() + self.assertEqual(len(rows), 1) + docs = conn.execute( + "SELECT * FROM documents WHERE content IS NOT NULL").fetchall() + self.assertTrue(docs, "search snippets should seed document content") + self.assertEqual(docs[0]["content_source"], "search-snippet") + conn.close() + + def test_rows_are_tagged_with_instance_and_mode(self): + """Fictional and real content must never be linked together.""" + with _Db() as db: + client, _ = db.client() + client.chat("checkout incident") + conn = flow.connect(db.path) + for table in ("sessions", "documents"): + row = conn.execute(f"SELECT * FROM {table} LIMIT 1").fetchone() + self.assertEqual(row["instance"], INSTANCE) + self.assertEqual(row["mode"], "mock") + conn.close() + + def test_capture_never_breaks_the_call(self): + with _Db() as db: + client, _ = db.client() + with mock.patch.object(flow, "_record", side_effect=RuntimeError("boom")): + resp = client.chat("still works?") + self.assertIn("messages", resp) + + def test_capture_off_records_nothing(self): + with _Db() as db: + cfg = Config(mode="mock", instance=INSTANCE, flow_capture="off") + GleanClient(cfg).chat("checkout incident") + self.assertEqual(flow.stats(db.path)["sessions"], 0) + + +class TestEnrichment(unittest.TestCase): + def test_enrich_fills_document_text(self): + with _Db() as db: + client, cfg = db.client() + client.chat("what happened in the checkout incident?") + before = flow.stats(db.path) + enriched, attempted = flow.enrich(client, cfg, db=db.path) + after = flow.stats(db.path) + self.assertGreater(enriched, 0) + self.assertEqual(attempted, before["documents"] - before["enriched"]) + self.assertEqual(after["enriched"], after["documents"]) + + def test_enrich_prefers_getdocuments(self): + """Mock /getdocuments returns body, so no fallback should be needed.""" + with _Db() as db: + client, cfg = db.client() + client.chat("checkout incident") + flow.enrich(client, cfg, db=db.path) + conn = flow.connect(db.path) + sources = {r["content_source"] for r in + conn.execute("SELECT content_source FROM documents")} + conn.close() + self.assertIn("getdocuments", sources) + + +class TestLinking(unittest.TestCase): + def _prepared(self, db): + client, cfg = db.client() + first = client.chat("what happened in the checkout incident?") + client.chat("who owned the fix?", chat_id=first["chatId"]) + client.chat("what are the risks going into the Northwind renewal?") + flow.enrich(client, cfg, db=db.path, limit=100) + flow.link_documents(db=db.path, instance=INSTANCE, mode="mock") + flow.link_sessions(db=db.path, instance=INSTANCE, mode="mock") + return client, cfg + + def test_identifier_links_are_exact(self): + with _Db() as db: + self._prepared(db) + conn = flow.connect(db.path) + rows = conn.execute( + "SELECT * FROM doc_links WHERE kind = 'identifier'").fetchall() + conn.close() + self.assertTrue(rows, "INC-1183 spans several documents") + self.assertTrue(all(r["score"] == 1.0 for r in rows)) + self.assertTrue(any("INC-" in r["evidence"] for r in rows)) + + def test_unrelated_sessions_are_connected_through_a_document(self): + """The case worth having: a QBR mentions an incident in passing. + + Neither conversation shares wording with the other, and the QBR names + no ticket — the link exists only because a document cited by one + investigation refers to the other's subject. + """ + with _Db() as db: + self._prepared(db) + summary = flow.get_flow_summary(db=db.path, instance=INSTANCE, mode="mock") + self.assertTrue(summary["connections"], "expected a cross-session link") + conn = summary["connections"][0] + self.assertEqual(conn["kind"], "linked-document") + self.assertIn("checkout", conn["why"].lower()) + + def test_link_evidence_is_specific_not_generic(self): + """A link nobody can explain is worse than no link.""" + with _Db() as db: + self._prepared(db) + c = flow.connect(db.path) + rows = c.execute("SELECT evidence FROM doc_links WHERE kind = 'phrase'").fetchall() + c.close() + generic = {"across", "percent", "customer", "going", "into"} + for r in rows: + terms = {t.strip() for t in r["evidence"].split(",")} + self.assertTrue(terms - generic, + f"evidence is entirely generic: {r['evidence']}") + + def test_title_anchoring_beats_bare_rarity(self): + """A word in the other document's title outranks a once-seen word.""" + docs = [ + {"doc_id": "a", "title": "Checkout Latency Incident", + "content": "A connection pool exhaustion pushed checkout latency up."}, + {"doc_id": "b", "title": "Customer QBR", + "content": "Risks: the checkout incident in June and a stray zebra."}, + ] + rows = [dict(d) for d in docs] + idf = flow.build_idf(rows) + # 'zebra' appears once and is therefore rarer than 'checkout'... + self.assertGreater(idf.get("zebra", 0), idf.get("checkout", 0)) + # ...but only 'checkout' is anchored in the other document's title. + a_title, _, _ = flow._doc_parts(rows[0]) + _, b_words, _ = flow._doc_parts(rows[1]) + self.assertIn("checkout", b_words & a_title) + self.assertNotIn("zebra", b_words & a_title) + + def test_links_never_cross_partitions(self): + with _Db() as db: + self._prepared(db) + written = flow.link_documents(db=db.path, instance="other-be.glean.com", + mode="mock") + self.assertEqual(written, 0) + + +class TestQueries(unittest.TestCase): + def test_summary_lists_questions_and_documents(self): + with _Db() as db: + client, _ = db.client() + client.chat("what happened in the checkout incident?") + s = flow.get_flow_summary(db=db.path, instance=INSTANCE, mode="mock") + self.assertEqual(len(s["sessions"]), 1) + self.assertIn("checkout", s["sessions"][0]["questions"][0]) + self.assertTrue(s["sessions"][0]["documents"]) + + def test_collapsed_counts_repeats(self): + with _Db() as db: + client, _ = db.client() + first = client.chat("same question") + client.chat("same question", chat_id=first["chatId"]) + c = flow.get_flow_collapsed(db=db.path, instance=INSTANCE, mode="mock") + counts = {q["text"]: q["count"] for q in c["threads"][0]["questions"]} + self.assertEqual(counts["same question"], 2) + + def test_get_flow_can_narrow_to_one_session(self): + with _Db() as db: + client, _ = db.client() + client.chat("first") + client.chat("second") + all_sessions = flow.get_flow(db=db.path, instance=INSTANCE, mode="mock") + one = flow.get_flow(db=db.path, session_id=all_sessions["sessions"][0]["id"], + instance=INSTANCE, mode="mock") + self.assertEqual(len(all_sessions["sessions"]), 2) + self.assertEqual(len(one["sessions"]), 1) + + def test_queries_are_json_serialisable(self): + with _Db() as db: + client, _ = db.client() + client.chat("checkout incident") + for fn in (flow.get_flow, flow.get_flow_summary, flow.get_flow_collapsed): + json.dumps(fn(db=db.path, instance=INSTANCE, mode="mock"), default=str) + + +class TestPurge(unittest.TestCase): + def test_purge_scoped_to_partition(self): + with _Db() as db: + client, _ = db.client() + client.chat("checkout incident") + removed = flow.purge(db=db.path, instance=INSTANCE, mode="mock") + self.assertEqual(removed, 1) + self.assertEqual(flow.stats(db.path)["sessions"], 0) + + def test_purge_everything(self): + with _Db() as db: + client, _ = db.client() + client.chat("one") + client.chat("two") + flow.purge(db=db.path) + st = flow.stats(db.path) + self.assertEqual((st["sessions"], st["turns"], st["documents"]), (0, 0, 0)) + + +class TestRendering(unittest.TestCase): + def _html(self, db): + client, cfg = db.client() + first = client.chat("what happened in the checkout incident?") + client.chat("who owned the fix?", chat_id=first["chatId"]) + client.chat("what are the risks going into the Northwind renewal?") + flow.enrich(client, cfg, db=db.path, limit=100) + flow.link_documents(db=db.path, instance=INSTANCE, mode="mock") + flow.link_sessions(db=db.path, instance=INSTANCE, mode="mock") + return flow.render_timeline(db=db.path, instance=INSTANCE, mode="mock") + + def test_html_is_self_contained(self): + with _Db() as db: + page = self._html(db) + self.assertNotRegex(page, r'src=|href="http|@import', + "the timeline must not reference anything external") + + def test_html_is_well_formed(self): + class Checker(html.parser.HTMLParser): + def __init__(self): + super().__init__(); self.stack = []; self.bad = [] + def handle_starttag(self, tag, attrs): + if tag not in ("meta", "br", "img", "link", "hr"): + self.stack.append(tag) + def handle_endtag(self, tag): + if self.stack and self.stack[-1] == tag: + self.stack.pop() + else: + self.bad.append(tag) + with _Db() as db: + c = Checker(); c.feed(self._html(db)) + self.assertEqual(c.bad, []) + self.assertEqual(c.stack, []) + + def test_mock_data_is_badged(self): + with _Db() as db: + self.assertIn("Mock mode", self._html(db)) + + def test_connections_are_rendered_with_their_evidence(self): + with _Db() as db: + page = self._html(db) + self.assertIn("Connects to", page) + self.assertIn("checkout", page.lower()) + + def test_escaping(self): + self.assertEqual(flow._esc('