diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md index 484c282683..a2a782a6be 100644 --- a/ts/docs/plans/vscode-devx/STATUS.md +++ b/ts/docs/plans/vscode-devx/STATUS.md @@ -5,11 +5,13 @@ > and what remains. Update it whenever a capability changes state. Branch: earlier feature work merged to `main` via PR #2468; ongoing work -continues on the `dev/talzacc/typeagent_studio_part*` stack (currently part6 — -corpus capture + Impact Report regression verdict, which **closes the headline -"find a regression" gate**: the shipped regression predicate — the detector, not -this change — agrees with blind human labels on a fixed benchmark of hand-built -deltas at ~92%, above the 80% bar). +continues on the `dev/talzacc/typeagent_studio_part*` stack (currently part7). +part6 delivered corpus capture + the Impact Report regression verdict, which +**closes the headline "find a regression" gate**: the shipped regression +predicate — the detector, not this change — agrees with blind human labels on a +fixed benchmark of hand-built deltas at ~92%, above the 80% bar. **part7 adds the +Trace Viewer** (Gate D / J5): an A/B resolution-pipeline debugger that drills into +a single utterance from an Impact Report row. ## Capability matrix @@ -17,25 +19,26 @@ Legend: ✅ done · 🟡 partial · ❌ not started. "Wired to dispatcher" = backed by the real TypeAgent dispatcher/engine rather than an in-memory stand-in. -| Capability | Core logic | UI | Wired to dispatcher | Tested | -| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ | -| Sandboxes (lifecycle, agent load/unload) | ✅ in-memory | ✅ tree view (channel-backed; agent runtime is source of truth) | ❌ in-memory only (no subprocess/isolated dispatcher) | ✅ | -| Sandbox persistence across reload/restart | ✅ | ✅ (auto-restore) | n/a | ✅ | -| Corpora (federation: in-repo / captures / external / feedback) | ✅ file-backed | ✅ tree view | n/a | ✅ | -| Event Log (structured event stream) | 🟡 in-memory ring buffer | ✅ tree view (+ channel-backed source) | ❌ most emit sites unwired | ✅ | -| Agent health (status bar + findings) | 🟡 heuristic/filesystem checks | ✅ status bar | ❌ no real schema parse / grammar compile | ✅ | -| Collisions (cross-schema grammar overlap) | ✅ real NFA scanner over compiled `.ag.json` | ✅ tree view + Skipped group + auto-scan (+ channel-backed source) | n/a (reads compiled grammars) | ✅ | -| Feedback (thumbs up/down → corpus) | ✅ | ✅ command | n/a | ✅ | -| Replay / compare engine | ✅ schema-enriched grammar (L1), construction-cache (L2), selectable two-mode (grammar/cache) + opt-in live wildcard validation (L4a) + red/green regression predicate (`likelyRegression`) | ✅ Impact Report webview (`ActionDelta[]`) | 🟡 grammar + live construction cache + working-tree wildcard validation; no two-version build-from-ref (L4b, deferred to P-7) | ✅ | -| Onboarding bridge (snapshot/restore, stale detection) | ✅ | ✅ commands | ❌ in-memory bridge | ✅ | -| Repo-root detection (find `packages/agents`) | ✅ | ✅ warn toast + status bar | n/a | ✅ | -| Webview infrastructure (`webviewKit`) | ✅ CSP/nonce host + typed protocol | ✅ singleton-panel host | — | ✅ | -| Regression verdict (red/green predicate) | ✅ `likelyRegression` (delta-shape + observation-scoped feedback-override); the detector agrees with blind human labels on a fixed benchmark of hand-built deltas at ~92% (bar 80%) | ✅ verdict banner + per-row Impact column | n/a (classifies replay rows) | ✅ | -| Impact Report webview | ✅ `replayCorpus` over channel | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state, verdict banner + Impact column, unified live-count filter chips, column sorting, utterance filter | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a) | ✅ | -| Corpus capture (real utterances → actions, any agent) | ✅ display/interaction-log → corpus transform + import, auto-promote to shared in-repo corpus | ✅ import command + Corpora tree (filename-based rows, auto-refresh) | ✅ reads real interaction logs | ✅ | -| Schema Studio | ❌ | ❌ | ❌ | ❌ | -| Live Trace | ❌ | ❌ | ❌ | ❌ | -| `agr-language` / `vscode-shell` refactor onto core | 🟡 dependency edge only | — | ❌ no behavioral integration | ❌ | +| Capability | Core logic | UI | Wired to dispatcher | Tested | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| Sandboxes (lifecycle, agent load/unload) | ✅ in-memory | ✅ tree view (channel-backed; agent runtime is source of truth) | ❌ in-memory only (no subprocess/isolated dispatcher) | ✅ | +| Sandbox persistence across reload/restart | ✅ | ✅ (auto-restore) | n/a | ✅ | +| Corpora (federation: in-repo / captures / external / feedback) | ✅ file-backed | ✅ tree view | n/a | ✅ | +| Event Log (structured event stream) | 🟡 in-memory ring buffer | ✅ tree view (+ channel-backed source) | ❌ most emit sites unwired | ✅ | +| Agent health (status bar + findings) | 🟡 heuristic/filesystem checks | ✅ status bar | ❌ no real schema parse / grammar compile | ✅ | +| Collisions (cross-schema grammar overlap) | ✅ real NFA scanner over compiled `.ag.json` | ✅ tree view + Skipped group + auto-scan (+ channel-backed source) | n/a (reads compiled grammars) | ✅ | +| Feedback (thumbs up/down → corpus) | ✅ | ✅ command | n/a | ✅ | +| Replay / compare engine | ✅ schema-enriched grammar (L1), construction-cache (L2), selectable two-mode (grammar/cache) + opt-in live wildcard validation (L4a) + red/green regression predicate (`likelyRegression`) | ✅ Impact Report webview (`ActionDelta[]`) | 🟡 grammar + live construction cache + working-tree wildcard validation; no two-version build-from-ref (L4b, deferred to P-7) | ✅ | +| Onboarding bridge (snapshot/restore, stale detection) | ✅ | ✅ commands | ❌ in-memory bridge | ✅ | +| Repo-root detection (find `packages/agents`) | ✅ | ✅ warn toast + status bar | n/a | ✅ | +| Webview infrastructure (`webviewKit`) | ✅ CSP/nonce host + typed protocol | ✅ singleton-panel host | — | ✅ | +| Regression verdict (red/green predicate) | ✅ `likelyRegression` (delta-shape + observation-scoped feedback-override); the detector agrees with blind human labels on a fixed benchmark of hand-built deltas at ~92% (bar 80%) | ✅ verdict banner + per-row Impact column | n/a (classifies replay rows) | ✅ | +| Impact Report webview | ✅ `replayCorpus` over channel | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state, verdict banner + Impact column, unified live-count filter chips, column sorting, utterance filter | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a) | ✅ | +| Trace Viewer webview | ✅ per-row `ReplayResolutionTrace` capture (all rows, changed-first, capped) + per-run persistence + single-utterance recompute; browser-neutral divergence view model (parity + cause attribution to the first diverging layer, rule-level culprit file via grammar hash) | ✅ A/B resolution-pipeline webview (cache → grammar → wildcard → result), cause bar + native A↔B file diff + per-side source jumps (grammar → `.agr`, action → schema); Impact Report entry points (right-click, telescope, arrow-nav) | 🟡 captured during real `replayCorpus` (grammar + construction cache + working-tree wildcard, L1–L4a) | ✅ view model unit-tested | +| Corpus capture (real utterances → actions, any agent) | ✅ display/interaction-log → corpus transform + import, auto-promote to shared in-repo corpus | ✅ import command + Corpora tree (filename-based rows, auto-refresh) | ✅ reads real interaction logs | ✅ | +| Schema Studio | ❌ | ❌ | ❌ | ❌ | +| Live Trace | ❌ | ❌ | ❌ | ❌ | +| `agr-language` / `vscode-shell` refactor onto core | 🟡 dependency edge only | — | ❌ no behavioral integration | ❌ | ## Roadmap at a glance — gate spine + backlog @@ -49,13 +52,13 @@ is clear why it waits. There is no second roadmap — depth (L4b) and breadth ### Gate spine (MVP critical path) -| Gate | Journey | Capability | Phase | Status | -| ----- | -------------------- | ------------------------------------------------------- | ----- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **A** | J1 Stand up an agent | New-Agent Wizard | P-2 | ❌ not started (onboarding bridge ✅) | -| **B** | J2 Tune schema | Schema Studio | P-4 | ❌ not started | -| **C** | J4 Find a regression | Impact Report ≥ 80% red/green on hand-labelled `player` | P-3 | 🟡 **long pole** — engine L1–L4a ✅; corpus ✅; predicate + measurement ✅ (detector agrees with human labels at 92% on a fixture-delta benchmark); Impact Report wiring ❌ | -| **D** | J5 Debug a trace | Trace Viewer | P-3 | ❌ not started | -| **E** | J6 Observe live | Live Trace + status bar | P-5 | ❌ not started | +| Gate | Journey | Capability | Phase | Status | +| ----- | -------------------- | ------------------------------------------------------- | ----- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **A** | J1 Stand up an agent | New-Agent Wizard | P-2 | ❌ not started (onboarding bridge ✅) | +| **B** | J2 Tune schema | Schema Studio | P-4 | ❌ not started | +| **C** | J4 Find a regression | Impact Report ≥ 80% red/green on hand-labelled `player` | P-3 | 🟡 **long pole** — engine L1–L4a ✅; corpus ✅; predicate + measurement ✅ (detector agrees with human labels at 92% on a fixture-delta benchmark); Impact Report wiring ❌ | +| **D** | J5 Debug a trace | Trace Viewer | P-3 | 🟡 A/B resolution-pipeline viewer built — one-click drill-in from an Impact Report row (right-click / telescope / arrow-nav), side-by-side vA/vB with cause attribution, grammar-step → AGR line and action → schema jumps, per-row trace capture + persistence + single-utterance recompute; **remaining for full pass:** cache-node → cache-entry inspect, and a surfaced replay-this-trace | +| **E** | J6 Observe live | Live Trace + status bar | P-5 | ❌ not started | **Critical path now: close Gate C** — corpus capture (the agent-agnostic capture path; `player` is just the anchor set the gate is scored on) → predicate tuning → diff --git a/ts/packages/grammarTools/ui/README.AUTOGEN.md b/ts/packages/grammarTools/ui/README.AUTOGEN.md index b8999ca6a9..b690a82929 100644 --- a/ts/packages/grammarTools/ui/README.AUTOGEN.md +++ b/ts/packages/grammarTools/ui/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # grammar-tools-ui — AI-generated documentation @@ -12,91 +12,100 @@ ## Overview -The `grammar-tools-ui` package provides a set of Lit web components for grammar visualization, editing, and debugging. These components are designed to work with the `grammar-tools-core` package, offering a user interface for interacting with grammar data. +The `grammar-tools-ui` package provides a collection of reusable Lit web components designed for grammar visualization, editing, and debugging. These components are tightly integrated with the `grammar-tools-core` package, enabling developers to create interactive user interfaces for working with grammar data. The package is used in other parts of the TypeAgent monorepo, such as the `agr-language` extension. ## What it does -The package includes several web components that facilitate various grammar-related tasks: +This package offers a suite of web components that support various grammar-related tasks, including visualization, debugging, and editing. These components are designed to work with a `GrammarBackend` interface, which acts as a bridge to the `grammar-tools-core` service. The key components include: -- `GtRuleList`: Displays a list of grammar rules. -- `GtSourceView`: Shows the source code of the grammar. -- `GtCompletionPanel`: Provides an interactive completion preview panel. -- `GtTraceTimeline`: Visualizes the trace of grammar matches. -- `GtCoverageHeatmap`: Displays a heatmap of grammar rule coverage. -- `GtDiffView`: Shows side-by-side grammar rule differences. -- `GtDebugPanel`: A composite debug panel that integrates multiple components for a comprehensive debugging experience. +- **`GtRuleList`**: Displays a list of grammar rules, allowing users to view and interact with individual rules. +- **`GtSourceView`**: Provides a detailed view of the grammar's source code, enabling users to explore its structure. +- **`GtCompletionPanel`**: An interactive panel for previewing completions based on partial input, helping users understand how different inputs are parsed. +- **`GtTraceTimeline`**: Visualizes the trace of grammar matches, offering insights into how the grammar processes input step by step. +- **`GtCoverageHeatmap`**: Displays a heatmap of grammar rule coverage, highlighting which rules are most frequently used. +- **`GtDiffView`**: Provides a side-by-side comparison of grammar rule differences, useful for analyzing changes between versions. +- **`GtDebugPanel`**: A composite panel that integrates multiple components, such as grammar loading, completion preview, trace visualization, coverage analysis, and diff comparison, into a single interface. -These components interact with a `GrammarBackend` interface, which defines methods for loading grammars, previewing completions, tracing matches, computing coverage, and more. +These components are designed to be modular and can be used individually or combined to create a comprehensive grammar debugging and visualization tool. ## Setup -To set up the `grammar-tools-ui` package, ensure you have the following environment variables configured: +To use the `grammar-tools-ui` package, ensure the following prerequisites are met: -- `GRAMMAR_TOOLS_CORE_PATH`: Path to the `grammar-tools-core` package. +1. **Install Dependencies**: This package depends on the `grammar-tools-core` package and the `lit` library. Ensure these dependencies are installed in your project. +2. **Environment Variables**: Set the `GRAMMAR_TOOLS_CORE_PATH` environment variable to the path of the `grammar-tools-core` package. This is required for the `GrammarBackend` interface to function correctly. +3. **Build the Package**: Use `pnpm` to install dependencies and build the package: + ```bash + pnpm install + pnpm build + ``` +4. **Integration**: Import the required components into your project and use them as custom elements in your HTML or JavaScript/TypeScript files. -For detailed setup instructions, see the hand-written README. +For additional setup details, refer to the hand-written README. ## Key Files -The package is structured as follows: +The `grammar-tools-ui` package is organized into several key files, each responsible for specific functionality: -- [src/index.ts](./src/index.ts): Exports the main components and types. -- [src/backend.ts](./src/backend.ts): Defines the `GrammarBackend` interface, which mirrors the `grammar-tools-core` service interface. -- src/fixture/index.ts: Exports the `FixtureBackend` for development and testing. -- [src/gt-completion-panel.ts](./src/gt-completion-panel.ts): Implements the `GtCompletionPanel` component. -- [src/gt-coverage-heatmap.ts](./src/gt-coverage-heatmap.ts): Implements the `GtCoverageHeatmap` component. -- [src/gt-debug-panel.ts](./src/gt-debug-panel.ts): Implements the `GtDebugPanel` component. -- [src/gt-diff-view.ts](./src/gt-diff-view.ts): Implements the `GtDiffView` component. -- [src/gt-rule-list.ts](./src/gt-rule-list.ts): Implements the `GtRuleList` component. -- [src/gt-source-view.ts](./src/gt-source-view.ts): Implements the `GtSourceView` component. +- **[src/index.ts](./src/index.ts)**: The main entry point of the package, exporting all components and types. +- **[src/backend.ts](./src/backend.ts)**: Defines the `GrammarBackend` interface, which mirrors the `grammar-tools-core` service interface. This interface includes methods for loading grammars, previewing completions, tracing matches, computing coverage, and more. +- **src/fixture/index.ts**: Provides the `FixtureBackend`, a mock implementation of the `GrammarBackend` interface for development and testing purposes. +- **[src/gt-completion-panel.ts](./src/gt-completion-panel.ts)**: Implements the `GtCompletionPanel` component for interactive completion previews. +- **[src/gt-coverage-heatmap.ts](./src/gt-coverage-heatmap.ts)**: Implements the `GtCoverageHeatmap` component for visualizing grammar rule coverage. +- **[src/gt-debug-panel.ts](./src/gt-debug-panel.ts)**: Implements the `GtDebugPanel` component, which integrates multiple debugging tools into a single interface. +- **[src/gt-diff-view.ts](./src/gt-diff-view.ts)**: Implements the `GtDiffView` component for side-by-side grammar rule comparisons. +- **[src/gt-rule-list.ts](./src/gt-rule-list.ts)**: Implements the `GtRuleList` component for displaying and interacting with a list of grammar rules. +- **[src/gt-source-view.ts](./src/gt-source-view.ts)**: Implements the `GtSourceView` component for exploring grammar source code. +- **[src/gt-trace-timeline.ts](./src/gt-trace-timeline.ts)**: Implements the `GtTraceTimeline` component for visualizing the trace of grammar matches. -### Component Details +## How to extend -- **GtRuleList**: Displays a list of grammar rules, allowing users to view and interact with individual rules. -- **GtSourceView**: Shows the source code of the grammar, providing a detailed view of the grammar's structure. -- **GtCompletionPanel**: Provides an interactive panel for previewing completions based on partial input, helping users understand how different inputs are parsed. -- **GtTraceTimeline**: Visualizes the trace of grammar matches, offering insights into how the grammar processes input. -- **GtCoverageHeatmap**: Displays a heatmap of grammar rule coverage, highlighting which rules are most frequently used. -- **GtDiffView**: Shows side-by-side differences between grammar rules, useful for comparing changes between versions. -- **GtDebugPanel**: Integrates multiple components into a single panel for comprehensive debugging, including grammar loading, completion preview, trace visualization, coverage analysis, and diff comparison. +To extend the `grammar-tools-ui` package, you can add new components or enhance existing ones. Below are the steps to guide you through the process: -## How to extend +1. **Create a New Component**: + + - Add a new TypeScript file in the `src/` directory. + - Use the Lit framework to define your custom element. For example: + + ```ts + import { LitElement, html, css } from "lit"; + import { customElement, property } from "lit/decorators.js"; + + @customElement("gt-new-component") + export class GtNewComponent extends LitElement { + static override styles = css` + /* Add your styles here */ + `; + + @property({ type: String }) + title = "New Component"; -To extend the `grammar-tools-ui` package, follow these steps: + override render() { + return html`
${this.title}
`; + } + } + ``` -1. **Add a new component**: Create a new TypeScript file in the `src/` directory and define your Lit component. -2. **Export the component**: Update [src/index.ts](./src/index.ts) to export your new component. -3. **Implement backend methods**: If your component requires new backend methods, update [src/backend.ts](./src/backend.ts) to include these methods. -4. **Test your component**: Ensure your component works correctly by writing tests and using the `FixtureBackend` for mock data. +2. **Export the Component**: -For example, to add a new component for visualizing grammar errors: + - Update [src/index.ts](./src/index.ts) to export your new component: + ```ts + export { GtNewComponent } from "./gt-new-component.js"; + ``` -```ts -// src/gt-error-view.ts -import { LitElement, html, css } from "lit"; -import { customElement, property } from "lit/decorators.js"; +3. **Update the Backend Interface**: -@customElement("gt-error-view") -export class GtErrorView extends LitElement { - static override styles = css` - /* Your styles here */ - `; + - If your component requires new backend methods, add them to [src/backend.ts](./src/backend.ts). Ensure these methods align with the `grammar-tools-core` service interface. - @property({ type: Array }) - errors = []; +4. **Test Your Component**: - override render() { - return html` -
${this.errors.map((error) => html`

${error.message}

`)}
- `; - } -} + - Use the `FixtureBackend` from src/fixture/index.ts to mock backend data for development and testing. + - Write unit tests for your component to ensure it behaves as expected. -// src/index.ts -export { GtErrorView } from "./gt-error-view.js"; -``` +5. **Integrate with Existing Components**: + - If your new component interacts with existing components, update the relevant files to include your component. For example, you might add your component to the `GtDebugPanel` if it provides debugging functionality. -By following these steps, you can extend the functionality of the `grammar-tools-ui` package to meet your specific needs. +By following these steps, you can enhance the `grammar-tools-ui` package to support additional grammar-related features or customize it for your specific use case. ## Reference @@ -104,7 +113,7 @@ By following these steps, you can extend the functionality of the `grammar-tools ### Entry points -- default → [./dist/index.js](./dist/index.js) +- default → `./dist/index.js` _(not found on disk)_ ### Dependencies @@ -124,6 +133,6 @@ External: `lit` --- -_Auto-generated against commit `127a36a95a15e918be533d6eaaf08adebe9070d9` on `2026-06-26T03:01:52.873Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter grammar-tools-ui docs:verify-links` to spot-check._ +_Auto-generated against commit `fbf54a8aff55bd1ef482ad8fbf2064bc3d38486c` on `2026-07-17T05:44:32.534Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter grammar-tools-ui docs:verify-links` to spot-check._ diff --git a/ts/packages/grammarTools/ui/src/gt-trace-timeline.ts b/ts/packages/grammarTools/ui/src/gt-trace-timeline.ts index b6c4b30db9..a54cd54d66 100644 --- a/ts/packages/grammarTools/ui/src/gt-trace-timeline.ts +++ b/ts/packages/grammarTools/ui/src/gt-trace-timeline.ts @@ -2,8 +2,10 @@ // Licensed under the MIT License. import { LitElement, html, css, nothing } from "lit"; +import type { PropertyValues } from "lit"; import { customElement, property, state } from "lit/decorators.js"; import type { + GrammarDebugInfo, LoadedGrammar, MatchTrace, TraceEvent, @@ -42,15 +44,6 @@ const EVENT_LABELS: Record = { backtrack: "back", }; -const EVENT_COLORS: Record = { - ruleEntered: "var(--vscode-editorInfo-foreground, #3794ff)", - ruleExited: "var(--vscode-editorInfo-foreground, #3794ff)", - partAttempted: "var(--vscode-descriptionForeground, #9d9d9d)", - partMatched: "#4ec9b0", - partFailed: "var(--vscode-errorForeground, #f48771)", - backtrack: "var(--vscode-editorWarning-foreground, #cca700)", -}; - interface DerivedTraceData { trace: MatchTrace; depths: number[]; @@ -249,6 +242,29 @@ export class GtTraceTimeline extends LitElement { font-size: 0.9em; color: var(--vscode-descriptionForeground, #9d9d9d); } + .evk-ruleEntered, + .evk-ruleExited { + color: var(--vscode-editorInfo-foreground, #3794ff); + } + .evk-partAttempted { + color: var(--vscode-descriptionForeground, #9d9d9d); + } + .evk-partMatched { + color: #4ec9b0; + } + .evk-partFailed { + color: var(--vscode-errorForeground, #f48771); + } + .evk-backtrack { + color: var(--vscode-editorWarning-foreground, #cca700); + } + .indent-unit { + display: inline-block; + width: 12px; + } + .status-line { + padding: 8px; + } `, ]; @@ -264,6 +280,19 @@ export class GtTraceTimeline extends LitElement { @property({ attribute: false }) onSourceClick: ((loc: SourceLocation) => void) | undefined; + /** + * Display-only inputs. When set, the component renders this stored trace + * directly and never recomputes via the backend: the input box, match-policy + * options, and Trace button are hidden. Source jumps resolve through + * displayDebugInfo instead of a live grammar, so a persisted trace can be + * shown with no grammar-tools backend or live grammar present. + */ + @property({ attribute: false }) + displayTrace: MatchTrace | undefined; + + @property({ attribute: false }) + displayDebugInfo: GrammarDebugInfo | undefined; + @state() private _input: string = ""; @@ -316,6 +345,36 @@ export class GtTraceTimeline extends LitElement { } } + /** True when the component shows a supplied stored trace instead of running + * its own live match. */ + private get _readOnly(): boolean { + return this.displayTrace !== undefined; + } + + /** The trace currently rendered: the supplied stored one in display-only + * mode, otherwise the one this component computed via the backend. */ + private get _activeTrace(): MatchTrace | undefined { + return this.displayTrace ?? this._trace; + } + + /** The debug info used to map events to source spans and part labels: the + * supplied serialized one in display-only mode, otherwise the live + * grammar's. */ + private get _activeDebugInfo(): GrammarDebugInfo | undefined { + return this._readOnly ? this.displayDebugInfo : this.grammar?.debugInfo; + } + + override willUpdate(changed: PropertyValues): void { + // A new stored trace is a fresh table: drop selection and expansion + // state that referenced the previous trace's row indices. + if (changed.has("displayTrace")) { + this._selectedRow = -1; + this._hoveredRow = -1; + this._expandedRows = new Set(); + this._collapsedGroups = new Set(); + } + } + private async _runTrace(): Promise { if (!this.backend || !this.grammar || !this._input) return; this._loading = true; @@ -377,11 +436,12 @@ export class GtTraceTimeline extends LitElement { groupSize: number; // >0 for ruleEntered with children attemptPos?: number; // inputPos from preceding partAttempted }> { - if (!this._trace) return []; + const trace = this._activeTrace; + if (!trace) return []; const derived = this._derivedTraceData(); const { depths, groupSize, attemptPosForEvent, onSuccessPath } = derived; - const events = this._trace.events; + const events = trace.events; const n = events.length; // Build the hidden set from collapsed groups. @@ -423,10 +483,10 @@ export class GtTraceTimeline extends LitElement { */ private _derivedTraceData(): DerivedTraceData { const cached = this._derivedCache; - if (cached !== undefined && cached.trace === this._trace) { + if (cached !== undefined && cached.trace === this._activeTrace) { return cached; } - const events = this._trace!.events; + const events = this._activeTrace!.events; const n = events.length; // Compute depth for every event. ruleEntered carries an @@ -523,7 +583,7 @@ export class GtTraceTimeline extends LitElement { } this._derivedCache = { - trace: this._trace!, + trace: this._activeTrace!, depths, groupSize, attemptPosForEvent, @@ -533,8 +593,9 @@ export class GtTraceTimeline extends LitElement { } private _highlightRange(): { start: number; end: number } | undefined { - if (!this._trace || this._hoveredRow < 0) return undefined; - const event = this._trace.events[this._hoveredRow]; + const trace = this._activeTrace; + if (!trace || this._hoveredRow < 0) return undefined; + const event = trace.events[this._hoveredRow]; if (!event) return undefined; const start = "inputPos" in event ? event.inputPos : 0; const end = "endPos" in event ? event.endPos : start; @@ -542,7 +603,7 @@ export class GtTraceTimeline extends LitElement { } private _renderInputDisplay() { - const text = this._trace?.input ?? this._input; + const text = this._activeTrace?.input ?? this._input; const range = this._highlightRange(); if (!range || range.start === range.end) { return html`
${text}
`; @@ -565,7 +626,7 @@ export class GtTraceTimeline extends LitElement { case "ruleExited": return event.result; case "partMatched": { - const input = this._trace?.input ?? ""; + const input = this._activeTrace?.input ?? ""; const start = attemptPos ?? 0; const end = event.endPos; let matchedSpan: string | undefined; @@ -596,6 +657,7 @@ export class GtTraceTimeline extends LitElement { override render() { const visible = this._visibleEvents(); + const trace = this._activeTrace; const allKinds: EventKindFilter[] = [ "ruleEntered", "ruleExited", @@ -606,142 +668,145 @@ export class GtTraceTimeline extends LitElement { ]; return html` -
- Match Options -
- - - -
-
- -
- { - this._input = (e.target as HTMLInputElement).value; - }} - @keydown=${this._onInputKeydown} - /> - -
- - ${this._trace ? this._renderInputDisplay() : nothing} + ${this._readOnly + ? nothing + : html`
+ Match Options +
+ + + +
+
+ +
+ { + this._input = ( + e.target as HTMLInputElement + ).value; + }} + @keydown=${this._onInputKeydown} + /> + +
`} + ${trace ? this._renderInputDisplay() : nothing} ${this._error - ? html`
- ${this._error} -
` + ? html`
${this._error}
` : nothing} ${this._loading - ? html`
Tracing...
` + ? html`
Tracing...
` : nothing} - ${this._trace + ${trace ? html`
- ${this._trace.events.length} events, - ${this._trace.events.filter( - (e) => e.kind === "ruleEntered", - ).length} + ${trace.events.length} events, + ${trace.events.filter((e) => e.kind === "ruleEntered") + .length} rules entered, - ${this._trace.events.filter( - (e) => e.kind === "backtrack", - ).length} + ${trace.events.filter((e) => e.kind === "backtrack") + .length} backtracks, result: - ${this._trace.result} - ${this._trace.matchValue !== undefined + ${trace.result} + ${trace.matchValue !== undefined ? html`, value: ${JSON.stringify( - this._trace.matchValue, + trace.matchValue, null, 2, )} this._toggleKind(kind)} > - ${EVENT_ICONS[kind]} ${kind} @@ -781,7 +846,7 @@ export class GtTraceTimeline extends LitElement { !this._showSuccessPath; }} > - ✓ + ✓ Success path
@@ -826,12 +888,15 @@ export class GtTraceTimeline extends LitElement { event.kind === "partAttempted" || event.kind === "partMatched" || event.kind === "partFailed"; - const partLabel = isPartEvent - ? this.grammar?.debugInfo?.partLabels.get( - (event as { part: number }) - .part, - ) + const partId = isPartEvent + ? (event as { part: number }).part : undefined; + const partLabel = + partId !== undefined + ? this._activeDebugInfo?.partLabels.get( + partId, + ) + : undefined; const ruleName = event.kind !== "backtrack" ? event.rule @@ -875,12 +940,13 @@ export class GtTraceTimeline extends LitElement { > ${index + 1} - ${isGroup + ${Array.from( + { length: depth }, + () => + html``, + )}${isGroup ? html`` : nothing} - ${partLabel ?? ""} + + ${this._renderPartLink( + partId, + partLabel, + )} + ${EVENT_ICONS[ event.kind ]}${ruleName}`; } - const loc = this.grammar.debugInfo.rules.get(ruleName); + const loc = debugInfo.rules.get(ruleName); if (!loc) return html`${ruleName}`; return html`${ruleName}`; } + + private _renderPartLink( + part: number | undefined, + label: string | undefined, + ) { + if (label === undefined) return html`${""}`; + const debugInfo = this._activeDebugInfo; + const loc = part !== undefined ? debugInfo?.parts.get(part) : undefined; + if (!this.onSourceClick || !loc) { + return html`${label}`; + } + return html` { + e.stopPropagation(); + this.onSourceClick!(loc); + }} + >${label}`; + } } declare global { diff --git a/ts/packages/grammarTools/ui/test/components/gt-trace-timeline.test.ts b/ts/packages/grammarTools/ui/test/components/gt-trace-timeline.test.ts index a2703ba332..7980d8a0c6 100644 --- a/ts/packages/grammarTools/ui/test/components/gt-trace-timeline.test.ts +++ b/ts/packages/grammarTools/ui/test/components/gt-trace-timeline.test.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { fixture, html, expect, waitUntil } from "@open-wc/testing"; +import type { SourceLocation } from "grammar-tools-core"; import type { GtTraceTimeline } from "../../src/gt-trace-timeline.js"; import { FixtureBackend } from "../fixture/fixtureBackend.js"; @@ -112,4 +113,86 @@ describe("gt-trace-timeline", () => { const filterBtns = el.shadowRoot!.querySelectorAll(".filter-btn"); expect(filterBtns.length).to.be.greaterThan(0); }); + + describe("display-only mode", () => { + it("renders a stored trace with no backend and hides live controls", async () => { + const result = await backend.loadGrammarFromFile("test.agr"); + if (!result.ok) throw new Error("load failed"); + const trace = await backend.traceMatch( + result.grammar, + "play songs by the beatles", + ); + + const el = await fixture( + html``, + ); + + // Live-mode controls are hidden. + expect(el.shadowRoot!.querySelector("input")).to.not.exist; + expect(el.shadowRoot!.querySelector(".options-panel")).to.not.exist; + expect(el.shadowRoot!.querySelector(".input-row")).to.not.exist; + + // The table and summary render immediately, without clicking Trace. + expect(el.shadowRoot!.querySelector("table")).to.exist; + expect(el.shadowRoot!.querySelector(".summary-bar")).to.exist; + const rows = el.shadowRoot!.querySelectorAll("tbody tr"); + expect(rows.length).to.be.greaterThan(0); + }); + + it("colors event icons via class, never inline style", async () => { + const result = await backend.loadGrammarFromFile("test.agr"); + if (!result.ok) throw new Error("load failed"); + const trace = await backend.traceMatch( + result.grammar, + "play songs by the beatles", + ); + + const el = await fixture( + html``, + ); + + const icons = + el.shadowRoot!.querySelectorAll(".event-icon"); + expect(icons.length).to.be.greaterThan(0); + Array.from(icons).forEach((icon) => { + expect(icon.getAttribute("style")).to.not.exist; + const classes = Array.from(icon.classList) as string[]; + expect(classes.some((c) => c.startsWith("evk-"))).to.equal( + true, + ); + }); + }); + + it("invokes onSourceClick from a rule link in display-only mode", async () => { + const result = await backend.loadGrammarFromFile("test.agr"); + if (!result.ok) throw new Error("load failed"); + const trace = await backend.traceMatch( + result.grammar, + "play songs by the beatles", + ); + + let clicked: SourceLocation | undefined; + const el = await fixture( + html` { + clicked = loc; + }} + >`, + ); + + const link = + el.shadowRoot!.querySelector(".rule-link"); + expect(link, "a rule link should render").to.exist; + link!.click(); + expect(clicked).to.exist; + }); + }); }); diff --git a/ts/packages/studio-service/README.AUTOGEN.md b/ts/packages/studio-service/README.AUTOGEN.md index 534e56a190..7707fe8abf 100644 --- a/ts/packages/studio-service/README.AUTOGEN.md +++ b/ts/packages/studio-service/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # studio-service — AI-generated documentation @@ -16,23 +16,23 @@ The `studio-service` package is a standalone, per-workspace host for the TypeAge ## What it does -The `studio-service` package provides the following core functionalities: +The `studio-service` package provides the following key functionalities: -- **Hosting the Studio Runtime**: It runs the Studio runtime, which manages workspace state and executes actions within the developer's workspace. -- **Typed Service Channel**: The package establishes a typed `agent-rpc` service channel, enabling communication between the Studio runtime and its clients, such as the `studio` agent and the `typeagent-studio` extension. -- **Workspace State Management**: The service uses a durable, file-backed `StudioWorkspaceState` to persist runtime state across service restarts, ensuring continuity. +- **Studio Runtime Hosting**: It runs the Studio runtime, which is responsible for managing the state of the developer's workspace and executing actions within it. +- **Typed Service Channel**: The package establishes a typed `agent-rpc` service channel, enabling structured communication between the Studio runtime and its clients, such as the `studio` agent and the `typeagent-studio` extension. +- **Workspace State Management**: The service uses a file-backed `StudioWorkspaceState` implementation to persist runtime state across service restarts, ensuring continuity and durability. - **Service Discovery and Registration**: The service includes mechanisms for announcing its availability, discovering its port and token, and enabling clients to connect securely. -- **Proxy Client**: A minimal client is provided for the `studio` agent to forward read-only actions to the standalone Studio service. +- **Proxy Client**: A lightweight proxy client is provided for the `studio` agent to forward read-only actions to the standalone Studio service. -The package supports a range of actions, including workspace state management and communication with the Studio runtime. These actions are exposed through typed `agent-rpc` handlers, which are defined in [studioRpcHandlers.ts](./src/studioRpcHandlers.ts). +The service supports a range of actions, including workspace state management and communication with the Studio runtime. These actions are exposed through typed `agent-rpc` handlers defined in [studioRpcHandlers.ts](./src/studioRpcHandlers.ts). ## Setup To configure and run the `studio-service` package, you need to set the following environment variable: -- `TYPEAGENT_STUDIO_REPO_ROOT`: Specifies the root directory of the repository that the Studio service should inspect. If this variable is not set, the service defaults to using the current working directory. You can set this variable in your shell or `.env` file. +- `TYPEAGENT_STUDIO_REPO_ROOT`: Specifies the root directory of the repository that the Studio service should inspect. If this variable is not set, the service defaults to using the current working directory. You can set this variable in your shell or in a `.env` file located in the `ts/` directory. -Ensure that this environment variable is correctly configured before launching the service. The service can be started using the `typeagent-studio serve` CLI or programmatically via its API. +Once the environment variable is set, you can launch the service using the `typeagent-studio serve` CLI or programmatically via its API. For more details on the setup process, refer to the hand-written README. ## Key Files @@ -68,7 +68,7 @@ After making changes, ensure that you run the package's tests to verify the func ### Entry points -- default → `./dist/index.js` _(not found on disk)_ +- default → [./dist/index.js](./dist/index.js) ### Dependencies @@ -87,7 +87,7 @@ External: `debug`, `ws` ### Files of interest -`./src/index.ts`, `./src/main.ts`, `./src/fileWorkspaceState.ts`, …and 7 more under `./src/`. +`./src/index.ts`, `./src/main.ts`, `./src/fileWorkspaceState.ts`, …and 8 more under `./src/`. ### Environment variables @@ -97,6 +97,6 @@ _1 environment variable referenced from `./src/` (set in `ts/.env` or your shell --- -_Auto-generated against commit `366aaf867a7e8e5d130b6c87a365516bab725269` on `2026-07-07T09:05:05.703Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter studio-service docs:verify-links` to spot-check._ +_Auto-generated against commit `fbf54a8aff55bd1ef482ad8fbf2064bc3d38486c` on `2026-07-17T05:44:32.534Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter studio-service docs:verify-links` to spot-check._ diff --git a/ts/packages/studio-service/src/studioRpcHandlers.ts b/ts/packages/studio-service/src/studioRpcHandlers.ts index f1b038e5f6..28485b0ba1 100644 --- a/ts/packages/studio-service/src/studioRpcHandlers.ts +++ b/ts/packages/studio-service/src/studioRpcHandlers.ts @@ -97,6 +97,9 @@ export function createStudioInvokeHandlers( ? { ...result, rows: result.rows.slice(0, MAX_ROWS) } : result; }, + async replayResolutionTrace(repoRoot, request) { + return conn.getRuntime(repoRoot).replayResolutionTrace(request); + }, async subscribeEvents(repoRoot) { // Subscribe this connection to the repo's event stream. Installing // replaces any prior subscription (idempotent — never stacks diff --git a/ts/packages/typeagent-core/README.AUTOGEN.md b/ts/packages/typeagent-core/README.AUTOGEN.md index 39accf79f1..7a005d74f1 100644 --- a/ts/packages/typeagent-core/README.AUTOGEN.md +++ b/ts/packages/typeagent-core/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/core — AI-generated documentation @@ -12,67 +12,67 @@ ## Overview -The `@typeagent/core` package is a shared TypeScript library that provides foundational functionality for TypeAgent Studio extensions. It acts as the engine for managing key subsystems such as sandbox lifecycles, federated corpora, structured event streams, feedback mechanisms, health monitoring, collision detection, corpus replay, and onboarding workflows. This package is consumed by other components in the TypeAgent ecosystem, including `typeagent-studio`, `agr-language`, and `vscode-shell`. +The `@typeagent/core` package is a foundational TypeScript library that provides shared functionality for the TypeAgent Studio ecosystem. It serves as the core engine for various extensions and subsystems, enabling features such as sandbox lifecycle management, federated corpus handling, structured event streaming, feedback collection, health monitoring, collision detection, corpus replay, and onboarding workflows. This package is a critical dependency for other components in the TypeAgent ecosystem, including `typeagent-studio`, `agr-language`, and `vscode-shell`. ## What it does -The package is modular, with each subsystem implemented in its own directory under `src/`. These subsystems provide the following capabilities: +The `@typeagent/core` package is organized into modular subsystems, each of which provides specific functionality to support the broader TypeAgent Studio platform. These subsystems include: -- **Events**: Manages structured event streams, including event creation, logging, and dispatching. -- **Sandbox**: Handles the lifecycle of sandboxes, such as creation, destruction, and state management. -- **Corpus**: Provides tools for managing federated corpora, including file-based corpus services, JSONL utilities, and unique identifier generation for corpus entries. -- **Feedback**: Implements mechanisms for collecting and processing user feedback. -- **Health**: Includes a rule engine for monitoring and maintaining the health of the system. -- **Collisions**: Detects and manages collision events, supporting both dispatcher-based and grammar-based collision detection. -- **Replay**: Facilitates corpus replay for testing and debugging purposes. -- **Onboarding Bridge**: Manages snapshot and restore operations to support onboarding workflows. +- **Events**: Manages structured event streams, enabling the creation, logging, and dispatching of events. This is essential for tracking system activities and interactions. +- **Sandbox**: Handles the lifecycle of sandboxes, including their creation, destruction, and state management. Sandboxes are isolated environments used for executing agents and other processes. +- **Corpus**: Provides tools for managing federated corpora, including file-based corpus services, JSONL utilities, and unique identifier generation for corpus entries. This subsystem is crucial for organizing and accessing large datasets. +- **Feedback**: Implements mechanisms for collecting and processing user feedback, which can be used to improve system behavior and user experience. +- **Health**: Includes a rule engine for monitoring the health of the system, ensuring stability and identifying potential issues. +- **Collisions**: Detects and manages collision events, supporting both dispatcher-based and grammar-based collision detection. This is particularly useful for identifying and resolving conflicts in agent grammars. +- **Replay**: Facilitates the replay of corpus data for testing and debugging purposes, allowing developers to simulate and analyze past interactions. +- **Onboarding Bridge**: Manages snapshot and restore operations to support onboarding workflows, ensuring a smooth user experience during setup and transitions. -These subsystems are designed to be lightweight, reusable, and easily integrated into other parts of the TypeAgent ecosystem. +These subsystems are designed to be modular, lightweight, and reusable, making them easy to integrate into other parts of the TypeAgent ecosystem. ## Setup To use the `@typeagent/core` package, you need to configure the following environment variable: -- `TYPEAGENT_USER_DATA_DIR`: Specifies the directory where user data is stored. Ensure that this directory exists and is accessible by the application. For more details on how to configure this variable, refer to the hand-written README. +- `TYPEAGENT_USER_DATA_DIR`: This environment variable specifies the directory where user data is stored. Ensure that the directory exists and is accessible by the application. For more details on how to configure this variable, refer to the hand-written README. No additional setup steps are required beyond setting this environment variable. ## Key Files -The `@typeagent/core` package is organized into several key directories and files, each responsible for specific functionalities: +The `@typeagent/core` package is organized into several key directories and files, each responsible for specific functionalities. Below is an overview of the most important files and their roles: ### Core Modules -- **[src/index.ts](./src/index.ts)**: The main entry point for the package, exporting shared types and services. -- **[src/events/index.ts](./src/events/index.ts)**: Manages structured event streams, including event creation and logging. -- **[src/sandbox/index.ts](./src/sandbox/index.ts)**: Handles the lifecycle of sandboxes, such as creation and destruction. -- **[src/corpus/index.ts](./src/corpus/index.ts)**: Provides tools for managing federated corpora, including file-based corpus services and utilities for handling JSONL data. -- **[src/feedback/index.ts](./src/feedback/index.ts)**: Implements feedback mechanisms for collecting and processing user input. -- **[src/health/index.ts](./src/health/index.ts)**: Contains the health rule engine for monitoring system health and status. -- **[src/collisions/index.ts](./src/collisions/index.ts)**: Detects and manages collision events, with support for both dispatcher and grammar-based collisions. +- **[src/index.ts](./src/index.ts)**: The main entry point for the package, exporting shared types and services across all subsystems. +- **[src/events/index.ts](./src/events/index.ts)**: Manages structured event streams, including event creation, logging, and dispatching. +- **[src/sandbox/index.ts](./src/sandbox/index.ts)**: Handles sandbox lifecycles, including creation, destruction, and state management. +- **[src/corpus/index.ts](./src/corpus/index.ts)**: Provides tools for managing federated corpora, including file-based corpus services, JSONL utilities, and unique identifier generation. +- **[src/feedback/index.ts](./src/feedback/index.ts)**: Implements feedback collection and processing mechanisms. +- **[src/health/index.ts](./src/health/index.ts)**: Contains the health rule engine for monitoring and maintaining system health. +- **[src/collisions/index.ts](./src/collisions/index.ts)**: Detects and manages collision events, supporting dispatcher and grammar-based collision detection. - **[src/replay/index.ts](./src/replay/index.ts)**: Facilitates corpus replay functionalities for testing and debugging. - **[src/onboardingBridge/index.ts](./src/onboardingBridge/index.ts)**: Manages snapshot and restore processes for onboarding workflows. ### Supporting Files -- **[src/collisions/scanner.ts](./src/collisions/scanner.ts)**: Implements a repository-backed grammar collision scanner that integrates with the `grammar-tools-core` library. +- **[src/collisions/scanner.ts](./src/collisions/scanner.ts)**: Implements a repository-backed grammar collision scanner that integrates with the `grammar-tools-core` library. This file is designed to be lightweight and modular, avoiding unnecessary dependencies when importing collision services. - **[src/corpus/fileCorpusService.ts](./src/corpus/fileCorpusService.ts)**: Provides a filesystem-backed service for managing corpora, including support for multiple data sources. - **[src/corpus/id.ts](./src/corpus/id.ts)**: Contains utilities for generating unique identifiers for corpus entries. - **[src/corpus/jsonl.ts](./src/corpus/jsonl.ts)**: Includes utilities for parsing and formatting JSONL files, which are used to store corpus data. ## How to extend -To extend the functionality of the `@typeagent/core` package, follow these steps: +To contribute to or extend the `@typeagent/core` package, follow these steps: 1. **Identify the subsystem**: Determine which subsystem (e.g., events, sandbox, corpus) you need to modify or extend. 2. **Locate the relevant files**: Navigate to the corresponding directory under `src/` (e.g., `src/events/` for event-related functionality). -3. **Follow existing patterns**: Review the existing code to understand the structure and conventions used in the package. For example, most subsystems have a clear separation between types, services, and utility functions. -4. **Add new functionality**: Implement your changes or new features in the appropriate files. Ensure that your code adheres to the project's coding standards and is consistent with the existing implementation. +3. **Understand the existing implementation**: Review the code in the relevant files to understand the structure, patterns, and conventions used. +4. **Implement new functionality**: Add your changes or new features in the appropriate files. Ensure your code is consistent with the existing implementation and adheres to the project's coding standards. 5. **Update exports**: If you add new functions, classes, or types, make sure to export them in the module's `index.ts` file. -6. **Write tests**: Add unit tests for your new functionality. Place the tests in the appropriate test directory, following the existing test structure. -7. **Run the test suite**: Execute the test suite to ensure that your changes do not introduce any regressions or break existing functionality. +6. **Write tests**: Create unit tests for your new functionality. Place the tests in the appropriate test directory, following the existing test structure. +7. **Run tests**: Execute the test suite to ensure that your changes do not introduce any regressions or break existing functionality. -By following these steps, you can effectively contribute to the development of the `@typeagent/core` package and extend its capabilities. +By following these guidelines, you can effectively contribute to the `@typeagent/core` package and enhance its capabilities. ## Reference @@ -80,19 +80,19 @@ By following these steps, you can effectively contribute to the development of t ### Entry points -- default → `./dist/index.js` _(not found on disk)_ -- `./events` → `./dist/events/index.js` _(not found on disk)_ -- `./sandbox` → `./dist/sandbox/index.js` _(not found on disk)_ -- `./corpus` → `./dist/corpus/index.js` _(not found on disk)_ -- `./feedback` → `./dist/feedback/index.js` _(not found on disk)_ -- `./health` → `./dist/health/index.js` _(not found on disk)_ -- `./collisions` → `./dist/collisions/index.js` _(not found on disk)_ -- `./collisionScanner` → `./dist/collisions/scanner.js` _(not found on disk)_ -- `./replay` → `./dist/replay/index.js` _(not found on disk)_ -- `./replayResolver` → `./dist/replay/grammarResolver.js` _(not found on disk)_ -- `./onboardingBridge` → `./dist/onboardingBridge/index.js` _(not found on disk)_ -- `./webview` → `./dist/webview/index.js` _(not found on disk)_ -- `./runtime` → `./dist/runtime/index.js` _(not found on disk)_ +- default → [./dist/index.js](./dist/index.js) +- `./events` → [./dist/events/index.js](./dist/events/index.js) +- `./sandbox` → [./dist/sandbox/index.js](./dist/sandbox/index.js) +- `./corpus` → [./dist/corpus/index.js](./dist/corpus/index.js) +- `./feedback` → [./dist/feedback/index.js](./dist/feedback/index.js) +- `./health` → [./dist/health/index.js](./dist/health/index.js) +- `./collisions` → [./dist/collisions/index.js](./dist/collisions/index.js) +- `./collisionScanner` → [./dist/collisions/scanner.js](./dist/collisions/scanner.js) +- `./replay` → [./dist/replay/index.js](./dist/replay/index.js) +- `./replayResolver` → [./dist/replay/grammarResolver.js](./dist/replay/grammarResolver.js) +- `./onboardingBridge` → [./dist/onboardingBridge/index.js](./dist/onboardingBridge/index.js) +- `./webview` → [./dist/webview/index.js](./dist/webview/index.js) +- `./runtime` → [./dist/runtime/index.js](./dist/runtime/index.js) ### Dependencies @@ -125,7 +125,7 @@ External: `debug` - [./src/replay/index.ts](./src/replay/index.ts) - [./src/runtime/index.ts](./src/runtime/index.ts) - [./src/sandbox/index.ts](./src/sandbox/index.ts) -- _…and 38 more under `./src/`._ +- _…and 44 more under `./src/`._ ### Environment variables @@ -135,6 +135,6 @@ _1 environment variable referenced from `./src/` (set in `ts/.env` or your shell --- -_Auto-generated against commit `366aaf867a7e8e5d130b6c87a365516bab725269` on `2026-07-07T09:05:05.703Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/core docs:verify-links` to spot-check._ +_Auto-generated against commit `d7d1b43eebfadec45d2f3199e628189b353920b9` on `2026-07-17T20:31:18.363Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/core docs:verify-links` to spot-check._ diff --git a/ts/packages/typeagent-core/src/replay/constructionCacheResolver.ts b/ts/packages/typeagent-core/src/replay/constructionCacheResolver.ts index 25ae023b82..c68fa25544 100644 --- a/ts/packages/typeagent-core/src/replay/constructionCacheResolver.ts +++ b/ts/packages/typeagent-core/src/replay/constructionCacheResolver.ts @@ -37,6 +37,7 @@ import { type MatchResult, } from "agent-cache"; import { normalizeAction } from "./replayActionShape.js"; +import type { ConstructionCacheEntryDto } from "./resolutionTrace.js"; /** * Inputs to the dispatcher's schema-file hash. Mirrors @@ -94,6 +95,13 @@ export interface ConstructionCacheLayer { * otherwise `undefined` (the caller falls back to the grammar path). */ match(utterance: string): Record | undefined; + /** + * As {@link match}, but returns the matched construction's inspectable + * identity (id, namespace, rendered parts, ranking scores) alongside the + * action, so a trace can show which cache entry resolved the utterance. + * `undefined` on the same conditions as {@link match}. + */ + matchEntry(utterance: string): ConstructionCacheEntryDto | undefined; } /** @@ -216,6 +224,7 @@ export async function loadConstructionCacheLayer( currentHash, cachedHash, match: () => undefined, + matchEntry: () => undefined, }); let json: ConstructionCacheFileJSON; @@ -239,6 +248,7 @@ export async function loadConstructionCacheLayer( currentHash, cachedHash: found.hash, match: () => undefined, + matchEntry: () => undefined, }; } @@ -256,6 +266,7 @@ export async function loadConstructionCacheLayer( const loadedCache = cache; const namespaceKeys = [found.namespaceName]; + const namespaceName = found.namespaceName; return { status: "valid", cacheFilePath, @@ -269,5 +280,31 @@ export async function loadConstructionCacheLayer( } return normalizeAction(schemaName, topConstructionAction(results)); }, + matchEntry(utterance: string): ConstructionCacheEntryDto | undefined { + const results = loadedCache.match(utterance, { namespaceKeys }); + const top = results[0]; + if (top === undefined) { + return undefined; + } + const action = normalizeAction( + schemaName, + topConstructionAction(results), + ); + if (action === undefined) { + return undefined; + } + return { + action, + constructionId: top.construction.id.toString(), + namespace: namespaceName, + parts: top.construction.parts.map((part) => part.toString()), + scores: { + matchedCount: top.matchedCount, + wildcardCharCount: top.wildcardCharCount, + nonOptionalCount: top.nonOptionalCount, + }, + cacheFileId: cacheFilePath, + }; + }, }; } diff --git a/ts/packages/typeagent-core/src/replay/grammarResolver.ts b/ts/packages/typeagent-core/src/replay/grammarResolver.ts index 70691129ed..96db2e1cf1 100644 --- a/ts/packages/typeagent-core/src/replay/grammarResolver.ts +++ b/ts/packages/typeagent-core/src/replay/grammarResolver.ts @@ -55,7 +55,7 @@ import { type AgentRootsInput, } from "../health/index.js"; import type { CorpusEntry } from "../corpus/types.js"; -import type { VersionSpec } from "./types.js"; +import type { VersionSpec, ReplayCacheState } from "./types.js"; import { actionsEqual, type ReplayActionResolver, @@ -67,6 +67,19 @@ import type { ConstructionCacheStatus, } from "./constructionCacheResolver.js"; import type { WildcardMatchValidator } from "./wildcardValidator.js"; +import { captureGrammarMatchTrace } from "./resolutionTraceCapture.js"; +import type { + ActionTraceNode, + CacheConsultTraceNode, + ConstructionCacheEntryDto, + GrammarMatchTraceNode, + ReplayResolutionTrace, + ReplaySideTrace, + TraceRealization, + WildcardValidationTraceNode, +} from "./resolutionTrace.js"; + +export { captureGrammarMatchTrace } from "./resolutionTraceCapture.js"; const execFileAsync = promisify(execFile); @@ -78,6 +91,11 @@ const execFileAsync = promisify(execFile); type BuiltGrammar = { store: GrammarStoreImpl; enriched: boolean; + /** The grammar source that was compiled, retained so a trace can re-run the + * tracing matcher over the exact same text this side matched against. */ + grammarText: string; + /** File identity used when re-loading the grammar for tracing. */ + grammarFileName: string; }; /** @@ -227,6 +245,57 @@ export interface GrammarReplayResolver extends ReplayActionResolver { * `false` when no validator was supplied. */ readonly wildcardValidationApplied: boolean; + /** + * Resolve one utterance for a side exactly as {@link resolve} does, and also + * capture the step-by-step {@link ReplaySideTrace} that explains the result + * (the fidelity layers exercised, the grammar-match event stream mapped to + * `.agr` spans, and the construction-cache entry when hit). The extra work + * — re-running the tracing matcher and resolving cache identity — is only + * paid here, so streaming a run through {@link resolve} stays cheap. Call + * with the live resolver right after a run so the working tree still matches + * what produced the row. + */ + resolveWithTrace( + entry: CorpusEntry, + version: VersionSpec, + side: "A" | "B", + ): Promise<{ + resolution: ReplayAgentResolution; + sideTrace: ReplaySideTrace; + }>; +} + +/** + * What one side's resolution touched, recorded cheaply during {@link + * GrammarReplayResolver.resolve} so a trace can be built later without re-running + * the resolution. Holds no grammar-tools objects — only the inputs a trace needs. + */ +interface SideResolutionArtifacts { + utterance: string; + version: VersionSpec; + realization: TraceRealization; + /** True when the live construction cache was consulted (working-tree side, + * valid cache). */ + cacheConsulted: boolean; + /** True when the construction cache resolved the utterance. */ + cacheHit: boolean; + /** True once the grammar match was reached (i.e. the cache did not short- + * circuit). */ + grammarReached: boolean; + /** True when the grammar failed to build for this side. */ + buildFailed: boolean; + /** The built grammar for this side, present when the grammar was reached. */ + grammar?: BuiltGrammar; + /** The normalized action the grammar match produced, when the grammar was + * reached and matched. Drives the trace's ranking-parity check. */ + grammarAction?: unknown; + /** Whether a wildcard validator was configured for this run. */ + validatorAvailable: boolean; + /** Outcome of the wildcard-validation pass for this row. */ + wildcard: "accepted" | "rejected" | "not-run"; + /** The action this side ultimately produced (cache or grammar), when any. */ + finalAction?: unknown; + cacheState: ReplayCacheState; } let entitiesRegistered = false; @@ -366,8 +435,10 @@ async function buildGrammar( } try { + const grammarFileName = + target.grammarFileName ?? path.basename(target.grammarFilePath); const grammar = loadGrammarRules( - target.grammarFileName ?? path.basename(target.grammarFilePath), + grammarFileName, content, target.start !== undefined ? { start: target.start } : undefined, ); @@ -382,7 +453,7 @@ async function buildGrammar( const store = new GrammarStoreImpl(undefined); store.setUseNFA(true); store.addGrammar(target.schemaName, grammar); - return { store, enriched }; + return { store, enriched, grammarText: content, grammarFileName }; } catch (err) { throw new ReplayVersionBuildError( side, @@ -507,98 +578,370 @@ export function createGrammarReplayResolver( version: VersionSpec, side: "A" | "B", ): Promise { - const t0 = now(); - // A rating is observation-scoped: it judges the action that was on - // screen in the run where it was recorded (the action co-captured on - // this entry), not the utterance in the abstract. Attach it to a side - // only when that side reproduces that exact action; a rating for a - // now-different action says nothing about what this side resolved, so - // it is dropped and the structural delta decides. - const feedbackFor = (resolved: unknown) => - entry.feedback !== undefined && - entry.expectedAction !== undefined && - actionsEqual(resolved, entry.expectedAction) - ? { feedback: entry.feedback } - : {}; - - // On the working-tree side, consult the live construction cache - // first — the dispatcher's real first check. A construction hit is a - // genuine cache `hit`; everything the cache doesn't resolve falls - // through to the grammar path below and is reported as a `miss` - // (deterministically resolvable, but not served from the cache). - const useConstruction = - constructionActive && version.kind === "workingTree"; - if (useConstruction) { - const cacheAction = constructionCache!.match(entry.utterance); - if (cacheAction !== undefined) { - return { + return (await resolveSide(entry, version, side)).resolution; + }, + + async resolveWithTrace( + entry: CorpusEntry, + version: VersionSpec, + side: "A" | "B", + ): Promise<{ + resolution: ReplayAgentResolution; + sideTrace: ReplaySideTrace; + }> { + const { resolution, artifacts } = await resolveSide( + entry, + version, + side, + ); + // Resolve the construction identity only now (not on the hot + // `resolve` path): a hit means the cache is live, so the lookup is + // faithful and cheap. + const cacheEntry = artifacts.cacheHit + ? constructionCache!.matchEntry(entry.utterance) + : undefined; + const sideTrace = buildSideTrace( + side, + target, + artifacts, + cacheEntry, + ); + return { resolution, sideTrace }; + }, + }; + + /** + * Resolve one side and record the {@link SideResolutionArtifacts} a trace + * needs. This is the single source of resolution logic; `resolve` returns + * just its resolution and `resolveWithTrace` additionally builds the trace, + * so the two entry points can never drift. + */ + async function resolveSide( + entry: CorpusEntry, + version: VersionSpec, + side: "A" | "B", + ): Promise<{ + resolution: ReplayAgentResolution; + artifacts: SideResolutionArtifacts; + }> { + const t0 = now(); + // A rating is observation-scoped: it judges the action that was on + // screen in the run where it was recorded (the action co-captured on + // this entry), not the utterance in the abstract. Attach it to a side + // only when that side reproduces that exact action; a rating for a + // now-different action says nothing about what this side resolved, so + // it is dropped and the structural delta decides. + const feedbackFor = (resolved: unknown) => + entry.feedback !== undefined && + entry.expectedAction !== undefined && + actionsEqual(resolved, entry.expectedAction) + ? { feedback: entry.feedback } + : {}; + + const useConstruction = + constructionActive && version.kind === "workingTree"; + const artifacts: SideResolutionArtifacts = { + utterance: entry.utterance, + version, + realization: + version.kind === "workingTree" ? "built-live" : "source", + cacheConsulted: useConstruction, + cacheHit: false, + grammarReached: false, + buildFailed: false, + validatorAvailable: wildcardValidator !== undefined, + wildcard: "not-run", + cacheState: "needs-explanation", + }; + + // On the working-tree side, consult the live construction cache + // first — the dispatcher's real first check. A construction hit is a + // genuine cache `hit`; everything the cache doesn't resolve falls + // through to the grammar path below and is reported as a `miss` + // (deterministically resolvable, but not served from the cache). + if (useConstruction) { + const cacheAction = constructionCache!.match(entry.utterance); + if (cacheAction !== undefined) { + artifacts.cacheHit = true; + artifacts.finalAction = cacheAction; + artifacts.cacheState = "hit"; + return { + resolution: { action: cacheAction, cacheState: "hit", latencyMs: now() - t0, ...feedbackFor(cacheAction), - }; - } + }, + artifacts, + }; } + } - let g: BuiltGrammar; - try { - g = await build(version, side); - } catch { - // prepare() is expected to have surfaced build failures as a - // run-level error already; degrade safely here rather than throw - // (a throw would hang the engine's row stream). No action - // resolved, so no rating can pertain to this side. - return { + let g: BuiltGrammar; + try { + g = await build(version, side); + } catch { + // prepare() is expected to have surfaced build failures as a + // run-level error already; degrade safely here rather than throw + // (a throw would hang the engine's row stream). No action + // resolved, so no rating can pertain to this side. + artifacts.buildFailed = true; + return { + resolution: { cacheState: "needs-explanation", latencyMs: now() - t0, - }; - } + }, + artifacts, + }; + } + artifacts.grammarReached = true; + artifacts.grammar = g; + + const results = g.store.match(entry.utterance); - const results = g.store.match(entry.utterance); - - // Wildcard validation: on the working-tree side, run the - // agent's real `validateWildcardMatch` over the ranked candidates and - // resolve the first accepted match — mirroring the dispatcher's - // post-match `getValidatedMatches`. Never on a git ref (we can't load - // arbitrary-ref agent code). When no validator is supplied this is a - // plain top-match resolution (unchanged behavior). - const validateHere = - wildcardValidator !== undefined && - version.kind === "workingTree"; - let rawAction: unknown; - if (validateHere) { - const validated = await selectValidatedMatchAction( - results, - wildcardValidator!, - ); - rawAction = validated.action; - if (validated.consulted) { - wildcardValidationApplied = true; - } - } else { - rawAction = - results.length > 0 ? topMatchAction(results) : undefined; + // Wildcard validation: on the working-tree side, run the + // agent's real `validateWildcardMatch` over the ranked candidates and + // resolve the first accepted match — mirroring the dispatcher's + // post-match `getValidatedMatches`. Never on a git ref (we can't load + // arbitrary-ref agent code). When no validator is supplied this is a + // plain top-match resolution (unchanged behavior). + const validateHere = + wildcardValidator !== undefined && version.kind === "workingTree"; + let rawAction: unknown; + if (validateHere) { + const validated = await selectValidatedMatchAction( + results, + wildcardValidator!, + ); + rawAction = validated.action; + if (validated.consulted) { + wildcardValidationApplied = true; + artifacts.wildcard = + validated.action !== undefined ? "accepted" : "rejected"; } + } else { + rawAction = + results.length > 0 ? topMatchAction(results) : undefined; + } - const latencyMs = now() - t0; - const action = - rawAction !== undefined - ? normalizeAction(target.schemaName, rawAction) - : undefined; + const latencyMs = now() - t0; + const action = + rawAction !== undefined + ? normalizeAction(target.schemaName, rawAction) + : undefined; + artifacts.grammarAction = action; - if (action === undefined) { - return { - cacheState: "needs-explanation", - latencyMs, - }; - } - // When the live construction cache is in play, a grammar-only - // resolution is reported as a `miss` (not in the cache) to keep the - // working-tree side's cache states faithful; otherwise it is a plain - // grammar `hit` (unchanged grammar-match semantics). - const cacheState = useConstruction ? "miss" : "hit"; - return { action, cacheState, latencyMs, ...feedbackFor(action) }; - }, + if (action === undefined) { + return { + resolution: { cacheState: "needs-explanation", latencyMs }, + artifacts, + }; + } + // When the live construction cache is in play, a grammar-only + // resolution is reported as a `miss` (not in the cache) to keep the + // working-tree side's cache states faithful; otherwise it is a plain + // grammar `hit` (unchanged grammar-match semantics). + const cacheState = useConstruction ? "miss" : "hit"; + artifacts.finalAction = action; + artifacts.cacheState = cacheState; + return { + resolution: { + action, + cacheState, + latencyMs, + ...feedbackFor(action), + }, + artifacts, + }; + } +} + +/** + * Build the construction-cache-consult node from the run's artifacts. The cache + * is a live-only concept: `not-applicable` on a git-ref side or when no live + * cache was supplied, otherwise `ran` with the hit/miss outcome. + */ +function cacheConsultNode( + artifacts: SideResolutionArtifacts, + cacheEntry: ConstructionCacheEntryDto | undefined, +): CacheConsultTraceNode { + if (!artifacts.cacheConsulted) { + const detail = + artifacts.version.kind === "workingTree" + ? "no live construction cache" + : "cache consulted only on the working-tree side"; + return { kind: "cache-consult", execution: "not-applicable", detail }; + } + if (artifacts.cacheHit) { + return { + kind: "cache-consult", + execution: "ran", + outcome: "hit", + ...(cacheEntry !== undefined ? { entry: cacheEntry } : {}), + }; + } + return { kind: "cache-consult", execution: "ran", outcome: "miss" }; +} + +/** + * Build the grammar-match node. When the grammar was reached and built, re-run + * the tracing matcher over its source to capture the full event stream and + * source spans; otherwise record why no trace exists (cache short-circuit or a + * build failure). + */ +function grammarMatchNode( + target: GrammarReplayTarget, + artifacts: SideResolutionArtifacts, +): GrammarMatchTraceNode { + if (!artifacts.grammarReached) { + return { + kind: "grammar-match", + execution: "not-reached", + input: artifacts.utterance, + rankingParity: "unavailable", + detail: "resolved from the construction cache", + }; + } + if (artifacts.buildFailed || artifacts.grammar === undefined) { + return { + kind: "grammar-match", + execution: "unavailable", + input: artifacts.utterance, + rankingParity: "unavailable", + detail: "grammar failed to build", + }; + } + return captureGrammarMatchTrace( + target.schemaName, + artifacts.grammar.grammarFileName, + artifacts.grammar.grammarText, + artifacts.utterance, + artifacts.grammarAction, + undefined, + target.grammarFilePath, + ); +} + +/** Build the wildcard-validation node. Live-only; `ran` only when a configured + * validator actually judged a wildcard match this row. */ +function wildcardValidationNode( + artifacts: SideResolutionArtifacts, +): WildcardValidationTraceNode { + if (artifacts.version.kind !== "workingTree") { + return { + kind: "wildcard-validation", + execution: "not-applicable", + detail: "validation runs only on the working tree", + }; + } + if (!artifacts.validatorAvailable) { + return { + kind: "wildcard-validation", + execution: "not-applicable", + detail: "no wildcard validator configured", + }; + } + if ( + artifacts.wildcard === "accepted" || + artifacts.wildcard === "rejected" + ) { + return { + kind: "wildcard-validation", + execution: "ran", + outcome: artifacts.wildcard, + }; + } + return { + kind: "wildcard-validation", + execution: "not-reached", + detail: "no wildcard match to validate", + }; +} + +/** Build the produced-action node, carrying the action JSON and its schema + * location for the jump-to-variant action. */ +function actionNode( + target: GrammarReplayTarget, + artifacts: SideResolutionArtifacts, +): ActionTraceNode { + const action = artifacts.finalAction; + const produced = action !== undefined; + const rec = + action !== null && typeof action === "object" + ? (action as Record) + : undefined; + const actionName = + typeof rec?.actionName === "string" ? rec.actionName : undefined; + // Only surface the schema source for a side that actually produced an + // action. On a miss there is no resolved action, so the viewer must not + // offer a schema jump or A/B compare for this stage. + const sourceFilePath = produced ? target.schema?.sourceFilePath : undefined; + const schema = + sourceFilePath !== undefined || actionName !== undefined + ? { + ...(sourceFilePath !== undefined ? { sourceFilePath } : {}), + ...(actionName !== undefined ? { actionName } : {}), + } + : undefined; + return { + kind: "action", + execution: "ran", + outcome: produced ? "hit" : "miss", + ...(produced ? { action } : {}), + ...(schema !== undefined ? { schema } : {}), + }; +} + +/** Assemble one side's ordered fidelity-layer trace from its resolution + * artifacts. Nodes are always present (as `not-applicable` when a layer did not + * run) so both sides render as aligned rows in the viewer. */ +function buildSideTrace( + side: "A" | "B", + target: GrammarReplayTarget, + artifacts: SideResolutionArtifacts, + cacheEntry: ConstructionCacheEntryDto | undefined, +): ReplaySideTrace { + return { + side, + version: artifacts.version, + realization: artifacts.realization, + nodes: [ + cacheConsultNode(artifacts, cacheEntry), + grammarMatchNode(target, artifacts), + wildcardValidationNode(artifacts), + actionNode(target, artifacts), + ], + ...(artifacts.finalAction !== undefined + ? { finalAction: artifacts.finalAction } + : {}), + cacheState: artifacts.cacheState, + }; +} + +/** + * Capture the full {@link ReplayResolutionTrace} for one utterance by resolving + * both sides with tracing. Call with the live resolver used for the run (right + * after it, before any working-tree edit) so the trace reflects exactly what + * produced the row. Keyed by `entry.id`, matching the `utteranceId` the engine + * stamps on rows. + */ +export async function captureResolutionTrace( + resolver: GrammarReplayResolver, + entry: CorpusEntry, + runId: string, + versionA: VersionSpec, + versionB: VersionSpec, + now: () => number = () => Date.now(), +): Promise { + const a = await resolver.resolveWithTrace(entry, versionA, "A"); + const b = await resolver.resolveWithTrace(entry, versionB, "B"); + return { + runId, + utteranceId: entry.id, + utterance: entry.utterance, + a: a.sideTrace, + b: b.sideTrace, + capturedAt: now(), }; } diff --git a/ts/packages/typeagent-core/src/replay/index.ts b/ts/packages/typeagent-core/src/replay/index.ts index d5cf212596..7768703c87 100644 --- a/ts/packages/typeagent-core/src/replay/index.ts +++ b/ts/packages/typeagent-core/src/replay/index.ts @@ -3,6 +3,7 @@ export * from "./types.js"; export * from "./engine.js"; +export * from "./resolutionTrace.js"; export * from "./predicate.js"; export * from "./missPolicyStore.js"; export * from "./traceViews.js"; diff --git a/ts/packages/typeagent-core/src/replay/resolutionTrace.ts b/ts/packages/typeagent-core/src/replay/resolutionTrace.ts new file mode 100644 index 0000000000..4c5efd323d --- /dev/null +++ b/ts/packages/typeagent-core/src/replay/resolutionTrace.ts @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Serializable resolution-trace types for the replay Trace Viewer. + * + * A replay row records only the *outcome* of resolving one utterance against two + * agent versions (the produced action, the cache state, latency). To let a + * developer drill into a red row and see *why* the two sides diverged, the + * resolver also captures a step-by-step {@link ReplayResolutionTrace}: the + * ordered fidelity layers each side exercised (cache consult, grammar match, + * wildcard validation, action), the grammar-match event stream mapped back to + * exact `.agr` source spans, and the construction-cache entry that was hit. + * + * Everything here is JSON-safe so a trace can be persisted with the run and sent + * across the extension/webview boundary. The grammar match reuses + * `grammar-tools-core`'s {@link MatchTrace} (its events are already plain data) + * and a {@link SerializedGrammarDebugInfo} projection of that package's + * `GrammarDebugInfo`, whose `ReadonlyMap`s are flattened to entry arrays so the + * viewer can rebuild the maps and feed the existing timeline / source + * components. + */ + +import type { + GrammarDebugInfo, + MatchTrace, + PartId, + RuleId, + SourceLocation, +} from "grammar-tools-core"; +import type { CorpusFilter } from "../corpus/types.js"; +import type { + ReplayCacheState, + ReplayMissPolicy, + VersionSpec, +} from "./types.js"; + +/** + * Whether a fidelity layer actually executed for a side, independent of its + * outcome. A layer can be present in the trace yet not have run — e.g. the + * construction cache is a live-only concept, so on a git-ref side it is + * `not-applicable` rather than a `miss`. Keeping execution separate from outcome + * stops an absent node from being read as a failure. + */ +export type TraceNodeExecution = + | "ran" + | "skipped-by-mode" + | "not-applicable" + | "unavailable" + | "not-reached"; + +/** The result of a layer that actually ran. Only meaningful when the node's + * {@link TraceNodeExecution} is `"ran"`. */ +export type TraceNodeOutcome = + | "hit" + | "miss" + | "accepted" + | "rejected" + | "errored"; + +/** How a side's version was materialized: the live working tree (real compiled + * agent code, so live-only layers can run) or grammar/schema text read at a git + * ref (no build, so the cache and wildcard validation can't run). Mirrors the + * runtime's fidelity realization without coupling the replay layer to it. */ +export type TraceRealization = "built-live" | "source"; + +/** A JSON-safe projection of `grammar-tools-core`'s `GrammarDebugInfo`. Its + * `ReadonlyMap`s are flattened to `[key, value]` entry arrays so the whole + * structure survives `JSON.stringify`; the viewer rebuilds the maps before + * handing them to the grammar components. */ +export interface SerializedGrammarDebugInfo { + grammarHash: string; + rules: Array<[RuleId, SourceLocation]>; + parts: Array<[PartId, SourceLocation]>; + partRules: Array<[PartId, RuleId]>; + partLabels: Array<[PartId, string]>; + filePaths: Array<[string, string]>; +} + +/** + * A construction-cache entry surfaced for inspection in the viewer. Built from + * the matched construction so the cache node can be read inline without any host + * round-trip. Never the in-memory construction class — only this flattened, + * serializable identity. + */ +export interface ConstructionCacheEntryDto { + /** The normalized action the cache entry produced. */ + action: unknown; + /** Stable identity of the matched construction within its namespace. */ + constructionId?: string; + /** The cache namespace the construction lives in. */ + namespace?: string; + /** Human-readable rendering of the construction's parts. */ + parts?: string[]; + /** Ranking counts that decided this construction won. */ + scores?: { + matchedCount?: number; + wildcardCharCount?: number; + nonOptionalCount?: number; + }; + /** The cache file the entry was read from, for a "reveal in cache" jump. */ + cacheFileId?: string; +} + +interface TraceNodeBase { + execution: TraceNodeExecution; + outcome?: TraceNodeOutcome; + /** Short human explanation shown as hover detail (e.g. why a layer was + * skipped or unavailable). */ + detail?: string; +} + +/** The construction-cache consult. Live-only: `ran` on a working-tree side, + * `not-applicable` on a git-ref side. */ +export interface CacheConsultTraceNode extends TraceNodeBase { + kind: "cache-consult"; + /** The matched cache entry, present when the consult was a `hit`. */ + entry?: ConstructionCacheEntryDto; +} + +/** + * The grammar match. Carries the captured recursive-matcher {@link MatchTrace} + * plus the serialized debug info needed to resolve each event to a `.agr` line, + * and the winning rule's source span for the one-click jump. + */ +export interface GrammarMatchTraceNode extends TraceNodeBase { + kind: "grammar-match"; + /** The utterance fed to the matcher. */ + input: string; + /** The captured step-by-step match trace, when tracing succeeded. */ + trace?: MatchTrace; + /** Serialized debug info to map trace events to source spans. */ + debugInfo?: SerializedGrammarDebugInfo; + /** The winning rule the chosen parse settled on. */ + chosenRule?: RuleId; + /** Source span of the winning rule/alternative for the jump-to-line action. */ + source?: SourceLocation; + /** + * Absolute path of the `.agr` grammar file this side matched against, the + * same real repo file on both sides (the git-ref side's content is read via + * `git show`). Recorded independently of {@link source} — whose `displayPath` + * is only the grammar's basename — so the viewer can open the exact file and + * diff it across the two versions. Absent when no grammar file backs the + * match (e.g. a cache short-circuit or a build failure). + */ + sourceFilePath?: string; + /** + * Whether the traced parse (recursive matcher) agreed with the resolver's + * ranked pick (NFA `sortMatches`). `diverged` flags that the captured trace + * explains a different action than the row recorded; `unavailable` means no + * trace could be produced (e.g. no debug info). + */ + rankingParity: "matched" | "diverged" | "unavailable"; +} + +/** The opt-in wildcard-value validation pass. Live-only. */ +export interface WildcardValidationTraceNode extends TraceNodeBase { + kind: "wildcard-validation"; + /** Fail-open reasons when validation degraded (e.g. validator threw). */ + diagnostics?: string[]; +} + +/** The produced action and where its type is declared, for the schema jump. */ +export interface ActionTraceNode extends TraceNodeBase { + kind: "action"; + /** The typed action JSON this side produced, when matched. */ + action?: unknown; + /** The action's schema location, for the jump-to-variant action. */ + schema?: { + sourceFilePath?: string; + actionName?: string; + }; +} + +export type ReplayTraceNode = + | CacheConsultTraceNode + | GrammarMatchTraceNode + | WildcardValidationTraceNode + | ActionTraceNode; + +/** One side's resolution trace: the ordered layers it exercised and the action + * it settled on. Nodes are aligned across sides so the viewer can render them + * as matching rows (a live-only layer shows as `not-applicable` on the ref + * side rather than being omitted). */ +export interface ReplaySideTrace { + side: "A" | "B"; + version: VersionSpec; + realization: TraceRealization; + nodes: ReplayTraceNode[]; + /** The action this side produced, when matched. */ + finalAction?: unknown; + cacheState: ReplayCacheState; +} + +/** The full drill-in for one utterance: both sides' traces, keyed to the run and + * row they explain. Persisted with the run for red rows and read back by the + * viewer, so it always reflects the exact resolution that produced the row. */ +export interface ReplayResolutionTrace { + runId: string; + utteranceId: string; + utterance: string; + a: ReplaySideTrace; + b: ReplaySideTrace; + /** Epoch ms the trace was captured (during the corpus run). */ + capturedAt: number; +} + +/** The replay modes a trace can be captured under; mirrors the runtime's replay + * mode without coupling the replay layer to the runtime package. */ +export type ReplayTraceMode = "nfa-grammar" | "completionBased-cache"; + +/** The concrete identity a side ran against, pinned at run time so a bare + * `HEAD`/branch label (which moves) or drifting working-tree content stays + * bound to what the report reflects. */ +export interface TraceVersionPin { + spec: VersionSpec; + /** The display label the run used (e.g. "HEAD (main)"). */ + label: string; + /** The resolved commit SHA, when the side is a git ref. */ + sha?: string; + /** True when the side is the live working tree (uncommitted edits). */ + workingTree: boolean; + /** + * Content hashes of the working-tree grammar/schema/cache inputs at run time, + * keyed by a stable input name. Only the working-tree side can drift, so this + * is how its original trace is later detected as reproducible or stale. + */ + contentHashes?: Record; +} + +/** + * The exact inputs a replay run used, persisted with the run so a stored trace + * can be reproduced (or flagged as drifted) later. Captures the pinned versions, + * the resolution settings, and the corpus identity — everything the shared + * resolution path needs to recompute a fresh trace for the same row. + */ +export interface ReplayRunDescriptor { + runId: string; + agent: string; + a: TraceVersionPin; + b: TraceVersionPin; + mode: ReplayTraceMode; + missPolicy: ReplayMissPolicy; + validateWildcards: boolean; + corpus: CorpusFilter; + /** Epoch ms the run was issued. */ + runAt: number; +} + +/** Flatten a `grammar-tools-core` `GrammarDebugInfo` into its JSON-safe + * projection so it can be persisted and sent to the viewer. */ +export function serializeGrammarDebugInfo( + debugInfo: GrammarDebugInfo, +): SerializedGrammarDebugInfo { + return { + grammarHash: debugInfo.grammarHash, + rules: [...debugInfo.rules], + parts: [...debugInfo.parts], + partRules: [...debugInfo.partRules], + partLabels: [...debugInfo.partLabels], + filePaths: [...debugInfo.filePaths], + }; +} + +/** Rebuild a `GrammarDebugInfo` (with real `Map`s) from its serialized form so + * the viewer can hand it to the grammar components. */ +export function deserializeGrammarDebugInfo( + serialized: SerializedGrammarDebugInfo, +): GrammarDebugInfo { + return { + grammarHash: serialized.grammarHash, + rules: new Map(serialized.rules), + parts: new Map(serialized.parts), + partRules: new Map(serialized.partRules), + partLabels: new Map(serialized.partLabels), + filePaths: new Map(serialized.filePaths), + }; +} + +/** Inputs for {@link buildTraceVersionPin}. The `spec` decides which of `sha` + * (git ref) and `contentHashes` (working tree) is meaningful; the builder keeps + * only the applicable one. */ +export interface TraceVersionPinInput { + spec: VersionSpec; + label: string; + sha?: string; + contentHashes?: Record; +} + +/** Assemble a {@link TraceVersionPin}, deriving `workingTree` from the spec and + * dropping fields that can't apply to that side: a commit SHA identifies a + * git-ref side (the working tree has none), and only the working tree can drift, + * so content hashes are recorded for it alone. */ +export function buildTraceVersionPin( + input: TraceVersionPinInput, +): TraceVersionPin { + const workingTree = input.spec.kind === "workingTree"; + const pin: TraceVersionPin = { + spec: input.spec, + label: input.label, + workingTree, + }; + if (!workingTree && input.sha !== undefined) { + pin.sha = input.sha; + } + if (workingTree && input.contentHashes !== undefined) { + pin.contentHashes = input.contentHashes; + } + return pin; +} + +/** Inputs for {@link buildReplayRunDescriptor}. Pins are supplied already + * resolved (SHA/content-hash resolution belongs to the runtime layer); this + * builder only assembles the JSON-safe descriptor and stamps `runAt`. */ +export interface ReplayRunDescriptorInput { + runId: string; + agent: string; + a: TraceVersionPin; + b: TraceVersionPin; + mode: ReplayTraceMode; + missPolicy: ReplayMissPolicy; + validateWildcards: boolean; + corpus: CorpusFilter; + /** Epoch ms the run was issued; defaults to the injected clock. */ + runAt?: number; +} + +/** Build the {@link ReplayRunDescriptor} persisted with a run so a stored trace + * can later be reproduced (or flagged as drifted) from the same inputs. */ +export function buildReplayRunDescriptor( + input: ReplayRunDescriptorInput, + now: () => number = Date.now, +): ReplayRunDescriptor { + return { + runId: input.runId, + agent: input.agent, + a: input.a, + b: input.b, + mode: input.mode, + missPolicy: input.missPolicy, + validateWildcards: input.validateWildcards, + corpus: input.corpus, + runAt: input.runAt ?? now(), + }; +} diff --git a/ts/packages/typeagent-core/src/replay/resolutionTraceCapture.ts b/ts/packages/typeagent-core/src/replay/resolutionTraceCapture.ts new file mode 100644 index 0000000000..44b441295d --- /dev/null +++ b/ts/packages/typeagent-core/src/replay/resolutionTraceCapture.ts @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Capture helpers that turn a live grammar match into the serializable trace + * nodes the Trace Viewer renders. These sit beside the grammar resolver: the + * resolver decides *what* action a side produced, and these functions record + * the step-by-step *why* by re-running the same grammar through + * `grammar-tools-core`'s tracing matcher and mapping the event stream back to + * `.agr` source spans. + */ + +import { + loadGrammarFromBuffer, + traceMatch, + type FileLoader, + type SourceLocation, + type TraceEvent, +} from "grammar-tools-core"; +import { actionsEqual } from "./engine.js"; +import { normalizeAction } from "./replayActionShape.js"; +import { + serializeGrammarDebugInfo, + type GrammarMatchTraceNode, +} from "./resolutionTrace.js"; + +/** + * Build the grammar-match trace node for one side. Loads the grammar text + * through `grammar-tools-core` (so its compile-time debug info maps parts/rules + * to source spans), runs the tracing matcher over the utterance, and records the + * event stream, the serialized debug info, a headline source span, and whether + * the traced parse agreed with the resolver's ranked pick. + * + * @param schemaName Schema stamped onto actions, used to normalize the traced + * value before comparing it to the resolver's chosen action. + * @param grammarFileName Identity used as the grammar's file id (drives the + * `displayPath` on resolved spans). + * @param grammarText The `.agr` source read at this side's version. + * @param utterance The utterance being resolved. + * @param chosenAction The action the resolver's ranked match settled on + * (already normalized), or `undefined` when it produced no + * action. Drives outcome and ranking parity. + * @param fileLoader Resolves `import ... from "./other.agr"` statements in the + * grammar text. Supply one that reads sibling grammars at the + * same version so imported spans resolve; omit for a + * self-contained grammar. + * @param grammarFilePath Absolute path of the `.agr` file the text was read from, + * recorded on the node so the viewer can open and diff the + * exact file across versions. Omit when no file backs the text. + */ +export function captureGrammarMatchTrace( + schemaName: string, + grammarFileName: string, + grammarText: string, + utterance: string, + chosenAction: unknown, + fileLoader?: FileLoader, + grammarFilePath?: string, +): GrammarMatchTraceNode { + const outcome = chosenAction !== undefined ? "hit" : "miss"; + const base: GrammarMatchTraceNode = { + kind: "grammar-match", + execution: "ran", + outcome, + input: utterance, + rankingParity: "unavailable", + ...(grammarFilePath !== undefined + ? { sourceFilePath: grammarFilePath } + : {}), + }; + + let loaded; + try { + loaded = loadGrammarFromBuffer( + grammarFileName, + grammarText, + fileLoader, + ); + } catch (err) { + return { ...base, detail: `trace unavailable: ${message(err)}` }; + } + if (!loaded.ok) { + const first = loaded.diagnostics[0]?.message ?? "grammar load failed"; + return { ...base, detail: `trace unavailable: ${first}` }; + } + + const { debugInfo } = loaded.grammar; + if (debugInfo === undefined) { + return { ...base, detail: "trace unavailable: no debug info" }; + } + + const trace = traceMatch(loaded.grammar, utterance); + const tracedAction = normalizeAction(schemaName, trace.matchValue); + const rankingParity: GrammarMatchTraceNode["rankingParity"] = + chosenAction === undefined + ? "unavailable" + : actionsEqual(tracedAction, chosenAction) + ? "matched" + : "diverged"; + + const anchor = headlineSpan(trace.events, debugInfo); + + return { + ...base, + trace, + debugInfo: serializeGrammarDebugInfo(debugInfo), + rankingParity, + ...(anchor?.rule !== undefined ? { chosenRule: anchor.rule } : {}), + ...(anchor?.source !== undefined ? { source: anchor.source } : {}), + }; +} + +/** + * Pick a single source span to anchor the viewer's "jump to the winning line" + * action. The last `partMatched` event lies on the accepted parse (matching + * completes left-to-right and the final token is matched last), so its part's + * span points at the concrete `.agr` line the match landed on; its owning rule + * is the headline rule. Falls back to nothing when no part with a known span was + * matched (e.g. a match made entirely of spanless synthetic parts). + */ +function headlineSpan( + events: readonly TraceEvent[], + debugInfo: { + parts: ReadonlyMap; + partRules: ReadonlyMap; + }, +): { rule?: string; source?: SourceLocation } | undefined { + for (let i = events.length - 1; i >= 0; i--) { + const event = events[i]; + if (event.kind !== "partMatched") { + continue; + } + const source = debugInfo.parts.get(event.part); + if (source === undefined) { + continue; + } + const rule = debugInfo.partRules.get(event.part); + return { source, ...(rule !== undefined ? { rule } : {}) }; + } + return undefined; +} + +function message(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/ts/packages/typeagent-core/src/runtime/studioProtocol.ts b/ts/packages/typeagent-core/src/runtime/studioProtocol.ts index 730a0e69c0..c81be7cf8a 100644 --- a/ts/packages/typeagent-core/src/runtime/studioProtocol.ts +++ b/ts/packages/typeagent-core/src/runtime/studioProtocol.ts @@ -12,6 +12,8 @@ import type { AvailableAgent, StudioReplayRequest, StudioReplayResult, + ReplayResolutionTraceRequest, + ReplayResolutionTraceResult, StudioCollisionScanRequest, StudioCollisionScanResult, StudioCorpusImportRequest, @@ -134,6 +136,14 @@ export type StudioServiceInvokeFunctions = { repoRoot: string | undefined, request: StudioReplayRequest, ): Promise; + /** + * Recompute a single utterance's resolution trace from a stored run + * descriptor (the drill-in "replay this trace" path). + */ + replayResolutionTrace( + repoRoot: string | undefined, + request: ReplayResolutionTraceRequest, + ): Promise; /** * Start pushing live `studioEvent` calls to *this* connection for the given * repo. Idempotent per connection: a second call replaces the connection's diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts index 7989134f29..6020eb2cef 100644 --- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts +++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts @@ -46,6 +46,8 @@ import { type ReplayActionResolver, type ReplayAgentResolution, type ReplayMissPolicy, + type ReplayResolutionTrace, + type ReplayRunDescriptor, type ReplaySummary, type VersionSpec, } from "../replay/index.js"; @@ -62,6 +64,7 @@ import { import { createGrammarReplayResolver, resolveGrammarReplayTarget, + captureResolutionTrace, ReplayVersionBuildError, type ReplayRunError, type GrammarReplayTarget, @@ -276,6 +279,23 @@ export interface StudioReplayRequest { validateWildcards?: boolean; } +/** Request to recompute one utterance's resolution trace from a stored run + * descriptor — the drill-in's "replay this trace" path, reusing the same + * grammar resolution a full run would. */ +export interface ReplayResolutionTraceRequest { + descriptor: ReplayRunDescriptor; + utteranceId: string; +} + +/** Outcome of {@link StudioRuntime.replayResolutionTrace}: a freshly recomputed + * trace, or why one couldn't be produced — the utterance is no longer in the + * corpus (`entry-missing`), or the descriptor's versions don't reconstruct into + * a grammar run (`unavailable`, e.g. identity baseline or a build failure). */ +export type ReplayResolutionTraceResult = + | { status: "recomputed"; trace: ReplayResolutionTrace } + | { status: "entry-missing" } + | { status: "unavailable" }; + /** * How a replay resolved corpus utterances into actions: * - `identity` — the deterministic baseline (surfaces each entry's captured @@ -370,6 +390,22 @@ export interface StudioReplayResult { * validation pass) so the Impact Report can show an honest "what ran" matrix. */ sideFidelity: SideFidelity; + /** + * Per-row resolution traces captured right after the run, while the live + * resolver still matches what produced each row. Present only for a grammar + * run (the identity baseline has nothing to trace) and bounded so a large + * corpus doesn't inflate the result — changed rows are captured first, then + * unchanged ones fill the remaining capacity; the viewer reads these back + * through the trace store. Omitted when nothing was captured. + */ + resolutionTraces?: ReplayResolutionTrace[]; + /** + * The utterances that captured a resolution trace. Mirrors the ids in + * `resolutionTraces` so a consumer can show a "trace available" affordance + * without shipping the (large) traces themselves. Omitted when none were + * captured. + */ + tracedUtteranceIds?: string[]; /** * with an empty summary rather than emitting fabricated regression rows. */ @@ -785,6 +821,16 @@ export interface StudioRuntime { * and summary. */ replayCorpus(request: StudioReplayRequest): Promise; + /** + * Recompute the resolution trace for a single utterance from a stored run + * descriptor, reusing the same grammar path the run used. Returns + * `entry-missing` when the utterance is no longer in the corpus and + * `unavailable` when the descriptor's versions can't be reconstructed into a + * grammar run (identity baseline, or a version build failure). + */ + replayResolutionTrace( + request: ReplayResolutionTraceRequest, + ): Promise; /** * Report a detected schema/grammar collision. Stores it and emits a * `collision.detected` event (visible in the Event Log and the Collisions @@ -1005,6 +1051,58 @@ interface ReplayResolution { aborted?: StudioReplayResult; } +/** Cap how many traces one run captures so a large corpus doesn't inflate the + * result or the persisted payload; the report still lists every row, and the + * viewer drills into the captured ones first. */ +const MAX_CAPTURED_TRACES = 100; + +/** Capture a {@link ReplayResolutionTrace} for each row, up to the cap, by + * re-resolving with tracing on the live resolver. Changed (red) rows are traced + * first so a corpus larger than the cap never drops a divergence in favour of an + * unchanged row; remaining capacity captures equal rows so the viewer can open + * any row, changed or not. Entries are listed only when there is at least one + * row to trace and mapped by id to the row's `utteranceId`; a row whose entry + * can't be found is skipped rather than failing the run. Call while the resolver + * is still live so the trace reflects exactly what produced the row. */ +export async function captureRowTraces( + resolver: GrammarReplayResolver, + listEntries: () => Promise, + rows: readonly ActionDelta[], + runId: string, + versionA: VersionSpec, + versionB: VersionSpec, +): Promise { + // Changed rows first, then unchanged, so the cap never crowds out a + // divergence — the row a developer is most likely to inspect. + const ordered = [ + ...rows.filter((row) => !row.equal), + ...rows.filter((row) => row.equal), + ].slice(0, MAX_CAPTURED_TRACES); + if (ordered.length === 0) { + return []; + } + const entriesById = new Map( + (await listEntries()).map((entry) => [entry.id, entry] as const), + ); + const traces: ReplayResolutionTrace[] = []; + for (const row of ordered) { + const entry = entriesById.get(row.utteranceId); + if (entry === undefined) { + continue; + } + traces.push( + await captureResolutionTrace( + resolver, + entry, + runId, + versionA, + versionB, + ), + ); + } + return traces; +} + export function createStudioRuntimeCore( context: StudioRuntimeContext, options: CreateStudioRuntimeOptions = {}, @@ -1539,10 +1637,24 @@ export function createStudioRuntimeCore( }); const rows: ActionDelta[] = []; + let resolutionTraces: ReplayResolutionTrace[] = []; try { for await (const row of handle.rows) { rows.push(row); } + // Capture per-row traces while the resolver (and any wildcard + // validator) is still live and the working tree still matches what + // produced the rows — the dispose below tears the validator down. + if (resolution.activeGrammarResolver !== undefined) { + resolutionTraces = await captureRowTraces( + resolution.activeGrammarResolver, + () => corpus.list(request.agent, replayOptions.corpus), + rows, + handle.runId, + replayOptions.versionA, + replayOptions.versionB, + ); + } } finally { await wildcardValidator?.dispose(); } @@ -1581,7 +1693,65 @@ export function createStudioRuntimeCore( ...(wildcardValidation !== undefined ? { wildcardValidation } : {}), + ...(resolutionTraces.length > 0 + ? { + resolutionTraces, + tracedUtteranceIds: resolutionTraces.map( + (t) => t.utteranceId, + ), + } + : {}), + }; + }, + async replayResolutionTrace(request) { + const { descriptor, utteranceId } = request; + const replayOptions = { + agent: descriptor.agent, + corpus: descriptor.corpus, + versionA: descriptor.a.spec, + versionB: descriptor.b.spec, + missPolicy: descriptor.missPolicy, + } satisfies Parameters[0]; + const synthRequest: StudioReplayRequest = { + agent: descriptor.agent, + corpus: descriptor.corpus, + versionA: descriptor.a.spec, + versionB: descriptor.b.spec, + missPolicy: descriptor.missPolicy, + mode: descriptor.mode, + validateWildcards: descriptor.validateWildcards, }; + const resolution = await resolveReplayActions( + replayOptions, + synthRequest, + descriptor.mode, + ); + try { + if ( + resolution.aborted !== undefined || + resolution.activeGrammarResolver === undefined + ) { + return { status: "unavailable" }; + } + const entry = ( + await corpus.list(descriptor.agent, descriptor.corpus) + ).find((candidate) => candidate.id === utteranceId); + if (entry === undefined) { + return { status: "entry-missing" }; + } + return { + status: "recomputed", + trace: await captureResolutionTrace( + resolution.activeGrammarResolver, + entry, + descriptor.runId, + descriptor.a.spec, + descriptor.b.spec, + ), + }; + } finally { + await resolution.wildcardValidator?.dispose(); + } }, reportCollision(event) { return collisions.report(event); diff --git a/ts/packages/typeagent-core/test/captureRowTraces.spec.ts b/ts/packages/typeagent-core/test/captureRowTraces.spec.ts new file mode 100644 index 0000000000..624b6f50dc --- /dev/null +++ b/ts/packages/typeagent-core/test/captureRowTraces.spec.ts @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { captureRowTraces } from "../src/runtime/studioRuntimeCore.js"; +import type { GrammarReplayResolver } from "../src/replay/grammarResolver.js"; +import type { + ActionDelta, + ReplaySideTrace, + VersionSpec, +} from "../src/replay/index.js"; +import type { CorpusEntry } from "../src/corpus/index.js"; + +const versionA: VersionSpec = { kind: "workingTree" }; +const versionB: VersionSpec = { kind: "git", ref: "HEAD" }; + +function entry(id: string): CorpusEntry { + return { + id, + agent: "player", + utterance: `utterance ${id}`, + source: "in-repo", + } as CorpusEntry; +} + +function row(id: string, equal: boolean): ActionDelta { + return { + utterance: `utterance ${id}`, + source: "in-repo", + utteranceId: id, + actionA: equal ? { a: 1 } : { a: 1 }, + actionB: equal ? { a: 1 } : { b: 2 }, + equal, + cacheStateA: "miss", + cacheStateB: "miss", + collisionsA: [], + collisionsB: [], + latencyA: 0, + latencyB: 0, + requestIdA: `${id}-a`, + requestIdB: `${id}-b`, + } as ActionDelta; +} + +/** A resolver that records how many times it traced and returns a trivial side + * trace, so the helper's ordering/mapping is what's under test — not the real + * grammar match. */ +function fakeResolver(): { + resolver: GrammarReplayResolver; + traced: string[]; +} { + const traced: string[] = []; + const resolver = { + async resolveWithTrace( + e: CorpusEntry, + version: VersionSpec, + side: "A" | "B", + ) { + traced.push(`${e.id}:${side}`); + const sideTrace: ReplaySideTrace = { + side, + version, + realization: "built-live", + nodes: [], + cacheState: "miss", + } as ReplaySideTrace; + return { resolution: {} as never, sideTrace }; + }, + } as unknown as GrammarReplayResolver; + return { resolver, traced }; +} + +describe("captureRowTraces", () => { + test("captures a trace for every row, changed rows first", async () => { + const { resolver, traced } = fakeResolver(); + const rows = [row("u1", true), row("u2", false), row("u3", false)]; + const entries = [entry("u1"), entry("u2"), entry("u3")]; + + const traces = await captureRowTraces( + resolver, + async () => entries, + rows, + "run-1", + versionA, + versionB, + ); + + // Changed rows (u2, u3) lead; the unchanged row (u1) is still captured. + expect(traces.map((t) => t.utteranceId)).toEqual(["u2", "u3", "u1"]); + expect(traces[0].runId).toBe("run-1"); + expect(traced).toEqual([ + "u2:A", + "u2:B", + "u3:A", + "u3:B", + "u1:A", + "u1:B", + ]); + }); + + test("does not list entries when there are no rows", async () => { + const { resolver } = fakeResolver(); + let listed = 0; + const traces = await captureRowTraces( + resolver, + async () => { + listed += 1; + return []; + }, + [], + "run-1", + versionA, + versionB, + ); + expect(traces).toEqual([]); + expect(listed).toBe(0); + }); + + test("skips a row whose corpus entry can't be found", async () => { + const { resolver } = fakeResolver(); + const traces = await captureRowTraces( + resolver, + async () => [entry("u1")], // u2's entry is missing + [row("u1", false), row("u2", false)], + "run-1", + versionA, + versionB, + ); + expect(traces.map((t) => t.utteranceId)).toEqual(["u1"]); + }); + + test("caps the number of captured traces at 100", async () => { + const { resolver } = fakeResolver(); + const rows: ActionDelta[] = []; + const entries: CorpusEntry[] = []; + for (let i = 0; i < 150; i++) { + rows.push(row(`u${i}`, false)); + entries.push(entry(`u${i}`)); + } + const traces = await captureRowTraces( + resolver, + async () => entries, + rows, + "run-1", + versionA, + versionB, + ); + expect(traces.length).toBe(100); + }); + + test("keeps every changed row when the cap is reached, dropping unchanged first", async () => { + const { resolver } = fakeResolver(); + const rows: ActionDelta[] = []; + const entries: CorpusEntry[] = []; + // 50 unchanged rows listed before 60 changed rows: the cap of 100 must + // still admit all 60 changed rows, filling the rest with unchanged. + for (let i = 0; i < 50; i++) { + rows.push(row(`e${i}`, true)); + entries.push(entry(`e${i}`)); + } + for (let i = 0; i < 60; i++) { + rows.push(row(`r${i}`, false)); + entries.push(entry(`r${i}`)); + } + const traces = await captureRowTraces( + resolver, + async () => entries, + rows, + "run-1", + versionA, + versionB, + ); + const captured = new Set(traces.map((t) => t.utteranceId)); + expect(traces.length).toBe(100); + for (let i = 0; i < 60; i++) { + expect(captured.has(`r${i}`)).toBe(true); + } + }); +}); diff --git a/ts/packages/typeagent-core/test/constructionCacheResolver.spec.ts b/ts/packages/typeagent-core/test/constructionCacheResolver.spec.ts index 54bdec5b53..dbe2e92ab7 100644 --- a/ts/packages/typeagent-core/test/constructionCacheResolver.spec.ts +++ b/ts/packages/typeagent-core/test/constructionCacheResolver.spec.ts @@ -162,6 +162,27 @@ describe("loadConstructionCacheLayer", () => { expect(typeof play?.actionName).toBe("string"); }); + test("valid: matchEntry surfaces the matched construction's identity", async () => { + const { name, hash } = fixtureNamespace(); + const layer = await loadConstructionCacheLayer({ + cacheFilePath: FIXTURE, + schemaName: "player", + currentHash: hash, + }); + const entry = layer.matchEntry("pause"); + expect(entry).toBeDefined(); + expect(entry?.action).toEqual({ + schemaName: "player", + actionName: "pause", + }); + // Identity is drawn from the matched construction and its namespace. + expect(entry?.namespace).toBe(name); + expect(entry?.constructionId).toMatch(/^\d+$/); + expect(entry?.parts?.length).toBeGreaterThan(0); + expect(entry?.scores?.matchedCount).toBeGreaterThanOrEqual(0); + expect(entry?.cacheFileId).toBe(FIXTURE); + }); + test("stale: a mismatched hash falls back (no cache match)", async () => { const layer = await loadConstructionCacheLayer({ cacheFilePath: FIXTURE, @@ -171,6 +192,7 @@ describe("loadConstructionCacheLayer", () => { expect(layer.status).toBe("stale"); expect(layer.cachedHash).toBeDefined(); expect(layer.match("pause")).toBeUndefined(); + expect(layer.matchEntry("pause")).toBeUndefined(); }); test("absent: no namespace for the schema", async () => { @@ -182,6 +204,7 @@ describe("loadConstructionCacheLayer", () => { }); expect(layer.status).toBe("absent"); expect(layer.match("pause")).toBeUndefined(); + expect(layer.matchEntry("pause")).toBeUndefined(); }); test("absent: missing cache file degrades cleanly (never throws)", async () => { diff --git a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts index 1c3ca5d1a8..5d21ef1dd1 100644 --- a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts +++ b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts @@ -16,6 +16,7 @@ import { createGrammarReplayResolver, normalizeGrammarAction, resolveGrammarReplayTarget, + captureResolutionTrace, ReplayVersionBuildError, selectValidatedMatchAction, type GrammarReplayTarget, @@ -236,6 +237,22 @@ describe("createGrammarReplayResolver (construction-cache layer, L2)", () => { ? { schemaName: "demo", actionName: "pause" } : undefined; }, + matchEntry(utterance: string) { + return utterance === "pause" + ? { + action: { schemaName: "demo", actionName: "pause" }, + constructionId: "1", + namespace: "demo,hash=", + parts: ["pause"], + scores: { + matchedCount: 1, + wildcardCharCount: 0, + nonOptionalCount: 1, + }, + cacheFileId: "/stub/constructions.json", + } + : undefined; + }, }; } @@ -415,6 +432,32 @@ maybeDescribe("createGrammarReplayResolver (git working-tree vs HEAD)", () => { expect(summary.changedCount).toBe(0); expect(summary.newMatchCount).toBe(0); }); + + it("captures a source-realized trace with live-only layers not-applicable on a git ref", async () => { + const resolver = createGrammarReplayResolver({ + target: target(grammarPath), + }); + await resolver.prepare( + { kind: "git", ref: "HEAD" }, + { kind: "workingTree" }, + ); + const { sideTrace } = await resolver.resolveWithTrace( + entry("g", "pause"), + { kind: "git", ref: "HEAD" }, + "A", + ); + + expect(sideTrace.realization).toBe("source"); + const cache = sideTrace.nodes.find((n) => n.kind === "cache-consult"); + expect(cache?.execution).toBe("not-applicable"); + const wildcard = sideTrace.nodes.find( + (n) => n.kind === "wildcard-validation", + ); + expect(wildcard?.execution).toBe("not-applicable"); + // HEAD still contains , so the grammar match runs and hits. + const grammar = sideTrace.nodes.find((n) => n.kind === "grammar-match"); + expect(grammar?.execution).toBe("ran"); + }); }); const DEMO_SCHEMA_TS = [ @@ -518,6 +561,51 @@ describe("resolveGrammarReplayTarget + schema enrichment (L1)", () => { expect(res.action).toEqual({ schemaName: "demo", actionName: "pause" }); }); + it("omits the action schema source for a side that produced no action", async () => { + const { agentRoots } = scaffoldAgentPackage(dir); + const target = await resolveGrammarReplayTarget( + agentRoots, + "demo", + dir, + ); + expect(target?.schema?.sourceFilePath).toBeDefined(); + const resolver = createGrammarReplayResolver({ target: target! }); + await resolver.prepare( + { kind: "workingTree" }, + { kind: "workingTree" }, + ); + + // "nonsense" matches no rule, so the action stage is a miss: with no + // resolved action the node must not carry a schema for the viewer to + // offer a jump or A/B compare against. + const miss = await resolver.resolveWithTrace( + entry("miss", "nonsense"), + { kind: "workingTree" }, + "A", + ); + const missAction = miss.sideTrace.nodes.find( + (n) => n.kind === "action", + ); + expect(missAction?.kind === "action" && missAction.outcome).toBe( + "miss", + ); + expect( + missAction?.kind === "action" && missAction.schema, + ).toBeUndefined(); + + // A produced action still carries its schema source. + const hit = await resolver.resolveWithTrace( + entry("hit", "pause"), + { kind: "workingTree" }, + "A", + ); + const hitAction = hit.sideTrace.nodes.find((n) => n.kind === "action"); + expect(hitAction?.kind === "action" && hitAction.outcome).toBe("hit"); + expect( + hitAction?.kind === "action" && hitAction.schema?.sourceFilePath, + ).toMatch(/demoSchema\.ts$/); + }); + it("falls back to a non-enriched resolver when no schema is discoverable", async () => { const grammarPath = path.join(dir, "schema.agr"); writeFileSync(grammarPath, GRAMMAR_V1, "utf8"); @@ -748,3 +836,163 @@ describe("selectValidatedMatchAction (ranked-list contract)", () => { expect(result.consulted).toBe(false); }); }); + +describe("resolveWithTrace / captureResolutionTrace", () => { + let dir: string; + let grammarPath: string; + + beforeEach(() => { + dir = tempDir(); + grammarPath = path.join(dir, "schema.agr"); + writeFileSync(grammarPath, GRAMMAR_V1, "utf8"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("captures an ordered four-layer trace for a working-tree match", async () => { + const resolver = createGrammarReplayResolver({ + target: target(grammarPath), + }); + const { resolution, sideTrace } = await resolver.resolveWithTrace( + entry("t1", "pause"), + { kind: "workingTree" }, + "A", + ); + + expect(resolution.action).toEqual({ + schemaName: "demo", + actionName: "pause", + }); + expect(sideTrace.side).toBe("A"); + expect(sideTrace.realization).toBe("built-live"); + expect(sideTrace.cacheState).toBe("hit"); + expect(sideTrace.nodes.map((n) => n.kind)).toEqual([ + "cache-consult", + "grammar-match", + "wildcard-validation", + "action", + ]); + + const grammar = sideTrace.nodes.find((n) => n.kind === "grammar-match"); + expect(grammar?.kind === "grammar-match" && grammar.execution).toBe( + "ran", + ); + expect(grammar?.kind === "grammar-match" && grammar.rankingParity).toBe( + "matched", + ); + expect( + grammar?.kind === "grammar-match" && grammar.source, + ).toBeDefined(); + + const action = sideTrace.nodes.find((n) => n.kind === "action"); + expect(action?.kind === "action" && action.action).toEqual({ + schemaName: "demo", + actionName: "pause", + }); + }); + + it("marks live-only layers not-applicable when no cache or validator is set", async () => { + const resolver = createGrammarReplayResolver({ + target: target(grammarPath), + }); + const { sideTrace } = await resolver.resolveWithTrace( + entry("t2", "resume"), + { kind: "workingTree" }, + "A", + ); + + const cache = sideTrace.nodes.find((n) => n.kind === "cache-consult"); + expect(cache?.execution).toBe("not-applicable"); + const wildcard = sideTrace.nodes.find( + (n) => n.kind === "wildcard-validation", + ); + expect(wildcard?.execution).toBe("not-applicable"); + }); + + it("records a construction hit and leaves the grammar node not-reached", async () => { + const cache: ConstructionCacheLayer = { + status: "valid", + cacheFilePath: "/stub/constructions.json", + schemaName: "demo", + currentHash: "h", + cachedHash: "h", + match: (u) => + u === "pause" + ? { schemaName: "demo", actionName: "pause" } + : undefined, + matchEntry: (u) => + u === "pause" + ? { + action: { schemaName: "demo", actionName: "pause" }, + constructionId: "7", + namespace: "demo,h", + parts: ["pause"], + } + : undefined, + }; + const resolver = createGrammarReplayResolver({ + target: target(grammarPath), + constructionCache: cache, + }); + const { sideTrace } = await resolver.resolveWithTrace( + entry("t3", "pause"), + { kind: "workingTree" }, + "A", + ); + + const cacheNode = sideTrace.nodes.find( + (n) => n.kind === "cache-consult", + ); + expect(cacheNode?.kind === "cache-consult" && cacheNode.execution).toBe( + "ran", + ); + expect(cacheNode?.kind === "cache-consult" && cacheNode.outcome).toBe( + "hit", + ); + expect( + cacheNode?.kind === "cache-consult" && + cacheNode.entry?.constructionId, + ).toBe("7"); + const grammar = sideTrace.nodes.find((n) => n.kind === "grammar-match"); + expect(grammar?.execution).toBe("not-reached"); + }); + + it("assembles both sides keyed by the entry id, with an injected clock", async () => { + const resolver = createGrammarReplayResolver({ + target: target(grammarPath), + }); + let clock = 100; + const trace = await captureResolutionTrace( + resolver, + entry("row-7", "pause"), + "run-1", + { kind: "workingTree" }, + { kind: "workingTree" }, + () => clock++, + ); + + expect(trace.runId).toBe("run-1"); + expect(trace.utteranceId).toBe("row-7"); + expect(trace.utterance).toBe("pause"); + expect(trace.a.side).toBe("A"); + expect(trace.b.side).toBe("B"); + expect(trace.capturedAt).toBe(100); + }); + + it("survives a JSON round-trip (persistable with the run)", async () => { + const resolver = createGrammarReplayResolver({ + target: target(grammarPath), + }); + const trace = await captureResolutionTrace( + resolver, + entry("row-8", "pause"), + "run-2", + { kind: "workingTree" }, + { kind: "workingTree" }, + () => 0, + ); + expect(JSON.parse(JSON.stringify(trace))).toEqual(trace); + }); +}); diff --git a/ts/packages/typeagent-core/test/resolutionTrace.spec.ts b/ts/packages/typeagent-core/test/resolutionTrace.spec.ts new file mode 100644 index 0000000000..9ad0abd041 --- /dev/null +++ b/ts/packages/typeagent-core/test/resolutionTrace.spec.ts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + buildReplayRunDescriptor, + buildTraceVersionPin, +} from "../src/replay/resolutionTrace.js"; + +describe("buildTraceVersionPin", () => { + it("keeps the SHA and drops content hashes for a git ref", () => { + const pin = buildTraceVersionPin({ + spec: { kind: "git", ref: "HEAD" }, + label: "HEAD (main)", + sha: "3637bd663", + contentHashes: { grammar: "abc" }, + }); + expect(pin.workingTree).toBe(false); + expect(pin.sha).toBe("3637bd663"); + expect(pin.contentHashes).toBeUndefined(); + }); + + it("keeps content hashes and drops the SHA for the working tree", () => { + const pin = buildTraceVersionPin({ + spec: { kind: "workingTree" }, + label: "Working tree", + sha: "should-be-ignored", + contentHashes: { grammar: "abc", schema: "def" }, + }); + expect(pin.workingTree).toBe(true); + expect(pin.sha).toBeUndefined(); + expect(pin.contentHashes).toEqual({ grammar: "abc", schema: "def" }); + }); + + it("omits optional fields when not supplied", () => { + const pin = buildTraceVersionPin({ + spec: { kind: "git", ref: "v1" }, + label: "v1", + }); + expect(pin).toEqual({ + spec: { kind: "git", ref: "v1" }, + label: "v1", + workingTree: false, + }); + }); +}); + +describe("buildReplayRunDescriptor", () => { + const a = buildTraceVersionPin({ + spec: { kind: "git", ref: "HEAD" }, + label: "HEAD", + sha: "aaa", + }); + const b = buildTraceVersionPin({ + spec: { kind: "workingTree" }, + label: "Working tree", + contentHashes: { grammar: "bbb" }, + }); + + it("assembles a JSON-safe descriptor and stamps runAt from the clock", () => { + const descriptor = buildReplayRunDescriptor( + { + runId: "run-1", + agent: "player", + a, + b, + mode: "nfa-grammar", + missPolicy: "needs-explanation", + validateWildcards: true, + corpus: { sources: ["in-repo"] }, + }, + () => 12345, + ); + expect(descriptor).toEqual({ + runId: "run-1", + agent: "player", + a, + b, + mode: "nfa-grammar", + missPolicy: "needs-explanation", + validateWildcards: true, + corpus: { sources: ["in-repo"] }, + runAt: 12345, + }); + expect(JSON.parse(JSON.stringify(descriptor))).toEqual(descriptor); + }); + + it("prefers an explicit runAt over the clock", () => { + const descriptor = buildReplayRunDescriptor( + { + runId: "run-2", + agent: "list", + a, + b, + mode: "completionBased-cache", + missPolicy: "strict-cache", + validateWildcards: false, + corpus: {}, + runAt: 999, + }, + () => 12345, + ); + expect(descriptor.runAt).toBe(999); + }); +}); diff --git a/ts/packages/typeagent-core/test/resolutionTraceCapture.spec.ts b/ts/packages/typeagent-core/test/resolutionTraceCapture.spec.ts new file mode 100644 index 0000000000..fef0e6dc2a --- /dev/null +++ b/ts/packages/typeagent-core/test/resolutionTraceCapture.spec.ts @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { captureGrammarMatchTrace } from "../src/replay/resolutionTraceCapture.js"; +import { deserializeGrammarDebugInfo } from "../src/replay/resolutionTrace.js"; + +const GRAMMAR = [ + " = | ;", + ' = pause -> { actionName: "pause" };', + ' = resume -> { actionName: "resume" };', +].join("\n"); + +const pauseAction = { schemaName: "demo", actionName: "pause" }; +const resumeAction = { schemaName: "demo", actionName: "resume" }; + +describe("captureGrammarMatchTrace", () => { + test("captures a matched trace with a resolvable source span", () => { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "pause", + pauseAction, + ); + + expect(node.kind).toBe("grammar-match"); + expect(node.execution).toBe("ran"); + expect(node.outcome).toBe("hit"); + expect(node.trace?.result).toBe("matched"); + expect(node.trace?.events.length).toBeGreaterThan(0); + expect(node.debugInfo).toBeDefined(); + // The headline span points at a real line of the grammar source. + expect(node.source).toBeDefined(); + expect(node.source?.range.start.line).toBeGreaterThanOrEqual(0); + }); + + test("flags ranking parity as matched when the traced action agrees", () => { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "pause", + pauseAction, + ); + expect(node.rankingParity).toBe("matched"); + }); + + test("flags ranking parity as diverged when the traced action differs", () => { + // The resolver's ranked pick (resumeAction) disagrees with what the + // recursive matcher produces for "pause" (a pause action). + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "pause", + resumeAction, + ); + expect(node.rankingParity).toBe("diverged"); + }); + + test("reports a miss with an unavailable parity when nothing resolved", () => { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "fizzbuzz", + undefined, + ); + expect(node.outcome).toBe("miss"); + expect(node.rankingParity).toBe("unavailable"); + expect(node.trace?.result).toBe("noMatch"); + }); + + test("degrades cleanly when the grammar fails to load", () => { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + " = { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "pause", + pauseAction, + ); + const roundTripped = JSON.parse(JSON.stringify(node)); + expect(roundTripped).toEqual(node); + // The serialized debug info rebuilds into real maps. + const debugInfo = deserializeGrammarDebugInfo(roundTripped.debugInfo); + expect(debugInfo.parts.size).toBeGreaterThan(0); + expect(debugInfo.rules.size).toBeGreaterThan(0); + }); + + test("records the absolute grammar file path when one is supplied", () => { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "pause", + pauseAction, + undefined, + "/repo/agents/demo/demo.agr", + ); + expect(node.sourceFilePath).toBe("/repo/agents/demo/demo.agr"); + }); + + test("records the grammar file path even when nothing matched", () => { + // The regressed side matches nothing, but must still record the file + // path so the viewer can diff it against the side that did match. + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "fizzbuzz", + undefined, + undefined, + "/repo/agents/demo/demo.agr", + ); + expect(node.outcome).toBe("miss"); + expect(node.sourceFilePath).toBe("/repo/agents/demo/demo.agr"); + }); + + test("omits the file path when none is supplied", () => { + const node = captureGrammarMatchTrace( + "demo", + "demo.agr", + GRAMMAR, + "pause", + pauseAction, + ); + expect(node.sourceFilePath).toBeUndefined(); + }); +}); diff --git a/ts/packages/typeagent-studio/README.AUTOGEN.md b/ts/packages/typeagent-studio/README.AUTOGEN.md index 7440489b6f..78dd939dc6 100644 --- a/ts/packages/typeagent-studio/README.AUTOGEN.md +++ b/ts/packages/typeagent-studio/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # typeagent-studio — AI-generated documentation @@ -12,28 +12,28 @@ ## Overview -The `typeagent-studio` package is a TypeScript library that powers the TypeAgent developer experience within Visual Studio Code. It provides a suite of tools for authoring, debugging, and optimizing TypeAgent agents. Key functionalities include schema and grammar tuning, compare-and-replay regression detection, trace investigation, and live observation of agent behavior. This package is integral to the development and validation workflows for TypeAgent agents, offering a streamlined and interactive environment for developers. +The `typeagent-studio` package is a TypeScript library that powers the TypeAgent developer experience within Visual Studio Code. It provides a suite of tools for authoring, debugging, and optimizing TypeAgent agents. This package supports workflows such as schema and grammar tuning, compare-and-replay regression detection, trace investigation, and live observation of agent behavior. It is designed to streamline the development and debugging process for TypeAgent agents, offering an integrated and interactive environment. ## What it does -The `typeagent-studio` package integrates with Visual Studio Code to deliver a comprehensive set of features for working with TypeAgent agents. These features are exposed through a combination of commands and views, enabling developers to manage, debug, and optimize their agents effectively. +The `typeagent-studio` package integrates with Visual Studio Code to provide a rich set of features for managing and debugging TypeAgent agents. These features are accessible through a combination of commands and views, enabling developers to efficiently manage their workflows. ### Commands -The package provides a rich set of commands accessible via the VS Code command palette. These commands are grouped into several categories: +The package provides a comprehensive set of commands accessible via the VS Code command palette. These commands are grouped into several categories: - **Onboarding and Sandbox Management**: - - Commands like `TypeAgent Studio: Start onboarding session`, `TypeAgent Studio: Run onboarding phase`, and `TypeAgent Studio: Advance onboarding phase` guide developers through the onboarding process for new agents. - - Sandbox-related commands such as `TypeAgent Studio: Start sandbox`, `TypeAgent Studio: Stop sandbox`, and `TypeAgent Studio: Refresh sandboxes` allow developers to manage and interact with sandboxes for testing and debugging. + - Commands such as `TypeAgent Studio: Start onboarding session`, `TypeAgent Studio: Run onboarding phase`, and `TypeAgent Studio: Advance onboarding phase` guide developers through the onboarding process for new agents. + - Sandbox management commands like `TypeAgent Studio: Start sandbox`, `TypeAgent Studio: Stop sandbox`, and `TypeAgent Studio: Refresh sandboxes` allow developers to manage and interact with sandboxes for testing and debugging. - **Agent and Corpus Management**: - - Commands like `TypeAgent Studio: Install latest onboarding session to sandbox` and `TypeAgent Studio: Refresh corpora` help manage agents and their associated corpora. + - Commands such as `TypeAgent Studio: Install latest onboarding session to sandbox` and `TypeAgent Studio: Refresh corpora` help manage agents and their associated corpora. - The `TypeAgent Studio: Replay corpus` command enables developers to replay a corpus through the `replayCorpus()` engine, facilitating regression testing and comparison of agent behavior across versions. - **Health and Diagnostics**: - - Commands such as `TypeAgent Studio: Check packaging health gate` and `TypeAgent Studio: Enforce packaging health gate` provide insights into the health and readiness of agents. + - Commands like `TypeAgent Studio: Check packaging health gate` and `TypeAgent Studio: Enforce packaging health gate` provide insights into the health and readiness of agents. - The `TypeAgent Studio: Export onboarding artifact...` command allows developers to export various artifacts, including summaries, health snapshots, and diagnostics bundles. ### Views @@ -50,21 +50,16 @@ The package introduces several specialized views within the **TypeAgent Studio** - Organizes and displays corpus entries for agents, grouped by source (e.g., in-repo, captures, external, feedback). - Allows users to explore and manage corpora associated with loaded agents. -3. **Feedback Capture**: - - - Offers a guided flow for recording feedback on agent performance, including ratings, comments, and categories. - - Feedback is integrated into the Corpora view for further analysis. - -4. **Event Log View**: +3. **Event Log View**: - Displays a chronological list of recent events related to agent activity, with timestamps, icons, and tooltips for quick reference. -5. **Collisions View**: +4. **Collisions View**: - Lists detected schema/grammar collisions, categorized by type (e.g., overlap, shadow, ambiguity). - Provides tools for investigating and resolving conflicts in agent definitions. -6. **Health Status Bar**: +5. **Health Status Bar**: - Summarizes the health of all agents loaded into running sandboxes. - Offers a quick visual indicator of the overall system health. @@ -101,7 +96,7 @@ For additional setup details, refer to the hand-written README or other relevant ## Key Files -The `typeagent-studio` package is structured into several key files and modules, each responsible for specific aspects of the extension's functionality: +The `typeagent-studio` package is organized into several key files and modules, each responsible for specific functionalities: - **[extension.ts](./src/extension.ts)**: The main entry point for the VS Code extension. It initializes the extension, registers commands, and sets up views. - **[commands.ts](./src/commands.ts)**: Implements the commands available in the VS Code command palette, such as managing sandboxes, running onboarding sessions, and exporting artifacts. @@ -167,15 +162,16 @@ Workspace: - [@typeagent/agent-server-client](../../packages/agentServer/client/README.md) - [@typeagent/core](../../packages/typeagent-core/README.md) - [@typeagent/websocket-utils](../../packages/utils/webSocketUtils/README.md) +- [default-agent-provider](../../packages/defaultAgentProvider/README.md) External: `debug`, `ws` ### Files of interest -`./src/baseTreeProvider.ts`, `./src/collisionsPresentation.ts`, `./src/collisionsSource.ts`, …and 54 more under `./src/`. +`./src/baseTreeProvider.ts`, `./src/collisionsPresentation.ts`, `./src/collisionsSource.ts`, …and 63 more under `./src/`. --- -_Auto-generated against commit `366aaf867a7e8e5d130b6c87a365516bab725269` on `2026-07-07T09:05:05.703Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter typeagent-studio docs:verify-links` to spot-check._ +_Auto-generated against commit `d7d1b43eebfadec45d2f3199e628189b353920b9` on `2026-07-17T20:31:18.363Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter typeagent-studio docs:verify-links` to spot-check._ diff --git a/ts/packages/typeagent-studio/esbuild.mjs b/ts/packages/typeagent-studio/esbuild.mjs index fc6292d0e0..b50cb3fc37 100644 --- a/ts/packages/typeagent-studio/esbuild.mjs +++ b/ts/packages/typeagent-studio/esbuild.mjs @@ -153,18 +153,37 @@ const webviewConfig = { minify: !watch, }; +/** + * The Trace Viewer webview client bundle. Same browser-only constraints as the + * Impact Report client; renders the side-by-side resolution trace for one row. + * @type {import('esbuild').BuildOptions} + */ +const traceViewerConfig = { + entryPoints: ["src/webviewKit/client/traceViewer.ts"], + bundle: true, + outfile: "dist/webview/traceViewer.js", + format: "iife", + platform: "browser", + target: "es2020", + sourcemap: true, + minify: !watch, +}; + if (watch) { const ctx = await esbuild.context(extensionConfig); const serviceCtx = await esbuild.context(serviceConfig); const webviewCtx = await esbuild.context(webviewConfig); + const traceViewerCtx = await esbuild.context(traceViewerConfig); await ctx.watch(); await serviceCtx.watch(); await webviewCtx.watch(); + await traceViewerCtx.watch(); copyRuntimeAssets(); console.log("typeagent-studio: watching…"); } else { await esbuild.build(extensionConfig); await esbuild.build(serviceConfig); await esbuild.build(webviewConfig); + await esbuild.build(traceViewerConfig); copyRuntimeAssets(); } diff --git a/ts/packages/typeagent-studio/media/impactReport.css b/ts/packages/typeagent-studio/media/impactReport.css index 9504d3a53c..dcde79a667 100644 --- a/ts/packages/typeagent-studio/media/impactReport.css +++ b/ts/packages/typeagent-studio/media/impactReport.css @@ -122,6 +122,9 @@ body { .codicon-settings-gear::before { content: "\eb51"; } +.codicon-telescope::before { + content: "\eb68"; +} /* --- Action bar (the native editor toolbar) ------------------------------ */ .action-bar { @@ -270,9 +273,26 @@ body { color: var(--vscode-descriptionForeground); } +/* The A/B selector badges: small filled tokens sharing the Trace Viewer's + blue(A)/purple(B) side language, so "A" and "B" mean the same thing and read + the same way across both panels. */ .picker-side { - font-weight: 600; - color: var(--vscode-descriptionForeground); + font-weight: 700; + font-size: 0.82em; + min-width: 1.3em; + height: 1.3em; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 3px; + color: #fff; + flex: none; +} +.picker-side-a { + background: var(--vscode-charts-blue, #4ea1ff); +} +.picker-side-b { + background: var(--vscode-charts-purple, #b48ead); } /* Toolbar icon buttons (run, swap, reconnect, close). */ @@ -612,6 +632,12 @@ body { font-size: 10px; letter-spacing: 0.04em; } +.fidelity-head-a { + color: var(--vscode-charts-blue, #4ea1ff); +} +.fidelity-head-b { + color: var(--vscode-charts-purple, #b48ead); +} .fidelity-grid .fidelity-head:not(:first-child), .fidelity-grid .fidelity-cell { @@ -1058,6 +1084,55 @@ tr.row-open td.latency { color: var(--vscode-list-activeSelectionForeground); } +/* --- Row context menu ---------------------------------------------------- */ +/* A small floating menu shown on right-click of a row; positioned at the cursor + from script. Mirrors a native menu so "Open in Trace Viewer" reads as the + row's action. */ +.context-menu { + position: fixed; + z-index: 20; + min-width: 168px; + padding: 4px; + border: 1px solid var(--vscode-menu-border, var(--vscode-panel-border)); + border-radius: 5px; + background: var( + --vscode-menu-background, + var(--vscode-editorWidget-background) + ); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.36); +} + +.context-menu-item { + display: block; + width: 100%; + padding: 5px 10px; + border: none; + border-radius: 3px; + background: none; + text-align: left; + font: inherit; + color: var(--vscode-menu-foreground, var(--vscode-foreground)); + cursor: pointer; +} + +.context-menu-item:hover:not(:disabled), +.context-menu-item:focus-visible { + outline: none; + background: var( + --vscode-menu-selectionBackground, + var(--vscode-list-activeSelectionBackground) + ); + color: var( + --vscode-menu-selectionForeground, + var(--vscode-list-activeSelectionForeground) + ); +} + +.context-menu-item.is-disabled { + color: var(--vscode-disabledForeground); + cursor: default; +} + /* --- Detail pane (docked action diff) ------------------------------------ */ /* Docked at the bottom of the content column so the selected row stays visible above it; the pane scrolls internally when the diff is tall. */ @@ -1087,46 +1162,17 @@ tr.row-open td.latency { ); } -.detail-title { +.detail-utterance { + flex: 1 1 auto; + min-width: 0; font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } -.detail-meta { - display: inline-flex; - align-items: center; - gap: 6px; - color: var(--vscode-descriptionForeground); - font-size: 12px; - white-space: nowrap; - font-variant-numeric: tabular-nums; -} - -.detail-meta .added { - color: var( - --vscode-charts-green, - var(--vscode-gitDecoration-addedResourceForeground) - ); -} - -.detail-meta .removed { - color: var(--vscode-errorForeground); -} - -.detail-close { - margin-left: auto; -} - -.detail-legend { - display: flex; - align-items: center; - gap: 6px; - padding: 4px 12px 0; - color: var(--vscode-descriptionForeground); - font-size: 11px; -} +/* The utterance's flex-grow claims the free space, so the close control sits at + the right edge. */ .detail-diff { margin: 0; diff --git a/ts/packages/typeagent-studio/media/traceViewer.css b/ts/packages/typeagent-studio/media/traceViewer.css new file mode 100644 index 0000000000..419adf1e94 --- /dev/null +++ b/ts/packages/typeagent-studio/media/traceViewer.css @@ -0,0 +1,562 @@ +/* Copyright (c) Microsoft Corporation. + Licensed under the MIT License. */ + +/* The codicon icon font ships next to this stylesheet (copied from + @vscode/codicons at build time) so the viewer can use VS Code's native icon + set under the webview's font-src policy. */ +@font-face { + font-family: "codicon"; + font-display: block; + src: url("codicon.ttf") format("truetype"); +} +.codicon { + font: normal normal normal 16px/1 codicon; + display: inline-block; + text-decoration: none; + text-rendering: auto; + text-align: center; + vertical-align: middle; + -webkit-font-smoothing: antialiased; + user-select: none; +} +.codicon-go-to-file::before { + content: "\ea94"; +} + +:root { + --tv-gap: 10px; + --tv-radius: 6px; + --tv-border: var(--vscode-panel-border, rgba(128, 128, 128, 0.35)); + --tv-muted: var(--vscode-descriptionForeground, #8a8a8a); + --tv-mono: var(--vscode-editor-font-family, monospace); +} + +html, +body { + height: 100%; +} + +body { + font-family: var(--vscode-font-family); + font-size: var(--vscode-font-size); + color: var(--vscode-foreground); + background: var(--vscode-editor-background); + margin: 0; + padding: 0; +} + +#root { + display: flex; + flex-direction: column; + min-height: 100vh; + padding: 14px 18px 28px; + box-sizing: border-box; + gap: var(--tv-gap); +} + +[hidden] { + display: none !important; +} + +code.mono, +.mono { + font-family: var(--tv-mono); + font-size: 0.92em; +} + +/* --- Header --------------------------------------------------------------- */ +.trace-header { + display: flex; + flex-direction: column; + gap: 8px; +} +.utterance { + font-family: var(--tv-mono); + font-size: 1.15em; + font-weight: 600; + line-height: 1.35; + overflow-wrap: anywhere; +} + +/* --- Verdict + subtitle -------------------------------------------------- */ +.header-top { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 12px; +} +.header-top .utterance { + flex: 1; + min-width: 0; +} +/* The one-word answer to "did A and B agree?", parked top-right of the header. */ +.verdict { + flex: none; + font-size: 0.8em; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + padding: 4px 10px; + border-radius: 999px; + white-space: nowrap; + border: 1px solid transparent; +} +.verdict.is-match { + background: rgba(70, 160, 90, 0.16); + color: var(--vscode-charts-green, #6bbf78); + border-color: rgba(70, 160, 90, 0.4); +} +.verdict.is-differ { + background: rgba(200, 150, 0, 0.16); + color: var(--vscode-charts-orange, #c8961e); + border-color: rgba(200, 150, 0, 0.45); +} +.verdict.is-neutral { + background: rgba(128, 128, 128, 0.16); + color: var(--tv-muted); + border-color: var(--tv-border); +} +.verdict.is-low-confidence { + border-style: dashed; +} + +/* The optional one-line reading under the header: only shown when it adds + something the verdict badge and cause stage don't already say. */ +.subtitle { + display: flex; + flex-direction: column; + gap: 3px; +} +.subtitle-text { + font-size: 1.02em; + line-height: 1.4; +} +.subtitle-note { + font-size: 0.85em; + color: var(--tv-muted); +} +.subtitle.is-low-confidence .subtitle-text { + font-style: italic; +} + +.note-banner { + font-size: 0.88em; + color: var(--tv-muted); + padding: 6px 10px; + border-radius: var(--tv-radius); + background: var(--vscode-editorWidget-background, rgba(128, 128, 128, 0.1)); +} + +/* --- Pipeline ------------------------------------------------------------- */ +/* Stages stack vertically in runtime order and hang off a single left rail so + the panel reads as one flow the utterance traveled down. */ +.pipeline { + position: relative; + display: flex; + flex-direction: column; + gap: 4px; + padding-left: 26px; +} +.pipeline::before { + content: ""; + position: absolute; + left: 7px; + top: 10px; + bottom: 14px; + width: 2px; + background: var(--tv-border); +} + +.stage { + position: relative; + border: 1px solid transparent; + border-radius: var(--tv-radius); + padding: 8px 12px; +} +/* Marker dot sitting on the rail; its fill encodes the stage's parity status. */ +.stage-marker { + position: absolute; + left: -26px; + top: 11px; + width: 16px; + height: 16px; + border-radius: 50%; + border: 2px solid var(--tv-border); + background: var(--vscode-editor-background); + box-sizing: border-box; +} +.stage.status-agree .stage-marker { + border-color: var(--vscode-charts-green, #6bbf78); + background: var(--vscode-charts-green, #6bbf78); +} +.stage.status-diverge .stage-marker, +.stage.is-cause .stage-marker { + border-color: var(--vscode-charts-orange, #c8961e); + background: var(--vscode-charts-orange, #c8961e); +} +.stage.status-one-sided .stage-marker { + border-color: var(--vscode-charts-blue, #4ea1ff); + background: var(--vscode-editor-background); +} +.stage.status-inapplicable .stage-marker { + border-style: dashed; +} + +.stage-head { + display: flex; + align-items: baseline; + gap: 8px; +} +.stage-name { + font-weight: 600; + font-size: 0.96em; +} +.stage-status { + font-size: 0.76em; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--tv-muted); +} +.stage.is-cause .stage-status { + color: var(--vscode-charts-orange, #c8961e); + font-weight: 600; +} +/* Directional status wording ("new in B" / "lost in B") reuses the outcome + palette: green for a match B gained, orange for one it lost. The green wins + over the cause card's orange so an improvement still reads as a win. */ +.stage-status.is-positive { + color: var(--vscode-charts-green, #6bbf78); + font-weight: 600; +} +.stage-status.is-negative { + color: var(--vscode-charts-orange, #c8961e); + font-weight: 600; +} +.stage.is-cause .stage-status.is-positive { + color: var(--vscode-charts-green, #6bbf78); +} + +/* The accented "this is where they split" card. */ +.stage.is-cause { + border-color: var(--vscode-charts-orange, rgba(200, 150, 0, 0.5)); + background: var( + --vscode-inputValidation-warningBackground, + rgba(200, 150, 0, 0.08) + ); +} +.stage.is-result { + border-color: var(--tv-border); + margin-top: 6px; +} +.stage.is-result.is-cause { + border-color: var(--vscode-charts-orange, rgba(200, 150, 0, 0.5)); +} +/* A converged Result collapses like the agreeing pre-action stages: no card + border, just the compact one-liner, so "same" reads the same way down the + whole pipeline. A diverged Result keeps its bordered card for the A/B split. */ +.stage.is-result.status-agree { + border-color: transparent; +} + +/* Compact (converged / one-sided / n/a) stage: a single summary line. */ +.stage-compact .stage-summary { + margin-top: 4px; + font-size: 0.88em; + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; +} +.stage-compact .stage-summary.is-muted { + color: var(--tv-muted); +} +.summary-content { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; + min-width: 0; +} +/* The single tone-colored outcome phrase every side card and compact row leads + with, so status reads the same way at a glance across all stages. */ +.outcome { + overflow-wrap: anywhere; +} +.outcome.is-positive { + color: var(--vscode-charts-green, #6bbf78); +} +.outcome.is-negative { + color: var(--vscode-charts-orange, #c8961e); +} +.outcome.is-neutral { + color: var(--vscode-foreground); +} +.outcome.is-muted { + color: var(--tv-muted); + font-style: italic; +} +.only-side { + font-weight: 700; + font-size: 0.82em; + color: var(--vscode-charts-blue, #4ea1ff); +} +.cache-id { + font-size: 0.82em; + color: var(--tv-muted); +} + +/* Expanded cause body: A/B columns + the "What changed" callout. */ +.stage-body { + margin-top: 8px; + display: flex; + flex-direction: column; + gap: 10px; +} +/* The cause bar: the single ⇄ diff chip naming the grammar/schema file a + divergence traces to, set on an accent rail so it reads as the answer inside + the tinted cause card. */ +.cause-bar { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 10px; + padding: 7px 12px; + border: 1px solid var(--tv-border); + border-left: 3px solid var(--vscode-charts-orange, #d9822b); + border-radius: var(--tv-radius); + background: var(--vscode-editor-background); +} +/* Names the cause bar's meaning in a word, tinted to the accent rail so the + file chip beside it reads as the divergence's attributed change. */ +.cause-label { + font-size: 0.7em; + text-transform: uppercase; + letter-spacing: 0.06em; + font-weight: 700; + color: var(--vscode-charts-orange, #d9822b); +} +.stage-sides { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} +.side-col { + border: 1px solid var(--tv-border); + border-radius: var(--tv-radius); + padding: 8px 10px; + display: flex; + flex-direction: column; + gap: 6px; + min-width: 0; + background: var(--vscode-editor-background); +} +.side-col-head { + display: flex; + align-items: center; + gap: 8px; +} +.side-tag { + font-weight: 700; + font-size: 0.85em; + width: 1.3em; + height: 1.3em; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 3px; + color: #fff; + flex: none; +} +.side-col.side-a .side-tag { + background: var(--vscode-charts-blue, #4ea1ff); +} +.side-col.side-b .side-tag { + background: var(--vscode-charts-purple, #b48ead); +} +/* The side card's primary outcome phrase, a step up in weight from the compact + rows so a diverging stage's two verdicts read first. */ +.side-col .outcome { + font-size: 0.95em; + font-weight: 600; +} +/* A side card's artifact on its own line under the outcome: the dim micro-label + and the mono value (matched rule / produced action / construction id). */ +.side-artifact { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; + min-width: 0; +} +.artifact-label { + font-size: 0.68em; + text-transform: uppercase; + letter-spacing: 0.06em; + font-weight: 700; + color: var(--tv-muted); +} +/* A small, secondary go-to-file icon beside a rule/action value; a distinct + affordance from the value token, brightening to a link tone on hover. */ +.open-source { + display: inline-flex; + align-items: center; + justify-content: center; + border: none; + background: none; + padding: 0; + margin: 0; + color: var(--tv-muted); + cursor: pointer; +} +.open-source .codicon { + font-size: 13px; +} +.open-source:hover { + color: var(--vscode-textLink-foreground); +} +.open-source:focus-visible { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: 2px; + border-radius: 2px; +} +.side-empty { + font-size: 0.85em; + color: var(--tv-muted); + font-style: italic; +} + +.compare-link { + font: inherit; + font-size: 0.9em; + display: inline-flex; + align-items: center; + gap: 6px; + border: none; + background: none; + padding: 0; + margin: 0; + color: var(--vscode-textLink-foreground); + cursor: pointer; +} +.compare-link:hover { + color: var( + --vscode-textLink-activeForeground, + var(--vscode-textLink-foreground) + ); +} +.compare-link:hover .compare-label { + text-decoration: underline; +} +.compare-link:focus-visible { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: 2px; + border-radius: 2px; +} +.compare-glyph { + font-size: 1.1em; + line-height: 1; +} + +/* Output difference: the unified JSON diff of the two produced actions, shown at + the Result whenever the actions differ. Mirrors the Impact Report's diff + colors so the two surfaces read the same. */ +.output-diff { + display: flex; + flex-direction: column; + gap: 4px; +} +.output-diff-label { + font-size: 0.72em; + text-transform: uppercase; + letter-spacing: 0.06em; + font-weight: 700; + color: var(--tv-muted); +} +.action-diff { + margin: 0; + padding: 8px 10px; + overflow-x: auto; + border: 1px solid var(--tv-border); + border-radius: var(--tv-radius); + background: var(--vscode-editor-background); + font-family: var(--tv-mono); + font-size: 0.86em; + line-height: 1.4; +} +.action-diff .diff-line { + display: block; + white-space: pre; + padding: 0 4px; + border-radius: 2px; +} +.action-diff .diff-added { + color: var( + --vscode-charts-green, + var(--vscode-gitDecoration-addedResourceForeground) + ); + background: var(--vscode-diffEditor-insertedTextBackground, transparent); +} +.action-diff .diff-removed { + color: var( + --vscode-errorForeground, + var(--vscode-gitDecoration-deletedResourceForeground) + ); + background: var(--vscode-diffEditor-removedTextBackground, transparent); +} +.action-diff .diff-context { + color: var(--vscode-descriptionForeground); +} + +.rule, +.action-name { + background: var(--vscode-textCodeBlock-background, rgba(128, 128, 128, 0.14)); + padding: 1px 6px; + border-radius: 3px; + overflow-wrap: anywhere; +} + +/* --- State screens -------------------------------------------------------- */ +.state-screen { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + padding: 40px 20px; +} +.state-inner { + display: flex; + flex-direction: column; + align-items: center; + gap: 14px; + max-width: 40ch; + text-align: center; + color: var(--tv-muted); + line-height: 1.5; +} +.spinner { + width: 22px; + height: 22px; + border-radius: 50%; + border: 2px solid var(--tv-border); + border-top-color: var(--vscode-focusBorder, #4ea1ff); + animation: tv-spin 0.8s linear infinite; +} +@keyframes tv-spin { + to { + transform: rotate(360deg); + } +} +@media (prefers-reduced-motion: reduce) { + .spinner { + animation: none; + } +} + +/* --- Narrow layout -------------------------------------------------------- */ +@media (max-width: 560px) { + /* Stack the A/B columns of a diverging stage when the panel gets too narrow + to keep them side-by-side and legible. */ + .stage-sides { + grid-template-columns: 1fr; + } +} diff --git a/ts/packages/typeagent-studio/src/impactReportStore.ts b/ts/packages/typeagent-studio/src/impactReportStore.ts index 835c8ea8c6..ac9ef4bd28 100644 --- a/ts/packages/typeagent-studio/src/impactReportStore.ts +++ b/ts/packages/typeagent-studio/src/impactReportStore.ts @@ -82,7 +82,7 @@ export async function savePersistedRun( versionA?: ResolvedVersion, versionB?: ResolvedVersion, ): Promise { - const bounded = + const rowsBounded = payload.rows.length > MAX_PERSISTED_ROWS ? { ...payload, @@ -92,6 +92,20 @@ export async function savePersistedRun( ), } : payload; + // Drop the (large) per-row traces; only the lightweight `tracedUtteranceIds` + // is needed to restore the "Open trace" affordance, and the full traces are + // persisted separately by the Trace Viewer store. Derive the ids from the + // traces when the payload didn't already carry them. + const { resolutionTraces, ...withoutTraces } = rowsBounded; + const bounded: StudioReplayResult = + resolutionTraces === undefined + ? withoutTraces + : { + ...withoutTraces, + tracedUtteranceIds: + rowsBounded.tracedUtteranceIds ?? + resolutionTraces.map((t) => t.utteranceId), + }; await state.update(runStoreKey(agent), { payload: bounded, runAt, diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts index 7f7aad9507..58158e32c8 100644 --- a/ts/packages/typeagent-studio/src/impactReportView.ts +++ b/ts/packages/typeagent-studio/src/impactReportView.ts @@ -11,6 +11,7 @@ import { import type { ResolvedVersion, RunProvenance, + VersionProvenance, } from "./webviewKit/replayViewModel.js"; import { defaultGitExec, @@ -22,6 +23,15 @@ import { import { StudioServiceClient } from "./studioServiceClient.js"; import type { StudioConnectionState } from "./studioServiceConnection.js"; import { loadPersistedRun, savePersistedRun } from "./impactReportStore.js"; +import { saveTraceRun } from "./traceStore.js"; +import { openTraceViewer, focusTraceViewer } from "./traceViewerView.js"; +import { + buildReplayRunDescriptor, + buildTraceVersionPin, + type ReplayTraceMode, + type TraceVersionPin, + type VersionSpec, +} from "@typeagent/core/replay"; import type { StudioReplayResult } from "@typeagent/core/runtime"; const VIEW_TYPE = "typeagentStudio.impactReport"; @@ -151,15 +161,34 @@ export function openImpactReport( const post = (message: HostToWebviewMessage) => panel.post(message); + // The webview only needs the set of traced utterances (carried by + // `tracedUtteranceIds`) to offer the "Open trace" affordance; the full + // traces are large (per-red-row grammar debug info) and are persisted + // separately for the Trace Viewer. Strip them from anything sent to the + // webview so a big changed set doesn't inflate the host→webview message. + const forWebview = (result: StudioReplayResult): StudioReplayResult => { + if (result.resolutionTraces === undefined) { + return result; + } + const { resolutionTraces, ...slim } = result; + return { + ...slim, + tracedUtteranceIds: + result.tracedUtteranceIds ?? + resolutionTraces.map((t) => t.utteranceId), + }; + }; + // Let an external Replay (launched from the Corpora view) refresh this open // report in place: adopt its result as this panel's last result (so a later // reload/ready re-push shows it) and, when the webview is listening, post it // as an external result the client accepts regardless of its own request-id // sequence. openReportRefreshers.set(agent, (update) => { + const slim = forWebview(update.payload); lastResult = { requestId: 0, - payload: update.payload, + payload: slim, runAt: update.runAt, versionA: update.versionA, versionB: update.versionB, @@ -170,7 +199,7 @@ export function openImpactReport( type: "result", requestId: 0, external: true, - payload: update.payload, + payload: slim, runAt: update.runAt, versionA: update.versionA, versionB: update.versionB, @@ -483,6 +512,19 @@ export function openImpactReport( await pickVersion(msg.side); return; } + if (msg.type === "openTrace") { + // Drill into the full resolution trace behind one red row, side-by- + // side with this report. The viewer reads the exact trace this run + // persisted, so it always reflects what produced the row. + openTraceViewer(context, repoRoot, msg.runId, msg.utteranceId); + return; + } + if (msg.type === "focusTrace") { + // The selected row changed: follow it into an already-open Trace + // Viewer, but don't open one if none is showing. + focusTraceViewer(msg.runId, msg.utteranceId); + return; + } if (msg.type === "searchUtterances") { // A live input box: each keystroke posts the current text back so the // report filters as the user types. Closing it (accept or Esc) keeps @@ -556,10 +598,11 @@ export function openImpactReport( missPolicy: "needs-explanation", }); const completedAt = Date.now(); + const slim = forWebview(payload); post({ type: "result", requestId: msg.requestId, - payload, + payload: slim, runAt: completedAt, versionA: msg.resolvedA, versionB: msg.resolvedB, @@ -567,7 +610,7 @@ export function openImpactReport( }); lastResult = { requestId: msg.requestId, - payload, + payload: slim, runAt: completedAt, versionA: msg.resolvedA, versionB: msg.resolvedB, @@ -582,6 +625,41 @@ export function openImpactReport( msg.resolvedA, msg.resolvedB, ); + // Persist the captured per-red-row traces + the run descriptor so the + // Trace Viewer can reopen the exact resolution behind a row and + // recompute a fresh one from the same pinned inputs. + if ( + payload.resolutionTraces && + payload.resolutionTraces.length > 0 + ) { + const pinFor = ( + spec: VersionSpec, + prov: VersionProvenance | undefined, + ): TraceVersionPin => + buildTraceVersionPin({ + spec, + label: + prov?.label ?? + (spec.kind === "git" ? spec.ref : "working tree"), + ...(prov?.sha !== undefined ? { sha: prov.sha } : {}), + }); + const descriptor = buildReplayRunDescriptor({ + runId: payload.runId, + agent, + a: pinFor(msg.versionA, provenance?.a), + b: pinFor(msg.versionB, provenance?.b), + mode: (msg.mode ?? "nfa-grammar") as ReplayTraceMode, + missPolicy: "needs-explanation", + validateWildcards: msg.validateWildcards === true, + corpus: {}, + runAt: completedAt, + }); + await saveTraceRun( + context.workspaceState, + descriptor, + payload.resolutionTraces, + ); + } } catch (e) { post({ type: "error", diff --git a/ts/packages/typeagent-studio/src/serviceRuntimeFacade.ts b/ts/packages/typeagent-studio/src/serviceRuntimeFacade.ts index d165ca7de2..46f0c5ba4c 100644 --- a/ts/packages/typeagent-studio/src/serviceRuntimeFacade.ts +++ b/ts/packages/typeagent-studio/src/serviceRuntimeFacade.ts @@ -5,6 +5,8 @@ import type { RepoRootResolution, StudioReplayRequest, StudioReplayResult, + ReplayResolutionTraceRequest, + ReplayResolutionTraceResult, StudioCorpusImportRequest, StudioCorpusImportResult, } from "@typeagent/core/runtime"; @@ -104,4 +106,10 @@ export class StudioServiceRuntimeFacade implements CorpusSource, HealthSource { ): Promise { return this.require().replayCorpus(request); } + + async replayResolutionTrace( + request: ReplayResolutionTraceRequest, + ): Promise { + return this.require().replayResolutionTrace(request); + } } diff --git a/ts/packages/typeagent-studio/src/studioServiceClient.ts b/ts/packages/typeagent-studio/src/studioServiceClient.ts index 041b6acfad..1558e408f6 100644 --- a/ts/packages/typeagent-studio/src/studioServiceClient.ts +++ b/ts/packages/typeagent-studio/src/studioServiceClient.ts @@ -11,6 +11,8 @@ import type { StudioClientCallFunctions, StudioReplayRequest, StudioReplayResult, + ReplayResolutionTraceRequest, + ReplayResolutionTraceResult, StudioCollisionScanRequest, StudioCollisionScanResult, StudioCorpusImportRequest, @@ -271,6 +273,13 @@ export class StudioServiceClient { return this.rpc.invoke("replayCorpus", this.repoRoot, request); } + /** Recompute one utterance's resolution trace from a stored run descriptor. */ + replayResolutionTrace( + request: ReplayResolutionTraceRequest, + ): Promise { + return this.rpc.invoke("replayResolutionTrace", this.repoRoot, request); + } + /** Start receiving live `studioEvent` pushes for this connection's repo. */ subscribeEvents(): Promise { return this.rpc.invoke("subscribeEvents", this.repoRoot); diff --git a/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts b/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts index 4ed754bd8b..0ac5978615 100644 --- a/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts +++ b/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts @@ -45,6 +45,7 @@ export function stubInvokeHandlers( summary: {} as never, rows: [], }), + replayResolutionTrace: async () => ({ status: "unavailable" }), subscribeEvents: async () => {}, unsubscribeEvents: async () => {}, listSandboxes: async () => [], diff --git a/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts b/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts index 57d2a104dc..b6ef93b868 100644 --- a/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts +++ b/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts @@ -20,6 +20,7 @@ import type { CorpusService, } from "@typeagent/core/corpus"; import type { ReplayActionResolver } from "@typeagent/core/replay"; +import type { ReplayRunDescriptor } from "@typeagent/core/replay"; import type { CollisionDetectedEvent } from "@typeagent/core/events"; import { createStudioRuntimeCore } from "@typeagent/core/runtime"; @@ -1274,6 +1275,11 @@ function spyCacheResolver(record: { consulted: boolean }) { ? { schemaName: "demo", actionName: "pause" } : undefined; }, + matchEntry(utterance: string) { + return utterance === "pause" + ? { action: { schemaName: "demo", actionName: "pause" } } + : undefined; + }, }; }; } @@ -1347,3 +1353,90 @@ test("replayCorpus consults the construction cache in completionBased-cache mode await fs.rm(repoRoot, { recursive: true, force: true }); } }); + +// `replayResolutionTrace` recomputes a single utterance's trace from a stored +// run descriptor, reusing the same grammar path a run uses. + +function workingTreeDescriptor(): ReplayRunDescriptor { + const pin = { + spec: { kind: "workingTree" as const }, + label: "working tree", + workingTree: true, + }; + return { + runId: "run-recompute", + agent: "demo", + a: pin, + b: pin, + mode: "nfa-grammar", + missPolicy: "needs-explanation", + validateWildcards: false, + corpus: {}, + runAt: 0, + }; +} + +test("replayResolutionTrace recomputes a trace for a known utterance", async () => { + const repoRoot = await scaffoldGatingRepo(); + try { + const corpus = new StubCorpusService([gatingEntry("p", "pause")]); + const runtime = createStudioRuntimeCore( + createContext([repoRoot]).context, + { corpus }, + ); + + const result = await runtime.replayResolutionTrace({ + descriptor: workingTreeDescriptor(), + utteranceId: "p", + }); + + assert.equal(result.status, "recomputed"); + if (result.status === "recomputed") { + assert.equal(result.trace.utteranceId, "p"); + assert.equal(result.trace.runId, "run-recompute"); + } + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } +}); + +test("replayResolutionTrace reports entry-missing for an unknown utterance", async () => { + const repoRoot = await scaffoldGatingRepo(); + try { + const corpus = new StubCorpusService([gatingEntry("p", "pause")]); + const runtime = createStudioRuntimeCore( + createContext([repoRoot]).context, + { corpus }, + ); + + const result = await runtime.replayResolutionTrace({ + descriptor: workingTreeDescriptor(), + utteranceId: "does-not-exist", + }); + + assert.equal(result.status, "entry-missing"); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } +}); + +test("replayResolutionTrace reports unavailable without a grammar path", async () => { + const { context } = createContext(); + const corpus = new StubCorpusService([corpusEntry("a")]); + const resolver: ReplayActionResolver = { + resolve(entry) { + return { action: { value: entry.id }, cacheState: "hit" }; + }, + }; + const runtime = createStudioRuntimeCore(context, { + corpus, + replayResolver: resolver, + }); + + const result = await runtime.replayResolutionTrace({ + descriptor: { ...workingTreeDescriptor(), agent: "player" }, + utteranceId: "a", + }); + + assert.equal(result.status, "unavailable"); +}); diff --git a/ts/packages/typeagent-studio/src/test/traceDivergenceViewModel.spec.ts b/ts/packages/typeagent-studio/src/test/traceDivergenceViewModel.spec.ts new file mode 100644 index 0000000000..b31c202d90 --- /dev/null +++ b/ts/packages/typeagent-studio/src/test/traceDivergenceViewModel.spec.ts @@ -0,0 +1,801 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import test from "node:test"; +import assert from "node:assert/strict"; +import type { + ReplayResolutionTrace, + ReplaySideTrace, + ReplayTraceNode, + VersionSpec, +} from "@typeagent/core/replay"; +import { toTraceDivergenceViewModel } from "../webviewKit/traceDivergenceViewModel.js"; + +const GIT: VersionSpec = { kind: "git", ref: "HEAD" }; +const WORKING: VersionSpec = { kind: "workingTree" }; + +/** A working-tree side that resolved through its grammar to `action`, matching + * the rule `chosenRule`. Cache consulted and missed; wildcard validation ran + * and accepted. */ +function liveSide( + side: "A" | "B", + action: unknown, + chosenRule: string, + overrides: { + cacheHit?: boolean; + wildcardRejected?: boolean; + rankingParity?: "matched" | "diverged" | "unavailable"; + hasTrace?: boolean; + grammarHash?: string; + grammarFilePath?: string; + ruleLocations?: Array< + [string, { fileId: string; displayPath: string }] + >; + filePaths?: Array<[string, string]>; + } = {}, +): ReplaySideTrace { + const cacheHit = overrides.cacheHit ?? false; + const grammarReached = !cacheHit; + const zeroPos = { line: 0, character: 0, offset: 0 }; + const zeroRange = { start: zeroPos, end: zeroPos }; + const nodes: ReplayTraceNode[] = [ + { + kind: "cache-consult", + execution: "ran", + outcome: cacheHit ? "hit" : "miss", + ...(cacheHit + ? { + entry: { + action, + constructionId: "c-1", + namespace: "player", + parts: ["play", ""], + }, + } + : {}), + }, + { + kind: "grammar-match", + execution: grammarReached ? "ran" : "not-reached", + ...(grammarReached ? { outcome: "hit" as const } : {}), + input: "play something", + chosenRule, + ...(overrides.grammarFilePath !== undefined + ? { sourceFilePath: overrides.grammarFilePath } + : {}), + rankingParity: overrides.rankingParity ?? "matched", + ...((overrides.hasTrace ?? true) && grammarReached + ? { + trace: { + input: "play something", + events: [], + result: "matched" as const, + }, + debugInfo: { + grammarHash: overrides.grammarHash ?? "h", + rules: (overrides.ruleLocations ?? []).map( + ([id, loc]) => [ + id, + { + fileId: loc.fileId, + displayPath: loc.displayPath, + range: zeroRange, + }, + ], + ), + parts: [], + partRules: [], + partLabels: [], + filePaths: overrides.filePaths ?? [], + }, + } + : {}), + }, + { + kind: "wildcard-validation", + execution: "ran", + outcome: overrides.wildcardRejected ? "rejected" : "accepted", + }, + { + kind: "action", + execution: grammarReached || cacheHit ? "ran" : "not-reached", + outcome: "hit", + action, + schema: { sourceFilePath: "/repo/player.ts", actionName: "play" }, + }, + ]; + return { + side, + version: WORKING, + realization: "built-live", + nodes, + finalAction: action, + cacheState: cacheHit ? "hit" : "miss", + }; +} + +/** A git-ref side: grammar text only, so cache + wildcard validation are + * not-applicable. */ +function refSide( + side: "A" | "B", + action: unknown, + chosenRule: string, +): ReplaySideTrace { + const nodes: ReplayTraceNode[] = [ + { + kind: "cache-consult", + execution: "not-applicable", + detail: "The construction cache runs only on the working tree.", + }, + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play something", + chosenRule, + rankingParity: "matched", + trace: { + input: "play something", + events: [], + result: "matched" as const, + }, + debugInfo: { + grammarHash: "h", + rules: [], + parts: [], + partRules: [], + partLabels: [], + filePaths: [], + }, + }, + { + kind: "wildcard-validation", + execution: "not-applicable", + }, + { + kind: "action", + execution: "ran", + outcome: "hit", + action, + schema: { sourceFilePath: "/repo/player.ts", actionName: "play" }, + }, + ]; + return { + side, + version: GIT, + realization: "source", + nodes, + finalAction: action, + cacheState: "miss", + }; +} + +function trace(a: ReplaySideTrace, b: ReplaySideTrace): ReplayResolutionTrace { + return { + runId: "run-1", + utteranceId: "u-1", + utterance: "play something", + a, + b, + capturedAt: 0, + }; +} + +test("matching actions on both live sides reads as parity, high confidence", () => { + const action = { actionName: "play", parameters: { song: "x" } }; + const vm = toTraceDivergenceViewModel( + trace(liveSide("A", action, "Play"), liveSide("B", action, "Play")), + ); + assert.equal(vm.conclusion.parity, "match"); + assert.equal(vm.conclusion.bothNoAction, false); + assert.equal(vm.conclusion.confidence, "high"); + assert.equal( + vm.conclusion.headline, + "Both versions produced the same action.", + ); + assert.equal(vm.conclusion.cause, undefined); + assert.equal(vm.divergingLayer, undefined); + assert.equal(vm.conclusion.pathNote, undefined); +}); + +test("neither side produces an action reads as bothNoAction", () => { + const a: ReplaySideTrace = { + side: "A", + version: WORKING, + realization: "built-live", + nodes: [ + { kind: "cache-consult", execution: "ran", outcome: "miss" }, + { + kind: "grammar-match", + execution: "ran", + outcome: "miss", + input: "zzz", + rankingParity: "unavailable", + }, + { kind: "wildcard-validation", execution: "not-reached" }, + { kind: "action", execution: "not-reached" }, + ], + cacheState: "miss", + }; + const b: ReplaySideTrace = { ...a, side: "B" }; + const vm = toTraceDivergenceViewModel(trace(a, b)); + assert.equal(vm.conclusion.parity, "match"); + assert.equal(vm.conclusion.bothNoAction, true); + assert.match(vm.conclusion.headline, /Neither version resolved/); +}); + +test("matching actions with a git-ref side surfaces a fidelity-path note", () => { + const action = { actionName: "play", parameters: { song: "x" } }; + const vm = toTraceDivergenceViewModel( + trace(refSide("A", action, "Play"), liveSide("B", action, "Play")), + ); + assert.equal(vm.conclusion.parity, "match"); + assert.ok(vm.conclusion.pathNote); + assert.match(vm.conclusion.pathNote!, /A ran from grammar source only/); +}); + +test("different rules attribute the difference to grammar match", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", { actionName: "play" }, "Play"), + liveSide("B", { actionName: "queue" }, "Queue"), + ), + ); + assert.equal(vm.conclusion.parity, "differ"); + assert.equal(vm.conclusion.cause?.kind, "grammar-differs"); + assert.equal(vm.divergingLayer, "grammar-match"); + assert.equal(vm.conclusion.confidence, "high"); + assert.match( + vm.conclusion.cause!.detail, + /A matched Play, B matched Queue/, + ); +}); + +test("a cache short-circuit on one side attributes to the cache", () => { + const cachedAction = { actionName: "play", parameters: { song: "cached" } }; + const grammarAction = { actionName: "play", parameters: { song: "fresh" } }; + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", cachedAction, "Play", { cacheHit: true }), + liveSide("B", grammarAction, "Play"), + ), + ); + assert.equal(vm.conclusion.parity, "differ"); + assert.equal(vm.conclusion.cause?.kind, "cache-decided"); + assert.equal(vm.conclusion.cause?.side, "A"); + assert.equal(vm.divergingLayer, "cache-consult"); +}); + +test("same rule with a one-sided wildcard rejection attributes to validation", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { q: "a" } }, + "Play", + ), + liveSide( + "B", + { actionName: "play", parameters: { q: "b" } }, + "Play", + { wildcardRejected: true }, + ), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "wildcard-validation"); + assert.equal(vm.conclusion.cause?.side, "B"); + assert.equal(vm.divergingLayer, "wildcard-validation"); +}); + +test("same rule with differing parameters falls back to action-payload", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { song: "a" } }, + "Play", + ), + liveSide( + "B", + { actionName: "play", parameters: { song: "b" } }, + "Play", + ), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "action-payload"); + assert.equal(vm.divergingLayer, "action"); + assert.equal(vm.conclusion.confidence, "high"); + assert.match(vm.conclusion.cause!.detail, /parameters differ/); +}); + +test("a diverged grammar parity demotes attribution to low confidence", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { song: "a" } }, + "Play", + { + rankingParity: "diverged", + }, + ), + liveSide( + "B", + { actionName: "play", parameters: { song: "b" } }, + "Queue", + ), + ), + ); + assert.equal(vm.conclusion.parity, "differ"); + // Grammar level is untrusted, so it must NOT claim grammar-differs. + assert.notEqual(vm.conclusion.cause?.kind, "grammar-differs"); + assert.equal(vm.conclusion.confidence, "low"); + assert.ok(vm.conclusion.confidenceNote); +}); + +test("a diverged parse with a differing grammar hash still blames the grammar", () => { + // The compiled-grammar hash is resolver-independent, so a genuine grammar + // change is named even when one side's diagnostic parse diverged. Two + // utterances carrying the same change must not flip between grammar- and + // action-level attribution just because their parses took different paths. + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { song: "a" } }, + "Play", + { + rankingParity: "diverged", + grammarHash: "h1", + }, + ), + liveSide( + "B", + { actionName: "play", parameters: { song: "a", album: "x" } }, + "Play", + { + grammarHash: "h2", + }, + ), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "grammar-differs"); + assert.equal(vm.divergingLayer, "grammar-match"); + assert.equal(vm.conclusion.confidence, "high"); +}); + +test("node summaries carry grammar, cache, and action extras", () => { + const cachedAction = { actionName: "play", parameters: { song: "cached" } }; + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", cachedAction, "Play", { cacheHit: true }), + liveSide("B", { actionName: "play" }, "Play"), + ), + ); + + // Side A: cache hit → cache node carries the inline entry. + const aCache = vm.a.nodes.find((n) => n.kind === "cache-consult"); + assert.equal(aCache?.outcomeLabel, "hit"); + assert.equal(aCache?.cache?.constructionId, "c-1"); + assert.deepEqual(aCache?.cache?.parts, ["play", ""]); + + // Side B: grammar ran with a trace → timeline available, parse matches pick. + const bGrammar = vm.b.nodes.find((n) => n.kind === "grammar-match"); + assert.equal(bGrammar?.grammar?.chosenRule, "Play"); + assert.equal(bGrammar?.grammar?.hasTimeline, true); + assert.equal(bGrammar?.grammar?.diagnosticOnly, false); + assert.equal(bGrammar?.grammar?.rankingParityLabel, "parse matches pick"); + + // Action node exposes the produced action's name and schema availability. + const bAction = vm.b.nodes.find((n) => n.kind === "action"); + assert.equal(bAction?.action?.actionName, "play"); + assert.equal(bAction?.action?.hasSchema, true); + + // Nodes are emitted in canonical layer order. + assert.deepEqual( + vm.a.nodes.map((n) => n.kind), + ["cache-consult", "grammar-match", "wildcard-validation", "action"], + ); +}); + +test("not-applicable git-ref layers are labelled, not read as failures", () => { + const vm = toTraceDivergenceViewModel( + trace( + refSide("A", { actionName: "play" }, "Play"), + liveSide("B", { actionName: "play" }, "Play"), + ), + ); + const aCache = vm.a.nodes.find((n) => n.kind === "cache-consult"); + assert.equal(aCache?.executionLabel, "not applicable"); + assert.equal(aCache?.outcomeLabel, undefined); +}); + +test("the pipeline lists the engaged stages in runtime order", () => { + const action = { actionName: "play", parameters: { song: "x" } }; + const vm = toTraceDivergenceViewModel( + trace(liveSide("A", action, "Play"), liveSide("B", action, "Play")), + ); + assert.deepEqual( + vm.stages.map((s) => s.kind), + ["cache-consult", "grammar-match", "wildcard-validation", "action"], + ); + // Matching actions: no stage is the attributed cause. + assert.equal( + vm.stages.some((s) => s.isCause), + false, + ); + // Both sides ran every stage → all agree. + assert.deepEqual( + vm.stages.map((s) => s.status), + ["agree", "agree", "agree", "agree"], + ); + // Both sides' nodes are paired onto each stage for side-by-side rendering. + assert.equal( + vm.stages.every((s) => s.a !== undefined && s.b !== undefined), + true, + ); +}); + +test("the diverging grammar stage is the sole cause; the rest agree", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", { actionName: "play" }, "Play"), + liveSide("B", { actionName: "queue" }, "Queue"), + ), + ); + assert.deepEqual( + vm.stages.filter((s) => s.isCause).map((s) => s.kind), + ["grammar-match"], + ); + const grammar = vm.stages.find((s) => s.kind === "grammar-match"); + assert.equal(grammar?.status, "diverge"); + assert.ok(grammar?.a); + assert.ok(grammar?.b); + // Layers that ran on both sides but aren't the cause read as agreement. + assert.equal( + vm.stages.find((s) => s.kind === "cache-consult")?.status, + "agree", + ); + assert.equal(vm.stages.find((s) => s.kind === "action")?.status, "agree"); +}); + +test("a cache short-circuit is the cause and leaves the other grammar one-sided", () => { + const cachedAction = { actionName: "play", parameters: { song: "cached" } }; + const grammarAction = { actionName: "play", parameters: { song: "fresh" } }; + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", cachedAction, "Play", { cacheHit: true }), + liveSide("B", grammarAction, "Play"), + ), + ); + const cache = vm.stages.find((s) => s.kind === "cache-consult"); + assert.equal(cache?.isCause, true); + assert.equal(cache?.status, "diverge"); + // A short-circuited through its cache so its grammar never ran; B's did. + const grammar = vm.stages.find((s) => s.kind === "grammar-match"); + assert.equal(grammar?.status, "one-sided"); +}); + +test("the action stage exposes a schema compare handle", () => { + const action = { actionName: "play", parameters: { song: "x" } }; + const vm = toTraceDivergenceViewModel( + trace(liveSide("A", action, "Play"), liveSide("B", action, "Play")), + ); + assert.equal(vm.stages.find((s) => s.kind === "action")?.compare, "action"); + // The grammar fixtures carry no source span → no grammar compare handle. + assert.equal( + vm.stages.find((s) => s.kind === "grammar-match")?.compare, + undefined, + ); +}); + +test("a grammar rule with a source span exposes a grammar compare handle", () => { + const withSource = (side: "A" | "B", rule: string): ReplaySideTrace => ({ + side, + version: WORKING, + realization: "built-live", + nodes: [ + { kind: "cache-consult", execution: "ran", outcome: "miss" }, + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play something", + chosenRule: rule, + rankingParity: "matched", + source: { + fileId: "f1", + displayPath: "/repo/player.agr", + range: { + start: { line: 1, character: 0, offset: 0 }, + end: { line: 1, character: 4, offset: 4 }, + }, + }, + }, + { + kind: "action", + execution: "ran", + outcome: "hit", + action: { actionName: rule.toLowerCase() }, + schema: { + sourceFilePath: "/repo/player.ts", + actionName: rule.toLowerCase(), + }, + }, + ], + finalAction: { actionName: rule.toLowerCase() }, + cacheState: "miss", + }); + const vm = toTraceDivergenceViewModel( + trace(withSource("A", "Play"), withSource("B", "Queue")), + ); + assert.equal( + vm.stages.find((s) => s.kind === "grammar-match")?.compare, + "grammar-match", + ); + // Neither side captured wildcard validation → it's omitted from the flow. + assert.equal( + vm.stages.some((s) => s.kind === "wildcard-validation"), + false, + ); +}); + +test("a side that missed grammar still exposes the compare handle via its recorded path", () => { + // The `resume`-style regression: A matched a rule, B matched nothing. B has + // no winning-rule span, but both sides record the absolute .agr path, so the + // diverging grammar stage is still diffable across A ↔ B. + const matched: ReplaySideTrace = { + side: "A", + version: WORKING, + realization: "built-live", + nodes: [ + { kind: "cache-consult", execution: "ran", outcome: "miss" }, + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "resume", + chosenRule: "Resume", + rankingParity: "matched", + sourceFilePath: "/repo/agents/player/player.agr", + source: { + fileId: "f1", + displayPath: "player.agr", + range: { + start: { line: 5, character: 0, offset: 0 }, + end: { line: 5, character: 6, offset: 6 }, + }, + }, + }, + { + kind: "wildcard-validation", + execution: "ran", + outcome: "accepted", + }, + { + kind: "action", + execution: "ran", + outcome: "hit", + action: { actionName: "resume" }, + schema: { + sourceFilePath: "/repo/player.ts", + actionName: "resume", + }, + }, + ], + finalAction: { actionName: "resume" }, + cacheState: "miss", + }; + const missed: ReplaySideTrace = { + side: "B", + version: WORKING, + realization: "built-live", + nodes: [ + { kind: "cache-consult", execution: "ran", outcome: "miss" }, + { + kind: "grammar-match", + execution: "ran", + outcome: "miss", + input: "resume", + rankingParity: "unavailable", + sourceFilePath: "/repo/agents/player/player.agr", + }, + { kind: "action", execution: "not-reached" }, + ], + cacheState: "miss", + }; + const vm = toTraceDivergenceViewModel(trace(matched, missed)); + assert.equal(vm.divergingLayer, "grammar-match"); + const grammar = vm.stages.find((s) => s.kind === "grammar-match"); + assert.equal(grammar?.isCause, true); + assert.equal(grammar?.compare, "grammar-match"); + // The missed side records the absolute path even with no winning-rule span. + assert.equal(grammar?.b?.grammar?.hasSource, true); +}); + +test("a git-ref side's live-only stages read as one-sided, not divergent", () => { + const action = { actionName: "play", parameters: { song: "x" } }; + const vm = toTraceDivergenceViewModel( + trace(refSide("A", action, "Play"), liveSide("B", action, "Play")), + ); + // Actions match → no stage is the cause. + assert.equal( + vm.stages.some((s) => s.isCause), + false, + ); + // Cache + wildcard validation ran only on the live side. + assert.equal( + vm.stages.find((s) => s.kind === "cache-consult")?.status, + "one-sided", + ); + assert.equal( + vm.stages.find((s) => s.kind === "wildcard-validation")?.status, + "one-sided", + ); + // Grammar + action ran on both sides → agreement. + assert.equal( + vm.stages.find((s) => s.kind === "grammar-match")?.status, + "agree", + ); + assert.equal(vm.stages.find((s) => s.kind === "action")?.status, "agree"); +}); + +test("a same-rule grammar-hash change still attributes to the grammar", () => { + // Both sides matched `Play`, but the compiled grammar hash differs — a rule + // body edit the rule-name comparison alone would miss. It's still the cause, + // and the copy speaks to the file changing rather than a rule swap. + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { song: "a" } }, + "Play", + { + grammarHash: "h1", + }, + ), + liveSide( + "B", + { actionName: "play", parameters: { song: "b" } }, + "Play", + { + grammarHash: "h2", + }, + ), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "grammar-differs"); + assert.equal(vm.divergingLayer, "grammar-match"); + assert.equal(vm.conclusion.confidence, "high"); + assert.match( + vm.conclusion.cause!.detail, + /changed between the two versions/, + ); +}); + +test("a grammar divergence names the changed .agr file and stays diffable", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", { actionName: "play" }, "Play", { + grammarFilePath: "/repo/agents/player/player.agr", + }), + liveSide("B", { actionName: "queue" }, "Queue", { + grammarFilePath: "/repo/agents/player/player.agr", + }), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "grammar-differs"); + assert.equal(vm.conclusion.cause?.fileName, "player.agr"); + assert.match(vm.conclusion.cause!.detail, /player\.agr/); + // The named file is diffable from the diverging grammar stage. + assert.equal( + vm.stages.find((s) => s.kind === "grammar-match")?.compare, + "grammar-match", + ); +}); + +test("a grammar divergence pinpoints the file defining the matched rule", () => { + // A's matched rule is defined in an imported grammar file; rule-level + // attribution names that file, not the top-level grammar, so a multi-file + // edit points at the one file on the match path. + const vm = toTraceDivergenceViewModel( + trace( + liveSide("A", { actionName: "play" }, "Play", { + grammarFilePath: "/repo/agents/player/player.agr", + ruleLocations: [ + [ + "Play", + { + fileId: "player/commands.agr", + displayPath: "player/commands.agr", + }, + ], + ], + filePaths: [ + ["player/commands.agr", "/repo/agents/player/commands.agr"], + ], + }), + liveSide("B", { actionName: "queue" }, "Queue", { + grammarFilePath: "/repo/agents/player/player.agr", + }), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "grammar-differs"); + assert.equal(vm.conclusion.cause?.fileName, "commands.agr"); +}); + +test("an action-payload divergence names the action schema file", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { song: "a" } }, + "Play", + ), + liveSide( + "B", + { actionName: "play", parameters: { song: "b" } }, + "Play", + ), + ), + ); + assert.equal(vm.conclusion.cause?.kind, "action-payload"); + assert.equal(vm.conclusion.cause?.fileName, "player.ts"); +}); +test("resultDiff reports identical actions as identical", () => { + const action = { actionName: "play", parameters: { song: "x" } }; + const vm = toTraceDivergenceViewModel( + trace(liveSide("A", action, "Play"), liveSide("B", action, "Play")), + ); + assert.equal(vm.resultDiff.identical, true); + assert.equal(vm.resultDiff.onlyA, false); + assert.equal(vm.resultDiff.onlyB, false); +}); + +test("resultDiff flags an action produced on only one side as onlyA", () => { + // A resolved `resume`; B produced nothing — the divergence the Result chip + // must read as a lost match, never as "same". + const a = liveSide("A", { actionName: "resume" }, "Resume"); + const b: ReplaySideTrace = { + side: "B", + version: WORKING, + realization: "built-live", + nodes: [ + { kind: "cache-consult", execution: "ran", outcome: "miss" }, + { + kind: "grammar-match", + execution: "ran", + outcome: "miss", + input: "resume", + rankingParity: "unavailable", + }, + { kind: "action", execution: "not-reached" }, + ], + cacheState: "miss", + }; + const vm = toTraceDivergenceViewModel(trace(a, b)); + assert.equal(vm.resultDiff.identical, false); + assert.equal(vm.resultDiff.onlyA, true); + assert.equal(vm.resultDiff.onlyB, false); +}); + +test("resultDiff reports two differing actions as a non-identical diff", () => { + const vm = toTraceDivergenceViewModel( + trace( + liveSide( + "A", + { actionName: "play", parameters: { song: "a" } }, + "Play", + ), + liveSide( + "B", + { actionName: "play", parameters: { song: "b" } }, + "Play", + ), + ), + ); + assert.equal(vm.resultDiff.identical, false); + assert.equal(vm.resultDiff.onlyA, false); + assert.equal(vm.resultDiff.onlyB, false); + assert.ok(vm.resultDiff.lines.length > 0); +}); diff --git a/ts/packages/typeagent-studio/src/test/traceProtocol.spec.ts b/ts/packages/typeagent-studio/src/test/traceProtocol.spec.ts new file mode 100644 index 0000000000..d573ed96ee --- /dev/null +++ b/ts/packages/typeagent-studio/src/test/traceProtocol.spec.ts @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseTraceMessage } from "../webviewKit/traceProtocol.js"; + +test("parseTraceMessage accepts well-formed messages", () => { + assert.deepEqual(parseTraceMessage({ type: "ready" }), { type: "ready" }); + + assert.deepEqual( + parseTraceMessage({ + type: "compare-source", + requestId: 7, + node: "grammar-match", + }), + { type: "compare-source", requestId: 7, node: "grammar-match" }, + ); + + assert.deepEqual( + parseTraceMessage({ + type: "open-source", + requestId: 2, + side: "a", + node: "grammar-match", + }), + { type: "open-source", requestId: 2, side: "a", node: "grammar-match" }, + ); + assert.deepEqual( + parseTraceMessage({ + type: "open-source", + requestId: 3, + side: "b", + node: "action", + }), + { type: "open-source", requestId: 3, side: "b", node: "action" }, + ); +}); + +test("parseTraceMessage rejects malformed or unknown messages", () => { + // Not an object / missing type. + assert.equal(parseTraceMessage(undefined), undefined); + assert.equal(parseTraceMessage(null), undefined); + assert.equal(parseTraceMessage("ready"), undefined); + assert.equal(parseTraceMessage({}), undefined); + assert.equal(parseTraceMessage({ type: "nope" }), undefined); + + // compare-source needs a numeric requestId and a valid node. + assert.equal(parseTraceMessage({ type: "compare-source" }), undefined); + assert.equal( + parseTraceMessage({ + type: "compare-source", + requestId: "1", + node: "action", + }), + undefined, + ); + assert.equal( + parseTraceMessage({ + type: "compare-source", + requestId: 1, + node: "cache-consult", + }), + undefined, + ); + + // open-source needs a numeric requestId, a valid side, and a valid node. + assert.equal( + parseTraceMessage({ type: "open-source", side: "a", node: "action" }), + undefined, + ); + assert.equal( + parseTraceMessage({ + type: "open-source", + requestId: 1, + side: "c", + node: "action", + }), + undefined, + ); + assert.equal( + parseTraceMessage({ + type: "open-source", + requestId: 1, + side: "a", + node: "cache-consult", + }), + undefined, + ); +}); diff --git a/ts/packages/typeagent-studio/src/test/traceSourceResolver.spec.ts b/ts/packages/typeagent-studio/src/test/traceSourceResolver.spec.ts new file mode 100644 index 0000000000..fc88d56fe3 --- /dev/null +++ b/ts/packages/typeagent-studio/src/test/traceSourceResolver.spec.ts @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import test from "node:test"; +import assert from "node:assert/strict"; +import type { + ReplayResolutionTrace, + ReplaySideTrace, + ReplayTraceNode, + VersionSpec, +} from "@typeagent/core/replay"; +import { sourceTargetFor } from "../traceSourceResolver.js"; + +const WORKING: VersionSpec = { kind: "workingTree" }; + +function range(startLine: number, endLine: number) { + return { + start: { line: startLine, character: 2, offset: 0 }, + end: { line: endLine, character: 8, offset: 0 }, + }; +} + +function side(nodes: ReplayTraceNode[]): ReplaySideTrace { + return { + side: "A", + version: WORKING, + realization: "built-live", + nodes, + } as ReplaySideTrace; +} + +function trace(a: ReplaySideTrace, b: ReplaySideTrace): ReplayResolutionTrace { + return { + utterance: "play despacito", + utteranceId: "u1", + a, + b, + } as ReplayResolutionTrace; +} + +test("resolves a grammar span from the node's debug-info file table", () => { + const a = side([ + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play despacito", + chosenRule: "PlayTrack", + rankingParity: "matched", + source: { + fileId: "player.agr", + displayPath: "player.agr", + range: range(10, 10), + }, + debugInfo: { + grammarHash: "h", + rules: [], + parts: [], + partRules: [], + partLabels: [], + filePaths: [["player.agr", "/repo/agents/player/player.agr"]], + }, + }, + ]); + const target = sourceTargetFor(trace(a, a), "a", "grammar-match"); + assert.deepEqual(target, { + absPath: "/repo/agents/player/player.agr", + range: { + start: { line: 10, character: 2 }, + end: { line: 10, character: 8 }, + }, + }); +}); + +test("falls back to the span display path when the file table lacks the id", () => { + const a = side([ + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play despacito", + chosenRule: "PlayTrack", + rankingParity: "matched", + source: { + fileId: "player.agr", + displayPath: "/abs/player.agr", + range: range(3, 4), + }, + }, + ]); + const target = sourceTargetFor(trace(a, a), "a", "grammar-match"); + assert.equal(target?.absPath, "/abs/player.agr"); + assert.deepEqual(target?.range, { + start: { line: 3, character: 2 }, + end: { line: 4, character: 8 }, + }); +}); + +test("returns undefined when the grammar node recorded no source span", () => { + const a = side([ + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play despacito", + rankingParity: "unavailable", + }, + ]); + assert.equal(sourceTargetFor(trace(a, a), "a", "grammar-match"), undefined); +}); + +test("prefers the node's recorded sourceFilePath over the span file table", () => { + const a = side([ + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play despacito", + chosenRule: "PlayTrack", + rankingParity: "matched", + sourceFilePath: "/repo/agents/player/player.agr", + source: { + fileId: "player.agr", + displayPath: "player.agr", + range: range(10, 10), + }, + debugInfo: { + grammarHash: "h", + rules: [], + parts: [], + partRules: [], + partLabels: [], + filePaths: [["player.agr", "/stale/other.agr"]], + }, + }, + ]); + const target = sourceTargetFor(trace(a, a), "a", "grammar-match"); + assert.deepEqual(target, { + absPath: "/repo/agents/player/player.agr", + range: { + start: { line: 10, character: 2 }, + end: { line: 10, character: 8 }, + }, + }); +}); + +test("yields the grammar file path without a range when the side matched no rule", () => { + // The regressed side matched nothing, so there's no winning-rule span — but + // the recorded absolute path still lets the host diff the file across A/B. + const a = side([ + { + kind: "grammar-match", + execution: "ran", + outcome: "miss", + input: "resume", + rankingParity: "unavailable", + sourceFilePath: "/repo/agents/player/player.agr", + }, + ]); + assert.deepEqual(sourceTargetFor(trace(a, a), "a", "grammar-match"), { + absPath: "/repo/agents/player/player.agr", + }); +}); + +test("resolves an action schema file with no range", () => { + const a = side([ + { + kind: "action", + execution: "ran", + outcome: "hit", + action: { actionName: "play" }, + schema: { + sourceFilePath: "/repo/agents/player/playerSchema.ts", + actionName: "play", + }, + }, + ]); + const target = sourceTargetFor(trace(a, a), "a", "action"); + assert.deepEqual(target, { + absPath: "/repo/agents/player/playerSchema.ts", + }); +}); + +test("returns undefined when the action node has no schema file", () => { + const a = side([ + { + kind: "action", + execution: "ran", + outcome: "hit", + action: { actionName: "play" }, + }, + ]); + assert.equal(sourceTargetFor(trace(a, a), "a", "action"), undefined); +}); + +test("reads the requested side independently", () => { + const a = side([ + { + kind: "action", + execution: "ran", + outcome: "hit", + schema: { sourceFilePath: "/a/schema.ts" }, + }, + ]); + const b = side([ + { + kind: "action", + execution: "ran", + outcome: "hit", + schema: { sourceFilePath: "/b/schema.ts" }, + }, + ]); + assert.equal( + sourceTargetFor(trace(a, b), "a", "action")?.absPath, + "/a/schema.ts", + ); + assert.equal( + sourceTargetFor(trace(a, b), "b", "action")?.absPath, + "/b/schema.ts", + ); +}); diff --git a/ts/packages/typeagent-studio/src/test/traceStore.spec.ts b/ts/packages/typeagent-studio/src/test/traceStore.spec.ts new file mode 100644 index 0000000000..884131ded9 --- /dev/null +++ b/ts/packages/typeagent-studio/src/test/traceStore.spec.ts @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import test from "node:test"; +import assert from "node:assert/strict"; +import type * as vscode from "vscode"; +import type { + ReplayResolutionTrace, + ReplayRunDescriptor, + ReplaySideTrace, + SerializedGrammarDebugInfo, +} from "@typeagent/core/replay"; +import { + loadResolutionTrace, + loadTraceRun, + saveTraceRun, +} from "../traceStore.js"; + +/** A minimal in-memory Memento mirroring VS Code's workspaceState. */ +function fakeMemento(): vscode.Memento & { raw: Map } { + const store = new Map(); + return { + keys(): readonly string[] { + return [...store.keys()]; + }, + get(key: string): T | undefined { + return store.get(key) as T | undefined; + }, + async update(key: string, value: unknown): Promise { + if (value === undefined) { + store.delete(key); + } else { + store.set(key, value); + } + }, + // Inspection handle for tests to assert on the stored shape directly. + raw: store, + } as vscode.Memento & { raw: Map }; +} + +function debugInfo(hash: string): SerializedGrammarDebugInfo { + return { + grammarHash: hash, + rules: [["rule.play" as never, { displayPath: "player.agr" } as never]], + parts: [], + partRules: [], + partLabels: [], + filePaths: [["player.agr", "/repo/player.agr"]], + }; +} + +function side( + label: "A" | "B", + hash: string, + action: unknown, +): ReplaySideTrace { + return { + side: label, + version: { kind: "workingTree" }, + realization: "built-live", + nodes: [ + { + kind: "grammar-match", + execution: "ran", + outcome: "hit", + input: "play despacito", + debugInfo: debugInfo(hash), + rankingParity: "matched", + }, + { kind: "action", execution: "ran", outcome: "hit", action }, + ], + finalAction: action, + cacheState: "not-consulted", + } as unknown as ReplaySideTrace; +} + +function trace( + utteranceId: string, + hash: string, + runId = "run-1", +): ReplayResolutionTrace { + return { + runId, + utteranceId, + utterance: `utterance ${utteranceId}`, + a: side("A", hash, { actionName: "playTrack", a: 1 }), + b: side("B", hash, { actionName: "playTrack", b: 2 }), + capturedAt: 100, + }; +} + +function descriptor(runId: string, agent: string): ReplayRunDescriptor { + return { + runId, + agent, + a: { + spec: { kind: "workingTree" }, + label: "working tree", + workingTree: true, + }, + b: { + spec: { kind: "git", ref: "HEAD" }, + label: "HEAD", + workingTree: false, + sha: "abc123", + }, + mode: "nfa-grammar", + missPolicy: "needs-explanation", + validateWildcards: false, + corpus: {}, + runAt: 1, + }; +} + +test("saveTraceRun round-trips a run's traces and descriptor", async () => { + const state = fakeMemento(); + await saveTraceRun(state, descriptor("run-1", "player"), [ + trace("u1", "h1"), + trace("u2", "h1"), + ]); + + const lookup = loadTraceRun(state, "run-1"); + assert.equal(lookup.status, "present"); + if (lookup.status !== "present") return; + assert.equal(lookup.descriptor.agent, "player"); + assert.equal(lookup.traces.length, 2); + const u1 = lookup.traces.find((t) => t.utteranceId === "u1"); + assert.ok(u1); + const node = u1.a.nodes.find((n) => n.kind === "grammar-match"); + assert.ok(node && node.kind === "grammar-match"); + // Debug info is rehydrated back onto the node from the deduped table. + assert.equal(node.debugInfo?.grammarHash, "h1"); + assert.equal(node.debugInfo?.rules.length, 1); +}); + +test("debug info is deduped to one blob per grammar hash", async () => { + const state = fakeMemento(); + await saveTraceRun(state, descriptor("run-1", "player"), [ + trace("u1", "h1"), + trace("u2", "h1"), + trace("u3", "h2"), + ]); + const stored = state.raw.get("traceRun.run-1") as { + debugInfos: Record; + }; + // Two distinct grammar hashes across three rows / six sides → two blobs. + assert.deepEqual(Object.keys(stored.debugInfos).sort(), ["h1", "h2"]); +}); + +test("loadResolutionTrace returns a single rehydrated row", async () => { + const state = fakeMemento(); + await saveTraceRun(state, descriptor("run-1", "player"), [ + trace("u1", "h1"), + trace("u2", "h2"), + ]); + const hit = loadResolutionTrace(state, "run-1", "u2"); + assert.ok(hit); + assert.equal(hit.trace.utteranceId, "u2"); + const node = hit.trace.b.nodes.find((n) => n.kind === "grammar-match"); + assert.ok(node && node.kind === "grammar-match"); + assert.equal(node.debugInfo?.grammarHash, "h2"); + assert.equal(loadResolutionTrace(state, "run-1", "nope"), undefined); +}); + +test("a new run for an agent evicts the agent's previous run", async () => { + const state = fakeMemento(); + await saveTraceRun(state, descriptor("run-1", "player"), [ + trace("u1", "h1"), + ]); + await saveTraceRun(state, descriptor("run-2", "player"), [ + trace("u1", "h1", "run-2"), + ]); + + // The superseded run reads back as evicted (it was recorded, then rotated). + assert.equal(loadTraceRun(state, "run-1").status, "evicted"); + assert.equal(loadTraceRun(state, "run-2").status, "present"); +}); + +test("runs for different agents are retained independently", async () => { + const state = fakeMemento(); + await saveTraceRun(state, descriptor("run-1", "player"), [ + trace("u1", "h1"), + ]); + await saveTraceRun(state, descriptor("run-2", "list"), [ + trace("u1", "h1", "run-2"), + ]); + assert.equal(loadTraceRun(state, "run-1").status, "present"); + assert.equal(loadTraceRun(state, "run-2").status, "present"); +}); + +test("a never-stored run reads back as missing", () => { + const state = fakeMemento(); + assert.equal(loadTraceRun(state, "ghost").status, "missing"); +}); + +test("the global cap evicts the oldest runs", async () => { + const state = fakeMemento(); + // Nine distinct agents; the cap keeps the eight most recent runs. + for (let i = 1; i <= 9; i++) { + await saveTraceRun(state, descriptor(`run-${i}`, `agent-${i}`), [ + trace("u1", "h1", `run-${i}`), + ]); + } + assert.equal(loadTraceRun(state, "run-1").status, "evicted"); + assert.equal(loadTraceRun(state, "run-2").status, "present"); + assert.equal(loadTraceRun(state, "run-9").status, "present"); +}); diff --git a/ts/packages/typeagent-studio/src/test/webviewBundle.spec.ts b/ts/packages/typeagent-studio/src/test/webviewBundle.spec.ts index 6d2727477b..ae3d08abd8 100644 --- a/ts/packages/typeagent-studio/src/test/webviewBundle.spec.ts +++ b/ts/packages/typeagent-studio/src/test/webviewBundle.spec.ts @@ -16,29 +16,46 @@ const bundlePath = path.join( "impactReport.js", ); +const traceViewerBundlePath = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "..", + "dist", + "webview", + "traceViewer.js", +); + // The webview client bundle is produced by `pnpm build` (esbuild browser // target). Skip gracefully if tests run before a build. const hasBundle = existsSync(bundlePath); +function assertBrowserSafe(src: string): void { + // No CommonJS requires of node/host-only modules leaking into the iframe. + assert.ok(!/require\(["']ws["']\)/.test(src), "must not bundle ws"); + assert.ok(!/require\(["']vscode["']\)/.test(src), "must not bundle vscode"); + assert.ok( + !/require\(["']node:/.test(src), + "must not bundle node: built-ins", + ); + assert.ok( + !/require\(["'](?:crypto|fs|os|path|net|http)["']\)/.test(src), + "must not bundle bare node built-ins", + ); +} + test( "webview bundle excludes node/vscode/ws (browser-safe)", { skip: !hasBundle }, () => { - const src = readFileSync(bundlePath, "utf8"); - // No CommonJS requires of node/host-only modules leaking into the iframe. - assert.ok(!/require\(["']ws["']\)/.test(src), "must not bundle ws"); - assert.ok( - !/require\(["']vscode["']\)/.test(src), - "must not bundle vscode", - ); - assert.ok( - !/require\(["']node:/.test(src), - "must not bundle node: built-ins", - ); - assert.ok( - !/require\(["'](?:crypto|fs|os|path|net|http)["']\)/.test(src), - "must not bundle bare node built-ins", - ); + assertBrowserSafe(readFileSync(bundlePath, "utf8")); + }, +); + +test( + "trace viewer bundle excludes node/vscode/ws (browser-safe)", + { skip: !existsSync(traceViewerBundlePath) }, + () => { + assertBrowserSafe(readFileSync(traceViewerBundlePath, "utf8")); }, ); diff --git a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts index a9b9e9a248..31589191a1 100644 --- a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts +++ b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts @@ -136,6 +136,50 @@ test("parseWebviewMessage accepts well-formed messages", () => { ); }); +test("parseWebviewMessage accepts an openTrace request", () => { + assert.deepEqual( + parseWebviewMessage({ + type: "openTrace", + runId: "run-1", + utteranceId: "u-7", + }), + { type: "openTrace", runId: "run-1", utteranceId: "u-7" }, + ); +}); + +test("parseWebviewMessage rejects an openTrace missing ids", () => { + assert.equal( + parseWebviewMessage({ type: "openTrace", runId: "run-1" }), + undefined, + ); + assert.equal( + parseWebviewMessage({ type: "openTrace", runId: 1, utteranceId: "u" }), + undefined, + ); +}); + +test("parseWebviewMessage accepts a focusTrace request", () => { + assert.deepEqual( + parseWebviewMessage({ + type: "focusTrace", + runId: "run-1", + utteranceId: "u-7", + }), + { type: "focusTrace", runId: "run-1", utteranceId: "u-7" }, + ); +}); + +test("parseWebviewMessage rejects a focusTrace missing ids", () => { + assert.equal( + parseWebviewMessage({ type: "focusTrace", runId: "run-1" }), + undefined, + ); + assert.equal( + parseWebviewMessage({ type: "focusTrace", runId: 1, utteranceId: "u" }), + undefined, + ); +}); + test("parseWebviewMessage rejects malformed / hostile input", () => { assert.equal(parseWebviewMessage(undefined), undefined); assert.equal(parseWebviewMessage(null), undefined); diff --git a/ts/packages/typeagent-studio/src/traceSourceResolver.ts b/ts/packages/typeagent-studio/src/traceSourceResolver.ts new file mode 100644 index 0000000000..1f667ea9f5 --- /dev/null +++ b/ts/packages/typeagent-studio/src/traceSourceResolver.ts @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Pure extraction of a jump target from a captured trace. Given a side and the + * node the webview asked to open, this reads the source span the trace recorded + * — the winning grammar rule's location, or the produced action's schema file — + * and returns the absolute path (plus a range for grammar rules). The host then + * resolves that path against the side's pinned version to open it; nothing here + * touches the filesystem or git, so it unit-tests without a workspace. + */ + +import type { + ReplayResolutionTrace, + GrammarMatchTraceNode, + ActionTraceNode, +} from "@typeagent/core/replay"; +import type { TraceSide, TraceSourceNode } from "./webviewKit/traceProtocol.js"; + +export interface TraceSourcePosition { + line: number; + character: number; +} +export interface TraceSourceRange { + start: TraceSourcePosition; + end: TraceSourcePosition; +} + +/** Where a jump should land: an absolute path, and (for grammar rules) the exact + * span to select. Action jumps carry no range — only the schema file. */ +export interface TraceSourceTarget { + absPath: string; + range?: TraceSourceRange; +} + +/** Resolve the grammar node's `.agr` to an absolute path: the node's recorded + * `sourceFilePath` (the real repo file, captured on both sides) when present, + * else — for older captures — the winning-rule span's file table (fileId → + * resolved path), falling back to the span's display path. */ +function grammarAbsolutePath(node: GrammarMatchTraceNode): string | undefined { + if (node.sourceFilePath !== undefined) { + return node.sourceFilePath; + } + const source = node.source; + if (source === undefined) { + return undefined; + } + const filePaths = node.debugInfo?.filePaths; + if (filePaths !== undefined) { + for (const [id, resolved] of filePaths) { + if (id === source.fileId) { + return resolved; + } + } + } + return source.displayPath; +} + +/** + * The source target for `{side, node}`, or `undefined` when the trace recorded + * no location (e.g. the grammar produced no winning-rule span, or the action had + * no schema file). The returned position is 0-based line/character, matching the + * editor's coordinate space. + */ +export function sourceTargetFor( + trace: ReplayResolutionTrace, + side: TraceSide, + node: TraceSourceNode, +): TraceSourceTarget | undefined { + const sideTrace = side === "a" ? trace.a : trace.b; + if (node === "grammar-match") { + const grammar = sideTrace.nodes.find( + (n) => n.kind === "grammar-match", + ) as GrammarMatchTraceNode | undefined; + if (grammar === undefined) { + return undefined; + } + const absPath = grammarAbsolutePath(grammar); + if (absPath === undefined) { + return undefined; + } + // The winning-rule span drives the jump-to-line selection; a side that + // matched no rule (or an older capture) still yields the file path for a + // cross-version diff, just without a range to select. + const source = grammar.source; + return { + absPath, + ...(source !== undefined + ? { + range: { + start: { + line: source.range.start.line, + character: source.range.start.character, + }, + end: { + line: source.range.end.line, + character: source.range.end.character, + }, + }, + } + : {}), + }; + } + const action = sideTrace.nodes.find((n) => n.kind === "action") as + | ActionTraceNode + | undefined; + const sourceFilePath = action?.schema?.sourceFilePath; + return sourceFilePath !== undefined + ? { absPath: sourceFilePath } + : undefined; +} diff --git a/ts/packages/typeagent-studio/src/traceStore.ts b/ts/packages/typeagent-studio/src/traceStore.ts new file mode 100644 index 0000000000..cc8bd2bb7a --- /dev/null +++ b/ts/packages/typeagent-studio/src/traceStore.ts @@ -0,0 +1,235 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type * as vscode from "vscode"; +import type { + ReplayResolutionTrace, + ReplayRunDescriptor, + ReplaySideTrace, + ReplayTraceNode, + SerializedGrammarDebugInfo, +} from "@typeagent/core/replay"; + +// Per-red-row resolution traces + the run descriptor are persisted here so the +// Trace Viewer can open the exact side-by-side resolution that produced a row +// long after the run streamed, and can recompute a fresh trace from the same +// pinned inputs. Retention mirrors the Impact Report: one run is kept per agent +// (a new run for the agent evicts the previous one), and a small global cap +// bounds how many agents' runs are retained at once. + +/** Keep the last run for at most this many agents so the persisted payload stays + * bounded even if the user cycles through many agents. */ +const MAX_RETAINED_RUNS = 8; + +const runKey = (runId: string): string => `traceRun.${runId}`; +const agentRunKey = (agent: string): string => `traceRun.byAgent.${agent}`; +const RUN_ORDER_KEY = "traceRun.order"; +const EVICTED_KEY = "traceRun.evicted"; + +/** Remember this many recently evicted run ids so the viewer can distinguish a + * run whose traces were rotated out ("evicted") from a run id it never stored + * ("missing"). */ +const MAX_EVICTED_TOMBSTONES = 32; + +/** A grammar-match node with its heavy debug info lifted into the run-level + * dedupe table, referenced by the grammar hash it was stored under. */ +type StoredNode = ReplayTraceNode & { debugInfoRef?: string }; + +/** The persisted form of a run's traces: the descriptor, every row's trace with + * grammar debug info stripped to a per-run reference, and the deduped debug-info + * blobs keyed by grammar hash (one copy per distinct grammar, not per row). */ +interface StoredTraceRun { + descriptor: ReplayRunDescriptor; + agent: string; + capturedAt: number; + debugInfos: Record; + rows: Record; +} + +/** The outcome of looking up a run's traces. `present` carries the rehydrated + * traces; `evicted` means the run existed but its traces were rotated out (the + * report may outlive them); `missing` means the run was never stored. */ +export type TraceRunLookup = + | { + status: "present"; + descriptor: ReplayRunDescriptor; + traces: ReplayResolutionTrace[]; + } + | { status: "evicted" } + | { status: "missing" }; + +/** JSON round-trips through the Memento anyway; clone up front so stripping the + * debug info never mutates the caller's traces. */ +function clone(value: T): T { + return JSON.parse(JSON.stringify(value)) as T; +} + +function grammarNodes(side: ReplaySideTrace): StoredNode[] { + return side.nodes as StoredNode[]; +} + +/** Move each grammar-match node's debug info into `debugInfos` (keyed by its + * grammar hash) and replace it on the node with a lightweight reference, so a + * grammar shared across rows is stored once per run rather than per row. */ +function stripDebugInfo( + trace: ReplayResolutionTrace, + debugInfos: Record, +): ReplayResolutionTrace { + const copy = clone(trace); + for (const side of [copy.a, copy.b]) { + for (const node of grammarNodes(side)) { + if (node.kind !== "grammar-match" || node.debugInfo === undefined) { + continue; + } + const hash = node.debugInfo.grammarHash; + if (debugInfos[hash] === undefined) { + debugInfos[hash] = node.debugInfo; + } + node.debugInfoRef = hash; + node.debugInfo = undefined; + } + } + return copy; +} + +/** Reverse {@link stripDebugInfo}: put each grammar-match node's debug info back + * from the run-level table so the returned trace is fully self-contained. */ +function rehydrateDebugInfo( + trace: ReplayResolutionTrace, + debugInfos: Record, +): ReplayResolutionTrace { + const copy = clone(trace); + for (const side of [copy.a, copy.b]) { + for (const node of grammarNodes(side)) { + if ( + node.kind !== "grammar-match" || + node.debugInfoRef === undefined + ) { + continue; + } + const info = debugInfos[node.debugInfoRef]; + if (info !== undefined) { + node.debugInfo = info; + } + node.debugInfoRef = undefined; + } + } + return copy; +} + +/** Persist a run's descriptor and its captured red-row traces. Evicts the + * agent's previous run and enforces the global retention cap so the workspace + * state stays bounded. Awaits the underlying writes so the data is durable + * before the caller proceeds. */ +export async function saveTraceRun( + state: vscode.Memento, + descriptor: ReplayRunDescriptor, + traces: readonly ReplayResolutionTrace[], +): Promise { + const debugInfos: Record = {}; + const rows: Record = {}; + for (const trace of traces) { + rows[trace.utteranceId] = stripDebugInfo(trace, debugInfos); + } + const stored: StoredTraceRun = { + descriptor, + agent: descriptor.agent, + capturedAt: Date.now(), + debugInfos, + rows, + }; + + // Evict this agent's previous run before recording the new one. + const previousForAgent = state.get(agentRunKey(descriptor.agent)); + if ( + previousForAgent !== undefined && + previousForAgent !== descriptor.runId + ) { + await evictRun(state, previousForAgent); + } + + await state.update(runKey(descriptor.runId), stored); + await state.update(agentRunKey(descriptor.agent), descriptor.runId); + + const order = (state.get(RUN_ORDER_KEY) ?? []).filter( + (id) => id !== descriptor.runId, + ); + order.push(descriptor.runId); + await state.update(RUN_ORDER_KEY, order); + + // evictRun rewrites the order list in state, so re-read it each pass rather + // than trusting the stale local copy. + let retained = order; + while (retained.length > MAX_RETAINED_RUNS) { + await evictRun(state, retained[0]); + retained = state.get(RUN_ORDER_KEY) ?? []; + } +} + +/** Remove a run's stored blob, drop it from the retained order, clear its agent + * pointer, and record a bounded tombstone so a later lookup reports `evicted` + * rather than `missing`. */ +async function evictRun(state: vscode.Memento, runId: string): Promise { + const stored = state.get(runKey(runId)); + await state.update(runKey(runId), undefined); + if (stored !== undefined) { + const pointer = state.get(agentRunKey(stored.agent)); + if (pointer === runId) { + await state.update(agentRunKey(stored.agent), undefined); + } + } + + const order = (state.get(RUN_ORDER_KEY) ?? []).filter( + (id) => id !== runId, + ); + await state.update(RUN_ORDER_KEY, order); + + const evicted = (state.get(EVICTED_KEY) ?? []).filter( + (id) => id !== runId, + ); + evicted.push(runId); + while (evicted.length > MAX_EVICTED_TOMBSTONES) { + evicted.shift(); + } + await state.update(EVICTED_KEY, evicted); +} + +/** Look up a run's captured traces, rehydrated with their grammar debug info. + * Returns `evicted` when the run was recorded but has since been rotated out, + * and `missing` when it was never stored. */ +export function loadTraceRun( + state: vscode.Memento, + runId: string, +): TraceRunLookup { + const stored = state.get(runKey(runId)); + if (stored === undefined) { + const evicted = state.get(EVICTED_KEY) ?? []; + return evicted.includes(runId) + ? { status: "evicted" } + : { status: "missing" }; + } + const traces = Object.values(stored.rows).map((row) => + rehydrateDebugInfo(row, stored.debugInfos), + ); + return { status: "present", descriptor: stored.descriptor, traces }; +} + +/** Look up a single row's trace by run and utterance, rehydrated with its + * grammar debug info. `undefined` when the run or row is not present. */ +export function loadResolutionTrace( + state: vscode.Memento, + runId: string, + utteranceId: string, +): + | { descriptor: ReplayRunDescriptor; trace: ReplayResolutionTrace } + | undefined { + const stored = state.get(runKey(runId)); + const row = stored?.rows[utteranceId]; + if (stored === undefined || row === undefined) { + return undefined; + } + return { + descriptor: stored.descriptor, + trace: rehydrateDebugInfo(row, stored.debugInfos), + }; +} diff --git a/ts/packages/typeagent-studio/src/traceViewerView.ts b/ts/packages/typeagent-studio/src/traceViewerView.ts new file mode 100644 index 0000000000..b857bc1883 --- /dev/null +++ b/ts/packages/typeagent-studio/src/traceViewerView.ts @@ -0,0 +1,489 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as vscode from "vscode"; +import * as path from "node:path"; +import { realpath } from "node:fs/promises"; +import { WebviewKitPanel } from "./webviewKit/host.js"; +import { + parseTraceMessage, + type HostToTraceMessage, + type TraceProvenanceSummary, + type TraceVersionSummary, + type TraceSide, + type TraceSourceNode, +} from "./webviewKit/traceProtocol.js"; +import { loadResolutionTrace, loadTraceRun } from "./traceStore.js"; +import { + sourceTargetFor, + type TraceSourceRange, +} from "./traceSourceResolver.js"; +import { defaultGitExec } from "./gitRefProvider.js"; +import type { + ReplayRunDescriptor, + TraceVersionPin, +} from "@typeagent/core/replay"; + +const VIEW_TYPE = "typeagentStudio.traceViewer"; + +/** URI scheme for the read-only virtual documents that show a grammar/schema + * file at a pinned git ref (the exact content the ref side resolved against). */ +const TRACE_SOURCE_SCHEME = "typeagent-trace-src"; + +/** The tab title is static: one viewer follows the selected row, and the + * utterance it currently shows is in the panel header. */ +const PANEL_TITLE = "Trace Viewer"; + +function toVersionSummary(pin: TraceVersionPin): TraceVersionSummary { + return { + label: pin.label, + workingTree: pin.workingTree, + ...(pin.sha !== undefined ? { sha: pin.sha } : {}), + }; +} + +function toProvenance(descriptor: ReplayRunDescriptor): TraceProvenanceSummary { + return { + agent: descriptor.agent, + a: toVersionSummary(descriptor.a), + b: toVersionSummary(descriptor.b), + runAt: descriptor.runAt, + }; +} + +/** Encode a git-pinned file into a virtual-document URI the content provider can + * re-open: the ref content is fetched lazily via `git show :`. */ +function traceSourceUri( + repoRoot: string, + sha: string, + relPath: string, +): vscode.Uri { + const query = new URLSearchParams({ repo: repoRoot, sha, path: relPath }); + return vscode.Uri.from({ + scheme: TRACE_SOURCE_SCHEME, + // A leading-slash path gives the tab a readable file name and lets the + // editor pick a language mode from the extension. + path: `/${relPath}`, + query: query.toString(), + }); +} + +/** Read-only view of a grammar/schema file at a pinned ref, so the ref side of a + * trace jumps to the exact bytes it resolved against rather than the (possibly + * drifted) working tree. A file absent at the ref reads as empty so a compare + * shows it as a clean addition/removal rather than an error line. */ +class TraceSourceContentProvider implements vscode.TextDocumentContentProvider { + async provideTextDocumentContent(uri: vscode.Uri): Promise { + const query = new URLSearchParams(uri.query); + const repoRoot = query.get("repo"); + const sha = query.get("sha"); + const relPath = query.get("path"); + if (!repoRoot || !sha || !relPath) { + return ""; + } + try { + const git = defaultGitExec(repoRoot); + return await git(["show", "--end-of-options", `${sha}:${relPath}`]); + } catch { + // The URI is only minted after the repo is validated, so the sole + // realistic failure here is "this path didn't exist at that ref". + // Return empty so a one-sided file reads as an add/remove in a diff. + return ""; + } + } +} + +/** Resolve a captured absolute source path to its git top-level and the path + * relative to that top-level, so `git show :` reads it from the + * right tree even when the workspace root is a subdirectory of the repo. The + * path is canonicalized (best-effort realpath) the same way the resolver + * recorded refs, so symlinked temp/working dirs resolve consistently. */ +async function gitRefRelPath( + absPath: string, +): Promise<{ gitRoot: string; relPath: string } | undefined> { + let real = absPath; + try { + real = await realpath(absPath); + } catch { + // The file may not exist in the working tree (only at the ref); the raw + // path's directory still locates the repository. + } + let gitRoot: string; + try { + gitRoot = ( + await defaultGitExec(path.dirname(real))([ + "rev-parse", + "--show-toplevel", + ]) + ).trim(); + } catch { + return undefined; + } + if (gitRoot.length === 0) { + return undefined; + } + const relPath = path.relative(gitRoot, real).split(path.sep).join("/"); + // A leading ".." *segment* means the path escapes the repo root. A segment + // that merely begins with two dots (e.g. "..foo") is a real in-repo file and + // must be allowed, so match the segment, not a raw "..". + const escapesRepo = relPath === ".." || relPath.startsWith("../"); + if (escapesRepo || path.isAbsolute(relPath)) { + return undefined; + } + return { gitRoot, relPath }; +} + +let sourceProviderRegistered = false; + +/** Register the virtual-document provider once per extension activation. */ +function ensureSourceProvider(context: vscode.ExtensionContext): void { + if (sourceProviderRegistered) { + return; + } + sourceProviderRegistered = true; + context.subscriptions.push( + vscode.workspace.registerTextDocumentContentProvider( + TRACE_SOURCE_SCHEME, + new TraceSourceContentProvider(), + ), + ); +} + +/** Open a document in `viewColumn` and, when a span is known, select and reveal + * it. The selection is best-effort: a document shorter than the recorded span + * (e.g. drifted working-tree content) still opens without throwing. */ +async function revealSource( + uri: vscode.Uri, + range: TraceSourceRange | undefined, + viewColumn: vscode.ViewColumn, +): Promise { + const doc = await vscode.workspace.openTextDocument(uri); + const selection = + range !== undefined + ? new vscode.Range( + new vscode.Position(range.start.line, range.start.character), + new vscode.Position(range.end.line, range.end.character), + ) + : undefined; + await vscode.window.showTextDocument(doc, { + viewColumn, + preview: true, + ...(selection !== undefined ? { selection } : {}), + }); +} + +/** The editor column to open a source file or diff in: the group immediately + * past the Trace Viewer's own, so opened sources land in a stable side panel + * beside the viewer — reused across opens — rather than ever replacing the + * webview in its own column. Falls back to `Beside` when the viewer's column + * isn't currently known (e.g. while it's hidden). */ +function sourceColumnBeside(panel: WebviewKitPanel): vscode.ViewColumn { + const col = panel.panel.viewColumn; + return col === undefined + ? vscode.ViewColumn.Beside + : (Math.min(col + 1, 9) as vscode.ViewColumn); +} + +/** Handle to the one live Trace Viewer: bring it forward, or point it at a + * different row. */ +interface TraceViewerHandle { + reveal(): void; + retarget(runId: string, utteranceId: string): void; +} + +/** The single live Trace Viewer, or undefined when none is open. One viewer + * follows the selected row rather than opening a tab per row. */ +let liveViewer: TraceViewerHandle | undefined; + +/** + * Open (or reveal) the Trace Viewer for one red row of a run, side-by-side with + * the Impact Report it drills in from. A single viewer follows the selection: + * opening it for another row re-targets the existing panel instead of stacking + * up a tab per row. + * + * The stored trace and its run descriptor are the source of truth: the viewer + * shows the exact resolution the row was produced from, and can diff a node's + * source across the two versions. When the run's traces have been rotated out of + * the durable store, the viewer shows an explicit evicted/missing state instead + * of a blank panel. + */ +export function openTraceViewer( + context: vscode.ExtensionContext, + repoRoot: string | undefined, + runId: string, + utteranceId: string, +): void { + if (liveViewer) { + liveViewer.retarget(runId, utteranceId); + liveViewer.reveal(); + return; + } + liveViewer = createTraceViewer(context, repoRoot, runId, utteranceId); +} + +/** + * Point an already-open Trace Viewer at a different row. A no-op when no viewer + * is open, so switching the selected row in the report follows into an existing + * viewer without ever opening a new one. + */ +export function focusTraceViewer(runId: string, utteranceId: string): void { + liveViewer?.retarget(runId, utteranceId); +} + +function createTraceViewer( + context: vscode.ExtensionContext, + repoRoot: string | undefined, + runId: string, + utteranceId: string, +): TraceViewerHandle { + // The row the viewer currently shows; retargeting swaps these so the reused + // panel re-pushes a different row's trace. + let currentRunId = runId; + let currentUtteranceId = utteranceId; + + let webviewReady = false; + + ensureSourceProvider(context); + + const panel = WebviewKitPanel.createOrReveal(context, { + viewType: VIEW_TYPE, + title: PANEL_TITLE, + // Sit next to the originating Impact Report instead of taking its column. + viewColumn: vscode.ViewColumn.Beside, + scriptPath: ["dist", "webview", "traceViewer.js"], + stylePath: ["media", "traceViewer.css"], + // The rendered trace is re-pushed only via the reveal `ready` handshake; + // retain context so navigating away and back keeps that state. + retainContextWhenHidden: true, + onMessage: (raw) => void handleMessage(raw), + onDispose: () => { + liveViewer = undefined; + }, + }); + + const post = (message: HostToTraceMessage) => panel.post(message); + + // Read the stored trace and push it, or explain why it can't be shown. The + // per-row lookup answers "present"; a miss is disambiguated at the run level + // into evicted (rotated out) vs missing (never captured). + const pushTrace = (): void => { + const found = loadResolutionTrace( + context.workspaceState, + currentRunId, + currentUtteranceId, + ); + if (found) { + post({ + type: "trace", + recorded: found.trace, + provenance: toProvenance(found.descriptor), + }); + return; + } + const run = loadTraceRun(context.workspaceState, currentRunId); + post({ + type: "trace-state", + state: run.status === "evicted" ? "evicted" : "missing", + }); + }; + + // Build the URI that shows one node's source at a given side's version: the + // live working-tree file, or a read-only virtual document at the pinned git + // ref. Returns undefined when the side has no openable version. + const uriForSide = ( + pin: TraceVersionPin, + gitRoot: string, + relPath: string, + ): vscode.Uri | undefined => { + if (pin.workingTree) { + return vscode.Uri.file(path.join(gitRoot, relPath)); + } + if (pin.sha !== undefined) { + return traceSourceUri(gitRoot, pin.sha, relPath); + } + return undefined; + }; + + // Jump to the source the row resolved against: the winning grammar rule's + // span, or the produced action's schema file. The webview names only the + // side and node; the host rederives the location from the stored trace and + // opens it against that side's pinned version — the live working-tree file, + // or a read-only virtual document at the pinned git ref. + const openSource = async ( + requestId: number, + side: TraceSide, + node: TraceSourceNode, + ): Promise => { + const unavailable = (message: string): void => + post({ + type: "source-result", + requestId, + status: "unavailable", + message, + }); + + const found = loadResolutionTrace( + context.workspaceState, + currentRunId, + currentUtteranceId, + ); + if (!found) { + unavailable("The recorded trace is no longer available."); + return; + } + const target = sourceTargetFor(found.trace, side, node); + if (!target) { + unavailable("This step didn't record a source location."); + return; + } + if (!path.isAbsolute(target.absPath)) { + // A relative or synthetic path (e.g. built-in entity grammar) can't + // be located on disk; the webview normally hides such links, but a + // stale one still lands here. + unavailable("This step's source file isn't available to open."); + return; + } + const pin = side === "a" ? found.descriptor.a : found.descriptor.b; + const column = sourceColumnBeside(panel); + try { + if (pin.workingTree) { + await revealSource( + vscode.Uri.file(target.absPath), + target.range, + column, + ); + } else if (pin.sha !== undefined) { + const resolved = await gitRefRelPath(target.absPath); + if (resolved === undefined) { + unavailable("The source file is outside the repository."); + return; + } + await revealSource( + traceSourceUri(resolved.gitRoot, pin.sha, resolved.relPath), + target.range, + column, + ); + } else { + unavailable("This version has no source to open."); + return; + } + post({ type: "source-result", requestId, status: "opened" }); + } catch { + unavailable("Couldn't open the source file."); + } + }; + + // Open a native side-by-side diff of one node's source file across the A and + // B versions. The same repo-relative path is diffed on both sides (each side + // resolved against its own pinned version), so a grammar/schema change shows + // as a concrete source delta rather than two separate file jumps. + const compareSource = async ( + requestId: number, + node: TraceSourceNode, + ): Promise => { + const unavailable = (message: string): void => + post({ + type: "source-result", + requestId, + status: "unavailable", + message, + }); + + const found = loadResolutionTrace( + context.workspaceState, + currentRunId, + currentUtteranceId, + ); + if (!found) { + unavailable("The recorded trace is no longer available."); + return; + } + // Either side may have recorded the span; use whichever resolves to a + // real file to name the path to diff across both versions. + const target = + sourceTargetFor(found.trace, "b", node) ?? + sourceTargetFor(found.trace, "a", node); + if (!target || !path.isAbsolute(target.absPath)) { + unavailable("This step's source file isn't available to compare."); + return; + } + const resolved = await gitRefRelPath(target.absPath); + if (resolved === undefined) { + unavailable("The source file is outside the repository."); + return; + } + const { gitRoot, relPath } = resolved; + const left = uriForSide(found.descriptor.a, gitRoot, relPath); + const right = uriForSide(found.descriptor.b, gitRoot, relPath); + if (!left || !right) { + unavailable("One version has no source to compare."); + return; + } + // When both versions pin the same working-tree file, the two sides of the + // diff would be byte-identical — VS Code renders that as a single pane, + // which reads as "the compare is broken". Say plainly that this file is + // shared, so the divergence lives in another stage. + if (left.toString() === right.toString()) { + unavailable( + "Both versions resolve to the same working-tree file, so there's no cross-version diff to show here.", + ); + return; + } + const name = relPath.split("/").pop() ?? relPath; + const title = `${name} — ${found.descriptor.a.label} ↔ ${found.descriptor.b.label}`; + try { + await vscode.commands.executeCommand( + "vscode.diff", + left, + right, + title, + { viewColumn: sourceColumnBeside(panel), preview: true }, + ); + post({ type: "source-result", requestId, status: "opened" }); + } catch { + unavailable("Couldn't open the source comparison."); + } + }; + + const handleMessage = async (raw: unknown): Promise => { + const message = parseTraceMessage(raw); + if (!message) { + return; + } + switch (message.type) { + case "ready": + webviewReady = true; + pushTrace(); + break; + case "open-source": + await openSource(message.requestId, message.side, message.node); + break; + case "compare-source": + await compareSource(message.requestId, message.node); + break; + } + }; + + // Swap the viewer to a different row and re-push its trace. Skips redundant + // work when the row is already showing (e.g. re-clicking the open row, or the + // button for the shown row). The tab title is static, so only the content + // changes. + const retarget = (nextRunId: string, nextUtteranceId: string): void => { + if ( + nextRunId === currentRunId && + nextUtteranceId === currentUtteranceId + ) { + return; + } + currentRunId = nextRunId; + currentUtteranceId = nextUtteranceId; + if (webviewReady) { + pushTrace(); + } + }; + + return { + reveal: () => panel.panel.reveal(), + retarget, + }; +} diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts index 5037e240ad..9bf925228e 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts @@ -115,6 +115,10 @@ let currentTotal = 0; // Raw deltas of the current result keyed by utterance id, so a row drill-in can // build the action A/B diff without the host re-sending the payload. let currentRawById = new Map(); +// Utterance ids that captured a resolution trace this run (changed rows first, +// then equal rows up to the capture cap), so the drill-in offers "Open trace" +// exactly for the rows the viewer can show. +let tracedIds: Set = new Set(); // The utterance id whose drill-in detail is open, or undefined when closed; kept // so a filter re-render can re-assert (or drop) the open detail. let openDetailId: string | undefined; @@ -122,6 +126,10 @@ let openDetailId: string | undefined; // from openDetailId so equal rows — which have no A/B diff to show — can still // read as "selected" while the detail pane stays closed. let selectedId: string | undefined; +// The utterance ids of the currently rendered clickable rows, in display order +// (after filtering and sorting), so Up/Down arrows can walk row-to-row. Rebuilt +// on every table render since sorting/filtering reorders the set. +let clickableOrder: string[] = []; // Row-level filter state: the set of hidden utterance ids. Chips over the same // rows overlap, so visibility is tracked per row rather than per chip. Empty // means nothing is hidden (the "All" pill is lit); toggling a chip hides or @@ -327,7 +335,31 @@ document.addEventListener("keydown", (event) => { if (event.key === "Escape" && !settingsPopover.hidden) { toggleSettingsPopover(false); settingsButton.focus(); + return; + } + // Up/Down walk the open row selection from anywhere in the report, so + // navigation survives focus leaving the row (e.g. a click on empty space or + // the detail pane). The settings popover, an open context menu, and text + // fields keep their own arrow behaviour. + if (event.key === "ArrowDown" || event.key === "ArrowUp") { + if (openDetailId === undefined) return; + if (!settingsPopover.hidden || openContextMenu !== undefined) return; + const tag = (event.target as HTMLElement | null)?.tagName; + if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return; + event.preventDefault(); + moveRowSelection(event.key === "ArrowDown" ? 1 : -1, openDetailId); + } +}); +// Reclaim keyboard focus when the pointer moves over the report while a row is +// selected but focus sits in another panel, so hovering back in resumes Up/Down +// without a click. `hasFocus` gates it to only grab focus when it's elsewhere. +document.addEventListener("mouseover", () => { + if (openDetailId === undefined || document.hasFocus()) { + return; } + tableWrap + .querySelector(".row-open") + ?.focus({ preventScroll: true }); }); // --- Messaging ------------------------------------------------------------ @@ -452,6 +484,12 @@ function runReplay(): void { filterChipsEl.textContent = ""; filtersEl.hidden = true; emptyStateEl.hidden = true; + // Drop the prior run's verdict headline and fidelity matrix so a stale + // judgment isn't left standing while the new replay is in flight. + verdictBannerEl.textContent = ""; + verdictBannerEl.hidden = true; + fidelityEl.textContent = ""; + fidelityEl.hidden = true; currentRows = []; hiddenRowIds = new Set(); resetSearchAndSort(); @@ -575,6 +613,10 @@ function renderResult( hiddenRowIds = new Set(); resetSearchAndSort(); currentRawById = new Map(result.rows.map((r) => [r.utteranceId, r])); + tracedIds = new Set( + result.tracedUtteranceIds ?? + (result.resolutionTraces ?? []).map((t) => t.utteranceId), + ); closeDetail(); // A run-level error (a version that failed to build) aborts with an empty // summary — surface the failure instead of a misleading zero-row success. @@ -742,8 +784,8 @@ function renderFidelity( const grid = el("div", "fidelity-grid"); grid.append( fidelityHeadCell("Layer"), - fidelityHeadCell("A"), - fidelityHeadCell("B"), + fidelityHeadCell("A", "a"), + fidelityHeadCell("B", "b"), ); for (const row of view.rows) { grid.append( @@ -758,8 +800,11 @@ function renderFidelity( fidelityEl.hidden = false; } -function fidelityHeadCell(label: string): HTMLElement { +function fidelityHeadCell(label: string, side?: "a" | "b"): HTMLElement { const cell = el("span", "fidelity-head"); + if (side !== undefined) { + cell.classList.add(`fidelity-head-${side}`); + } cell.textContent = label; return cell; } @@ -1016,9 +1061,30 @@ function renderTable(): void { table.appendChild(tbody); tableWrap.appendChild(table); + // Record the clickable rows in display order so arrow-key navigation walks + // exactly the rows a click could open, in the order they're shown. + clickableOrder = rows + .filter((r) => currentRawById.has(r.utteranceId)) + .map((r) => r.utteranceId); + appendTableNotes(rows); } +/** Move the selection to the clickable row `delta` steps from `fromId` (a no-op + * at either end). `openDetail` re-homes focus to the newly-rendered row, so a + * run of Up/Down walks the table and carries an open Trace Viewer along. */ +function moveRowSelection(delta: 1 | -1, fromId: string): void { + const idx = clickableOrder.indexOf(fromId); + if (idx === -1) { + return; + } + const nextIdx = idx + delta; + if (nextIdx < 0 || nextIdx >= clickableOrder.length) { + return; + } + openDetail(clickableOrder[nextIdx]); +} + interface HeaderSpec { label: string; title: string; @@ -1141,9 +1207,9 @@ function buildTableRow(row: ImpactRow): HTMLTableRowElement { cell(row.resolutionB, "resolution", resolutionTooltip(row.resolutionB)), ); tr.appendChild(latencyCell(row)); - // Every row is clickable. Difference rows drill into an action A/B diff; - // equal rows have nothing to compare, so a click just clears any open - // detail pane. + // Every row with a backing delta is clickable: difference rows drill into an + // action A/B diff, equal rows show the resolved action JSON. Right-clicking a + // row offers to open its resolution in the Trace Viewer. if (!currentRawById.has(row.utteranceId)) { return tr; } @@ -1156,9 +1222,8 @@ function buildTableRow(row: ImpactRow): HTMLTableRowElement { tr.setAttribute("role", "button"); tr.title = isDiff ? "Show the action A/B diff for this utterance." - : "Equal row — click to close the detail diff."; - const activate = () => - isDiff ? openDetail(row.utteranceId) : selectEqualRow(row.utteranceId); + : "Show the resolved action for this utterance."; + const activate = () => openDetail(row.utteranceId); tr.addEventListener("click", activate); tr.addEventListener("keydown", (e: KeyboardEvent) => { if (e.key === "Enter" || e.key === " ") { @@ -1166,9 +1231,98 @@ function buildTableRow(row: ImpactRow): HTMLTableRowElement { activate(); } }); + tr.addEventListener("contextmenu", (e: MouseEvent) => + openRowContextMenu(e, row.utteranceId), + ); return tr; } +/** The floating row context menu, when open. Only one shows at a time. */ +let openContextMenu: HTMLElement | undefined; + +/** Tear down the row context menu and the listeners that dismiss it. */ +function closeRowContextMenu(): void { + if (openContextMenu === undefined) { + return; + } + openContextMenu.remove(); + openContextMenu = undefined; + document.removeEventListener("pointerdown", onContextMenuPointerDown, true); + document.removeEventListener("keydown", onContextMenuKeydown, true); + window.removeEventListener("blur", closeRowContextMenu); + window.removeEventListener("resize", closeRowContextMenu); +} + +function onContextMenuPointerDown(e: PointerEvent): void { + if (openContextMenu && !openContextMenu.contains(e.target as Node)) { + closeRowContextMenu(); + } +} + +function onContextMenuKeydown(e: KeyboardEvent): void { + if (e.key === "Escape") { + e.preventDefault(); + closeRowContextMenu(); + } +} + +/** Right-click menu on a drillable row: a single "Open in Trace Viewer" action. + * It is disabled when the run captured no resolution trace for the row, so the + * affordance is consistent across every row even when a trace isn't available + * (e.g. a row beyond the per-run capture cap, or a run predating the trace). */ +function openRowContextMenu(event: MouseEvent, utteranceId: string): void { + event.preventDefault(); + closeRowContextMenu(); + if (!lastRenderedResult) { + return; + } + const runId = lastRenderedResult.runId; + const menu = el("div", "context-menu"); + const item = document.createElement("button"); + item.type = "button"; + item.className = "context-menu-item"; + item.textContent = "Open in Trace Viewer"; + if (tracedIds.has(utteranceId)) { + item.addEventListener("click", () => { + closeRowContextMenu(); + openDetail(utteranceId); + vscode.postMessage({ type: "openTrace", runId, utteranceId }); + }); + } else { + item.disabled = true; + item.classList.add("is-disabled"); + item.title = "No resolution trace was captured for this row."; + } + menu.appendChild(item); + document.body.appendChild(menu); + openContextMenu = menu; + + // Keep the menu inside the viewport, nudging it back from the right/bottom + // edges when the cursor is near them. + const rect = menu.getBoundingClientRect(); + const x = Math.max( + 4, + Math.min(event.clientX, window.innerWidth - rect.width - 4), + ); + const y = Math.max( + 4, + Math.min(event.clientY, window.innerHeight - rect.height - 4), + ); + menu.style.left = `${x}px`; + menu.style.top = `${y}px`; + + document.addEventListener("pointerdown", onContextMenuPointerDown, true); + document.addEventListener("keydown", onContextMenuKeydown, true); + window.addEventListener("blur", closeRowContextMenu); + window.addEventListener("resize", closeRowContextMenu); + if (item.disabled) { + menu.tabIndex = -1; + menu.focus(); + } else { + item.focus(); + } +} + /** Append the empty-state message and any truncation notes below the table. */ function appendTableNotes(rows: ImpactRow[]): void { if (rows.length === 0) { @@ -1215,22 +1369,17 @@ function openDetail(utteranceId: string): void { renderDetail(delta); // Re-render the table so the open row gets its highlight. renderTable(); - // Keep the selected row visible above the docked detail pane (scrolling the - // pane itself would push the row out of view on a long table). - const openRow = tableWrap.querySelector(".row-open"); + // Focus the selected row so a run of Up/Down keeps walking the table: the + // re-render above replaces the row node, so without re-homing focus here the + // arrow keydown would land on the document and just scroll. Keep the row + // visible above the docked detail pane (scrolling the pane itself would push + // the row out of view on a long table); `preventScroll` leaves that to + // `scrollIntoView` so focus doesn't fight the controlled scroll. + const openRow = tableWrap.querySelector(".row-open"); + openRow?.focus({ preventScroll: true }); openRow?.scrollIntoView({ block: "nearest" }); } -/** Select an equal row: it has no A/B diff, so just highlight it and close any - * open detail pane. */ -function selectEqualRow(utteranceId: string): void { - selectedId = utteranceId; - openDetailId = undefined; - detailEl.hidden = true; - detailEl.textContent = ""; - renderTable(); -} - /** Hide the drill-in detail pane and clear the row selection. */ function closeDetail(): void { openDetailId = undefined; @@ -1239,46 +1388,52 @@ function closeDetail(): void { detailEl.textContent = ""; } -/** Paint the detail pane: a header (utterance + close) and the unified A/B diff - * of the two resolved actions. */ +/** Paint the detail pane: a header (utterance, an optional trace drill-in, and + * close) and, below it, the two resolved actions as a unified A/B diff. For an + * equal row that diff is the resolved action JSON shown as all-context lines. */ function renderDetail(delta: ActionDelta): void { detailEl.textContent = ""; const diff = toActionDiff(delta); const header = el("div", "detail-header"); - const title = el("span", "detail-title"); - title.textContent = collapseWhitespace(delta.utterance); - title.title = delta.utterance; - const meta = el("span", "detail-meta"); - if (diff.onlyB) { - meta.append(codicon("diff-added"), text("new match (no action on A)")); - } else if (diff.onlyA) { - meta.append( - codicon("diff-removed"), - text("lost match (no action on B)"), - ); - } else if (diff.identical) { - meta.append(codicon("pass"), text("actions identical")); - } else { - const added = el("span", "added"); - added.textContent = `+${diff.addedCount}`; - const removed = el("span", "removed"); - removed.textContent = `\u2212${diff.removedCount}`; - meta.append(added, removed); - } + const utterance = el("span", "detail-utterance"); + utterance.textContent = collapseWhitespace(delta.utterance); + utterance.title = delta.utterance; const close = toolButton("close", "Close detail", () => { closeDetail(); renderTable(); }); close.classList.add("detail-close"); - header.append(title, meta, close); - - const legend = el("div", "detail-legend"); - legend.append( - text("Base (A)"), - codicon("arrow-right"), - text("Compare (B)"), - ); + // A trace shortcut sits beside the close control when this row captured a + // resolution trace, putting the pipeline breakdown one click from the diff. + if (tracedIds.has(delta.utteranceId) && lastRenderedResult) { + const runId = lastRenderedResult.runId; + const openTrace = toolButton( + "telescope", + "Open in Trace Viewer", + () => { + vscode.postMessage({ + type: "openTrace", + runId, + utteranceId: delta.utteranceId, + }); + }, + ); + header.append(utterance, openTrace, close); + } else { + header.append(utterance, close); + } + // Follow the selection into an already-open Trace Viewer (a no-op when none + // is showing, so selecting a row never opens one on its own). Only rows that + // captured a trace can be shown; the viewer itself is opened from a row's + // right-click menu. + if (tracedIds.has(delta.utteranceId) && lastRenderedResult) { + vscode.postMessage({ + type: "focusTrace", + runId: lastRenderedResult.runId, + utteranceId: delta.utteranceId, + }); + } const body = el("pre", "detail-diff"); for (const line of diff.lines) { @@ -1290,7 +1445,7 @@ function renderDetail(delta: ActionDelta): void { body.appendChild(span); } - detailEl.append(header, legend, body); + detailEl.append(header, body); detailEl.hidden = false; } @@ -1548,7 +1703,8 @@ interface ToolButtonOptions { } /** A toolbar button carrying a codicon (and optional label). `icon` matches a - * `.codicon-` rule; `label` is the accessible name and hover title. */ + * `.codicon-` rule, or pass an empty string for a text-only button; + * `label` is the accessible name and hover title. */ function toolButton( icon: string, label: string, @@ -1558,7 +1714,9 @@ function toolButton( const b = document.createElement("button"); b.type = "button"; b.className = opts.primary ? "tool-button is-primary" : "tool-button"; - b.append(codicon(icon)); + if (icon) { + b.append(codicon(icon)); + } if (opts.text) { b.append(text(opts.text)); } @@ -1617,6 +1775,9 @@ function picker( button.setAttribute("aria-label", description); if (sideTag) { const tag = el("span", "picker-side"); + tag.classList.add( + sideTag.toLowerCase() === "a" ? "picker-side-a" : "picker-side-b", + ); tag.textContent = sideTag; button.appendChild(tag); } diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/traceViewer.ts b/ts/packages/typeagent-studio/src/webviewKit/client/traceViewer.ts new file mode 100644 index 0000000000..bcdbc70678 --- /dev/null +++ b/ts/packages/typeagent-studio/src/webviewKit/client/traceViewer.ts @@ -0,0 +1,888 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * The Trace Viewer webview client. Renders one row's resolution as a top-to- + * bottom **pipeline** a developer reads to answer "where did version A and + * version B diverge on this utterance, and what changed there?" The stages run + * in resolution order (construction cache → grammar match → wildcard validation + * → Result). Stages the two versions agree on are compact one-liners; the stage + * where they diverged expands into a side-by-side A vs B card accented as the + * cause, and carries its own dig-in — including a native A↔B diff of the grammar + * or schema file behind the divergence, driven from that stage rather than a + * disconnected top button. + * + * Everything divergence-related is derived by the shared, browser-neutral + * {@link toTraceDivergenceViewModel}; this file only turns that view model into + * DOM. No inline styles (the CSP forbids them) — every visual is a CSS class. + */ + +import type { ReplayResolutionTrace } from "@typeagent/core/replay"; +import { + toTraceDivergenceViewModel, + TRACE_LAYER_NAME, + type TraceDivergenceViewModel, + type TraceNodeSummary, + type TraceStageView, +} from "../traceDivergenceViewModel.js"; +import type { ActionDiff } from "../replayViewModel.js"; +import type { + HostToTraceMessage, + TraceToHostMessage, + TraceUnavailableState, + TraceSide, + TraceSourceNode, +} from "../traceProtocol.js"; + +interface VsCodeApi { + postMessage(message: TraceToHostMessage): void; +} +declare function acquireVsCodeApi(): VsCodeApi; + +const vscode = acquireVsCodeApi(); + +// The recorded trace is the single source of truth for what the row resolved. +let recorded: ReplayResolutionTrace | undefined; +let unavailable: TraceUnavailableState | undefined = "loading"; +let traceError: string | undefined; + +// Transient feedback from the last source open/compare (e.g. a file that can't +// be located); cleared when the row changes or a jump succeeds. +let sourceNote: string | undefined; + +// Monotonic id for source open/compare requests so a stale reply is ignored. +let sourceRequestId = 0; + +// Post a source-jump request for one node on one side; the host opens that +// side's version of the backing file, scrolled to the recorded span. +function requestSource(side: TraceSide, node: TraceSourceNode): void { + sourceRequestId += 1; + sourceNote = undefined; + render(); + vscode.postMessage({ + type: "open-source", + requestId: sourceRequestId, + side, + node, + }); +} + +// Post a compare request: the host opens a native A↔B diff of the node's file. +function requestCompare(node: TraceSourceNode): void { + sourceRequestId += 1; + sourceNote = undefined; + render(); + vscode.postMessage({ + type: "compare-source", + requestId: sourceRequestId, + node, + }); +} + +const root = document.getElementById("root") as HTMLElement; + +window.addEventListener("message", (event: MessageEvent) => { + const msg = event.data as HostToTraceMessage; + switch (msg.type) { + case "trace": + recorded = msg.recorded; + unavailable = undefined; + traceError = undefined; + // Reset transient per-row state so a previous row's source-open + // error doesn't bleed into the new row. + sourceNote = undefined; + render(); + break; + case "trace-state": + unavailable = msg.state; + render(); + break; + case "trace-error": + traceError = msg.message; + unavailable = undefined; + render(); + break; + case "source-result": + // Ignore a stale reply superseded by a newer request. + if (msg.requestId !== sourceRequestId) { + break; + } + // A successful open/compare cleared the view; drop any stale note. + // A failure surfaces the host's explanation. + if (msg.status === "opened") { + sourceNote = undefined; + } else if (msg.message !== undefined) { + sourceNote = msg.message; + } + render(); + break; + } +}); + +vscode.postMessage({ type: "ready" }); + +// --- Rendering ------------------------------------------------------------ + +function render(): void { + clear(root); + + if (unavailable === "loading") { + root.appendChild(centeredState("Loading trace…", "loading")); + return; + } + if (traceError !== undefined) { + root.appendChild(centeredState(traceError, "error")); + return; + } + if (unavailable === "evicted") { + root.appendChild( + centeredState( + "This run's traces have been rotated out of the store. Re-run the report to capture them again.", + "evicted", + ), + ); + return; + } + if (unavailable === "missing") { + root.appendChild( + centeredState("No trace was captured for this row.", "missing"), + ); + return; + } + + if (!recorded) { + root.appendChild(centeredState("No trace to show.", "missing")); + return; + } + + let vm: TraceDivergenceViewModel; + try { + vm = toTraceDivergenceViewModel(recorded); + } catch { + root.appendChild( + centeredState( + "The captured trace is malformed and can't be interpreted.", + "error", + ), + ); + return; + } + + root.appendChild(header(vm)); + if (sourceNote !== undefined) { + root.appendChild(noteBanner(sourceNote)); + } + const sub = subtitle(vm); + if (sub !== undefined) { + root.appendChild(sub); + } + root.appendChild(pipeline(vm)); +} + +/** Utterance on the left, the divergence verdict on the right. The A/B version + * provenance is deliberately not repeated here — it lives in the Impact Report + * toolbar this panel is opened from. */ +function header(vm: TraceDivergenceViewModel): HTMLElement { + const head = el("div", "trace-header"); + const top = el("div", "header-top"); + const utter = el("div", "utterance"); + utter.textContent = vm.utterance; + utter.title = vm.utterance; + top.appendChild(utter); + top.appendChild(verdictBadge(vm)); + head.appendChild(top); + return head; +} + +const LOW_CONFIDENCE_NOTE = + "Low confidence: the captured trace can't pin the difference to a single layer."; + +/** The headline badge on the top row: whether the two versions ended at the same + * action and — when they diverged — the stage the split is attributed to. */ +function verdictBadge(vm: TraceDivergenceViewModel): HTMLElement { + const { conclusion } = vm; + const badge = el("span", "verdict"); + if (conclusion.parity === "match") { + badge.classList.add( + conclusion.bothNoAction ? "is-neutral" : "is-match", + ); + badge.textContent = conclusion.bothNoAction + ? "No action" + : "Same result"; + } else { + badge.classList.add("is-differ"); + badge.textContent = + vm.divergingLayer !== undefined + ? `Diverged · ${TRACE_LAYER_NAME[vm.divergingLayer]}` + : "Diverged"; + } + if (conclusion.confidence === "low") { + badge.classList.add("is-low-confidence"); + badge.title = LOW_CONFIDENCE_NOTE; + } + return badge; +} + +/** An optional line under the header, shown only when it adds something the + * verdict badge and the cause stage don't already say: the "same action" reading + * when the versions agree, a fidelity-path caveat, a low-confidence note, or — + * when no single stage could be blamed — the cause sentence itself (there's no + * cause card to carry it). For a normal single-stage divergence this returns + * nothing, so the pipeline speaks for itself. */ +function subtitle(vm: TraceDivergenceViewModel): HTMLElement | undefined { + const { conclusion } = vm; + const notes: string[] = []; + if (conclusion.parity === "match") { + notes.push(conclusion.headline); + } else if ( + vm.divergingLayer === undefined && + conclusion.cause !== undefined + ) { + notes.push(conclusion.cause.detail); + } + if (conclusion.pathNote !== undefined) { + notes.push(conclusion.pathNote); + } + if (conclusion.confidenceNote !== undefined) { + notes.push(conclusion.confidenceNote); + } else if (conclusion.confidence === "low") { + notes.push(LOW_CONFIDENCE_NOTE); + } + if (notes.length === 0) { + return undefined; + } + const line = el("div", "subtitle"); + if (conclusion.confidence === "low") { + line.classList.add("is-low-confidence"); + } + notes.forEach((noteText, i) => { + const span = el("span", i === 0 ? "subtitle-text" : "subtitle-note"); + span.textContent = noteText; + line.appendChild(span); + }); + return line; +} + +// --- Pipeline ------------------------------------------------------------- + +const STAGE_STATUS_LABEL: Record = { + agree: "same", + diverge: "diverges", + "one-sided": "one side", + inapplicable: "n/a", +}; + +const STAGE_STATUS_HELP: Record = { + agree: "Both versions ran this stage the same way.", + diverge: "This is where the two versions parted ways.", + "one-sided": + "Only one version engaged this stage — typically a live-only step a git-ref side can't run.", + inapplicable: "Neither version actively ran this stage here.", +}; + +/** The resolution pipeline: the pre-action stages (cache → grammar → wildcard) + * as a top-to-bottom flow, each compact unless it's the divergence (which + * expands side-by-side), capped by the terminal Result so the produced action + * is always shown for both versions. */ +function pipeline(vm: TraceDivergenceViewModel): HTMLElement { + const wrap = el("div", "pipeline"); + for (const stage of vm.stages) { + if (stage.kind === "action") { + continue; + } + wrap.appendChild(stageRow(vm, stage)); + } + wrap.appendChild(resultBlock(vm)); + return wrap; +} + +/** One pipeline stage: a header (name + status) then its body — a compact + * one-liner when the two sides agree, or an expanded side-by-side card when + * this is the divergence. */ +function stageRow( + vm: TraceDivergenceViewModel, + stage: TraceStageView, +): HTMLElement { + const row = el("div", "stage"); + row.classList.add(`status-${stage.status}`); + if (stage.isCause) { + row.classList.add("is-cause"); + } + row.appendChild(stageHead(stage.layerName, stageStatusChip(stage))); + row.appendChild(stage.isCause ? causeBody(vm, stage) : compactBody(stage)); + return row; +} + +/** The text, tone, and hover for a stage's status chip. */ +interface StatusChip { + label: string; + tone?: OutcomeTone; + help: string; +} + +/** The status chip for a pre-action stage. A grammar divergence that gained or + * lost a rule match in B is stated directionally — "new in B" (green) for a + * match B introduced, "lost in B" (orange) for one it dropped — the same + * vocabulary and tone the Result chip uses; every other status keeps its plain + * convergence label. */ +function stageStatusChip(stage: TraceStageView): StatusChip { + if (stage.status === "diverge" && stage.kind === "grammar-match") { + const aMatched = stage.a?.grammar?.chosenRule !== undefined; + const bMatched = stage.b?.grammar?.chosenRule !== undefined; + if (!aMatched && bMatched) { + return { + label: "new in B", + tone: "positive", + help: "Only B matched a rule — a new match B introduced.", + }; + } + if (aMatched && !bMatched) { + return { + label: "lost in B", + tone: "negative", + help: "Only A matched a rule — a match B lost.", + }; + } + } + return { + label: STAGE_STATUS_LABEL[stage.status], + help: STAGE_STATUS_HELP[stage.status], + }; +} + +function stageHead(name: string, chip: StatusChip): HTMLElement { + const head = el("div", "stage-head"); + head.appendChild(el("span", "stage-marker")); + const label = el("span", "stage-name"); + label.textContent = name; + head.appendChild(label); + head.appendChild(statusChip(chip)); + return head; +} + +/** A stage-status chip element, tinted green/orange when the status carries an + * improvement/regression direction. */ +function statusChip(chip: StatusChip): HTMLElement { + const node = el("span", "stage-status"); + if (chip.tone !== undefined) { + node.classList.add(`is-${chip.tone}`); + } + node.textContent = chip.label; + node.title = chip.help; + return node; +} + +/** The representative node for a compact stage: the side that actually ran, + * preferring the live B side, so the one-liner describes what happened. */ +function representative( + stage: TraceStageView, +): { node: TraceNodeSummary; side: TraceSide } | undefined { + if (stage.b?.executionLabel === "ran") { + return { node: stage.b, side: "b" }; + } + if (stage.a?.executionLabel === "ran") { + return { node: stage.a, side: "a" }; + } + if (stage.b !== undefined) { + return { node: stage.b, side: "b" }; + } + if (stage.a !== undefined) { + return { node: stage.a, side: "a" }; + } + return undefined; +} + +/** A converged / one-sided / inapplicable stage as a single line: what happened, + * with the winning rule still a source jump where recorded. One-sided stages + * name the side that ran. */ +function compactBody(stage: TraceStageView): HTMLElement { + const body = el("div", "stage-compact"); + const rep = representative(stage); + const summary = el("div", "stage-summary"); + if (rep === undefined) { + summary.classList.add("is-muted"); + summary.textContent = "not applicable to either version"; + body.appendChild(summary); + return body; + } + summary.appendChild(compactContent(stage, rep.node)); + if (stage.status === "one-sided") { + const only = el("span", "only-side"); + only.textContent = `${rep.side.toUpperCase()} only`; + only.title = + "Only this version engaged the stage; the other ran a different fidelity path."; + summary.appendChild(only); + } + body.appendChild(summary); + return body; +} + +/** The single outcome phrase for one stage on one side, plus the tone used to + * color it. A side that didn't actively run the stage reads as its muted + * execution state ("Not applicable", "Not reached"). */ +type OutcomeTone = "positive" | "negative" | "neutral" | "muted"; + +function stageOutcome( + kind: TraceStageView["kind"], + node: TraceNodeSummary, +): { text: string; tone: OutcomeTone } { + if (node.executionLabel !== "ran") { + return { text: capitalize(node.executionLabel), tone: "muted" }; + } + switch (kind) { + case "cache-consult": + return node.outcomeLabel === "hit" + ? { text: "Cache hit", tone: "positive" } + : { text: "Cache miss", tone: "neutral" }; + case "grammar-match": + return node.grammar?.chosenRule !== undefined + ? { text: "Matched", tone: "positive" } + : { text: "No match", tone: "negative" }; + case "wildcard-validation": + return node.outcomeLabel === "rejected" + ? { text: "Rejected", tone: "negative" } + : { text: "Accepted", tone: "positive" }; + default: + return node.action?.actionName !== undefined + ? { text: "Produced action", tone: "positive" } + : { text: "No action", tone: "negative" }; + } +} + +/** The outcome phrase as a tone-colored span, with the node's raw detail on + * hover. The single consistent primary line every side card and compact row + * leads with. */ +function outcomeSpan( + kind: TraceStageView["kind"], + node: TraceNodeSummary, +): HTMLElement { + const { text, tone } = stageOutcome(kind, node); + const span = el("span", "outcome"); + span.classList.add(`is-${tone}`); + span.textContent = text; + if (node.detail !== undefined) { + span.title = node.detail; + } + return span; +} + +/** The one mono value identifying what a stage produced on a side: the matched + * rule, the produced action, or the construction id. A read-only value paired at + * the call site with {@link artifactLabel} so its meaning is explicit; the file + * behind a divergence is reached through the diverging stage's diff chip, not + * from here. Absent for stages with no such artifact or when the side produced + * none. */ +function stageArtifact( + kind: TraceStageView["kind"], + node: TraceNodeSummary, +): HTMLElement | undefined { + switch (kind) { + case "grammar-match": + return node.grammar?.chosenRule !== undefined + ? ruleChip(node.grammar) + : undefined; + case "action": + return node.action?.actionName !== undefined + ? actionChip(node.action) + : undefined; + case "cache-consult": + return node.cache?.constructionId !== undefined + ? cacheChip(node.cache) + : undefined; + default: + return undefined; + } +} + +/** The dim micro-label naming what a side card's artifact value is, so a bare + * token (a rule name) can't be mistaken for a file or an action. */ +function artifactLabel(kind: TraceStageView["kind"]): string | undefined { + switch (kind) { + case "grammar-match": + return "rule"; + case "action": + return "action"; + case "cache-consult": + return "construction"; + default: + return undefined; + } +} + +/** A side card's artifact as a labeled value: the dim micro-label, the mono + * value, and — when the trace recorded a location — a small go-to-file icon that + * opens that side's version of the backing file. Nothing when the side produced + * no such artifact. */ +function labeledArtifact( + kind: TraceStageView["kind"], + side: TraceSide, + node: TraceNodeSummary, +): HTMLElement | undefined { + const value = stageArtifact(kind, node); + if (value === undefined) { + return undefined; + } + const line = el("div", "side-artifact"); + const label = artifactLabel(kind); + if (label !== undefined) { + const tag = el("span", "artifact-label"); + tag.textContent = label; + line.appendChild(tag); + } + line.appendChild(value); + const open = openSourceButton(kind, side, node); + if (open !== undefined) { + line.appendChild(open); + } + return line; +} + +/** The source node a stage's artifact can open, when the trace recorded its + * location: the matched rule's `.agr` for a grammar stage, the produced action's + * schema for an action stage. Undefined when nothing is openable. */ +function sourceNodeFor( + kind: TraceStageView["kind"], + node: TraceNodeSummary, +): TraceSourceNode | undefined { + if (kind === "grammar-match" && node.grammar?.hasSource === true) { + return "grammar-match"; + } + if (kind === "action" && node.action?.hasSchema === true) { + return "action"; + } + return undefined; +} + +/** A small go-to-file icon beside a rule/action value that opens that side's + * version of the backing file, scrolled to the rule or schema span. A distinct + * affordance from the value itself, so the token stays a plain identifier. */ +function openSourceButton( + kind: TraceStageView["kind"], + side: TraceSide, + node: TraceNodeSummary, +): HTMLElement | undefined { + const source = sourceNodeFor(kind, node); + if (source === undefined) { + return undefined; + } + const what = + source === "grammar-match" ? "grammar source" : "action schema"; + const title = `Open ${side.toUpperCase()}'s ${what}`; + const btn = el("button", "open-source") as HTMLButtonElement; + btn.type = "button"; + btn.title = title; + btn.setAttribute("aria-label", title); + btn.appendChild(el("span", "codicon codicon-go-to-file")); + btn.addEventListener("click", () => requestSource(side, source)); + return btn; +} + +/** A converged / one-sided / inapplicable stage as one inline line: the + * representative side's outcome phrase and its artifact value. */ +function compactContent( + stage: TraceStageView, + node: TraceNodeSummary, +): HTMLElement { + const wrap = el("span", "summary-content"); + wrap.appendChild(outcomeSpan(stage.kind, node)); + const artifact = stageArtifact(stage.kind, node); + if (artifact !== undefined) { + wrap.appendChild(artifact); + } + return wrap; +} + +// --- Diverging (expanded) stage & terminal Result ------------------------- + +/** The expanded body for the stage the divergence is attributed to: the two + * sides side-by-side, then — for a file-backed cause — the ⇄ diff chip of the + * grammar or schema file the split traces to. */ +function causeBody( + vm: TraceDivergenceViewModel, + stage: TraceStageView, +): HTMLElement { + const body = el("div", "stage-body"); + body.appendChild(sideBySide(stage.a, stage.b, stage.kind)); + const bar = causeBar(vm, stage); + if (bar !== undefined) { + body.appendChild(bar); + } + return body; +} + +/** The A and B nodes for one stage laid out side-by-side. */ +function sideBySide( + a: TraceNodeSummary | undefined, + b: TraceNodeSummary | undefined, + kind: TraceStageView["kind"], +): HTMLElement { + const sides = el("div", "stage-sides"); + sides.appendChild(sideColumn("a", a, kind)); + sides.appendChild(sideColumn("b", b, kind)); + return sides; +} + +/** One version's column within an expanded stage, in the shared anatomy every + * side card uses: the A/B badge, one tone-colored outcome phrase, and — when the + * side produced one — the labeled read-only value (matched rule, produced + * action, or construction id) that names what it settled on. A side that didn't + * run the stage reads as a muted "did not run this stage". */ +function sideColumn( + side: TraceSide, + node: TraceNodeSummary | undefined, + kind: TraceStageView["kind"], +): HTMLElement { + const col = el("div", "side-col"); + col.classList.add(side === "a" ? "side-a" : "side-b"); + + const head = el("div", "side-col-head"); + const tag = el("span", "side-tag"); + tag.textContent = side.toUpperCase(); + head.appendChild(tag); + col.appendChild(head); + + if (node === undefined) { + const none = el("div", "side-empty"); + none.textContent = "did not run this stage"; + col.appendChild(none); + return col; + } + + col.appendChild(outcomeSpan(kind, node)); + const artifact = labeledArtifact(kind, side, node); + if (artifact !== undefined) { + col.appendChild(artifact); + } + return col; +} + +/** The single visual attribution under a diverging stage: a short accented label + * naming it as the likely (or, at low confidence, possible) cause, then a ⇄ diff + * chip for the grammar/schema file the split traces to, opening its A↔B diff. + * Rendered only for a file-backed cause with a diffable path recorded — cache + * and wildcard divergences (no file changed) and older captures without a path + * show none, since the accented side cards and the output diff carry those. */ +function causeBar( + vm: TraceDivergenceViewModel, + stage: TraceStageView, +): HTMLElement | undefined { + if (stage.compare === undefined) { + return undefined; + } + const fileName = vm.conclusion.cause?.fileName; + const noun = stage.compare === "grammar-match" ? "grammar" : "schema"; + const bar = el("div", "cause-bar"); + const label = el("span", "cause-label"); + label.textContent = + vm.conclusion.confidence === "low" ? "Possible cause" : "Likely cause"; + bar.appendChild(label); + bar.appendChild(compareLink(fileName ?? noun, stage.compare)); + return bar; +} + +/** The two produced actions as a unified JSON diff (the Impact Report's action + * diff, reused): the ground-truth "what differs in the output". Only rendered + * when the actions actually differ. */ +function actionDiffBlock(diff: ActionDiff): HTMLElement { + const wrap = el("div", "output-diff"); + const label = el("span", "output-diff-label"); + label.textContent = "Output difference"; + wrap.appendChild(label); + const pre = el("pre", "action-diff"); + for (const line of diff.lines) { + const span = document.createElement("span"); + span.className = `diff-line diff-${line.kind}`; + const sign = + line.kind === "added" ? "+" : line.kind === "removed" ? "-" : " "; + span.textContent = `${sign} ${line.text}\n`; + pre.appendChild(span); + } + wrap.appendChild(pre); + return wrap; +} + +/** The ⇄ compare affordance: an inline link (not a chrome button) that asks the + * host to open the two versions of the backing file side-by-side. */ +function compareLink(label: string, node: TraceSourceNode): HTMLButtonElement { + const link = el("button", "compare-link") as HTMLButtonElement; + link.type = "button"; + const glyph = el("span", "compare-glyph"); + glyph.textContent = "⇄"; + link.appendChild(glyph); + const text = el("span", "compare-label"); + text.textContent = label; + link.appendChild(text); + link.addEventListener("click", () => requestCompare(node)); + return link; +} + +/** The terminal Result: the two versions' produced actions side-by-side plus, + * whenever they differ, the unified JSON diff of the output — the ground-truth + * "what came out". Its status reads from the actions themselves (same / differs + * / a new or lost match), not from whether both sides merely ran, so a real + * output divergence never mislabels as "same". When the action payload is the + * attributed cause, this block is the accented cause and carries the schema diff + * chip. */ +function resultBlock(vm: TraceDivergenceViewModel): HTMLElement { + const stage = vm.stages.find((s) => s.kind === "action"); + const diff = vm.resultDiff; + const block = el("div", "stage is-result"); + block.classList.add(diff.identical ? "status-agree" : "status-diverge"); + if (stage?.isCause === true) { + block.classList.add("is-cause"); + } + + const head = el("div", "stage-head"); + head.appendChild(el("span", "stage-marker")); + const name = el("span", "stage-name"); + name.textContent = "Result"; + head.appendChild(name); + head.appendChild( + statusChip({ + label: resultStatusLabel(diff), + tone: resultStatusTone(diff), + help: resultStatusHelp(diff), + }), + ); + block.appendChild(head); + + // An unchanged result collapses to the same compact one-liner the agreeing + // pre-action stages use, so a converged Result reads consistently with them + // rather than expanding a redundant A/B split of one identical action. A + // changed result keeps the side-by-side, its output diff, and — when the + // action payload is the cause — the schema diff chip. + if (diff.identical && stage !== undefined) { + block.appendChild(compactBody(stage)); + return block; + } + const body = el("div", "stage-body"); + body.appendChild(sideBySide(stage?.a, stage?.b, "action")); + if (!diff.identical) { + body.appendChild(actionDiffBlock(diff)); + } + if (stage?.isCause === true) { + const bar = causeBar(vm, stage); + if (bar !== undefined) { + body.appendChild(bar); + } + } + block.appendChild(body); + return block; +} + +/** The Result status chip text, from the action comparison: identical actions, + * a new match (only B produced one), a lost match (only A), or a plain diff. */ +function resultStatusLabel(diff: ActionDiff): string { + if (diff.identical) { + return "same"; + } + if (diff.onlyB) { + return "new in B"; + } + if (diff.onlyA) { + return "lost in B"; + } + return "differs"; +} + +/** The Result chip tone: green when B introduced an action A lacked, orange when + * B lost one A had; a plain differing or identical result stays untinted. */ +function resultStatusTone(diff: ActionDiff): OutcomeTone | undefined { + if (diff.onlyB) { + return "positive"; + } + if (diff.onlyA) { + return "negative"; + } + return undefined; +} + +function resultStatusHelp(diff: ActionDiff): string { + if (diff.identical) { + return "Both versions produced the same action."; + } + if (diff.onlyB) { + return "Only B produced an action — a new match B introduced."; + } + if (diff.onlyA) { + return "Only A produced an action — a match B lost."; + } + return "The two versions produced different actions."; +} + +/** The matched rule as a read-only mono value. Labeled `rule` by its side card so + * the bare token can't be read as a file or an action; the file behind a + * divergence is reached through the diverging stage's diff chip, not here. */ +function ruleChip( + grammar: NonNullable, +): HTMLElement { + const code = el("code", "mono rule"); + code.textContent = grammar.chosenRule ?? "grammar"; + return code; +} + +/** The produced action as a read-only mono value, labeled `action` by its side + * card. */ +function actionChip( + action: NonNullable, +): HTMLElement { + const code = el("code", "mono action-name"); + code.textContent = action.actionName ?? "action"; + return code; +} + +/** The matched construction id as a mono chip, with its namespace and pattern on + * hover so a cache stage stays inspectable without a second panel. */ +function cacheChip(cache: NonNullable): HTMLElement { + const id = el("code", "mono cache-id"); + id.textContent = `#${cache.constructionId}`; + const tip: string[] = []; + if (cache.namespace !== undefined) { + tip.push(`Namespace: ${cache.namespace}`); + } + if (cache.parts !== undefined && cache.parts.length > 0) { + tip.push(`Pattern: ${cache.parts.join(" ")}`); + } + id.title = tip.length > 0 ? tip.join("\n") : "Matched construction id."; + return id; +} + +// --- State screens & bits ------------------------------------------------- + +function centeredState(message: string, kind: string): HTMLElement { + const box = el("div", "state-screen"); + box.classList.add(`state-${kind}`); + const inner = el("div", "state-inner"); + if (kind === "loading") { + inner.appendChild(el("div", "spinner")); + } + const text = el("div", "state-text"); + text.textContent = message; + inner.appendChild(text); + box.appendChild(inner); + return box; +} + +function noteBanner(message: string): HTMLElement { + const banner = el("div", "note-banner"); + banner.textContent = message; + return banner; +} + +// --- DOM helpers ---------------------------------------------------------- + +function el(tag: string, className: string): HTMLElement { + const node = document.createElement(tag); + node.className = className; + return node; +} + +function clear(node: HTMLElement): void { + while (node.firstChild) { + node.removeChild(node.firstChild); + } +} + +function capitalize(value: string): string { + return value.length === 0 ? value : value[0].toUpperCase() + value.slice(1); +} diff --git a/ts/packages/typeagent-studio/src/webviewKit/host.ts b/ts/packages/typeagent-studio/src/webviewKit/host.ts index ec4c359db9..033f648553 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/host.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/host.ts @@ -24,6 +24,13 @@ export interface WebviewKitPanelOptions { * handshake. Costs memory while hidden. */ retainContextWhenHidden?: boolean; + /** + * Which editor column to open the panel in. Defaults to + * {@link vscode.ViewColumn.Active}. A drill-in panel opened from another + * webview passes {@link vscode.ViewColumn.Beside} so it sits next to its + * originator instead of stealing its column. + */ + viewColumn?: vscode.ViewColumn; /** Path segments (from the extension root) to the client script bundle. */ scriptPath: string[]; /** Path segments (from the extension root) to the stylesheet. */ @@ -87,7 +94,7 @@ export class WebviewKitPanel { const panel = vscode.window.createWebviewPanel( options.viewType, options.title, - vscode.ViewColumn.Active, + options.viewColumn ?? vscode.ViewColumn.Active, { enableScripts: true, retainContextWhenHidden: diff --git a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts index 2f2ec0fbd8..fc392047b1 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts @@ -113,7 +113,14 @@ export type WebviewToHostMessage = | { type: "pickVersion"; side: ReplaySide } /** Ask the host to open a native input box to edit the utterance filter, * seeded with the filter currently applied in the report. */ - | { type: "searchUtterances"; current: string }; + | { type: "searchUtterances"; current: string } + /** Ask the host to open the Trace Viewer for one red row, drilling into the + * full side-by-side resolution trace behind it. */ + | { type: "openTrace"; runId: string; utteranceId: string } + /** Tell the host the selected row changed, so an already-open Trace Viewer + * can follow the selection. Unlike `openTrace`, this never opens a new + * viewer when none is showing. */ + | { type: "focusTrace"; runId: string; utteranceId: string }; function narrowSide(value: unknown): ReplaySide | undefined { return value === "a" || value === "b" ? value : undefined; @@ -176,6 +183,28 @@ export function parseWebviewMessage( current: typeof current === "string" ? current : "", }; } + case "openTrace": { + const m = value as { runId?: unknown; utteranceId?: unknown }; + return typeof m.runId === "string" && + typeof m.utteranceId === "string" + ? { + type: "openTrace", + runId: m.runId, + utteranceId: m.utteranceId, + } + : undefined; + } + case "focusTrace": { + const m = value as { runId?: unknown; utteranceId?: unknown }; + return typeof m.runId === "string" && + typeof m.utteranceId === "string" + ? { + type: "focusTrace", + runId: m.runId, + utteranceId: m.utteranceId, + } + : undefined; + } case "run": { const m = value as { requestId?: unknown; diff --git a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts index 348109feb8..d02907f966 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts @@ -884,12 +884,16 @@ function diffLines(a: string[], b: string[]): DiffLine[] { } /** - * Build the A→B action diff for a row drill-in from the {@link ActionDelta} we - * already have — no engine round-trip. A side that resolved no action (a new or - * lost match) is rendered against the `(no action)` placeholder so the diff still - * reads as a clean add/remove block. + * Build the A→B action diff from the two resolved actions — no engine round-trip. + * Accepts anything carrying `actionA`/`actionB` (a full {@link ActionDelta} row, + * or a bare pair built from a resolution trace's two `finalAction`s), so the same + * canonical diff is reused wherever two actions are compared. A side that resolved + * no action (a new or lost match) is rendered against the `(no action)` + * placeholder so the diff still reads as a clean add/remove block. */ -export function toActionDiff(row: ActionDelta): ActionDiff { +export function toActionDiff( + row: Pick, +): ActionDiff { const aPresent = row.actionA !== undefined; const bPresent = row.actionB !== undefined; const aText = aPresent ? stableStringify(row.actionA) : NO_ACTION_TEXT; diff --git a/ts/packages/typeagent-studio/src/webviewKit/traceDivergenceViewModel.ts b/ts/packages/typeagent-studio/src/webviewKit/traceDivergenceViewModel.ts new file mode 100644 index 0000000000..cd796b46fb --- /dev/null +++ b/ts/packages/typeagent-studio/src/webviewKit/traceDivergenceViewModel.ts @@ -0,0 +1,872 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Browser-neutral view model for the Trace Viewer's divergence conclusion. + * + * Given a captured {@link ReplayResolutionTrace} for one utterance, this derives + * the two things the viewer leads with: + * + * - **Resolution parity** — whether the two sides produced the *same canonical + * final action*. This is the headline, and it is decided by comparing the + * actions themselves, never by comparing per-layer outcomes (two sides can + * both `hit` yet produce different actions, and a live-side cache hit vs a + * git-ref grammar match are different branches, not the same step). + * - **Cause attribution** — only when the actions differ, an ordered walk that + * names the first layer that explains the difference: a cache short-circuit, + * a different grammar rule, a wildcard-validation rejection, or an + * action-parameter difference under the same rule. + * + * Two guard rails come straight from how the trace is captured: + * - Capability asymmetry (a live-only layer that is `not-applicable` on a + * git-ref side) is never the headline. When the final actions match but the + * sides ran different fidelity paths, that is stated as a note, not a + * divergence. + * - When a grammar node's `rankingParity` is `diverged`, its captured recursive + * trace explains a *different* parse than the resolver's ranked pick, so the + * grammar level cannot be trusted to explain the difference. Attribution + * falls back to the action level and the conclusion is marked low-confidence. + * + * Everything here is a pure function of the trace DTO, so it unit-tests without a + * webview and bundles into the browser client. + */ + +import type { + ReplayResolutionTrace, + ReplaySideTrace, + ReplayTraceNode, + TraceRealization, + CacheConsultTraceNode, + GrammarMatchTraceNode, + ActionTraceNode, +} from "@typeagent/core/replay"; +import { stableStringify, toActionDiff } from "./replayViewModel.js"; +import type { ActionDiff } from "./replayViewModel.js"; + +/** The four fidelity layers, in the order they run, for aligned rendering. */ +export const TRACE_LAYER_ORDER: ReplayTraceNode["kind"][] = [ + "cache-consult", + "grammar-match", + "wildcard-validation", + "action", +]; + +/** Human layer names for the node cards and callout copy. */ +export const TRACE_LAYER_NAME: Record = { + "cache-consult": "Construction cache", + "grammar-match": "Grammar match", + "wildcard-validation": "Wildcard validation", + action: "Action", +}; + +const EXECUTION_LABEL: Record = { + ran: "ran", + "skipped-by-mode": "skipped", + "not-applicable": "not applicable", + unavailable: "unavailable", + "not-reached": "not reached", +}; + +const OUTCOME_LABEL: Record = { + hit: "hit", + miss: "miss", + accepted: "accepted", + rejected: "rejected", + errored: "errored", +}; + +const RANKING_PARITY_LABEL: Record< + GrammarMatchTraceNode["rankingParity"], + string +> = { + matched: "parse matches pick", + diverged: "parse diverges", + unavailable: "no trace", +}; + +/** Whether the two sides settled on the same action. */ +export type TraceParity = "match" | "differ"; + +/** The layer an action difference is attributed to. `unattributed` means the + * actions differ but the captured trace names no single explaining layer. */ +export type DivergenceCauseKind = + | "cache-decided" + | "grammar-differs" + | "wildcard-validation" + | "action-payload" + | "unattributed"; + +export interface DivergenceCause { + kind: DivergenceCauseKind; + /** The side whose layer explains the difference, when the cause is + * one-sided (e.g. only A short-circuited through its cache). */ + side?: "A" | "B"; + /** Human sentence for the callout. */ + detail: string; + /** Basename of the source file this cause is backed by — the `.agr` grammar + * for a grammar divergence, the action schema `.ts` for a payload one — so + * the viewer can name the changed file and offer its A↔B diff. Absent for + * cache/wildcard causes, which are runtime state rather than a file edit. */ + fileName?: string; +} + +export interface DivergenceConclusion { + parity: TraceParity; + /** Headline copy: the parity statement the callout leads with. */ + headline: string; + /** True when neither side produced an action at all. */ + bothNoAction: boolean; + /** How trustworthy the cause attribution is. `low` when a diverged grammar + * parity forced a fallback or no layer could be attributed. */ + confidence: "high" | "low"; + /** Present only when the actions differ. */ + cause?: DivergenceCause; + /** Present when the actions match but the sides ran different fidelity + * paths (working-tree vs git-ref); explains the expected asymmetry. */ + pathNote?: string; + /** Present when a diverged grammar parity makes the grammar level a + * diagnostic-only explanation rather than the resolver-selected parse. */ + confidenceNote?: string; +} + +/** A compact, display-ready summary of one fidelity node. */ +export interface TraceNodeSummary { + kind: ReplayTraceNode["kind"]; + layerName: string; + executionLabel: string; + outcomeLabel?: string; + detail?: string; + /** Grammar-match extras: the winning rule, ranking-parity confidence, and + * whether an expandable timeline is available. */ + grammar?: { + input: string; + chosenRule?: string; + rankingParity: GrammarMatchTraceNode["rankingParity"]; + rankingParityLabel: string; + /** True when a step-by-step timeline can be expanded for this node. */ + hasTimeline: boolean; + /** True when the winning rule carries a source span to jump to. */ + hasSource: boolean; + /** True when `rankingParity === "diverged"`: the timeline is a + * diagnostic parse, not the resolver-selected one. */ + diagnosticOnly: boolean; + }; + /** Cache extras rendered inline (no host round-trip). */ + cache?: { + constructionId?: string; + namespace?: string; + parts?: string[]; + }; + /** Action extras: the produced action's name and whether a schema jump is + * available. */ + action?: { + actionName?: string; + hasSchema: boolean; + }; +} + +export interface SideDivergenceView { + side: "A" | "B"; + realization: TraceRealization; + nodes: TraceNodeSummary[]; +} + +/** How the two sides compared at one pipeline stage. + * - `agree`: both sides ran this stage and it is not the attributed cause. + * - `diverge`: this stage is where the resolution split (the cause). + * - `one-sided`: exactly one side engaged this stage (typically a live-only + * cache/validation step a git-ref side can't run). + * - `inapplicable`: neither side actively ran the stage (e.g. grammar after a + * cache short-circuit). */ +export type TraceStageStatus = + | "agree" + | "diverge" + | "one-sided" + | "inapplicable"; + +/** One stage of the resolution pipeline, pairing the A and B nodes for a single + * fidelity layer so the viewer can render the flow top-to-bottom and expand the + * stage where the two versions diverged. */ +export interface TraceStageView { + kind: ReplayTraceNode["kind"]; + layerName: string; + /** The A-side node for this layer, when the side captured it. */ + a?: TraceNodeSummary; + /** The B-side node for this layer, when the side captured it. */ + b?: TraceNodeSummary; + status: TraceStageStatus; + /** True when the divergence is attributed to this stage; the viewer expands + * it side-by-side and accents it as the cause. */ + isCause: boolean; + /** When this stage is backed by a source file that can be diffed across the + * two versions, the node kind to drive a `compare-source` request from. */ + compare?: "grammar-match" | "action"; +} + +export interface TraceDivergenceViewModel { + runId: string; + utteranceId: string; + utterance: string; + conclusion: DivergenceConclusion; + a: SideDivergenceView; + b: SideDivergenceView; + /** The layer to accent as the divergence point, when attributable. */ + divergingLayer?: ReplayTraceNode["kind"]; + /** The resolution pipeline: one entry per fidelity layer at least one side + * engaged, in runtime order (cache → grammar → wildcard → action). Drives + * the top-to-bottom pipeline rendering and its per-stage dig-in. */ + stages: TraceStageView[]; + /** Canonical A→B JSON diff of the two produced actions — the ground-truth + * "what differs in the output", shown at the Result regardless of which + * earlier stage caused the split. */ + resultDiff: ActionDiff; +} + +function isActionObject(action: unknown): boolean { + return typeof action === "object" && action !== null; +} + +function sameAction(a: unknown, b: unknown): boolean { + const hasA = isActionObject(a); + const hasB = isActionObject(b); + if (!hasA && !hasB) return true; + if (!hasA || !hasB) return false; + return stableStringify(a) === stableStringify(b); +} + +function actionName(action: unknown): string | undefined { + if (!isActionObject(action)) return undefined; + const name = (action as { actionName?: unknown }).actionName; + return typeof name === "string" ? name : undefined; +} + +function nodeOf( + side: ReplaySideTrace, + kind: K, +): Extract | undefined { + return side.nodes.find((n) => n.kind === kind) as + | Extract + | undefined; +} + +/** True when a side short-circuited through its construction cache: the cache + * consult ran and hit, and the grammar consequently never ran. */ +function cacheDecided(side: ReplaySideTrace): boolean { + const cache = nodeOf(side, "cache-consult"); + const grammar = nodeOf(side, "grammar-match"); + return ( + cache?.execution === "ran" && + cache.outcome === "hit" && + grammar?.execution === "not-reached" + ); +} + +function sideLabel(side: "A" | "B"): string { + return side === "A" ? "A" : "B"; +} + +function otherSide(side: "A" | "B"): "A" | "B" { + return side === "A" ? "B" : "A"; +} + +function grammarDiverged(node: GrammarMatchTraceNode | undefined): boolean { + return node?.rankingParity === "diverged"; +} + +/** Browser-safe basename (handles POSIX `/` and Windows `\`), for naming the + * changed source file in the divergence copy without pulling in node:path. */ +function basename(p: string): string { + const normalized = p.replace(/\\/g, "/"); + const slash = normalized.lastIndexOf("/"); + return slash >= 0 ? normalized.slice(slash + 1) : normalized; +} + +/** The `.agr` file's display name for a grammar node: the basename of its + * recorded absolute path, else the winning-rule span's display path. */ +function grammarFileName( + node: GrammarMatchTraceNode | undefined, +): string | undefined { + if (node === undefined) { + return undefined; + } + if (node.sourceFilePath !== undefined) { + return basename(node.sourceFilePath); + } + if (node.source?.displayPath !== undefined) { + return basename(node.source.displayPath); + } + return undefined; +} + +/** The `.agr` file that defines the rule a side matched, resolved through the + * compiled grammar's rule→location and file tables. This pinpoints the single + * grammar source on the match path even when a grammar imports several files, so + * a multi-file edit still attributes to the one file that owns the matched rule. + * Undefined when the trace didn't record the rule's location. */ +function ruleDefiningFile( + node: GrammarMatchTraceNode | undefined, +): string | undefined { + const rule = node?.chosenRule; + const debug = node?.debugInfo; + if (rule === undefined || debug === undefined) { + return undefined; + } + const location = debug.rules.find(([id]) => id === rule)?.[1]; + if (location === undefined) { + return undefined; + } + const absolute = debug.filePaths.find( + ([id]) => id === location.fileId, + )?.[1]; + if (absolute !== undefined) { + return basename(absolute); + } + return location.displayPath !== undefined + ? basename(location.displayPath) + : undefined; +} + +/** The grammar file a divergence is attributed to: the file defining the rule the + * matching side landed on (rule-level, so an import is pinpointed among several + * changed files), falling back to a side's top-level grammar file when the + * rule→file map isn't recorded. */ +function grammarCulpritFile( + a: GrammarMatchTraceNode | undefined, + b: GrammarMatchTraceNode | undefined, +): string | undefined { + return ( + ruleDefiningFile(a) ?? + ruleDefiningFile(b) ?? + grammarFileName(a) ?? + grammarFileName(b) + ); +} + +/** The action schema's file name for an action node, when the trace recorded a + * schema source path. */ +function actionSchemaFileName( + node: ActionTraceNode | undefined, +): string | undefined { + const p = node?.schema?.sourceFilePath; + return p !== undefined ? basename(p) : undefined; +} + +/** Whether the two sides compiled different grammars, decided by their + * `grammarHash`: `true` when the hashes differ, `false` when they match, + * `undefined` when either side didn't record one. The hash covers the whole + * compiled grammar, so it flags a changed rule body even when the winning rule + * name is unchanged — a case the rule-name comparison alone would miss. */ +function grammarHashDiffers( + a: GrammarMatchTraceNode | undefined, + b: GrammarMatchTraceNode | undefined, +): boolean | undefined { + const ha = a?.debugInfo?.grammarHash; + const hb = b?.debugInfo?.grammarHash; + if (ha === undefined || hb === undefined) { + return undefined; + } + return ha !== hb; +} + +/** Name each side's matched rule for the grammar-change copy, reading a side + * that settled on no rule as "matched no rule". */ +function ruleClause(aRule?: string, bRule?: string): string { + const a = aRule !== undefined ? `A matched ${aRule}` : "A matched no rule"; + const b = bRule !== undefined ? `B matched ${bRule}` : "B matched no rule"; + return `${a}, ${b}`; +} + +/** The grammar-change sentence: names the `.agr` file when known and the two + * sides' rules when they differ (a rule body-only change reads as a plain + * "changed between the two versions"). */ +function grammarChangeDetail( + fileName: string | undefined, + aRule: string | undefined, + bRule: string | undefined, + rulesDiffer: boolean, +): string { + const subject = + fileName !== undefined ? `The grammar file ${fileName}` : "The grammar"; + const tail = rulesDiffer + ? `: ${ruleClause(aRule, bRule)}.` + : " between the two versions."; + return `${subject} changed${tail}`; +} + +type CauseAttribution = { + cause: DivergenceCause; + confidence: "high" | "low"; + confidenceNote?: string; +}; + +/** Shared inputs for the per-layer attribution steps, computed once from the + * trace so the individual steps stay simple. */ +interface AttributionContext { + a: ReplaySideTrace; + b: ReplaySideTrace; + aGrammar: GrammarMatchTraceNode | undefined; + bGrammar: GrammarMatchTraceNode | undefined; + /** True when a grammar node's captured parse diverged from the resolver's + * ranked pick, so the grammar level cannot be trusted to name the cause. */ + parityDiverged: boolean; + diagnosticNote: string; +} + +/** + * The ordered causal walk, run only when the two sides' final actions differ. + * Consults each layer in runtime order and returns the first that explains the + * difference plus a confidence, honoring the ranking-parity guard: a diverged + * grammar parse cannot be trusted to name the cause, so attribution falls back + * to the action level (low confidence). + */ +function attributeCause(trace: ReplayResolutionTrace): CauseAttribution { + const { a, b } = trace; + const fromCache = cacheCause(a, b); + if (fromCache !== undefined) { + return fromCache; + } + + const aGrammar = nodeOf(a, "grammar-match"); + const bGrammar = nodeOf(b, "grammar-match"); + const ctx: AttributionContext = { + a, + b, + aGrammar, + bGrammar, + parityDiverged: grammarDiverged(aGrammar) || grammarDiverged(bGrammar), + diagnosticNote: + "The captured grammar trace is a diagnostic parse that may differ from the resolver-selected one, so the cause is attributed at the action level.", + }; + return ( + grammarCause(ctx) ?? + wildcardCause(ctx) ?? + actionPayloadCause(ctx) ?? + unattributedCause(ctx) + ); +} + +/** Step 1: a cache short-circuit on one side means its action came from a + * different source than the other side's grammar match. */ +function cacheCause( + a: ReplaySideTrace, + b: ReplaySideTrace, +): CauseAttribution | undefined { + const aCacheDecided = cacheDecided(a); + const bCacheDecided = cacheDecided(b); + if (aCacheDecided && bCacheDecided) { + return { + cause: { + kind: "cache-decided", + detail: "Both versions were resolved from their construction cache.", + }, + confidence: "high", + }; + } + if (aCacheDecided || bCacheDecided) { + const side = aCacheDecided ? "A" : "B"; + return { + cause: { + kind: "cache-decided", + side, + detail: `${sideLabel(side)}'s construction cache returned a cached action, so its grammar never ran; ${sideLabel(otherSide(side))} resolved through its grammar.`, + }, + confidence: "high", + }; + } + return undefined; +} + +/** Step 2: the grammars matched differently. The compiled-grammar hash is + * resolver-independent, so a differing hash names the grammar as the cause even + * when a side's captured parse diverged from the resolver's ranked pick, so the + * same source change reads the same way whichever diagnostic path an utterance + * took. The rule-name comparison reads off that diagnostic parse, so it is only + * consulted when neither side diverged (a rule swap, or a body edit that flips + * the hash under an unchanged winning rule name). */ +function grammarCause(ctx: AttributionContext): CauseAttribution | undefined { + const { aGrammar, bGrammar, parityDiverged } = ctx; + const aRule = aGrammar?.chosenRule; + const bRule = bGrammar?.chosenRule; + const bothRan = + aGrammar?.execution === "ran" && bGrammar?.execution === "ran"; + const rulesDiffer = !parityDiverged && bothRan && aRule !== bRule; + const hashDiffers = grammarHashDiffers(aGrammar, bGrammar); + if (!rulesDiffer && !(bothRan && hashDiffers === true)) { + return undefined; + } + const fileName = grammarCulpritFile(aGrammar, bGrammar); + return { + cause: { + kind: "grammar-differs", + ...(fileName !== undefined ? { fileName } : {}), + detail: grammarChangeDetail(fileName, aRule, bRule, rulesDiffer), + }, + confidence: "high", + }; +} + +/** Step 3: the grammars agree on the rule, but one side's wildcard validation + * rejected the otherwise-selected value. Read off the parse, so only consulted + * when neither side's parse diverged. */ +function wildcardCause(ctx: AttributionContext): CauseAttribution | undefined { + const { a, b, parityDiverged } = ctx; + if (parityDiverged) { + return undefined; + } + const aWild = nodeOf(a, "wildcard-validation"); + const bWild = nodeOf(b, "wildcard-validation"); + const aRejected = + aWild?.execution === "ran" && aWild.outcome === "rejected"; + const bRejected = + bWild?.execution === "ran" && bWild.outcome === "rejected"; + if (aRejected === bRejected) { + return undefined; + } + const side = aRejected ? "A" : "B"; + return { + cause: { + kind: "wildcard-validation", + side, + detail: `${sideLabel(side)}'s wildcard validation rejected the value the other side accepted.`, + }, + confidence: "high", + }; +} + +/** Step 4: same rule and path, differing parameters. Also where a diverged + * parity lands: the actions differ but the grammar level cannot be trusted to + * say why. The action's schema is the file to diff, since the grammar matched + * the same and the payload still changed. */ +function actionPayloadCause( + ctx: AttributionContext, +): CauseAttribution | undefined { + const { a, b, parityDiverged, diagnosticNote } = ctx; + const aName = actionName(a.finalAction); + const bName = actionName(b.finalAction); + if ( + aName === undefined || + bName === undefined || + aName !== bName || + sameAction(a.finalAction, b.finalAction) + ) { + return undefined; + } + const schemaFile = + actionSchemaFileName(nodeOf(a, "action")) ?? + actionSchemaFileName(nodeOf(b, "action")); + return { + cause: { + kind: "action-payload", + ...(schemaFile !== undefined ? { fileName: schemaFile } : {}), + detail: `Both versions produced a ${aName} action, but its parameters differ.`, + }, + confidence: parityDiverged ? "low" : "high", + ...(parityDiverged ? { confidenceNote: diagnosticNote } : {}), + }; +} + +/** Step 5: the actions differ but the captured trace names no single explaining + * layer. */ +function unattributedCause(ctx: AttributionContext): CauseAttribution { + const { parityDiverged, diagnosticNote } = ctx; + return { + cause: { + kind: "unattributed", + detail: "The versions produced different actions, but the captured trace does not attribute the difference to a single layer.", + }, + confidence: "low", + ...(parityDiverged ? { confidenceNote: diagnosticNote } : {}), + }; +} + +/** The layer a cause maps to, for accenting the diverging node row. */ +function divergingLayerFor( + cause: DivergenceCause, +): ReplayTraceNode["kind"] | undefined { + switch (cause.kind) { + case "cache-decided": + return "cache-consult"; + case "grammar-differs": + return "grammar-match"; + case "wildcard-validation": + return "wildcard-validation"; + case "action-payload": + return "action"; + case "unattributed": + return undefined; + } +} + +/** Build the note shown when final actions match but the sides ran different + * fidelity paths (a git-ref side can't run the cache or wildcard validation). */ +function realizationPathNote( + a: ReplaySideTrace, + b: ReplaySideTrace, +): string | undefined { + if (a.realization === b.realization) return undefined; + const sourceSide = a.realization === "source" ? "A" : "B"; + return `Final actions match; ${sideLabel(sourceSide)} ran from grammar source only, so its construction cache and wildcard validation did not run.`; +} + +/** Browser-safe absolute-path test (POSIX `/…`, Windows `C:\…`/`C:/…`, UNC + * `\\…`). Captured paths may be Windows or POSIX regardless of where the viewer + * runs, so this can't use node:path. */ +function isAbsolutePath(p: string): boolean { + return ( + p.startsWith("/") || p.startsWith("\\\\") || /^[A-Za-z]:[\\/]/.test(p) + ); +} + +/** The matched grammar's `.agr` file resolved to an absolute path: the node's + * recorded absolute `sourceFilePath` when present (the real repo file, captured + * on both sides), else — for older captures without it — the winning rule span's + * file table (fileId → resolved path) or an already-absolute display path. + * Returns undefined when only a relative/synthetic path is available (e.g. + * built-in entity grammar), which the host can neither open nor diff. */ +function grammarSourceAbsPath(node: GrammarMatchTraceNode): string | undefined { + if ( + node.sourceFilePath !== undefined && + isAbsolutePath(node.sourceFilePath) + ) { + return node.sourceFilePath; + } + const source = node.source; + if (source === undefined) { + return undefined; + } + const resolved = node.debugInfo?.filePaths?.find( + ([id]) => id === source.fileId, + )?.[1]; + const candidate = resolved ?? source.displayPath; + return isAbsolutePath(candidate) ? candidate : undefined; +} + +function summarizeNode(node: ReplayTraceNode): TraceNodeSummary { + const summary: TraceNodeSummary = { + kind: node.kind, + layerName: TRACE_LAYER_NAME[node.kind], + executionLabel: EXECUTION_LABEL[node.execution] ?? node.execution, + }; + if (node.outcome !== undefined) { + summary.outcomeLabel = OUTCOME_LABEL[node.outcome] ?? node.outcome; + } + if (node.detail !== undefined) { + summary.detail = node.detail; + } + switch (node.kind) { + case "grammar-match": { + const g = node as GrammarMatchTraceNode; + summary.grammar = { + input: g.input, + rankingParity: g.rankingParity, + rankingParityLabel: RANKING_PARITY_LABEL[g.rankingParity], + hasTimeline: g.trace !== undefined && g.debugInfo !== undefined, + hasSource: grammarSourceAbsPath(g) !== undefined, + diagnosticOnly: g.rankingParity === "diverged", + ...(g.chosenRule !== undefined + ? { chosenRule: g.chosenRule } + : {}), + }; + break; + } + case "cache-consult": { + const c = node as CacheConsultTraceNode; + if (c.entry !== undefined) { + const entry = c.entry; + summary.cache = { + ...(entry.constructionId !== undefined + ? { constructionId: entry.constructionId } + : {}), + ...(entry.namespace !== undefined + ? { namespace: entry.namespace } + : {}), + ...(entry.parts !== undefined + ? { parts: entry.parts } + : {}), + }; + } + break; + } + case "action": { + const act = node as ActionTraceNode; + const schemaPath = act.schema?.sourceFilePath; + summary.action = { + hasSchema: + schemaPath !== undefined && isAbsolutePath(schemaPath), + ...(actionName(act.action) !== undefined + ? { actionName: actionName(act.action) } + : {}), + }; + break; + } + } + return summary; +} + +/** Order a side's nodes by the canonical layer order and summarize each. Missing + * layers are omitted (the viewer renders the fixed four-row grid and fills gaps + * with a placeholder), so only captured nodes are summarized here. */ +function summarizeSide(side: ReplaySideTrace): SideDivergenceView { + const byKind = new Map(side.nodes.map((n) => [n.kind, n] as const)); + const nodes: TraceNodeSummary[] = []; + for (const kind of TRACE_LAYER_ORDER) { + const node = byKind.get(kind); + if (node !== undefined) { + nodes.push(summarizeNode(node)); + } + } + return { side: side.side, realization: side.realization, nodes }; +} + +/** True when a summarized node actively ran (as opposed to being skipped, + * not-applicable, unavailable, or not-reached). */ +function stageRan(node: TraceNodeSummary | undefined): boolean { + return node?.executionLabel === EXECUTION_LABEL.ran; +} + +/** Classify how the two sides compared at one stage. The attributed cause stage + * is always `diverge`; otherwise the stage is `agree` when both sides ran it, + * `one-sided` when exactly one did, and `inapplicable` when neither did. */ +function stageStatusFor( + a: TraceNodeSummary | undefined, + b: TraceNodeSummary | undefined, + isCause: boolean, +): TraceStageStatus { + if (isCause) { + return "diverge"; + } + const aRan = stageRan(a); + const bRan = stageRan(b); + if (aRan && bRan) { + return "agree"; + } + if (aRan || bRan) { + return "one-sided"; + } + return "inapplicable"; +} + +/** The compare handle for a stage backed by a diffable source file, when either + * side exposes one: grammar rules diff their `.agr`, actions diff their schema + * `.ts`. Undefined for cache/wildcard stages, which have no file to diff. */ +function stageCompareFor( + kind: ReplayTraceNode["kind"], + a: TraceNodeSummary | undefined, + b: TraceNodeSummary | undefined, +): "grammar-match" | "action" | undefined { + if ( + kind === "grammar-match" && + (a?.grammar?.hasSource === true || b?.grammar?.hasSource === true) + ) { + return "grammar-match"; + } + if ( + kind === "action" && + (a?.action?.hasSchema === true || b?.action?.hasSchema === true) + ) { + return "action"; + } + return undefined; +} + +/** Assemble the pipeline: one stage per fidelity layer that at least one side + * engaged, in runtime order, each pairing the two sides' nodes and marking the + * diverging stage as the cause. Layers neither side captured are omitted so the + * flow shows only steps this utterance actually went through. */ +function buildStages( + aView: SideDivergenceView, + bView: SideDivergenceView, + divergingLayer: ReplayTraceNode["kind"] | undefined, +): TraceStageView[] { + const aByKind = new Map(aView.nodes.map((n) => [n.kind, n] as const)); + const bByKind = new Map(bView.nodes.map((n) => [n.kind, n] as const)); + const stages: TraceStageView[] = []; + for (const kind of TRACE_LAYER_ORDER) { + const a = aByKind.get(kind); + const b = bByKind.get(kind); + if (a === undefined && b === undefined) { + continue; + } + const isCause = divergingLayer === kind; + const compare = stageCompareFor(kind, a, b); + stages.push({ + kind, + layerName: TRACE_LAYER_NAME[kind], + ...(a !== undefined ? { a } : {}), + ...(b !== undefined ? { b } : {}), + status: stageStatusFor(a, b, isCause), + isCause, + ...(compare !== undefined ? { compare } : {}), + }); + } + return stages; +} + +/** + * Build the full divergence view model for one utterance's captured trace. + * Pure: the same trace always yields the same conclusion and node summaries. + */ +export function toTraceDivergenceViewModel( + trace: ReplayResolutionTrace, +): TraceDivergenceViewModel { + const { a, b } = trace; + const aHasAction = isActionObject(a.finalAction); + const bHasAction = isActionObject(b.finalAction); + const bothNoAction = !aHasAction && !bHasAction; + const parity: TraceParity = sameAction(a.finalAction, b.finalAction) + ? "match" + : "differ"; + + let conclusion: DivergenceConclusion; + let divergingLayer: ReplayTraceNode["kind"] | undefined; + + if (parity === "match") { + if (bothNoAction) { + conclusion = { + parity, + headline: + "Neither version resolved this utterance to an action.", + bothNoAction: true, + confidence: "high", + }; + } else { + const pathNote = realizationPathNote(a, b); + conclusion = { + parity, + headline: "Both versions produced the same action.", + bothNoAction: false, + confidence: "high", + ...(pathNote !== undefined ? { pathNote } : {}), + }; + } + } else { + const { cause, confidence, confidenceNote } = attributeCause(trace); + divergingLayer = divergingLayerFor(cause); + conclusion = { + parity, + headline: "A and B produced different actions.", + bothNoAction: false, + confidence, + cause, + ...(confidenceNote !== undefined ? { confidenceNote } : {}), + }; + } + + const aView = summarizeSide(a); + const bView = summarizeSide(b); + const resultDiff = toActionDiff({ + actionA: a.finalAction, + actionB: b.finalAction, + }); + return { + runId: trace.runId, + utteranceId: trace.utteranceId, + utterance: trace.utterance, + conclusion, + a: aView, + b: bView, + ...(divergingLayer !== undefined ? { divergingLayer } : {}), + stages: buildStages(aView, bView, divergingLayer), + resultDiff, + }; +} diff --git a/ts/packages/typeagent-studio/src/webviewKit/traceProtocol.ts b/ts/packages/typeagent-studio/src/webviewKit/traceProtocol.ts new file mode 100644 index 0000000000..14f842db64 --- /dev/null +++ b/ts/packages/typeagent-studio/src/webviewKit/traceProtocol.ts @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * The Trace Viewer's own message contract, kept separate from the Impact + * Report's protocol so the two panels evolve independently. A viewer panel is + * scoped to a single `{runId, utteranceId}`, so the webview never sends those + * ids back — it names only a side and a node, and the host rederives the rest + * from the run descriptor it holds. Every message the webview posts is untrusted + * input, so {@link parseTraceMessage} narrows it before the host acts on it. + */ + +import type { ReplayResolutionTrace } from "@typeagent/core/replay"; + +/** Which replayed version a webview interaction refers to. */ +export type TraceSide = "a" | "b"; + +/** The node whose source a jump/diff targets. Only these two carry a source + * span. */ +export type TraceSourceNode = "grammar-match" | "action"; + +/** A compact, display-only summary of one pinned version, derived host-side from + * the run descriptor for the header's provenance line. */ +export interface TraceVersionSummary { + label: string; + /** The resolved commit SHA, when the side is a git ref. */ + sha?: string; + /** True when the side is the live working tree (uncommitted edits). */ + workingTree: boolean; +} + +/** The provenance the header shows: the agent and which A/B versions ran. */ +export interface TraceProvenanceSummary { + agent: string; + a: TraceVersionSummary; + b: TraceVersionSummary; + /** Epoch ms the originating run was issued. */ + runAt: number; +} + +/** Why a stored trace can't be shown: its run's traces were rotated out + * (`evicted`, the report outlived them) or the row was never captured + * (`missing`). `loading` is the transient state before the lookup resolves. */ +export type TraceUnavailableState = "loading" | "evicted" | "missing"; + +/** Outcome of a source jump/diff the host attempted on the webview's behalf. */ +export type TraceSourceStatus = "opened" | "unavailable" | "stale"; + +export type HostToTraceMessage = + | { + type: "trace"; + recorded: ReplayResolutionTrace; + provenance: TraceProvenanceSummary; + } + | { type: "trace-state"; state: TraceUnavailableState } + | { type: "trace-error"; message: string } + | { + type: "source-result"; + requestId: number; + status: TraceSourceStatus; + message?: string; + }; + +export type TraceToHostMessage = + | { type: "ready" } + | { + type: "open-source"; + requestId: number; + side: TraceSide; + node: TraceSourceNode; + } + | { + // Open a side-by-side diff of one node's source file across the A and + // B versions, so a divergence shows as a concrete source change. + type: "compare-source"; + requestId: number; + node: TraceSourceNode; + }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isSide(value: unknown): value is TraceSide { + return value === "a" || value === "b"; +} + +function isSourceNode(value: unknown): value is TraceSourceNode { + return value === "grammar-match" || value === "action"; +} + +/** + * Narrow an untrusted webview message to a known {@link TraceToHostMessage}, or + * `undefined` when it doesn't match any known shape. Only the fields the host + * uses are validated; ids the panel already owns (runId/utteranceId) are never + * accepted from the webview. + */ +export function parseTraceMessage( + value: unknown, +): TraceToHostMessage | undefined { + if (!isRecord(value) || typeof value.type !== "string") { + return undefined; + } + switch (value.type) { + case "ready": + return { type: "ready" }; + case "open-source": + return typeof value.requestId === "number" && + isSide(value.side) && + isSourceNode(value.node) + ? { + type: "open-source", + requestId: value.requestId, + side: value.side, + node: value.node, + } + : undefined; + case "compare-source": + return typeof value.requestId === "number" && + isSourceNode(value.node) + ? { + type: "compare-source", + requestId: value.requestId, + node: value.node, + } + : undefined; + default: + return undefined; + } +}