diff --git a/ts/packages/agentServer/client/package.json b/ts/packages/agentServer/client/package.json index 23816610b..ba4d37817 100644 --- a/ts/packages/agentServer/client/package.json +++ b/ts/packages/agentServer/client/package.json @@ -32,6 +32,7 @@ }, "dependencies": { "@typeagent/agent-rpc": "workspace:*", + "@typeagent/agent-sdk": "workspace:*", "@typeagent/agent-server-protocol": "workspace:*", "@typeagent/dispatcher-rpc": "workspace:*", "debug": "^4.4.0", diff --git a/ts/packages/agentServer/client/src/agentServerClient.ts b/ts/packages/agentServer/client/src/agentServerClient.ts index e5388b6c9..af6def6e1 100644 --- a/ts/packages/agentServer/client/src/agentServerClient.ts +++ b/ts/packages/agentServer/client/src/agentServerClient.ts @@ -4,6 +4,8 @@ import { createChannelProviderAdapter } from "@typeagent/agent-rpc/channel"; import type { ChannelProviderAdapter } from "@typeagent/agent-rpc/channel"; import { createRpc } from "@typeagent/agent-rpc/rpc"; +import { createAgentRpcServer } from "@typeagent/agent-rpc/server"; +import type { AppAgent, AppAgentManifest } from "@typeagent/agent-sdk"; import { createClientIORpcServer } from "@typeagent/dispatcher-rpc/clientio/server"; import { createDispatcherRpcClient, @@ -132,6 +134,23 @@ export type AgentServerConnection = { * isn't configured, so callers can hide/disable the mic affordance. */ getSpeechToken(): Promise; + /** + * Register a client-hosted app agent with a joined conversation on the + * server. The agent's handlers run in this process; the server installs an + * rpc proxy that forwards calls back over the connection. Pass + * `conversationId` to target a specific joined conversation, or omit it when + * exactly one conversation is joined. Re-registering the same `name` (e.g. + * after {@link reconnect}) replaces the previous registration. Rejects if + * another client already registered `name` on the target conversation. + */ + registerClientAgent( + name: string, + manifest: AppAgentManifest, + agent: AppAgent, + conversationId?: string, + ): Promise; + /** Unregister an agent previously registered via registerClientAgent. */ + unregisterClientAgent(name: string, conversationId?: string): Promise; /** * Reopen the underlying transport and rebind the control rpc onto it, * reusing this connection object instead of building a new one. Returns @@ -203,6 +222,11 @@ export function createAgentServerConnection( { dispatcher: Dispatcher; connectionId: string } >(); + // Client-hosted agents registered on the server, name → agent-rpc server + // closeFn. Used to tear down the local rpc server when unregistering, + // re-registering, or closing the connection. + const clientAgentServers = new Map void>(); + let closed = false; const connection: AgentServerConnection = { @@ -338,6 +362,51 @@ export function createAgentServerConnection( return rpc.invoke("getSpeechToken"); }, + async registerClientAgent( + name: string, + manifest: AppAgentManifest, + agent: AppAgent, + conversationId?: string, + ): Promise { + // Drop any previous rpc server for this name (e.g. re-registering + // after a reconnect, where the old server sat on a stale channel). + clientAgentServers.get(name)?.(); + clientAgentServers.delete(name); + + const { closeFn, agentInterface } = createAgentRpcServer( + name, + agent, + currentChannel, + ); + try { + await rpc.invoke("registerClientAgent", { + name, + manifest, + agentInterface, + ...(conversationId !== undefined ? { conversationId } : {}), + }); + } catch (e) { + closeFn(); + throw e; + } + clientAgentServers.set(name, closeFn); + }, + + async unregisterClientAgent( + name: string, + conversationId?: string, + ): Promise { + try { + await rpc.invoke("unregisterClientAgent", { + name, + ...(conversationId !== undefined ? { conversationId } : {}), + }); + } finally { + clientAgentServers.get(name)?.(); + clientAgentServers.delete(name); + } + }, + async reconnect(): Promise { if (closed || reopenTransport === undefined) { return false; @@ -351,6 +420,12 @@ export function createAgentServerConnection( // The prior transport's per-conversation channels are gone; drop // our local bookkeeping so the caller re-joins cleanly. joinedConversations.clear(); + // Client-agent rpc servers were bound to the old channel; drop them + // so the caller re-registers them on the new channel after re-join. + for (const closeFn of clientAgentServers.values()) { + closeFn(); + } + clientAgentServers.clear(); return true; }, @@ -360,6 +435,10 @@ export function createAgentServerConnection( } closed = true; debug("Closing agent server connection"); + for (const closeFn of clientAgentServers.values()) { + closeFn(); + } + clientAgentServers.clear(); closeTransport(); }, }; diff --git a/ts/packages/agentServer/client/test/conversation-stubConnection.ts b/ts/packages/agentServer/client/test/conversation-stubConnection.ts index 3b257cc0b..895a0b475 100644 --- a/ts/packages/agentServer/client/test/conversation-stubConnection.ts +++ b/ts/packages/agentServer/client/test/conversation-stubConnection.ts @@ -206,6 +206,8 @@ export function makeStubConnection( async getSpeechToken() { return undefined; }, + async registerClientAgent() {}, + async unregisterClientAgent() {}, async reconnect() { return true; }, diff --git a/ts/packages/agentServer/protocol/package.json b/ts/packages/agentServer/protocol/package.json index aff613f6b..d0b213416 100644 --- a/ts/packages/agentServer/protocol/package.json +++ b/ts/packages/agentServer/protocol/package.json @@ -26,6 +26,8 @@ "tsc": "tsc -b" }, "dependencies": { + "@typeagent/agent-rpc": "workspace:*", + "@typeagent/agent-sdk": "workspace:*", "@typeagent/dispatcher-rpc": "workspace:*", "@typeagent/dispatcher-types": "workspace:*" }, diff --git a/ts/packages/agentServer/protocol/src/index.ts b/ts/packages/agentServer/protocol/src/index.ts index a94c6e73a..97a5904ea 100644 --- a/ts/packages/agentServer/protocol/src/index.ts +++ b/ts/packages/agentServer/protocol/src/index.ts @@ -18,6 +18,8 @@ export { ConversationNameCollisionBehavior, JoinConversationResult, RenameConversationOptions, + RegisterClientAgentParams, + UnregisterClientAgentParams, UserIdentity, DefaultUserIdentity, SpeechToken, diff --git a/ts/packages/agentServer/protocol/src/protocol.ts b/ts/packages/agentServer/protocol/src/protocol.ts index 94179335e..3b847c664 100644 --- a/ts/packages/agentServer/protocol/src/protocol.ts +++ b/ts/packages/agentServer/protocol/src/protocol.ts @@ -3,6 +3,8 @@ import type { PendingInteractionRequest } from "@typeagent/dispatcher-types"; import type { QueueSnapshot } from "@typeagent/dispatcher-types"; +import type { AppAgentManifest } from "@typeagent/agent-sdk"; +import type { AgentInterfaceFunctionName } from "@typeagent/agent-rpc/server"; export type DispatcherConnectOptions = { filter?: boolean; // filter to message for own request. Default is false (no filtering) @@ -124,6 +126,42 @@ export type AgentServerInvokeFunctions = { * mic affordance. */ getSpeechToken: () => Promise; + /** + * Register a client-hosted app agent with a joined conversation's + * dispatcher. The agent's handlers run in the connecting client's process + * (over agent-rpc on the connection's channel provider); the server builds + * an rpc proxy from `agentInterface` and installs it as a dynamic agent. + * The agent is removed automatically when the connection drops or leaves + * the conversation. + * + * The client must create its agent-rpc server on the `agent:` + * channel (via createAgentRpcServer over the connection channel provider) + * before calling this. Rejects if an agent with `name` is already + * registered on the target conversation (e.g. a second client trying to + * register the same singleton agent). + */ + registerClientAgent: (param: RegisterClientAgentParams) => Promise; + /** Unregister a previously registered client-hosted agent. */ + unregisterClientAgent: ( + param: UnregisterClientAgentParams, + ) => Promise; +}; + +export type RegisterClientAgentParams = { + name: string; + manifest: AppAgentManifest; + agentInterface: AgentInterfaceFunctionName[]; + /** + * Target conversation. If omitted, the server uses the connection's single + * joined conversation (and errors if the connection has joined none or + * more than one). + */ + conversationId?: string; +}; + +export type UnregisterClientAgentParams = { + name: string; + conversationId?: string; }; /** diff --git a/ts/packages/agentServer/server/package.json b/ts/packages/agentServer/server/package.json index 4f9c67e86..0f47decbf 100644 --- a/ts/packages/agentServer/server/package.json +++ b/ts/packages/agentServer/server/package.json @@ -41,6 +41,7 @@ "dependencies": { "@azure/identity": "^4.10.0", "@typeagent/agent-rpc": "workspace:*", + "@typeagent/agent-sdk": "workspace:*", "@typeagent/agent-server-client": "workspace:*", "@typeagent/agent-server-protocol": "workspace:*", "@typeagent/common-utils": "workspace:*", diff --git a/ts/packages/agentServer/server/src/connectionHandler.ts b/ts/packages/agentServer/server/src/connectionHandler.ts index bd347a98f..02fde1221 100644 --- a/ts/packages/agentServer/server/src/connectionHandler.ts +++ b/ts/packages/agentServer/server/src/connectionHandler.ts @@ -4,6 +4,7 @@ import { createDispatcherRpcServer } from "@typeagent/dispatcher-rpc/dispatcher/server"; import { createClientIORpcClient } from "@typeagent/dispatcher-rpc/clientio/client"; import { createRpc } from "@typeagent/agent-rpc/rpc"; +import { createAgentRpcClient } from "@typeagent/agent-rpc/client"; import type { ChannelProvider } from "@typeagent/agent-rpc/channel"; import { AgentServerInvokeFunctions, @@ -103,6 +104,39 @@ export function createAgentServerConnectionHandler( { dispatcher: Dispatcher; connectionId: string } >(); + // Client-hosted agents this connection registered, per conversation. + // conversationId → set of agent names. Used to tear them down when the + // connection drops so they don't linger on the (longer-lived) shared + // dispatcher. + const clientAgents = new Map>(); + + // Resolve the conversation a client-agent operation targets. When no id + // is given, use the single joined conversation; error if there are zero + // or many so the caller must disambiguate. + const resolveClientAgentConversation = ( + conversationId?: string, + ): string => { + if (conversationId !== undefined) { + if (!joinedConversations.has(conversationId)) { + throw new Error( + `Not joined to conversation: ${conversationId}`, + ); + } + return conversationId; + } + if (joinedConversations.size === 1) { + return joinedConversations.keys().next().value as string; + } + if (joinedConversations.size === 0) { + throw new Error( + "Cannot register client agent: no conversation joined", + ); + } + throw new Error( + "Cannot register client agent: multiple conversations joined; specify conversationId", + ); + }; + // Warn this connection about a stale server build at most once, even // if it joins several conversations. let notifiedStale = false; @@ -288,11 +322,71 @@ export function createAgentServerConnectionHandler( }, getUserIdentity: async () => getUserIdentity(), getSpeechToken: async () => getSpeechToken(), + registerClientAgent: async (param) => { + const conversationId = resolveClientAgentConversation( + param.conversationId, + ); + const { name, manifest, agentInterface } = param; + if (clientAgents.get(conversationId)?.has(name)) { + throw new Error( + `Client agent '${name}' is already registered on conversation '${conversationId}'`, + ); + } + // Build the rpc proxy on the connection's own channel provider + // (the client hosts the real agent via createAgentRpcServer on + // the matching agent: channel). + const appAgent = await createAgentRpcClient( + name, + channelProvider, + agentInterface, + ); + try { + await conversationManager.addClientAgent( + conversationId, + name, + manifest, + appAgent, + ); + } catch (e) { + channelProvider.deleteChannel(`agent:${name}`); + throw e; + } + let set = clientAgents.get(conversationId); + if (set === undefined) { + set = new Set(); + clientAgents.set(conversationId, set); + } + set.add(name); + }, + unregisterClientAgent: async (param) => { + const conversationId = resolveClientAgentConversation( + param.conversationId, + ); + const { name } = param; + await conversationManager.removeClientAgent( + conversationId, + name, + ); + channelProvider.deleteChannel(`agent:${name}`); + clientAgents.get(conversationId)?.delete(name); + }, }; // Clean up all conversations on disconnect channelProvider.on("disconnect", () => { onDisconnect?.(); + // Remove client-hosted agents first so they don't linger on the + // shared dispatcher after this connection's socket is gone. + for (const [conversationId, names] of clientAgents.entries()) { + for (const name of names) { + conversationManager + .removeClientAgent(conversationId, name) + .catch(() => { + // Best effort on disconnect + }); + } + } + clientAgents.clear(); for (const [ conversationId, { connectionId }, diff --git a/ts/packages/agentServer/server/src/conversationManager.ts b/ts/packages/agentServer/server/src/conversationManager.ts index e93c1f302..10322b825 100644 --- a/ts/packages/agentServer/server/src/conversationManager.ts +++ b/ts/packages/agentServer/server/src/conversationManager.ts @@ -13,6 +13,7 @@ import { RenameConversationOptions, } from "@typeagent/agent-server-protocol"; import { ClientIO, Dispatcher, DispatcherOptions } from "agent-dispatcher"; +import type { AppAgent, AppAgentManifest } from "@typeagent/agent-sdk"; import type { DisplayLogEntry, PendingInteractionRequest, @@ -166,6 +167,19 @@ export type ConversationManager = { importCopilotMirror( params: ImportCopilotMirrorParams, ): Promise; + /** + * Install a client-hosted agent (typically an agent-rpc proxy) as a dynamic + * agent on a conversation's dispatcher. The conversation must already have a + * dispatcher (i.e. a client has joined). Rejects if `name` already exists. + */ + addClientAgent( + conversationId: string, + name: string, + manifest: AppAgentManifest, + appAgent: AppAgent, + ): Promise; + /** Remove a client-hosted agent added via {@link addClientAgent}. */ + removeClientAgent(conversationId: string, name: string): Promise; close(): Promise; }; @@ -846,6 +860,39 @@ export async function createConversationManager( } }, + async addClientAgent( + conversationId: string, + name: string, + manifest: AppAgentManifest, + appAgent: AppAgent, + ): Promise { + const record = conversations.get(conversationId); + if (record === undefined) { + throw new Error(`Conversation not found: ${conversationId}`); + } + const sharedDispatcher = await ensureDispatcher(record); + await sharedDispatcher.addDynamicAgent(name, manifest, appAgent); + debugConversation( + `Registered client agent "${name}" on conversation "${record.name}" (${conversationId})`, + ); + }, + + async removeClientAgent( + conversationId: string, + name: string, + ): Promise { + const record = conversations.get(conversationId); + // If the conversation or its dispatcher is already gone, there is + // nothing to remove. + if (record?.sharedDispatcher === undefined) { + return; + } + await record.sharedDispatcher.removeDynamicAgent(name); + debugConversation( + `Removed client agent "${name}" from conversation "${record.name}" (${conversationId})`, + ); + }, + listConversations(name?: string): ConversationInfo[] { const result: ConversationInfo[] = []; for (const record of conversations.values()) { diff --git a/ts/packages/agentServer/server/src/sharedDispatcher.ts b/ts/packages/agentServer/server/src/sharedDispatcher.ts index 804da2d4d..b2fb8a1c4 100644 --- a/ts/packages/agentServer/server/src/sharedDispatcher.ts +++ b/ts/packages/agentServer/server/src/sharedDispatcher.ts @@ -13,6 +13,7 @@ import { ClientIO, RequestId, } from "agent-dispatcher"; +import type { AppAgent, AppAgentManifest } from "@typeagent/agent-sdk"; import type { PendingInteractionRequest, PendingInteractionResponse, @@ -716,6 +717,30 @@ export async function createSharedDispatcher( cancelQueued(requestId: string, reason: QueueCancelReason): boolean { return context.requestQueue.cancelQueued(requestId, reason); }, + async addDynamicAgent( + name: string, + manifest: AppAgentManifest, + appAgent: AppAgent, + ): Promise { + // Serialize with command processing and other agent mutations. + // Mirrors SessionContext.addDynamicAgent: install the agent, then + // recompute enable state so it becomes usable. + await context.commandLock(async () => { + await context.agents.addDynamicAgent(name, manifest, appAgent); + await context.agents.setState( + context, + context.session.getConfig(), + ); + }); + }, + async removeDynamicAgent(name: string): Promise { + await context.commandLock(async () => { + await context.agents.removeAgent( + name, + context.agentCache.grammarStore, + ); + }); + }, __testSetNoClientsGraceMs(ms: number): void { noClientsGraceMs = ms; }, @@ -754,6 +779,18 @@ export type SharedDispatcher = { isQueueIdle(): boolean; /** Cancel a queued (not running) entry by requestId. */ cancelQueued(requestId: string, reason: QueueCancelReason): boolean; + /** + * Install a client-hosted agent as a dynamic agent on this conversation's + * dispatcher. `appAgent` is typically an agent-rpc proxy that forwards + * calls back to the client. Rejects if an agent with `name` already exists. + */ + addDynamicAgent( + name: string, + manifest: AppAgentManifest, + appAgent: AppAgent, + ): Promise; + /** Remove a previously added dynamic agent. No-op if it doesn't exist. */ + removeDynamicAgent(name: string): Promise; /** @internal Test-only: tighten the no-clients grace window. */ __testSetNoClientsGraceMs(ms: number): void; }; diff --git a/ts/packages/agents/browser/src/extension/serviceWorker/dispatcherConnection.ts b/ts/packages/agents/browser/src/extension/serviceWorker/dispatcherConnection.ts index 1a068bcf3..17cec7171 100644 --- a/ts/packages/agents/browser/src/extension/serviceWorker/dispatcherConnection.ts +++ b/ts/packages/agents/browser/src/extension/serviceWorker/dispatcherConnection.ts @@ -577,6 +577,10 @@ function makeConnectionAdapter(): AgentServerConnection { // In-place rebind reconnect isn't driven through this adapter; the // service worker reconnects via its own doConnect path. reconnect: async () => false, + registerClientAgent: () => + notSupported("registerClientAgent") as Promise, + unregisterClientAgent: () => + notSupported("unregisterClientAgent") as Promise, close: () => notSupported("close"), }; } diff --git a/ts/packages/shell/src/main/agent.ts b/ts/packages/shell/src/main/agent.ts index 1e4ef6f6a..bf3b83242 100644 --- a/ts/packages/shell/src/main/agent.ts +++ b/ts/packages/shell/src/main/agent.ts @@ -36,11 +36,10 @@ export type ShellContext = { shellWindow: ShellWindow; }; -const config: AppAgentManifest = { +const baseManifest = { emojiChar: "🐚", description: "Shell", - localView: true, -}; +} as const; class ShellShowSettingsCommandHandler implements CommandHandlerNoParams { public readonly description = "Show shell settings"; @@ -416,18 +415,44 @@ const handlers: CommandHandlerTable = { }, }; -export function createShellAgentProvider(shellWindow: ShellWindow) { +export type ShellAgentOptions = { + // When false, the agent does not serve a local chat view: the manifest + // omits `localView` and initializeAgentContext does not start the local + // chat server. Used in connect mode, where the agent runs in the shell + // process over agent-rpc and the chat view is not served by the agent. + // Default is true (in-process/standalone shell). + serveLocalView?: boolean; +}; + +// Build the shell's own AppAgent and manifest. Shared by the in-process +// provider (standalone) and the connect-mode registration so the two paths +// can't drift. +export function createShellAgent( + shellWindow: ShellWindow, + options?: ShellAgentOptions, +): { manifest: AppAgentManifest; agent: AppAgent } { + const serveLocalView = options?.serveLocalView ?? true; + const manifest: AppAgentManifest = serveLocalView + ? { ...baseManifest, localView: true } + : { ...baseManifest }; const agent: AppAgent = { async initializeAgentContext( - settings: AppAgentInitSettings, + settings?: AppAgentInitSettings, ): Promise { - shellWindow.startChatServer(settings.localHostPort ?? -1); + if (serveLocalView) { + shellWindow.startChatServer(settings?.localHostPort ?? -1); + } return { shellWindow, }; }, ...getCommandInterface(handlers), }; + return { manifest, agent }; +} + +export function createShellAgentProvider(shellWindow: ShellWindow) { + const { manifest, agent } = createShellAgent(shellWindow); const shellAgentProvider: AppAgentProvider = { getAppAgentNames: () => { @@ -437,7 +462,7 @@ export function createShellAgentProvider(shellWindow: ShellWindow) { if (appAgentName !== "shell") { throw new Error(`Unknown app agent: ${appAgentName}`); } - return config; + return manifest; }, loadAppAgent: async (appAgentName: string) => { if (appAgentName !== "shell") { diff --git a/ts/packages/shell/src/main/instance.ts b/ts/packages/shell/src/main/instance.ts index f3a4e6e2d..3c2c1778e 100644 --- a/ts/packages/shell/src/main/instance.ts +++ b/ts/packages/shell/src/main/instance.ts @@ -23,7 +23,7 @@ import { getIndexingServiceRegistry, } from "default-agent-provider"; import { getTraceId } from "agent-dispatcher/helpers/data"; -import { createShellAgentProvider } from "./agent.js"; +import { createShellAgent, createShellAgentProvider } from "./agent.js"; import { createInlineBrowserControl } from "./inlineBrowserControl.js"; import { BrowserAgentIpc } from "./browserIpc.js"; import { @@ -280,6 +280,33 @@ async function initializeDispatcher( ); const url = `ws://localhost:${connect}`; + // Register the shell's own agent (@shell commands) with the remote + // dispatcher. The agent's handlers run here in the Electron main + // process (against the live ShellWindow) via agent-rpc over the + // connection; the remote dispatcher only holds a proxy. serveLocalView + // is false in connect mode: the agent does not serve a local chat view + // and needs no assigned localHostPort. Must be called after each + // (re)join, since a fresh dispatcher has no shell agent. + const registerShellAgent = async ( + conn: AgentServerConnection, + ): Promise => { + const { manifest, agent } = createShellAgent(shellWindow, { + serveLocalView: false, + }); + try { + await conn.registerClientAgent("shell", manifest, agent); + debugShellInit("Registered shell agent with remote server"); + } catch (e) { + // A second shell on the same conversation, or an older + // server without support, shouldn't block startup - the + // shell still works, just without @shell commands. + debugShellError( + "Failed to register shell agent with remote server:", + e instanceof Error ? e.message : e, + ); + } + }; + // Reconnect state. When the WebSocket drops we attempt a few // backoff retries before giving up and surfacing the modal // "Disconnected" dialog. This keeps the shell alive across @@ -425,6 +452,9 @@ async function initializeDispatcher( ); } rebindDispatcher(freshConversation.dispatcher); + // The reconnected server has a fresh dispatcher with + // no shell agent; re-register it. + await registerShellAgent(conn); reconnectAttempt = 0; broadcastReconnect(undefined); debugShellInit("Reconnected to dispatcher"); @@ -503,6 +533,7 @@ async function initializeDispatcher( // Find-or-create the default "Shell" conversation, matching CLI // behavior. Shared with the standalone (embedded) path. await restoreOrJoinShellConversation(connection); + await registerShellAgent(connection); // Note: connection.close() is called by closeDispatcher() on // shutdown, so no override here — it would double-close the WebSocket. diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index 2c50e1188..2c1ee2076 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -1580,6 +1580,9 @@ importers: '@typeagent/agent-rpc': specifier: workspace:* version: link:../../agentRpc + '@typeagent/agent-sdk': + specifier: workspace:* + version: link:../../agentSdk '@typeagent/agent-server-protocol': specifier: workspace:* version: link:../protocol @@ -1626,6 +1629,12 @@ importers: packages/agentServer/protocol: dependencies: + '@typeagent/agent-rpc': + specifier: workspace:* + version: link:../../agentRpc + '@typeagent/agent-sdk': + specifier: workspace:* + version: link:../../agentSdk '@typeagent/dispatcher-rpc': specifier: workspace:* version: link:../../dispatcher/rpc @@ -1651,6 +1660,9 @@ importers: '@typeagent/agent-rpc': specifier: workspace:* version: link:../../agentRpc + '@typeagent/agent-sdk': + specifier: workspace:* + version: link:../../agentSdk '@typeagent/agent-server-client': specifier: workspace:* version: link:../client