diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index a99f8acff20c..0d0526cd07b6 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -4,6 +4,8 @@ import { PermissionV1 } from "@opencode-ai/core/v1/permission" import { Provider } from "@/provider/provider" import { SessionV1 } from "@opencode-ai/core/v1/session" import { serviceUse } from "@opencode-ai/core/effect/service-use" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { ModelV2 } from "@opencode-ai/core/model" import { Context, Effect, Layer } from "effect" import * as Stream from "effect/Stream" import { streamText, wrapLanguageModel, type ModelMessage, type Tool } from "ai" @@ -51,6 +53,21 @@ export type StreamRequest = StreamInput & { abort: AbortSignal } +function currentUserMessageText(messages: ModelMessage[]): string { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i] + if (message.role !== "user") continue + const content = message.content + if (typeof content === "string") return content + const text = content + .filter((part) => part.type === "text" && "text" in part) + .map((part) => (part as { text: string }).text) + .join("\n") + if (text) return text + } + return "" +} + export interface Interface { readonly stream: (input: StreamInput) => Stream.Stream } @@ -83,9 +100,45 @@ const live: Layer.Layer< const flags = yield* RuntimeFlags.Service const run = Effect.fn("LLM.run")(function* (input: StreamRequest) { + const routeOutput: { model?: { providerID: string; modelID: string }; reason?: string } = {} + const route = yield* plugin.trigger( + "chat.model", + { + sessionID: input.sessionID, + agent: input.agent.name, + model: { + providerID: input.model.providerID, + modelID: input.model.id, + }, + message: { + id: input.user.id, + text: currentUserMessageText(input.messages), + }, + }, + routeOutput, + ) + const model = route.model + ? yield* provider.getModel( + ProviderV2.ID.make(route.model.providerID), + ModelV2.ID.make(route.model.modelID), + ) + : input.model + const user = route.model + ? { + ...input.user, + model: { + providerID: model.providerID, + modelID: model.id, + }, + } + : input.user + yield* Effect.logInfo("stream", { - providerID: input.model.providerID, - modelID: input.model.id, + providerID: model.providerID, + modelID: model.id, + ...(route.model + ? { routedFrom: `${input.model.providerID}/${input.model.id}`, reason: route.reason } + : {}), "session.id": input.sessionID, small: (input.small ?? false).toString(), agent: input.agent.name, @@ -94,10 +147,10 @@ const live: Layer.Layer< const [language, cfg, item, info] = yield* Effect.all( [ - provider.getLanguage(input.model), + provider.getLanguage(model), config.get(), - provider.getProvider(input.model.providerID), - auth.get(input.model.providerID), + provider.getProvider(model.providerID), + auth.get(model.providerID), ], { concurrency: "unbounded" }, ) @@ -105,6 +158,8 @@ const live: Layer.Layer< const isWorkflow = language instanceof GitLabWorkflowLanguageModel const prepared = yield* LLMRequestPrep.prepare({ ...input, + user, + model, provider: item, auth: info, plugin, @@ -225,7 +280,7 @@ const live: Layer.Layer< // either returns a ready LLMEvent stream or a concrete fallback reason. if (flags.experimentalNativeLlm) { const native = LLMNativeRuntime.stream({ - model: input.model, + model: model, provider: item, auth: info, llmClient, @@ -243,8 +298,8 @@ const live: Layer.Layer< if (native.type === "supported") { yield* Effect.logInfo("llm runtime selected", { "llm.runtime": "native", - "llm.provider": input.model.providerID, - "llm.model": input.model.id, + "llm.provider": model.providerID, + "llm.model": model.id, }) return { type: "native" as const, @@ -253,13 +308,13 @@ const live: Layer.Layer< } yield* Effect.logInfo("llm runtime selected", { "llm.runtime": "ai-sdk", - "llm.provider": input.model.providerID, - "llm.model": input.model.id, + "llm.provider": model.providerID, + "llm.model": model.id, "llm.native_unsupported_reason": native.reason, }) yield* Effect.logInfo("native runtime unavailable; falling back to ai-sdk", { - providerID: input.model.providerID, - modelID: input.model.id, + providerID: model.providerID, + modelID: model.id, "session.id": input.sessionID, small: (input.small ?? false).toString(), agent: input.agent.name, @@ -270,8 +325,8 @@ const live: Layer.Layer< yield* Effect.logInfo("llm runtime selected", { "llm.runtime": "ai-sdk", - "llm.provider": input.model.providerID, - "llm.model": input.model.id, + "llm.provider": model.providerID, + "llm.model": model.id, }) // Default runtime path: AI SDK owns provider execution and tool dispatch; // LLMAISDK.toLLMEvents below normalizes fullStream parts for the processor. @@ -281,8 +336,8 @@ const live: Layer.Layer< onError(error) { bridge.fork( Effect.logError("stream error", { - providerID: input.model.providerID, - modelID: input.model.id, + providerID: model.providerID, + modelID: model.id, "session.id": input.sessionID, small: (input.small ?? false).toString(), agent: input.agent.name, @@ -292,7 +347,7 @@ const live: Layer.Layer< ) }, // Copilot returns the authoritative billed amount only in provider-specific response fields. - includeRawChunks: input.model.providerID.includes("github-copilot"), + includeRawChunks: model.providerID.includes("github-copilot"), async experimental_repairToolCall(failed) { const lower = failed.toolCall.toolName.toLowerCase() if (lower !== failed.toolCall.toolName && prepared.tools[lower]) { @@ -313,7 +368,7 @@ const live: Layer.Layer< temperature: prepared.params.temperature, topP: prepared.params.topP, topK: prepared.params.topK, - providerOptions: ProviderTransform.providerOptions(input.model, prepared.params.options), + providerOptions: ProviderTransform.providerOptions(model, prepared.params.options), activeTools: Object.keys(prepared.tools).filter((x) => x !== "invalid"), tools: prepared.tools, toolChoice: input.toolChoice, @@ -332,7 +387,7 @@ const live: Layer.Layer< // @ts-expect-error args.params.prompt = ProviderTransform.message( args.params.prompt, - input.model, + model, prepared.messageTransformOptions, ) } diff --git a/packages/opencode/test/session/llm.test.ts b/packages/opencode/test/session/llm.test.ts index 3bfc722e2bec..d3b87f8efc58 100644 --- a/packages/opencode/test/session/llm.test.ts +++ b/packages/opencode/test/session/llm.test.ts @@ -13,6 +13,7 @@ import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route" import { Provider } from "@/provider/provider" import { ProviderTransform } from "@/provider/transform" import { ModelsDev } from "@opencode-ai/core/models-dev" +import { Plugin } from "@/plugin" import { testEffect } from "../lib/effect" import type { Agent } from "../../src/agent/agent" @@ -2113,3 +2114,212 @@ describe("session.llm.stream", () => { }, ) }) + +function pluginHookLayer(hook: Plugin.Interface["trigger"]): Layer.Layer { + return Layer.succeed( + Plugin.Service, + Plugin.Service.of({ + trigger: hook, + list: Effect.fn("TestPlugin.list")(() => Effect.succeed([])), + init: Effect.fn("TestPlugin.init")(() => Effect.void), + }), + ) +} + +describe("session.llm.chat.model", () => { + const vivgridFixture = { providerID: "vivgrid", modelID: "gemini-3.1-pro-preview" } + const routedModelID = "gpt-5.4-mini" + const testConfig = () => ({ + enabled_providers: [vivgridFixture.providerID], + provider: { + [vivgridFixture.providerID]: { + options: { apiKey: "test-key", baseURL: `${state.server!.url.origin}/v1` }, + }, + }, + }) + + function itWithPlugin(hook: Plugin.Interface["trigger"]) { + return testEffect( + AppNodeBuilder.build( + LayerNode.group([LLM.node, Provider.node]), + [[Plugin.node, pluginHookLayer(hook)]], + ), + ) + } + + itWithPlugin( + Effect.fn("TestPlugin.trigger")(function* (name, input, output) { + if (name === "chat.headers") { + const hookInput = input as { model: Provider.Model } + ;(output as { headers: Record }).headers["x-chat-model-test"] = "noop" + } + return output + }), + ).instance( + "uses the original model when chat.model does not route", + () => + Effect.gen(function* () { + const fixture = loadFixture(vivgridFixture.providerID, vivgridFixture.modelID) + const request = waitRequest( + "/chat/completions", + new Response(createChatStream("Hello"), { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }), + ) + const resolved = yield* Provider.use.getModel( + ProviderV2.ID.make(vivgridFixture.providerID), + ModelV2.ID.make(fixture.model.id), + ) + const sessionID = SessionID.make("session-test-chat-model-noop") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + } satisfies Agent.Info + + yield* drain({ + user: { + id: MessageID.make("msg_user-chat-model-noop"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderV2.ID.make(vivgridFixture.providerID), modelID: resolved.id }, + } satisfies SessionV1.User, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Hello" }], + tools: {}, + }) + + const capture = yield* Effect.promise(() => request) + expect(capture.body.model).toBe(fixture.model.id) + expect(capture.headers.get("x-chat-model-test")).toBe("noop") + }), + { config: testConfig }, + ) + + itWithPlugin( + Effect.fn("TestPlugin.trigger")(function* (name, input, output) { + if (name === "chat.model") { + const hookInput = input as { message: { text: string } } + if (hookInput.message.text === "Route me") { + const routeOutput = output as { model?: { providerID: string; modelID: string }; reason?: string } + routeOutput.model = { providerID: vivgridFixture.providerID, modelID: routedModelID } + routeOutput.reason = "test-routing" + } + } + return output + }), + ).instance( + "routes to the model returned by chat.model", + () => + Effect.gen(function* () { + const fixture = loadFixture(vivgridFixture.providerID, vivgridFixture.modelID) + const routed = yield* Provider.use.getModel( + ProviderV2.ID.make(vivgridFixture.providerID), + ModelV2.ID.make(routedModelID), + ) + const request = waitRequest( + "/chat/completions", + new Response(createChatStream("Hello"), { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }), + ) + const resolved = yield* Provider.use.getModel( + ProviderV2.ID.make(vivgridFixture.providerID), + ModelV2.ID.make(fixture.model.id), + ) + const sessionID = SessionID.make("session-test-chat-model-route") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + } satisfies Agent.Info + + yield* drain({ + user: { + id: MessageID.make("msg_user-chat-model-route"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderV2.ID.make(vivgridFixture.providerID), modelID: resolved.id }, + } satisfies SessionV1.User, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Route me" }], + tools: {}, + }) + + const capture = yield* Effect.promise(() => request) + expect(capture.body.model).toBe(routed.api.id) + }), + { config: testConfig }, + ) + + itWithPlugin( + Effect.fn("TestPlugin.trigger")(function* (name, input, output) { + if (name === "chat.model") { + const hookInput = input as { sessionID: string; agent: string; message: { id: string; text: string } } + expect(hookInput.sessionID).toBe("session-test-chat-model-ctx") + expect(hookInput.agent).toBe("test") + expect(hookInput.message.text).toBe("Hello") + expect(hookInput.message.id).toBe("msg_user-chat-model-ctx") + } + return output + }), + ).instance( + "passes session, agent, and message context to chat.model", + () => + Effect.gen(function* () { + const fixture = loadFixture(vivgridFixture.providerID, vivgridFixture.modelID) + const request = waitRequest( + "/chat/completions", + new Response(createChatStream("Hello"), { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }), + ) + const resolved = yield* Provider.use.getModel( + ProviderV2.ID.make(vivgridFixture.providerID), + ModelV2.ID.make(fixture.model.id), + ) + const sessionID = SessionID.make("session-test-chat-model-ctx") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + } satisfies Agent.Info + + yield* drain({ + user: { + id: MessageID.make("msg_user-chat-model-ctx"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderV2.ID.make(vivgridFixture.providerID), modelID: resolved.id }, + } satisfies SessionV1.User, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Hello" }], + tools: {}, + }) + + yield* Effect.promise(() => request) + }), + { config: testConfig }, + ) +}) diff --git a/packages/plugin/src/index.ts b/packages/plugin/src/index.ts index edfa0139dfca..74535aaf446d 100644 --- a/packages/plugin/src/index.ts +++ b/packages/plugin/src/index.ts @@ -241,6 +241,26 @@ export interface Hooks { }, output: { message: UserMessage; parts: Part[] }, ) => Promise + /** + * Route the model used for this LLM request before provider/model/auth resolution. + * This does not mutate the selected session model; later request hooks receive + * the routed model for this request. + */ + "chat.model"?: ( + input: { + sessionID: string + agent: string + model: { providerID: string; modelID: string } + message: { + id: string + text: string + } + }, + output: { + model?: { providerID: string; modelID: string } + reason?: string + }, + ) => Promise /** * Modify parameters sent to LLM */ diff --git a/packages/web/src/content/docs/plugins.mdx b/packages/web/src/content/docs/plugins.mdx index a8be798217a8..b9b0d963ee62 100644 --- a/packages/web/src/content/docs/plugins.mdx +++ b/packages/web/src/content/docs/plugins.mdx @@ -335,6 +335,39 @@ Levels: `debug`, `info`, `warn`, `error`. See [SDK documentation](https://openco --- +### Model routing hooks + +Route the model used for an LLM request before provider/model/auth resolution: + +```ts title=".opencode/plugins/model-router.ts" +import type { Plugin } from "@opencode-ai/plugin" + +export const ModelRouterPlugin: Plugin = async (ctx) => { + // Models come from what the user configured, not hardcoded names: + // e.g. OPENCODE_LOCAL_MODEL="ollama/qwen3:4b" in ~/.config/environment.d/ + const localModel = process.env.OPENCODE_LOCAL_MODEL?.split("/") as + | [providerID: string, modelID: string] + | undefined + + return { + "chat.model": async (input, output) => { + if (localModel && input.agent === "linux-helper") { + const [providerID, modelID] = localModel + output.model = { providerID, modelID } + output.reason = "small task, use the user-configured local model" + } + }, + } +} +``` + +The `chat.model` hook fires for every LLM request. Setting `output.model` +replaces the model for this request only; the session's selected model is +unchanged. `output.reason` is included in logs for observability. Setting no +`output.model` leaves the request untouched. The `providerID`/`modelID` pair +must exist in the user's provider configuration — read models from config or +environment, never hardcode them in the plugin. + ### Compaction hooks Customize the context included when a session is compacted: