diff --git a/src/plugin/index.ts b/src/plugin/index.ts index 888e61e..8cf637b 100644 --- a/src/plugin/index.ts +++ b/src/plugin/index.ts @@ -11,11 +11,15 @@ import { categorizeModel, } from '../utils/format-model-name' import type { LiteLLMModel, LiteLLMModelInfo } from '../types' +import { readModelCache, writeModelCache, readModelCacheSavedAt } from '../utils/model-cache' const CHAT_PROVIDER_ID = 'litellm' // Covers the sequential 3 s health check plus the parallel 15 s // models/model-info fetch phase, with headroom. const DISCOVERY_TIMEOUT_MS = 20000 +// Don't revalidate a baseURL's cache more often than this, so a burst +// of `session.created` events can't generate repeated discovery traffic. +const REFRESH_MIN_INTERVAL_MS = 5 * 60 * 1000 // 5 minutes /** * OpenCode invokes the `config` hook several times per run with a @@ -25,6 +29,37 @@ const DISCOVERY_TIMEOUT_MS = 20000 */ const injectedModelIds = new Map>() +/** + * Per-baseURL fetch context captured during the `config` hook, so the + * `event` hook can revalidate the cache in the background (SWR) without + * re-deriving auth/headers. + */ +interface RefreshContext { + apiKey?: string + customHeaders?: Record + providerId: string +} +const refreshContexts = new Map() + +/** baseURLs with an in-flight background refresh, to avoid pile-ups. */ +const refreshInFlight = new Set() + +/** + * Race a promise against a timeout, resolving to `null` if the timeout + * wins. Clears the timer either way so a resolved discovery can't keep + * a short-lived process alive waiting on a pending `setTimeout`. + */ +function withTimeout( + promise: Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), timeoutMs) + }) + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)) +} + /** * Helper to determine if a provider ID or its configured options indicate * compatibility with LiteLLM. @@ -151,6 +186,161 @@ function toConfigModel( return entry } +/** + * Fetch and build OpenCode model entries from a LiteLLM proxy. + * + * Pure with respect to plugin config: it performs the network calls, + * classifies + formats each model, and returns a `{ id -> entry }` map. + * Returns `null` when the proxy is unreachable/unauthorized or exposes + * no models, so callers can distinguish "no data" from "empty result". + */ +async function discoverModels( + baseURL: string, + apiKey: string | undefined, + customHeaders: Record | undefined, + providerId: string, +): Promise | null> { + if (!(await checkLiteLLMHealth(baseURL, apiKey, customHeaders))) { + console.warn( + `[opencode-litellm] LiteLLM appears offline or unauthorized for provider "${providerId}" at ${baseURL}`, + ) + return null + } + + // `/v1/models` omits `mode` and capability metadata for + // database-defined models, so fetch `/v1/model/info` alongside + // it. The info call is best-effort: without it, classification + // falls back to id heuristics. + const [modelsResult, infoResult] = await Promise.allSettled([ + discoverLiteLLMModels(baseURL, apiKey, customHeaders), + discoverLiteLLMModelInfo(baseURL, apiKey, customHeaders), + ]) + + if (modelsResult.status === 'rejected') { + const error = modelsResult.reason + console.warn( + `[opencode-litellm] Model discovery failed for provider "${providerId}":`, + error instanceof Error ? error.message : String(error), + ) + return null + } + + const discovered = modelsResult.value + let infoByName: Map | null = null + if (infoResult.status === 'fulfilled') { + infoByName = infoResult.value + } else { + const reason = infoResult.reason + console.warn( + `[opencode-litellm] /v1/model/info unavailable for provider "${providerId}"; non-chat model filtering will use id heuristics only:`, + reason instanceof Error ? reason.message : String(reason), + ) + } + + if (discovered.length === 0) { + console.warn( + `[opencode-litellm] LiteLLM responded for provider "${providerId}" but exposed zero models.`, + ) + return null + } + + const built: Record = {} + let skipped = 0 + let wildcards = 0 + const unmatched: string[] = [] + for (const model of discovered) { + // Wildcard entries (`deepseek/*`) are access rules, not + // callable models — invoking one sends a literal `*` upstream. + if (model.id.includes('*')) { + wildcards++ + continue + } + const info = infoByName?.get(model.id) + if (infoByName && !info) unmatched.push(model.id) + const entry = toConfigModel(info ? enrichModel(model, info) : model, info) + if (!entry) { + skipped++ + continue + } + built[model.id] = entry + } + + if (unmatched.length > 0) { + console.warn( + `[opencode-litellm] /v1/model/info has no entry for ${unmatched.length} model(s) on provider "${providerId}"; ` + + `classification uses id heuristics for: ${unmatched.slice(0, 5).join(', ')}` + + (unmatched.length > 5 ? `, +${unmatched.length - 5} more` : ''), + ) + } + + console.log( + `[opencode-litellm] Discovered ${discovered.length} models for provider "${providerId}" from ${baseURL} ` + + `(${Object.keys(built).length} built` + + (skipped > 0 ? `, ${skipped} non-chat hidden` : '') + + (wildcards > 0 ? `, ${wildcards} wildcard ignored` : '') + + ')', + ) + + return built +} + +/** + * Merge freshly built model entries into a provider's `models` map + * without clobbering user-curated (or previously injected) entries. + * Returns the ids actually added by this call. + */ +function mergeModels( + models: Record, + built: Record, +): string[] { + const added: string[] = [] + for (const [id, entry] of Object.entries(built)) { + if (Object.hasOwn(models, id)) continue + models[id] = entry + added.push(id) + } + // Remove the seed placeholder if real models were merged in. + if (Object.hasOwn(models, '_') && Object.keys(models).length > 1) { + delete models['_'] + } + return added +} + +/** + * Revalidate a baseURL's model cache off the critical path (SWR). The + * refreshed entries land in the on-disk cache and surface on the next + * OpenCode start — OpenCode only reads provider config at startup, so + * we can't mutate the live picker here. + */ +async function backgroundRefresh(baseURL: string): Promise { + if (refreshInFlight.has(baseURL)) return + const ctx = refreshContexts.get(baseURL) + if (!ctx) return + // Skip if the cache was refreshed recently — a burst of new sessions + // shouldn't hammer the proxy with health checks and discovery calls. + const savedAt = readModelCacheSavedAt(baseURL) + if (savedAt !== null && Date.now() - savedAt < REFRESH_MIN_INTERVAL_MS) { + return + } + refreshInFlight.add(baseURL) + try { + const built = await withTimeout( + discoverModels(baseURL, ctx.apiKey, ctx.customHeaders, ctx.providerId), + DISCOVERY_TIMEOUT_MS, + ) + if (built && Object.keys(built).length > 0) { + writeModelCache(baseURL, built) + console.log( + `[opencode-litellm] Background-refreshed model cache for ${baseURL} (${Object.keys(built).length} models)`, + ) + } + } catch { + // Best-effort — a failed refresh just leaves the stale cache in place. + } finally { + refreshInFlight.delete(baseURL) + } +} + /** * LiteLLM Plugin for OpenCode. * @@ -265,117 +455,54 @@ export const LiteLLMPlugin: Plugin = async (_input: PluginInput) => { const models = actualProvider.models as Record - // Discover models with timeout - const work = async () => { - const alreadyInjected = injectedModelIds.get(baseURL!) - if ( - alreadyInjected && - [...alreadyInjected].every((id) => models[id]) - ) { - return - } - - if (!(await checkLiteLLMHealth(baseURL!, apiKey, customHeaders))) { - console.warn( - `[opencode-litellm] LiteLLM appears offline or unauthorized for provider "${providerId}" at ${baseURL}`, - ) - return - } - - // `/v1/models` omits `mode` and capability metadata for - // database-defined models, so fetch `/v1/model/info` alongside - // it. The info call is best-effort: without it, classification - // falls back to id heuristics. - const [modelsResult, infoResult] = await Promise.allSettled([ - discoverLiteLLMModels(baseURL!, apiKey, customHeaders), - discoverLiteLLMModelInfo(baseURL!, apiKey, customHeaders), - ]) - - if (modelsResult.status === 'rejected') { - const error = modelsResult.reason - console.warn( - `[opencode-litellm] Model discovery failed for provider "${providerId}":`, - error instanceof Error ? error.message : String(error), - ) - return - } - - const discovered = modelsResult.value - let infoByName: Map | null = null - if (infoResult.status === 'fulfilled') { - infoByName = infoResult.value - } else { - const reason = infoResult.reason - console.warn( - `[opencode-litellm] /v1/model/info unavailable for provider "${providerId}"; non-chat model filtering will use id heuristics only:`, - reason instanceof Error ? reason.message : String(reason), - ) - } - - if (discovered.length === 0) { - console.warn( - `[opencode-litellm] LiteLLM responded for provider "${providerId}" but exposed zero models.`, - ) - return - } - - let added = 0 - let skipped = 0 - let wildcards = 0 - const unmatched: string[] = [] - for (const model of discovered) { - // Wildcard entries (`deepseek/*`) are access rules, not - // callable models — invoking one sends a literal `*` upstream. - if (model.id.includes('*')) { - wildcards++ - continue - } - // Don't overwrite user-curated entries - if (models[model.id]) continue - const info = infoByName?.get(model.id) - if (infoByName && !info) unmatched.push(model.id) - const entry = toConfigModel( - info ? enrichModel(model, info) : model, - info, - ) - if (!entry) { - skipped++ - continue - } - models[model.id] = entry - added++ - } - - if (unmatched.length > 0) { - console.warn( - `[opencode-litellm] /v1/model/info has no entry for ${unmatched.length} model(s) on provider "${providerId}"; ` + - `classification uses id heuristics for: ${unmatched.slice(0, 5).join(', ')}` + - (unmatched.length > 5 ? `, +${unmatched.length - 5} more` : ''), - ) - } - - // Remove the seed placeholder if real models were discovered - if (models['_'] && Object.keys(models).length > 1) { - delete models['_'] - } - - injectedModelIds.set(baseURL!, new Set(Object.keys(models))) + // Remember how to reach this proxy so the `event` hook can + // revalidate its cache in the background on new sessions. + refreshContexts.set(baseURL, { apiKey, customHeaders, providerId }) + + // Repeat config-hook invocations within a run are a no-op once + // we've injected this baseURL's models. + const alreadyInjected = injectedModelIds.get(baseURL) + if ( + alreadyInjected && + [...alreadyInjected].every((id) => Object.hasOwn(models, id)) + ) { + continue + } + // SWR fast path: serve cached entries synchronously so startup + // isn't blocked on the network. A background refresh (see the + // `event` hook) keeps the cache fresh for the next launch. + const cached = readModelCache(baseURL) + if (cached && Object.keys(cached).length > 0) { + const added = mergeModels(models, cached) + injectedModelIds.set(baseURL, new Set(added)) console.log( - `[opencode-litellm] Discovered ${discovered.length} models for provider "${providerId}" from ${baseURL} ` + - `(${added} added` + - (skipped > 0 ? `, ${skipped} non-chat hidden` : '') + - (wildcards > 0 ? `, ${wildcards} wildcard ignored` : '') + - ')', + `[opencode-litellm] Loaded ${Object.keys(cached).length} models from cache for provider "${providerId}" (${baseURL}); refresh happens in the background on new sessions.`, ) + continue } - await Promise.race([ - work(), - new Promise((resolve) => - setTimeout(resolve, DISCOVERY_TIMEOUT_MS), - ), - ]) + // Cold cache: do a live fetch (slow first run only), inject, and + // persist for subsequent startups. Capped by a timeout so a slow + // proxy never blocks boot. + const built = await withTimeout( + discoverModels(baseURL, apiKey, customHeaders, providerId), + DISCOVERY_TIMEOUT_MS, + ) + if (built && Object.keys(built).length > 0) { + const added = mergeModels(models, built) + injectedModelIds.set(baseURL, new Set(added)) + writeModelCache(baseURL, built) + } + } + }, + event: async ({ event }) => { + // Revalidate model caches off the critical path when a new session + // opens. Fresh data lands in the cache and surfaces on the next + // OpenCode start (SWR). + if (event.type !== 'session.created') return + for (const baseURL of refreshContexts.keys()) { + void backgroundRefresh(baseURL) } }, } diff --git a/src/utils/model-cache.ts b/src/utils/model-cache.ts new file mode 100644 index 0000000..b5a76dc --- /dev/null +++ b/src/utils/model-cache.ts @@ -0,0 +1,123 @@ +import { createHash } from 'node:crypto' +import { readFileSync, mkdirSync, writeFileSync, renameSync, rmSync } from 'node:fs' +import { homedir, tmpdir } from 'node:os' +import { join } from 'node:path' + +/** + * Bump when the shape of a cached model entry changes so stale caches + * from older plugin versions are ignored rather than deserialized into + * an incompatible structure. + */ +const CACHE_VERSION = 1 + +/** + * Maximum age of a cache entry before it's treated as a miss. Without a + * bound, a proxy that goes away permanently would keep serving the same + * stale model list on every launch. A background refresh normally keeps + * entries fresh well within this window. + */ +const CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000 // 7 days + +/** The on-disk shape of a per-baseURL model cache file. */ +interface ModelCacheFile { + version: number + savedAt: number + models: Record +} + +/** + * Directory where SWR model caches live. Honours `XDG_CACHE_HOME`, + * falling back to `~/.cache`, and finally the OS temp dir if the home + * directory is somehow unavailable. + */ +function cacheDir(): string { + const home = homedir() + const base = + process.env.XDG_CACHE_HOME || (home ? join(home, '.cache') : tmpdir()) + return join(base, 'opencode-litellm') +} + +/** Stable, filesystem-safe filename derived from the normalized baseURL. */ +function cacheFile(baseURL: string): string { + const hash = createHash('sha256').update(baseURL).digest('hex').slice(0, 16) + return join(cacheDir(), `models-${hash}.json`) +} + +/** + * Read cached model entries for a baseURL. Returns `null` on any + * problem (missing file, parse error, version mismatch, or an entry + * older than `CACHE_MAX_AGE_MS`) — the caller treats that as a cache + * miss and falls back to a live fetch. + */ +export function readModelCache( + baseURL: string, +): Record | null { + try { + const raw = readFileSync(cacheFile(baseURL), 'utf8') + const parsed = JSON.parse(raw) as ModelCacheFile + if (parsed.version !== CACHE_VERSION) return null + if (!parsed.models || typeof parsed.models !== 'object') return null + if ( + typeof parsed.savedAt !== 'number' || + Date.now() - parsed.savedAt > CACHE_MAX_AGE_MS + ) { + return null + } + return parsed.models + } catch { + return null + } +} + +/** + * Return the `savedAt` timestamp (epoch ms) of a baseURL's cache entry, + * or `null` if there's no readable/valid cache. Used to throttle + * background refreshes so still-fresh caches aren't re-fetched. + */ +export function readModelCacheSavedAt(baseURL: string): number | null { + try { + const raw = readFileSync(cacheFile(baseURL), 'utf8') + const parsed = JSON.parse(raw) as ModelCacheFile + if (parsed.version !== CACHE_VERSION) return null + if (typeof parsed.savedAt !== 'number') return null + return parsed.savedAt + } catch { + return null + } +} + +/** + * Persist model entries for a baseURL. Best-effort: never throws, so a + * read-only or full filesystem can't break discovery. + */ +export function writeModelCache( + baseURL: string, + models: Record, +): void { + try { + mkdirSync(cacheDir(), { recursive: true }) + const payload: ModelCacheFile = { + version: CACHE_VERSION, + savedAt: Date.now(), + models, + } + // Write to a temp file and rename so concurrent OpenCode processes + // never observe a partially-written JSON file (rename is atomic on + // the same filesystem). + const target = cacheFile(baseURL) + const tmp = `${target}.${process.pid}.tmp` + try { + writeFileSync(tmp, JSON.stringify(payload), 'utf8') + renameSync(tmp, target) + } catch (err) { + try { + rmSync(tmp, { force: true }) + } catch { + // Ignore cleanup failure. + } + throw err + } + } catch { + // Ignore — the cache is an optimization, not a requirement. + } +}