diff --git a/libraries/llm/.gitignore b/libraries/llm/.gitignore new file mode 100644 index 0000000..a7c4753 --- /dev/null +++ b/libraries/llm/.gitignore @@ -0,0 +1 @@ +types/ diff --git a/libraries/llm/README.md b/libraries/llm/README.md new file mode 100644 index 0000000..0e7a68c --- /dev/null +++ b/libraries/llm/README.md @@ -0,0 +1,116 @@ +# @patchwork/llm + +An LLM toolkit for Patchwork tools, extracted from `chat` and enriched with +`rlm`'s teaching telemetry. It gives you: + +- **`popup()` / `dom()`** — a `
` model picker (Browser / OpenRouter + / Ollama + sampling parameters, prompts, and tools). `popup()` is framed + (header + Cancel/Done); `dom()` is the bare panel to embed. Writes the choice + to the user's **account settings doc**, so the model + API key are shared + across every tool and synced across devices. +- **A refresh-surviving `SharedWorker`** that runs all three providers off the + main thread (cross-tab, survives reload — keyed by an optional `sessionKey`). +- **A streaming API** (`generate` callback-style, `stream` async-iterator-style) + that carries **rich telemetry alongside the text** so you can build UIs that + show *how the model thinks*: + - **`prediction`** events — the model's top-k next-token distribution at each + step (`[{token, p}]`). Works for **local** (via a transformers.js + `logits_processor`) **and OpenRouter** (via `logprobs`/`top_logprobs`). + - **`stats`** events — prompt/gen token counts, time-to-first-token, + tokens/sec, and the exact decode settings used (`temperature`, `top_p`, + greedy, …). + +Plain vanilla JS, no build step. Its only npm dependency is +`@inkandswitch/patchwork-providers` (the request/provide config plumbing); +transformers.js — the model runtime — is imported from a CDN inside the worker, +only when the local provider is used. + +## Install / consume + +The package lives at `libraries/llm` and is named `@patchwork/llm`. + +- **Bundled tools (vite):** add a resolve alias and the bundler inlines it, + worker included (vite understands `new URL("./worker.js", import.meta.url)`): + + ```js + // vite.config.js + resolve: { + alias: { + "@patchwork/llm": fileURLToPath(new URL("../libraries/llm/index.js", import.meta.url)), + }, + } + ``` + +- **Bundleless tools:** import by relative path, or add `@patchwork/llm` to the + host importmap to share one copy across tools. + +## Usage + +```js +import { popup, stream, generate, readConfig } from "@patchwork/llm" + +// 1. Let the user choose a model / paste their OpenRouter key. +const el = popup() +document.body.append(el) +el.showPopover() +await el.result // resolves to the config on close (null if cancelled) + +// 2a. Stream with telemetry (async iterator): +let text = "" +for await (const ev of stream(messages, { topk: 5 })) { + switch (ev.type) { + case "status": setStatus(ev.message); break // model loading… + case "token": text += ev.delta; render(text); break + case "prediction": renderCandidates(ev.step, ev.candidates); break + case "stats": renderStats(ev); break // ttftMs, tokPerSec, decode… + case "done": finish(ev.text); break + } +} + +// 2b. …or callback style: +const { text, stats } = await generate(messages, { + topk: 5, + temperature: 0.7, + onToken: (delta, full) => render(full), + onPrediction: (candidates, step) => renderCandidates(step, candidates), + onStats: (s) => renderStats(s), + onStatus: (m) => setStatus(m), + signal, // AbortSignal +}) +``` + +### Config (on the account doc) + +Everything lives under `accountDoc.llm`: + +```js +{ + provider: "local" | "openrouter" | "ollama", + temperature: 0.7, + local: { model }, + openrouter: { apiKey, model, contextLength, maxCompletionTokens }, + ollama: { url, model }, +} +``` + +`readConfig()` / `writeConfig(patch)` read/write it (defaulting missing fields); +`popup()` / `dom()` are the UI over them. + +### Resume after refresh + +Pass a stable `sessionKey` (e.g. a doc URL) to `generate`/`stream`; after a +reload, `resume(sessionKey, { onToken, onDone })` re-attaches to the still-running +stream in the worker. + +## Events reference + +| event | fields | local | openrouter | ollama | +|--------------|------------------------------------------------------------------|:-----:|:----------:|:------:| +| `token` | `delta`, `text` | ✓ | ✓ | ✓ | +| `prediction` | `step`, `candidates: [{token, p}]` | ✓ | ✓ | — | +| `stats` | `promptTokens`, `genTokens`, `ttftMs`, `totalMs`, `tokPerSec`, `decode` | ✓ | ✓ | ✓* | +| `status` | `message` (model download / shader compile) | ✓ | — | — | +| `done` | `text`, `stats` | ✓ | ✓ | ✓ | + +\* Ollama stats come from its final `done` chunk (`eval_count`, `eval_duration`). +OpenRouter needs a model that supports `logprobs` for `prediction` events. diff --git a/libraries/llm/builtin.js b/libraries/llm/builtin.js new file mode 100644 index 0000000..d2c7490 --- /dev/null +++ b/libraries/llm/builtin.js @@ -0,0 +1,135 @@ +/** + * Chrome built-in AI (the Prompt API — on-device Gemini Nano). + * + * Runs on the MAIN thread: the `LanguageModel` global is a window API, not a + * worker one, so this path bypasses the SharedWorker. No next-token logprobs are + * exposed, so `predict()` (the typing popup) doesn't work with built-in — + * generation + streaming do. + */ + +function getLM() { + if (typeof self === "undefined") return null + return self.LanguageModel || (self.ai && self.ai.languageModel) || null +} + +/** Is the Prompt API present at all (i.e. show the option)? */ +export function builtinSupported() { + return !!getLM() +} + +/** "available" | "downloadable" | "downloading" | "unavailable" */ +export async function builtinAvailability() { + const LM = getLM() + if (!LM) return "unavailable" + try { + if (LM.availability) return await LM.availability() + if (LM.capabilities) { + const c = await LM.capabilities() + return c?.available === "readily" + ? "available" + : c?.available === "after-download" + ? "downloadable" + : "unavailable" + } + } catch {} + return "unavailable" +} + +/** @param {import("./config.js").ChatMessage[]} messages */ +function messagesToText(messages) { + return messages + .filter((m) => m.role !== "system") + .map((m) => m.content) + .join("\n\n") +} + +/** @param {ReadableStream} stream */ +async function* readStream(stream) { + const reader = stream.getReader() + try { + while (true) { + const {done, value} = await reader.read() + if (done) break + yield value + } + } finally { + try { + reader.releaseLock() + } catch {} + } +} + +/** + * @typedef {Object} BuiltinOpts + * @property {number} [temperature] + * @property {number} [topK] + * @property {string} [system] + * @property {(delta: string, full: string) => void} [onToken] + * @property {(status: string) => void} [onStatus] + * @property {AbortSignal} [signal] + */ + +/** + * Generate via the Prompt API. `onToken(delta, full)` per chunk; returns full + * text. Handles both the old (cumulative chunk) and new (delta chunk) shapes. + * @param {string|import("./config.js").ChatMessage[]} input + * @param {BuiltinOpts} [opts] + */ +export async function builtinGenerate(input, opts = {}) { + const {temperature, topK, system, onToken, onStatus, signal} = opts + const LM = getLM() + if (!LM) + throw new Error( + "Built-in AI isn't available here (needs Chrome with the Prompt API)." + ) + /** @type {any} */ + const createOpts = {} + if (typeof temperature === "number") { + createOpts.temperature = Math.min(2, Math.max(0, temperature)) + createOpts.topK = topK && topK > 0 ? topK : 8 // Prompt API needs topK alongside temperature + } + if (system) createOpts.initialPrompts = [{role: "system", content: system}] + createOpts.monitor = (/** @type {any} */ m) => { + try { + m.addEventListener("downloadprogress", (/** @type {any} */ e) => + onStatus?.( + "Downloading built-in model… " + Math.round((e.loaded || 0) * 100) + "%" + ) + ) + } catch {} + } + let session + try { + session = await LM.create(createOpts) + } catch { + // retry without sampling params (version differences) + session = await LM.create(system ? {initialPrompts: createOpts.initialPrompts} : {}) + } + try { + const text = typeof input === "string" ? input : messagesToText(input) + let full = "" + if (session.promptStreaming) { + const stream = session.promptStreaming(text, signal ? {signal} : undefined) + for await (const chunk of readStream(stream)) { + if (typeof chunk !== "string") continue + let delta + if (chunk.startsWith(full)) { + delta = chunk.slice(full.length) + full = chunk + } else { + delta = chunk + full += chunk + } + if (delta) onToken?.(delta, full) + } + } else { + full = await session.prompt(text) + onToken?.(full, full) + } + return full + } finally { + try { + session.destroy?.() + } catch {} + } +} diff --git a/libraries/llm/client.js b/libraries/llm/client.js new file mode 100644 index 0000000..f9b4076 --- /dev/null +++ b/libraries/llm/client.js @@ -0,0 +1,883 @@ +/** + * Main-thread client for the @patchwork/llm worker. + * + * Connects to a dedicated Worker (one per page), routes + * messages back to the right in-flight generation, and exposes two call styles: + * + * const { text, stats } = await generate(messages, { onToken, onPrediction, onStats }) + * for await (const ev of stream(messages, { topk: 5 })) { ... } + * + * Provider / model / key / temperature come from the account-doc config + * (see config.js) unless overridden via opts. + */ + +import {readConfig, ensureConfig, callConfig, applyPrompts, effectiveSystem} from "./config.js" +import {builtinGenerate} from "./builtin.js" +import {resolveTools, toToolSchemas, buildToolsSystem, parseToolCalls, runTool, resolveCfgPrompts, sanitizeToolName} from "./tools.js" + +/** + * Per-call options. A superset of every option any exported function accepts; + * individual functions document the subset they use. Open-ended on purpose. + * @typedef {Object} GenOpts + * @property {import("./config.js").LLMConfig} [config] + * @property {import("./config.js").Scope} [scope] + * @property {boolean} [continuation] + * @property {number} [topk] + * @property {number} [temperature] + * @property {string} [model] + * @property {string} [provider] + * @property {number} [maxNewTokens] + * @property {string} [sessionKey] + * @property {string} [system] + * @property {any[]} [tools] + * @property {boolean} [sandbox] + * @property {number} [maxRounds] + * @property {AbortSignal} [signal] + * @property {(delta:string, full:string, round?:number)=>void} [onToken] + * @property {(candidates:{token:string,p:number}[], step:number)=>void} [onPrediction] + * @property {(stats:any)=>void} [onStats] + * @property {(message:string)=>void} [onStatus] + * @property {(info:{name?:string,tool?:string,args?:any,result?:any,error?:any})=>void} [onToolCall] + */ + +/** + * A message coming back from the worker (or built locally). Open-ended: + * the discriminant is `type` and the rest depends on it. Routing fields are + * typed non-optional because the dispatch guards (`msg.id != null`, etc.) + * already ensure they're present before use. + * @typedef {Object} WorkerMsg + * @property {string} type + * @property {string} id + * @property {string|number} sessionKey + * @property {string} text + * @property {string} delta + * @property {string} message + * @property {any[]} args + * @property {any} candidates + * @property {number} step + * @property {any} scores + * @property {any} spans + * @property {any} decoded + * @property {number} total + * @property {any} strings + * @property {any[]|null} toolCalls + * @property {string} [toolMode] + */ + +/** @typedef {{post: (m:any)=>void}} Connection */ + +/** CallConfig plus the extra mutable fields the client tacks on per-call. + * @typedef {import("./config.js").CallConfig & {tools?:any, toolSystem?:string, continuation?:boolean}} CallConfigExt */ + +/** @typedef {{type:string, id:string, sessionKey:string, provider:import("./config.js").ProviderId, config:import("./config.js").CallConfig, text?:string, messages?:any}} GeneratePayload */ + +// Providers with real function-calling APIs (the worker passes tool schemas and +// parses structured tool_calls). Everything else (local transformers, Chrome +// built-in) uses the XML prompt convention, parsed from the text. +const NATIVE_TOOL_PROVIDERS = new Set(["openrouter", "ollama", "webllm"]) +const TEMPLATE_TOOL_PROVIDERS = new Set(["local"]) + +/** @type {Connection|null} */ +let connection = null +let idSeq = 0 +/** @type {Mapvoid>} */ +const handlers = new Map() // generation id -> (msg) => void +/** @type {Mapvoid, onDone?:(t?:string)=>void, onError?:(m?:string)=>void, onNone?:()=>void}>} */ +const resumeHandlers = new Map() // sessionKey -> { onToken, onDone, onError, onNone } +/** @type {Set<(message:string)=>void>} */ +const statusListeners = new Set() // (message) => void + +function nextId() { + return "llm-" + ++idSeq + "-" + (performance.now() | 0) +} + +// Main-thread diagnostics. The worker forwards its own logs via {type:"log"} +// (see dispatch); this is for client-side events — aborts and worker errors — +// so a caller (e.g. loom) that only surfaces a generic AbortError still leaves a +// trail in the console explaining what actually happened. +/** @param {...any} args */ +function clog(...args) { + try { + console.log("[llm]", ...args) + } catch {} +} + +/** @param {WorkerMsg} msg */ +function dispatch(msg) { + if (!msg) return + // Worker diagnostics: the Worker's own console is separate, so re-print its + // logs here on the main thread where the tool's devtools can see them. + if (msg.type === "log") { + try { + // Stringify object args inline so the console shows the actual values + // (a bare object logs as a collapsed "Object" and hides what we need). + console.log( + "[llm worker]", + ...(msg.args || []).map((a) => + a && typeof a === "object" ? JSON.stringify(a) : a + ) + ) + } catch {} + return + } + if (msg.type === "status") { + for (const f of statusListeners) f(msg.message) + return + } + // Resume replies are keyed by sessionKey (the reconnecting tab never knew the + // original generation id) and tell us the live id to adopt for what follows. + if ( + msg.type === "resumed" || + msg.type === "resume-result" || + msg.type === "no-active-generation" + ) { + const rh = msg.sessionKey != null && resumeHandlers.get(msg.sessionKey) + if (!rh) return + if (msg.type === "no-active-generation") { + resumeHandlers.delete(msg.sessionKey) + rh.onNone?.() + } else if (msg.type === "resume-result") { + resumeHandlers.delete(msg.sessionKey) + rh.onToken?.(msg.text) + rh.onDone?.(msg.text) + } else { + rh.onToken?.(msg.text) + handlers.set(msg.id, (/** @type {WorkerMsg} */ m) => { + if (m.type === "token") rh.onToken?.(m.text) + else if (m.type === "result") { + handlers.delete(msg.id) + resumeHandlers.delete(msg.sessionKey) + rh.onDone?.(m.text) + } else if (m.type === "error") { + handlers.delete(msg.id) + resumeHandlers.delete(msg.sessionKey) + rh.onError?.(m.message) + } + }) + } + return + } + const h = msg.id != null && handlers.get(msg.id) + if (h) h(msg) +} + +function getConnection() { + if (connection) return connection + // A dedicated Worker (one per page), NOT a SharedWorker. A SharedWorker is a + // single instance shared across every tab/tool on the origin — which serialises + // all generation through one model and keeps running stale code until every + // page closes. A per-page Worker reloads with the page, is isolated, and dies + // with it; a page that wants many models running at once gets its own worker + // rather than fighting over one. (Trade-off: loses resume-after-refresh, which + // relied on the worker outliving the page — `resume()` now just reports + // no-active-generation, handled gracefully.) + // NOTE: `new URL("./worker.js", import.meta.url)` MUST stay inline inside the + // constructor — that's the exact pattern bundlers (vite) statically detect to + // emit the worker chunk; hoisting it to a variable silently breaks bundling. + const w = new Worker(new URL("./worker.js", import.meta.url), {type: "module"}) + w.onmessage = (/** @type {MessageEvent} */ ev) => dispatch(ev.data) + connection = {post: (m) => w.postMessage(m)} + return connection +} + +/** + * Generate. Resolves to `{ text, stats }`. + * + * Two intents: + * - CHAT (default): pass chat messages, or a string (wrapped as a user turn). + * The instruct/chat model responds; the system prompt applies. + * - CONTINUATION: pass a string with `{ continuation: true }` to *continue* the + * text rather than answer it. local/webllm/ollama feed it raw; chat-only + * OpenRouter is framed with a "continue, output only the continuation" + * instruction. (This is what Loom uses; everyone else gets plain chat.) + * + * @param {Array|string} messages chat messages, or a string + * @param {GenOpts} [opts] + * @returns {Promise<{text:string, toolCalls:object[]|null, toolMode?:string, stats:object|null}>} + */ +export async function generate(messages, opts = {}) { + const cfg0 = opts.config ?? (await ensureConfig(opts.scope)) + // Resolve the selected system/pre prompt docs → their text. repo.find is + // cached, so this is cheap after first load. + const cfg = await resolveCfgPrompts(cfg0) + /** @type {CallConfigExt} */ + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + + // Tools: native providers get JSON schemas on `config.tools`; the rest get the + // XML convention prepended to the system prompt (parsed from text). + const hasTools = Array.isArray(opts.tools) && opts.tools.length > 0 + const native = hasTools && NATIVE_TOOL_PROVIDERS.has(config.provider) + const templated = hasTools && TEMPLATE_TOOL_PROVIDERS.has(config.provider) + if (native || templated) config.tools = toToolSchemas(opts.tools) + if (templated) config.toolSystem = buildToolsSystem(opts.tools) + const extraSystem = + hasTools && !native && !templated + ? [buildToolsSystem(opts.tools), opts.system].filter(Boolean).join("\n\n") + : opts.system + + // Built-in (Chrome Prompt API) runs on the main thread, not the worker. + if (config.provider === "builtin") { + const pre = cfg.resolved?.pre || "" + const text = + typeof messages === "string" + ? pre + ? pre + "\n\n" + messages + : messages + : messages + return builtinGenerate(text, { + temperature: config.temperature, + topK: config.topK, + system: effectiveSystem(/** @type {any} */ (cfg), extraSystem), + onToken: opts.onToken, + onStatus: opts.onStatus, + signal: opts.signal, + }).then((t) => ({text: t, toolCalls: null, stats: null})) + } + + // A string input is CHAT by default — wrapped as a user turn, so instruct/chat + // models respond normally and the system prompt applies. It's a raw + // CONTINUATION only when opts.continuation is set: raw-fed for + // local/webllm/ollama, and CONTINUE_SYS-framed for chat-only OpenRouter (see + // the worker). Loom passes continuation:true; other callers get plain chat. + const asContinuation = !!opts.continuation && typeof messages === "string" + const prepared = asContinuation + ? messages + : typeof messages === "string" + ? [{role: "user", content: messages}] + : messages + // Prepend the configured system + pre-prompt (and any tool-supplied system). + const input = applyPrompts(prepared, /** @type {any} */ (cfg), extraSystem) + const conn = getConnection() + const id = nextId() + const sessionKey = opts.sessionKey || id + /** @type {any} */ + let stats = null + + return new Promise((resolve, reject) => { + const onStatus = opts.onStatus + if (onStatus) statusListeners.add(onStatus) + const cleanup = () => { + handlers.delete(id) + if (onStatus) statusListeners.delete(onStatus) + if (opts.signal) opts.signal.removeEventListener("abort", onAbort) + } + function onAbort() { + clog("generate: aborted by caller", {provider: config.provider, model: config.model}) + conn.post({type: "abort", sessionKey}) + cleanup() + reject(new DOMException("Aborted", "AbortError")) + } + + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + switch (msg.type) { + case "token": + opts.onToken?.(msg.delta, msg.text) + break + case "prediction": + opts.onPrediction?.(msg.candidates, msg.step) + break + case "stats": + stats = msg + opts.onStats?.(msg) + break + case "result": + cleanup() + resolve({text: msg.text, toolCalls: msg.toolCalls || null, toolMode: msg.toolMode, stats}) + break + case "error": + clog("generate: worker error", msg.message) + cleanup() + reject(new Error(msg.message)) + break + } + }) + + if (opts.signal) { + if (opts.signal.aborted) return onAbort() + opts.signal.addEventListener("abort", onAbort) + } + // A string is a raw continuation prompt; an array is chat messages. + /** @type {GeneratePayload} */ + const payload = {type: "generate", id, sessionKey, provider: config.provider, config} + if (typeof input === "string") payload.text = input + else payload.messages = input + conn.post(payload) + }) +} + +/** + * Stream a completion as an async iterable of telemetry events: + * { type:"token", delta, text } + * { type:"prediction", candidates:[{token,p}], step } + * { type:"stats", ... } + * { type:"status", message } + * { type:"done", text, stats } + * + * @returns {AsyncGenerator} + */ +/** + * @param {Array|string} messages + * @param {GenOpts} [opts] + */ +export async function* stream(messages, opts = {}) { + /** @type {any[]} */ + const queue = [] + /** @type {((v?:any) => void)|null} */ + let wake = null + let finished = false + /** @type {any} */ + let error = null + const push = (/** @type {any} */ ev) => { + queue.push(ev) + wake?.() + } + + generate(messages, { + ...opts, + onToken: (delta, text) => push({type: "token", delta, text}), + onPrediction: (candidates, step) => + push({type: "prediction", candidates, step}), + onStats: (s) => push({type: "stats", ...s}), + onStatus: (message) => push({type: "status", message}), + }) + .then((r) => push({type: "done", text: r.text, stats: r.stats})) + .catch((e) => (error = e)) + .finally(() => { + finished = true + wake?.() + }) + + while (true) { + if (queue.length) { + yield queue.shift() + continue + } + if (finished) break + await new Promise((r) => (wake = r)) + } + if (error) throw error +} + +/** + * Predict the next-token distribution after `text` — one forward pass, no + * generation. Resolves to `[{token, p}]` (top-k, highest p first). Powers + * "predict as you type". Local reads real logits; OpenRouter is best-effort + * (chat-only models return `[]`); Ollama returns `[]`. + * + * @param {string} text + * @param {GenOpts} [opts] + * @returns {Promise<{token:string,p:number}[]>} + */ +export async function predict(text, opts = {}) { + const base = opts.config ?? (await ensureConfig(opts.scope)) + // A config provider can carry an in-memory handler that intercepts the request + // on the main thread (e.g. choochoo runs a base model and adds a live LoRA + // delta). Opt-in — no handler means normal worker dispatch below. + if (typeof base?.handler?.predict === "function") return base.handler.predict(text, opts) + const cfg = await resolveCfgPrompts(base) + /** @type {CallConfigExt} */ + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ ({...opts, topk: opts.topk || 10})) + // continuation → frame chat-only providers (OpenRouter) to predict the + // *continuation's* next token, not a chat reply's. Opt-in (Loom sets it). + config.continuation = !!opts.continuation + // Built-in exposes no next-token logprobs. + if (config.provider === "builtin") return Promise.resolve([]) + const promptedText = applyPrompts(text, /** @type {any} */ (cfg), opts.system) // prepends pre-prompt (no system in raw) + const conn = getConnection() + const id = nextId() + const sessionKey = id // so an abort can reach the in-flight request in the worker + return new Promise((resolve, reject) => { + const cleanup = () => { + handlers.delete(id) + if (opts.signal) opts.signal.removeEventListener("abort", onAbort) + } + function onAbort() { + conn.post({type: "abort", sessionKey}) // cancel the worker-side fetch (OpenRouter/WebLLM) + cleanup() + reject(new DOMException("Aborted", "AbortError")) + } + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "predictions") { + cleanup() + resolve(msg.candidates) + } else if (msg.type === "error") { + cleanup() + reject(new Error(msg.message)) + } + }) + if (opts.signal) { + if (opts.signal.aborted) return onAbort() + opts.signal.addEventListener("abort", onAbort) + } + conn.post({type: "predict", id, sessionKey, provider: config.provider, text: promptedText, config}) + }) +} + +/** + * Score every token in `text` — runs a forward pass at each position and + * returns the model's probability, rank, entropy, and top-k alternatives for + * the actual next token. Powers the surprisal and info-gain overlays ("how + * surprised was the model by what you actually wrote?" and "how much did each + * token reduce its uncertainty?"). Only works for local models. + * + * Yields progress events and a final result: + * { type:"progress", step, total } + * { type:"done", scores:[{token, p, rank, entropy, topk:[{token,p}]}] } + * + * @param {string} text + * @param {GenOpts} [opts] + * @returns {AsyncGenerator} + */ +export async function* scoreTokens(text, opts = {}) { + const cfg = await resolveCfgPrompts(opts.config ?? (await ensureConfig(opts.scope))) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + if (config.provider !== "local") return // only local models expose raw logits + const conn = getConnection() + const id = nextId() + const sessionKey = id + + /** @type {any[]} */ + const queue = [] + /** @type {((v?:any) => void)|null} */ + let wake = null + let finished = false + /** @type {any} */ + let error = null + const push = (/** @type {any} */ ev) => { queue.push(ev); wake?.() } + + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "score-progress") push({type: "progress", step: msg.step, total: msg.total}) + else if (msg.type === "token-scores") { + push({type: "done", scores: msg.scores, spans: msg.spans, decoded: msg.decoded}) + finished = true + handlers.delete(id) + wake?.() + } else if (msg.type === "error") { + error = new Error(msg.message) + finished = true + handlers.delete(id) + wake?.() + } + }) + + function onAbort() { + conn.post({type: "abort", sessionKey}) + handlers.delete(id) + error = new DOMException("Aborted", "AbortError") + finished = true + wake?.() + } + if (opts.signal) { + if (opts.signal.aborted) { onAbort(); return } + opts.signal.addEventListener("abort", onAbort) + } + + conn.post({type: "score-tokens", id, sessionKey, provider: config.provider, text, config}) + + while (true) { + if (queue.length) { yield queue.shift(); continue } + if (finished) break + await new Promise((r) => (wake = r)) + } + if (opts.signal) opts.signal.removeEventListener("abort", onAbort) + if (error) throw error +} + +/** + * Compute per-token importance for `text` by erasure-based attribution: a + * baseline forward pass, then one pass per token with that token masked out, + * scoring each token by how much its removal shifts the model's next-token + * distribution (Jensen–Shannon divergence). N+1 forward passes on the local + * model — genuine importance, not an attention proxy. See Li, Chen, Zhu & + * Rudin 2016, "Understanding Neural Networks through Representation Erasure". + * + * This is erasure-based importance, NOT attention weights — for real attention + * see computeAttentionWeights. Returns `{decoded, spans}` where decoded is the + * tokenizer's round-tripped text and spans is `[{from, to, importance}]` + * (importance normalized to [0,1]) with positions relative to `decoded`. Only + * works for local models (others resolve to null). + * + * @param {string} text + * @param {GenOpts} [opts] + * @returns {Promise<{decoded:string, spans:Array}|null>} + */ +export async function computeImportance(text, opts = {}) { + const cfg = await resolveCfgPrompts(opts.config ?? (await ensureConfig(opts.scope))) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + if (config.provider !== "local") return null + const conn = getConnection() + const id = nextId() + return new Promise((resolve, reject) => { + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "importance-scores") { + handlers.delete(id) + resolve({decoded: msg.decoded, spans: msg.spans || []}) + } else if (msg.type === "error") { + handlers.delete(id) + reject(new Error(msg.message)) + } + }) + conn.post({type: "compute-importance", id, sessionKey: id, provider: config.provider, text, config}) + }) +} + +/** + * Compute REAL attention weights for `text` (not the erasure proxy that + * computeImportance returns). Only works for local models exported with an + * `attentions` output (see glomper-tuning/onnx_attn.py). One forward pass. + * + * Resolves to: + * { supported:true, dims:{layers,heads,seq}, received, fromLast, spans, tokens, decoded } + * where `received` and `fromLast` are Float32Arrays of length layers*heads*seq + * laid out as [(layer*heads + head)*seq + key]: + * received[…] = mean attention key j gets across all queries i≥j (causal) + * fromLast[…] = attention the final token places on key j + * `spans` is [{from,to,index}] (char positions in `decoded`, keyed by token + * index). Resolves to { supported:false } when the model has no attention + * output, or null for non-local providers. + * + * @param {string} text + * @param {GenOpts} [opts] + * @returns {Promise} + */ +export async function computeAttentionWeights(text, opts = {}) { + const cfg = await resolveCfgPrompts(opts.config ?? (await ensureConfig(opts.scope))) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + if (config.provider !== "local") return null + const conn = getConnection() + const id = nextId() + return new Promise((resolve, reject) => { + const onAbort = () => { clog("computeAttentionWeights: aborted by caller", {model: config.model}); handlers.delete(id); reject(new DOMException("Aborted", "AbortError")) } + if (opts.signal) { + if (opts.signal.aborted) return onAbort() + opts.signal.addEventListener("abort", onAbort, {once: true}) + } + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "attention-weights") { + handlers.delete(id) + opts.signal?.removeEventListener("abort", onAbort) + resolve(msg) + } else if (msg.type === "error") { + clog("computeAttentionWeights: worker error", msg.message) + handlers.delete(id) + opts.signal?.removeEventListener("abort", onAbort) + reject(new Error(msg.message)) + } + }) + conn.post({type: "compute-attention-weights", id, sessionKey: id, provider: config.provider, text, config}) + }) +} + +/** + * Extract per-position features from a local model for LoRA-on-head training: + * a single forward pass returning, for every token position, the final hidden + * state `h` and the base logits, plus token ids/spans. Requires a model + * exported with a `last_hidden_state` output (glomper-tuning/onnx_hidden.py). + * + * Resolves to { supported, seq, H, V, ids, tokens, spans, decoded, hidden, logits } + * where `hidden` is a Float32Array(seq*H) and `logits` is a Float32Array(seq*V), + * both row-major over positions. Returns null for non-local providers, and + * { supported:false, message } if the model lacks the hidden-state output. + * + * @param {string} text + * @param {GenOpts} [opts] + */ +export async function extractFeatures(text, opts = {}) { + const cfg = await resolveCfgPrompts(opts.config ?? (await ensureConfig(opts.scope))) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + if (config.provider !== "local") return null + const conn = getConnection() + const id = nextId() + return new Promise((resolve, reject) => { + const onAbort = () => { handlers.delete(id); reject(new DOMException("Aborted", "AbortError")) } + if (opts.signal) { + if (opts.signal.aborted) return onAbort() + opts.signal.addEventListener("abort", onAbort, {once: true}) + } + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "features") { + handlers.delete(id) + opts.signal?.removeEventListener("abort", onAbort) + resolve(msg) + } else if (msg.type === "error") { + handlers.delete(id) + opts.signal?.removeEventListener("abort", onAbort) + reject(new Error(msg.message)) + } + }) + conn.post({type: "extract-features", id, sessionKey: id, provider: config.provider, text, config}) + }) +} + +/** + * Decode vocab ids to token strings using the loaded local model's tokenizer. + * Used to label the next-token bars when training a LoRA adapter. Returns a + * string[] aligned with `ids`, or null for non-local providers. + * + * @param {any} ids + * @param {GenOpts} [opts] + */ +export async function decodeTokens(ids, opts = {}) { + const cfg = await resolveCfgPrompts(opts.config ?? (await ensureConfig(opts.scope))) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + if (config.provider !== "local") return null + const conn = getConnection() + const id = nextId() + return new Promise((resolve, reject) => { + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "decoded-tokens") { handlers.delete(id); resolve(msg.strings) } + else if (msg.type === "error") { handlers.delete(id); reject(new Error(msg.message)) } + }) + conn.post({type: "decode-tokens", id, sessionKey: id, provider: config.provider, ids, config}) + }) +} + +/** + * Like extractFeatures, but returns the `cut_hidden` state (the residual just + * before the last block's MLP) — for training a LoRA adapter on that MLP (rung 2). + * Requires a model exported with onnx_block.py. Resolves to + * { supported, seq, d, ids, tokens, spans, decoded, hidden: Float32Array(seq*d) }. + * + * @param {string} text + * @param {GenOpts} [opts] + */ +export async function extractCutFeatures(text, opts = {}) { + const cfg = await resolveCfgPrompts(opts.config ?? (await ensureConfig(opts.scope))) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + if (config.provider !== "local") return null + const conn = getConnection() + const id = nextId() + return new Promise((resolve, reject) => { + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + if (msg.type === "cut-features") { handlers.delete(id); resolve(msg) } + else if (msg.type === "error") { handlers.delete(id); reject(new Error(msg.message)) } + }) + conn.post({type: "extract-cut-features", id, sessionKey: id, provider: config.provider, text, config}) + }) +} + +/** + * Diagnostic: probe the loaded model for attention weight support. Posts a + * `probe-attention` message to the worker and returns the result — includes + * the ONNX session output names, forward-pass output keys, and whether any + * attention tensors are available. Only works for local models. + * + * @param {string} [text] + * @param {GenOpts} [opts] + */ +export async function probeAttention(text = "Hello world", opts = {}) { + const cfg = opts.config ?? (await ensureConfig(opts.scope)) + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + const conn = getConnection() + const id = nextId() + return new Promise((resolve) => { + handlers.set(id, (/** @type {WorkerMsg} */ msg) => { + handlers.delete(id) + resolve(msg) + }) + conn.post({type: "probe-attention", id, text, config}) + }) +} + +/** + * Chat with the user's configured tools available. Tells the model what tools + * exist, runs an agentic loop: generate → parse `tool-call` blocks → run each + * handler → feed the result back → generate again, until the model stops calling + * tools (or maxRounds). Folder-tool handlers run in the MAIN thread (full page + * access) by default; pass `sandbox` (or set the config's `toolSandbox`) to run + * them in an isolated Worker instead. Inline tools (with their own `handler` fn) + * always run as given. + * + * @param {Array|string} messages chat messages (or a string → one user turn) + * @param {GenOpts} [opts] same as generate(), plus onToolCall / maxRounds / sandbox + * @returns {Promise<{text:string, messages:Array}>} + */ +export async function generateWithTools(messages, opts = {}) { + const cfg = opts.config ?? (await ensureConfig(opts.scope)) + // Folder-tool handlers run sandboxed when the call or the config asks for it. + const sandbox = opts.sandbox ?? cfg.toolSandbox ?? false + // Inline tools (each with a `handler(args)` fn) + the user's folder tools. + const inline = (opts.tools || []).map((/** @type {any} */ t) => ({...t})) + const folder = await resolveTools(cfg) + const tools = [...inline, ...folder] + const convo = Array.isArray(messages) + ? [...messages] + : [{role: "user", content: String(messages)}] + const maxRounds = opts.maxRounds ?? 6 + let finalText = "" + + // Find a tool by name, matching either the original or sanitized name. + const findTool = (/** @type {string} */ name) => + tools.find((/** @type {any} */ t) => t.name === name || sanitizeToolName(t.name) === name) + + // Execute a single tool call, returning the result text. + const execTool = async (/** @type {any} */ call) => { + const tool = findTool(call.name) + if (!tool) { + opts.onToolCall?.({name: call.name, args: call.args, error: "unknown tool"}) + return `Error: no tool named "${call.name}"` + } + try { + const result = tool.handler + ? await tool.handler(call.args || {}) + : await runTool(tool, call.args, {sandbox}) + opts.onToolCall?.({name: call.name, args: call.args, result}) + return typeof result === "string" ? result : JSON.stringify(result) + } catch (e) { + const err = /** @type {any} */ (e) + opts.onToolCall?.({name: call.name, args: call.args, error: err?.message || String(e)}) + return "Error: " + (err?.message || String(e)) + } + } + + // Whether to use native tool schemas vs text-based XML convention. + // Starts true for native providers, falls back to false on error. + const provider = (callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts))).provider + let useNative = + tools.length > 0 && + (NATIVE_TOOL_PROVIDERS.has(provider) || TEMPLATE_TOOL_PROVIDERS.has(provider)) + // Text-based tool system prompt, built once and reused across rounds. + const textToolSystem = tools.length + ? [buildToolsSystem(tools), opts.system].filter(Boolean).join("\n\n") || undefined + : opts.system + + for (let round = 0; round < maxRounds; round++) { + const cb = opts.onToken + /** @type {GenOpts} */ + const genOpts = { + ...opts, + config: cfg, + onToken: cb + ? (/** @type {string} */ delta, /** @type {string} */ full) => cb(delta, full, round) + : undefined, + } + if (useNative) { + // Native: pass tools for generate() to convert to schemas + genOpts.tools = tools + } else { + // Text-based: inject tool descriptions into system prompt ourselves; + // don't pass tools so generate() won't attempt native for a provider + // we've already fallen back from. + genOpts.tools = undefined + genOpts.system = textToolSystem + } + + let res + try { + res = await generate(convo, genOpts) + } catch (err) { + // If native tool calling failed on the first round, fall back to + // text-based tool descriptions injected into the system prompt. + if (useNative && round === 0) { + useNative = false + res = await generate(convo, { + ...genOpts, + tools: undefined, + system: textToolSystem, + }) + } else { + throw err + } + } + finalText = res.text + + // Native structured tool_calls if the provider returned them; otherwise + // parse the model's text (XML / fenced / bare JSON). + const nativeCalls = res.toolCalls && res.toolCalls.length > 0 + const calls = /** @type {any[]} */ (nativeCalls ? res.toolCalls : parseToolCalls(res.text)) + if (!calls.length) break + + if (nativeCalls) { + // OpenAI format: assistant message includes tool_calls array, each + // tool result is role:"tool" with a matching tool_call_id. + convo.push({ + role: "assistant", + content: res.text || null, + tool_calls: calls.map((call, i) => ({ + id: call.id || "call_" + round + "_" + i, + type: "function", + function: { + name: call.name, + arguments: JSON.stringify(call.args || {}), + }, + })), + }) + for (let i = 0; i < calls.length; i++) { + const call = calls[i] + const callId = call.id || "call_" + round + "_" + i + convo.push({role: "tool", tool_call_id: callId, content: await execTool(call)}) + } + } else if (res.toolMode === "template") { + convo.push({ + role: "assistant", + content: "", + tool_calls: calls.map((call) => ({ + type: "function", + function: {name: call.name, arguments: call.args || {}}, + })), + }) + for (const call of calls) { + convo.push({role: "tool", content: await execTool(call)}) + } + } else { + // Text-based fallback: assistant text + user message with results. + convo.push({role: "assistant", content: res.text}) + for (const call of calls) { + convo.push({role: "user", content: `Tool "${call.name}" returned:\n${await execTool(call)}`}) + } + } + } + return {text: finalText, messages: convo} +} + +/** + * Warm the model/connection ahead of the first real call. + * @param {GenOpts} [opts] + */ +export function preload(opts = {}) { + const cfg = opts.config ?? readConfig() + const config = callConfig(/** @type {any} */ (cfg), /** @type {any} */ (opts)) + getConnection().post({type: "preload", provider: config.provider, config}) +} + +/** + * Subscribe to worker status messages (model download / shader compile / etc.). + * Returns an unsubscribe function. + * @param {(message:string)=>void} cb + */ +export function onStatus(cb) { + getConnection() + statusListeners.add(cb) + return () => statusListeners.delete(cb) +} + +/** Abort an in-flight generation by its sessionKey. + * @param {string|number} sessionKey */ +export function abort(sessionKey) { + getConnection().post({type: "abort", sessionKey}) +} + +/** + * Register a local ONNX model uploaded from disk so the worker can load it as + * `local/`. `files` is `[{path, blob}]` in transformers.js layout + * (config.json, tokenizer.json, onnx/model_.onnx, …). Session-only — the + * files aren't persisted, so they must be re-registered after a reload. + * + * @param {string} id e.g. "local/my-model" + * @param {{path:string, blob:Blob}[]} files + * @param {string} [dtype="q4f16"] + */ +export function registerLocalModel(id, files, dtype = "q4f16") { + getConnection().post({type: "register-local-model", id, files, dtype}) +} + +/** + * Resume a generation that may have survived a refresh, by sessionKey. + * Handlers: { onToken(full), onDone(text), onError(msg), onNone() }. + * @param {string|number} sessionKey + * @param {{onToken?:(t?:string)=>void, onDone?:(t?:string)=>void, onError?:(m?:string)=>void, onNone?:()=>void}} [handlers2] + */ +export function resume(sessionKey, handlers2 = {}) { + resumeHandlers.set(sessionKey, handlers2) + getConnection().post({type: "resume", sessionKey}) +} diff --git a/libraries/llm/config.js b/libraries/llm/config.js new file mode 100644 index 0000000..da11334 --- /dev/null +++ b/libraries/llm/config.js @@ -0,0 +1,951 @@ +/** + * Per-user LLM config, held in a private settings doc (its body IS the config) + * requested from the `patchwork:tool-storage` provider (see + * `patchwork-base/providers`) under the shared id `"llm"` — every LLM-touching + * tool resolves the same doc, the same way any other tool could park its own + * settings under a different id. The provider lazily creates the doc and + * scopes it to the current account, so this needs no `window.accountDocHandle` + * global: just a DOM node inside a mounted `` subtree (any + * element passed to `ensureSettingsDoc`/`subscribeConfig`/etc). See + * `ensureSettingsDoc()` for resolution/creation. + * + * @typedef {"local"|"openrouter"|"ollama"|"webllm"|"builtin"} ProviderId + * + * @typedef {Object} CustomModel + * @property {string} model_id HuggingFace repo of a self-compiled MLC model + * @property {string} model_lib URL/name of its compiled wasm lib + * + * @typedef {Object} RecentModel + * @property {ProviderId} provider + * @property {string|null} model + * + * @typedef {Object} ResolvedPrompts + * @property {string} [system] resolved system-prompt text + * @property {string} [pre] resolved pre-prompt text + * + * @typedef {Object} LLMConfig + * @property {ProviderId} provider + * @property {{predict?:Function}|null} [handler] in-memory request handler attached programmatically by a config provider (never persisted) to intercept calls on the main thread before the worker + * @property {number} temperature default sampling temperature (0 = greedy) + * @property {number} topP nucleus sampling (1 = off) + * @property {number} topK top-k sampling (0 = off) + * @property {number} minP min-p sampling (0 = off) + * @property {number} repetitionPenalty 1 = off + * @property {number} frequencyPenalty -2..2 + * @property {number} presencePenalty -2..2 + * @property {number|null} seed fixed seed (null = random) + * @property {number|null} maxTokens output cap (null = provider default) + * @property {boolean} outputAttentions request per-token attention scores + * @property {{model:string,dtype:string|null}} local + * @property {{apiKey:string,model:string,contextLength:number|null,maxCompletionTokens:number|null}} openrouter + * @property {{url:string,model:string}} ollama + * @property {{model:string,custom:CustomModel[]}} webllm + * @property {Object} builtin + * @property {string|null} tools URL of a folder doc of llm:tool DocLinks + * @property {boolean} toolSandbox run folder-tool handlers in an isolated Worker + * @property {string|null} prompts URL of a folder doc of prompt DocLinks + * @property {string|null} systemUrl selected llm:system-prompt doc + * @property {string|null} preUrl selected llm:pre-prompt doc + * @property {RecentModel[]} recentModels most-recently-chosen, newest first + * @property {Record} [toolToggles] {toolName: false} for host-tool built-ins the user disabled + * @property {ResolvedPrompts} [resolved] prompt TEXT, filled in by resolveCfgPrompts + * @property {Record}>} [pertool] per-tool / per-doc whole-config overrides + * + * @typedef {{toolId:string, docId?:string}} Scope config resolution scope (per-tool / per-doc override target) + * + * The flat per-provider shape the worker's `generate` message wants. Built by + * {@link callConfig} from an {@link LLMConfig} plus per-call overrides. + * @typedef {Object} CallConfig + * @property {ProviderId} provider + * @property {number} temperature + * @property {number} topP + * @property {number} topK + * @property {number} minP + * @property {number} repetitionPenalty + * @property {number} frequencyPenalty + * @property {number} presencePenalty + * @property {number|null} seed + * @property {number} topk how many candidate logprobs to stream (viz) + * @property {number} [maxNewTokens] + * @property {string} [apiKey] + * @property {string} [model] + * @property {number|null} [contextLength] + * @property {number|null} [maxCompletionTokens] + * @property {string} [url] + * @property {string} [dtype] + * @property {CustomModel[]} [custom] + * + * @typedef {import("@automerge/automerge-repo").DocHandle} DocHandle + */ + +import {subscribe, request} from "@inkandswitch/patchwork-providers" + +// Shared id every LLM-touching tool requests its settings doc under (see +// `patchwork:tool-storage`, patchwork-base/providers) — one config, not one +// per calling tool. +export const TOOL_STORAGE_ID = "llm" +export const CONFIG_SELECTOR = {type: "patchwork:llm-config"} + +export const DEFAULTS = { + provider: "local", + // --- sampling / decoding parameters --- + temperature: 0.7, + topP: 0.9, // nucleus sampling (1 = off) + topK: 0, // top-k sampling (0 = off) + minP: 0, // min-p sampling (0 = off) + repetitionPenalty: 1.1, // 1 = off (transformers / ollama / openrouter) + frequencyPenalty: 0, // -2..2 (openrouter / ollama / webllm) + presencePenalty: 0, // -2..2 + seed: null, // fixed seed for reproducibility (null = random) + maxTokens: null, // output cap (null = provider default / per-call) + outputAttentions: false, // request per-token attention scores (only some providers) + local: {model: "onnx-community/Qwen3-0.6B-ONNX", dtype: null}, // dtype null = auto (catalogue default / q4f16) + openrouter: { + apiKey: "", + model: "anthropic/claude-sonnet-4", + contextLength: null, + maxCompletionTokens: null, + }, + ollama: {url: "http://localhost:11434", model: "llama3.2"}, + webllm: {model: "Qwen2.5-1.5B-Instruct-q4f16_1-MLC", custom: []}, // MLC WebLLM (WebGPU); custom = self-compiled model records + builtin: {}, // Chrome built-in AI (Gemini Nano) — one model, no config + tools: null, // URL of a "folder" doc — its .docs are the llm:tool DocLinks + toolSandbox: false, // run folder-tool handlers in an isolated Worker (no page access) + prompts: null, // URL of a "folder" doc — its .docs are the prompt DocLinks + systemUrl: null, // selected llm:system-prompt doc + preUrl: null, // selected llm:pre-prompt doc + recentModels: [], // most-recently-chosen {provider, model}, newest first + toolToggles: {}, // {toolName: false} — a host tool's built-in tools the user turned off + // Per-tool / per-doc whole-config overrides. Shape: + // pertool[toolId] = { config?: , perdoc?: { [docId]: } } + // Resolution (see scopedRaw): most-specific present wins — doc → tool → default. + pertool: {}, +} + +// Sampling/decoding params reset together by the picker's "Reset to defaults". +export const PARAM_KEYS = [ + "temperature", + "topP", + "topK", + "minP", + "repetitionPenalty", + "frequencyPenalty", + "presencePenalty", + "seed", + "maxTokens", + "outputAttentions", +] + +// What each provider's runtime actually READS, not what it accepts. transformers.js +// in particular declares `top_p`/`typical_p` on its GenerationConfig and then never +// looks at them again — its multinomial sampler consults `top_k` and nothing else — +// so those read `false` here even though passing them throws no error. +export const PROVIDER_CAPS = { + local: { logprobs: true, attention: true, topP: false, topK: true, minP: false, typicalP: false, repetitionPenalty: true, noRepeatNgramSize: true, frequencyPenalty: false, presencePenalty: false, seed: false, maxTokens: true }, + openrouter: { logprobs: true, attention: false, topP: true, topK: true, minP: true, typicalP: false, repetitionPenalty: true, noRepeatNgramSize: false, frequencyPenalty: true, presencePenalty: true, seed: true, maxTokens: true }, + ollama: { logprobs: false, attention: false, topP: true, topK: true, minP: true, typicalP: true, repetitionPenalty: true, noRepeatNgramSize: false, frequencyPenalty: false, presencePenalty: false, seed: true, maxTokens: true }, + webllm: { logprobs: true, attention: false, topP: true, topK: false, minP: false, typicalP: false, repetitionPenalty: false, noRepeatNgramSize: false, frequencyPenalty: true, presencePenalty: true, seed: true, maxTokens: true }, + builtin: { logprobs: false, attention: false, topP: false, topK: true, minP: false, typicalP: false, repetitionPenalty: false, noRepeatNgramSize: false, frequencyPenalty: false, presencePenalty: false, seed: false, maxTokens: false }, +} + +// Why a param is greyed out, per provider. "Not supported" is true but useless; +// these say what the underlying runtime actually does with the value. +const TRANSFORMERS_NO_FIELD = + "transformers.js's GenerationConfig has no field for this — it drops the value silently, so the slider would lie to you." +const TRANSFORMERS_DEAD_FIELD = + "transformers.js declares this on its GenerationConfig but never reads it: its sampler filters by top-k and nothing else. Passing a value is a no-op." +const WEBLLM_MINIMAL = + "WebLLM's MLC runtime exposes only temperature, top_p and the two OpenAI-style penalties." +const NANO_MINIMAL = "Chrome's built-in Gemini Nano API exposes temperature and topK only." +export const CAP_NOTES = { + local: { + topP: TRANSFORMERS_DEAD_FIELD + " Use top-k and temperature to shape the tail.", + typicalP: TRANSFORMERS_DEAD_FIELD, + minP: TRANSFORMERS_NO_FIELD, + frequencyPenalty: TRANSFORMERS_NO_FIELD + " Use repetition penalty instead.", + presencePenalty: TRANSFORMERS_NO_FIELD + " Use repetition penalty instead.", + seed: "transformers.js samples from the global RNG and exposes no seed, so runs can't be made reproducible.", + }, + webllm: { + topK: WEBLLM_MINIMAL, + minP: WEBLLM_MINIMAL, + typicalP: WEBLLM_MINIMAL, + noRepeatNgramSize: WEBLLM_MINIMAL, + repetitionPenalty: + "WebLLM takes frequency/presence penalties instead of a multiplicative repetition penalty.", + }, + builtin: { + topP: NANO_MINIMAL, + minP: NANO_MINIMAL, + typicalP: NANO_MINIMAL, + repetitionPenalty: NANO_MINIMAL, + noRepeatNgramSize: NANO_MINIMAL, + frequencyPenalty: NANO_MINIMAL, + presencePenalty: NANO_MINIMAL, + seed: NANO_MINIMAL, + maxTokens: "Chrome's built-in Gemini Nano API caps output itself and takes no limit.", + }, + ollama: { + logprobs: "Ollama's API returns no per-token logprobs, so there are no next-token predictions to show.", + frequencyPenalty: "Ollama takes a single repeat_penalty rather than the OpenAI-style pair.", + presencePenalty: "Ollama takes a single repeat_penalty rather than the OpenAI-style pair.", + noRepeatNgramSize: + "Ollama has no n-gram ban; its repeat_penalty (with repeat_last_n) is the nearest thing.", + }, + openrouter: { + attention: "Attention scores need the raw model; a hosted API only returns text.", + typicalP: "No provider behind OpenRouter accepts typical_p — it isn't in any model's supported_parameters.", + noRepeatNgramSize: + "OpenAI-shaped APIs have no n-gram ban; repetition/frequency penalties are the nearest thing.", + }, +} + +/** + * Why `key` is unavailable under `provider`, or null if it is available. + * @param {string} provider + * @param {string} key + * @returns {string|null} + */ +export function capNote(provider, key) { + const caps = /** @type {Record} */ (PROVIDER_CAPS)[provider] || {} + if (caps[key] !== false) return null + return ( + /** @type {Record} */ (CAP_NOTES)[provider]?.[key] || + "Not supported by this provider." + ) +} + +function repoRef() { + return (typeof window !== "undefined" && window.repo) || null +} + +// --- settings doc ----------------------------------------------------------- +// The config lives in its own doc (its body IS the config), requested from the +// `patchwork:tool-storage` provider under `TOOL_STORAGE_ID`. That provider +// owns the account-doc pointer and the lazy-create; this module just resolves +// it and caches the handle so reads/writes stay synchronous afterwards. +// +// Resolving needs *some* DOM node inside a mounted `` subtree +// (to dispatch the `patchwork:subscribe` request against) — most callers here +// (client.js, tools.js, picker.js) have no element of their own. Rather than +// fall back to a global, we remember the most recent element any caller *did* +// supply (`lastElement`, warmed by whichever tool's UI mounted first — chat +// view, loom, the picker, …) and let elementless callers piggyback on that +// bootstrap. Until one has happened, resolution simply isn't ready yet — the +// same as any other not-yet-loaded doc. + +/** @type {DocHandle|null} */ +let settingsHandle = null +/** @type {Promise|null} */ +let settingsReady = null // de-dupes concurrent ensureSettingsDoc() calls +/** @type {HTMLElement|null} */ +let lastElement = null + +/** The cached settings DocHandle, or null until ensureSettingsDoc() resolves. */ +export function settingsDocHandle() { + return settingsHandle +} + +/** + * Resolve (or lazily create, via the `patchwork:tool-storage` provider) the + * settings doc and cache its handle. Idempotent and concurrency-safe. Pass an + * `element` the first time it's available (any node inside a mounted + * ``); later elementless calls reuse whichever element a + * caller most recently supplied. + * @param {HTMLElement|null} [element] + * @returns {Promise} + */ +export function ensureSettingsDoc(element) { + if (element) lastElement = element + if (settingsHandle) return Promise.resolve(settingsHandle) + if (settingsReady) return settingsReady + const el = element ?? lastElement + const repo = repoRef() + if (!repo || !el) return Promise.resolve(null) // not bootstrapped yet; don't cache — a later call with an element can still resolve + settingsReady = (async () => { + const url = await request(el, {type: "patchwork:tool-storage", toolId: TOOL_STORAGE_ID}) + if (!url) { + settingsReady = null + return null + } + settingsHandle = await repo.find(/** @type {any} */ (url)) + return settingsHandle + })() + return settingsReady +} + +/** + * Ensure the settings doc is resolved, then return the normalized config. + * @param {Scope} [scope] + * @param {HTMLElement|null} [element] + */ +export async function ensureConfig(scope, element) { + await ensureSettingsDoc(element) + return scope ? readScopedConfig(scope) : readConfig() +} + +/** + * Read the normalized LLM config. Reads the cached settings doc, or defaults + * if it hasn't resolved yet. Pass a settings-doc snapshot to normalize that + * instead. + * @param {Record} [snapshot] + * @returns {LLMConfig} + */ +export function readConfig(snapshot) { + if (snapshot !== undefined) return normalizeConfig(snapshot) + return normalizeConfig(settingsHandle?.doc() ?? {}) +} + +/** + * Fill in defaults for any missing fields of a raw `llm` config object. + * @param {any} [raw] + * @returns {LLMConfig} + */ +export function normalizeConfig(raw = {}) { + return { + provider: raw.provider ?? DEFAULTS.provider, + // Optional in-memory request handler. A config provider can attach + // { predict, ... } to intercept calls on the main thread before they reach + // the worker — e.g. choochoo runs a base model and adds a live LoRA delta. + // Never stored in a doc (it holds functions); only set programmatically via + // a provider element, and carried through normalize so predict() can find it. + handler: + raw.handler && typeof raw.handler === "object" ? raw.handler : null, + temperature: + typeof raw.temperature === "number" ? raw.temperature : DEFAULTS.temperature, + topP: typeof raw.topP === "number" ? raw.topP : DEFAULTS.topP, + topK: typeof raw.topK === "number" ? raw.topK : DEFAULTS.topK, + minP: typeof raw.minP === "number" ? raw.minP : DEFAULTS.minP, + repetitionPenalty: + typeof raw.repetitionPenalty === "number" + ? raw.repetitionPenalty + : DEFAULTS.repetitionPenalty, + frequencyPenalty: + typeof raw.frequencyPenalty === "number" + ? raw.frequencyPenalty + : DEFAULTS.frequencyPenalty, + presencePenalty: + typeof raw.presencePenalty === "number" + ? raw.presencePenalty + : DEFAULTS.presencePenalty, + seed: raw.seed ?? DEFAULTS.seed, + maxTokens: raw.maxTokens ?? DEFAULTS.maxTokens, + outputAttentions: raw.outputAttentions ?? DEFAULTS.outputAttentions, + local: {...DEFAULTS.local, ...(raw.local ?? {})}, + openrouter: {...DEFAULTS.openrouter, ...(raw.openrouter ?? {})}, + ollama: {...DEFAULTS.ollama, ...(raw.ollama ?? {})}, + webllm: { + ...DEFAULTS.webllm, + ...(raw.webllm ?? {}), + // Self-compiled MLC models: just {model_id, model_lib}. The weights URL is + // derived from the model_id (its HuggingFace repo) in the worker. Plain + // copies — these get re-assigned into the doc, and automerge rejects + // re-inserting its own proxy objects. + custom: Array.isArray(raw.webllm?.custom) + ? raw.webllm.custom + .map((/** @type {any} */ c) => ({ + model_id: c.model_id ?? "", + model_lib: c.model_lib ?? "", + })) + .filter((/** @type {CustomModel} */ c) => c.model_id || c.model_lib) + : [], + }, + builtin: {...DEFAULTS.builtin, ...(raw.builtin ?? {})}, + // Folders (URLs). The legacy array/object shapes resolve to null here; the + // one-time migrateConfig() converts them and rewrites the account doc. + tools: typeof raw.tools === "string" ? raw.tools : null, + toolSandbox: !!raw.toolSandbox, + prompts: typeof raw.prompts === "string" ? raw.prompts : null, + systemUrl: + raw.systemUrl ?? + (raw.prompts && typeof raw.prompts === "object" ? raw.prompts.systemUrl : null) ?? + null, + preUrl: + raw.preUrl ?? + (raw.prompts && typeof raw.prompts === "object" ? raw.prompts.preUrl : null) ?? + null, + // Plain copies (see webllm.custom note) — re-assigned into the doc on save. + recentModels: Array.isArray(raw.recentModels) + ? raw.recentModels.map((/** @type {any} */ r) => ({ + provider: r.provider, + model: r.model ?? null, + })) + : [], + // {toolName: false} for any host-tool built-in tool the user disabled. + // Plain copy (see webllm.custom note) — re-assigned into the doc on save. + toolToggles: + raw.toolToggles && typeof raw.toolToggles === "object" + ? {...raw.toolToggles} + : {}, + // Per-tool / per-doc whole-config overrides. Deep plain copy; each nested + // config is normalized on resolve (scopedRaw → normalizeConfig). + pertool: + raw.pertool && typeof raw.pertool === "object" + ? JSON.parse(JSON.stringify(raw.pertool)) + : {}, + } +} + +// Pick the raw config for a scope: the most-specific override present wins — +// per-doc → per-tool → the top-level default. `scope` is {toolId, docId?}. +// Whole-scope semantics: an override is a complete config, not a partial. +/** + * @param {any} raw + * @param {Scope} [scope] + */ +export function scopedRaw(raw, scope) { + if (!raw || !scope || !scope.toolId) return raw + const pt = raw.pertool && raw.pertool[scope.toolId] + if (!pt) return raw + if (scope.docId && pt.perdoc && pt.perdoc[scope.docId]) return pt.perdoc[scope.docId] + if (pt.config) return pt.config + return raw +} + +// Does THIS exact scope level hold its own override? (Used by the picker to show +// create-vs-remove and the active scope.) docId omitted → checks the tool level. +/** + * @param {any} raw + * @param {Scope} [scope] + */ +export function hasScopeOverride(raw, scope) { + if (!raw || !scope || !scope.toolId) return false + const pt = raw.pertool && raw.pertool[scope.toolId] + if (!pt) return false + return scope.docId ? !!(pt.perdoc && pt.perdoc[scope.docId]) : !!pt.config +} + +// The raw settings-doc body, used by scope read/write helpers. +function rawSettings() { + return settingsHandle?.doc() ?? {} +} + +// Read a scope's effective config (normalized). Falls back through tool → default. +/** @param {Scope} [scope] */ +export function readScopedConfig(scope) { + return normalizeConfig(scopedRaw(rawSettings(), scope)) +} + +// Create/replace a scope's whole-config override (writes a full normalized config +// into pertool). `cfgObj` defaults to the current default config (seed a fork). +/** + * @param {Scope} scope + * @param {any} [cfgObj] + */ +export function writeScopeOverride(scope, cfgObj) { + if (!scope || !scope.toolId) return + const full = JSON.parse(JSON.stringify(normalizeConfig(cfgObj ?? readConfig()))) + delete full.pertool // overrides never nest + const apply = (/** @type {DocHandle|null} */ handle) => { + if (!handle) return + handle.change((/** @type {any} */ d) => { + if (!d.pertool) d.pertool = {} + if (!d.pertool[scope.toolId]) d.pertool[scope.toolId] = {} + if (scope.docId) { + if (!d.pertool[scope.toolId].perdoc) d.pertool[scope.toolId].perdoc = {} + d.pertool[scope.toolId].perdoc[scope.docId] = full + } else { + d.pertool[scope.toolId].config = full + } + }) + } + if (settingsHandle) apply(settingsHandle) + else ensureSettingsDoc().then(apply) +} + +// Remove a scope's override (fall back to the less-specific scope / default). +/** @param {Scope} scope */ +export function clearScopeOverride(scope) { + if (!scope || !scope.toolId || !settingsHandle) return + settingsHandle.change((/** @type {any} */ d) => { + const pt = d.pertool && d.pertool[scope.toolId] + if (!pt) return + if (scope.docId) { + if (pt.perdoc) delete pt.perdoc[scope.docId] + } else { + delete pt.config + } + // Drop now-empty containers so the tool falls all the way back to default + // and the settings doc doesn't accumulate dead `pertool` entries. + if (pt.perdoc && Object.keys(pt.perdoc).length === 0) delete pt.perdoc + if (!pt.config && !pt.perdoc) delete d.pertool[scope.toolId] + }) +} + +/** + * Combine the configured system prompt with any tool-supplied one. Tools may + * append their own instructions to the user's system prompt. + * @param {LLMConfig} [cfg] + * @param {string} [extraSystem] + * @returns {string} + */ +export function effectiveSystem(cfg, extraSystem) { + // `cfg.resolved.{system,pre}` is the prompt TEXT, filled in by resolveCfgPrompts + // (which reads the selected prompt docs). Absent in a bare cfg → empty. + return [cfg?.resolved?.system, extraSystem].filter(Boolean).join("\n\n") +} + +/** + * @typedef {{role:string, content:string}} ChatMessage + */ + +/** + * Apply the configured pre-prompt + system prompt to a generation input. + * - string input (raw continuation): prefixes `system\n\npre\n\n…` + * - chat messages: prepends a system message. + * @param {string|ChatMessage[]} input + * @param {LLMConfig} [cfg] + * @param {string} [extraSystem] + * @returns {string|ChatMessage[]} + */ +export function applyPrompts(input, cfg, extraSystem) { + const sys = effectiveSystem(cfg, extraSystem) + const pre = cfg?.resolved?.pre || "" + if (typeof input === "string") { + // Raw completion has no `system` role, so the system prompt is OMITTED + // here — only the pre-prompt (literal text before your input) is prepended. + void sys + return pre ? pre + "\n\n" + input : input + } + // Chat: the system prompt becomes a real `system` turn; the pre-prompt is + // glued onto the front of the first user message (part of the user's input). + let msgs = [...input] + if (pre) { + const i = msgs.findIndex((m) => m.role === "user") + if (i === -1) msgs.unshift({role: "user", content: pre}) + else msgs[i] = {...msgs[i], content: pre + "\n\n" + msgs[i].content} + } + return sys ? [{role: "system", content: sys}, ...msgs] : msgs +} + +/** + * Resolve the *active* LLM config reactively, calling `callback(config)` now and + * whenever it changes. + * + * Resolution order: a `patchwork:llm-config` provider in `element`'s subtree + * (request/provide — lets a future provider element scope config per tool/view) + * wins; if no provider answers within `timeoutMs`, we fall back to this tool's + * `patchwork:tool-storage` settings doc (and keep it live by listening for + * changes to it). If a provider appears later, it takes over. Always pass a + * real `element` when you have one — it's also how the settings doc itself + * gets resolved (see `ensureSettingsDoc`), even for later callers that don't. + * + * @param {HTMLElement} element a node inside a + * @param {(config: import("./config.js").LLMConfig) => void} callback + * @returns {() => void} unsubscribe + */ +export function subscribeConfig(element, callback, {timeoutMs = 50} = {}) { + if (!element) throw new TypeError("subscribeConfig requires an element") + let providerAnswered = false + /** @type {(() => void)|null} */ + let fallbackOff = null + let providerOff = () => {} + /** @type {ReturnType|undefined} */ + let timer = undefined + + let cancelled = false + const startFallback = () => { + if (providerAnswered || cancelled) return + callback(readConfig()) // immediate sync value (defaults) so UI isn't blank + ensureSettingsDoc(element).then((handle) => { + if (providerAnswered || cancelled || !handle) return + callback(readConfig()) // real config now that the settings doc resolved + const onChange = () => { + if (!providerAnswered && !cancelled) callback(readConfig()) + } + handle.on("change", onChange) + fallbackOff = () => handle.off("change", onChange) + }) + } + + providerOff = subscribe(element, CONFIG_SELECTOR, (raw) => { + providerAnswered = true + clearTimeout(timer) + fallbackOff?.() + fallbackOff = null + callback(normalizeConfig(raw)) + }) + timer = setTimeout(startFallback, timeoutMs) + + return () => { + cancelled = true + clearTimeout(timer) + providerOff() + fallbackOff?.() + } +} + +/** + * One-shot resolve of the active config (request + account-doc fallback). + * @param {HTMLElement} element + * @param {{timeoutMs?: number}} [opts] + * @returns {Promise} + */ +export function resolveConfig(element, opts) { + return new Promise((resolve) => { + /** @type {(() => void)|null} */ + let off = null + let done = false + off = subscribeConfig( + element, + (cfg) => { + if (done) return + done = true + queueMicrotask(() => off && off()) + resolve(cfg) + }, + opts + ) + }) +} + +/** + * Merge a partial config into the settings doc. `undefined` values are + * skipped; `null` is stored (e.g. an unknown context length). + * @param {Partial & Record} next + */ +export function writeConfig(next) { + /** @param {DocHandle|null} handle */ + const apply = (handle) => { + if (!handle) return + handle.change((/** @type {any} */ d) => { + if (next.provider !== undefined) d.provider = next.provider + for (const k of [ + "temperature", + "topP", + "topK", + "minP", + "repetitionPenalty", + "frequencyPenalty", + "presencePenalty", + "seed", + "maxTokens", + "outputAttentions", + "tools", // folder URL + "toolSandbox", // run folder-tool handlers in an isolated Worker + "prompts", // folder URL + "systemUrl", // selected prompt docs + "preUrl", + "recentModels", // [{provider, model}], newest first + "toolToggles", // {toolName: false} host-tool built-ins turned off + ]) { + if (next[k] !== undefined) d[k] = next[k] + } + for (const group of ["local", "openrouter", "ollama", "webllm", "builtin"]) { + if (!next[group]) continue + if (!d[group]) d[group] = {} + for (const [field, value] of Object.entries(next[group])) { + if (value === undefined) continue + d[group][field] = value // null is allowed in automerge + } + } + }) + } + // Settings doc is usually already resolved (the picker awaits it). If a write + // races ahead of that, resolve first, then apply. + if (settingsHandle) apply(settingsHandle) + else ensureSettingsDoc().then(apply) +} + +/** + * Resolve the flat call config for a given provider from a full LLMConfig — the + * shape the worker's `generate` message wants. + * @param {LLMConfig} cfg + * @param {Partial} [overrides] + * @returns {CallConfig} + */ +export function callConfig(cfg, overrides = {}) { + const provider = overrides.provider ?? cfg.provider + const base = { + provider, + temperature: + overrides.temperature != null ? overrides.temperature : cfg.temperature, + topP: overrides.topP != null ? overrides.topP : cfg.topP, + topK: cfg.topK, // sampling top-k (distinct from `topk`, the prediction-viz count) + minP: cfg.minP, + repetitionPenalty: cfg.repetitionPenalty, + frequencyPenalty: cfg.frequencyPenalty, + presencePenalty: cfg.presencePenalty, + seed: cfg.seed, + topk: (overrides.topk ?? 0) | 0, // how many candidate logprobs to stream (viz) + maxNewTokens: overrides.maxNewTokens ?? cfg.maxTokens ?? undefined, + } + if (provider === "openrouter") { + return { + ...base, + apiKey: overrides.apiKey ?? cfg.openrouter.apiKey, + model: overrides.model ?? cfg.openrouter.model, + contextLength: cfg.openrouter.contextLength, + maxCompletionTokens: cfg.openrouter.maxCompletionTokens, + } + } + if (provider === "ollama") { + return { + ...base, + url: overrides.url ?? cfg.ollama.url, + model: overrides.model ?? cfg.ollama.model, + } + } + if (provider === "webllm") { + return { + ...base, + model: overrides.model ?? cfg.webllm.model, + custom: cfg.webllm.custom || [], // self-compiled MLC model records + } + } + return {...base, model: overrides.model ?? cfg.local.model, dtype: overrides.dtype ?? cfg.local.dtype ?? undefined} +} + +// --------------------------------------------------------------------------- +// Model catalogues (for the picker) +// --------------------------------------------------------------------------- + +/** In-browser (WebGPU/WASM) models, mirroring chat's catalogue. */ +export const LOCAL_MODELS = [ + {id: "LiquidAI/LFM2.5-2.6B-ONNX", name: "LFM2.5 2.6B", canUseTool: true}, + {id: "onnx-community/Qwen3-4B-ONNX", name: "Qwen3 4B", canUseTool: true}, + {id: "onnx-community/Qwen3-1.7B-ONNX", name: "Qwen3 1.7B", canUseTool: true}, + {id: "onnx-community/Qwen3-0.6B-ONNX", name: "Qwen3 0.6B", canUseTool: true}, + { + id: "onnx-community/Llama-3.2-1B-Instruct-ONNX", + name: "Llama 3.2 1B", + canUseTool: true, + }, + {id: "onnx-community/gemma-3-1b-it-ONNX", name: "Gemma 3 1B", canUseTool: false}, + { + id: "onnx-community/gemma-3-270m-it-ONNX", + name: "Gemma 3 270M (tiny)", + canUseTool: false, + }, + { + id: "onnx-community/DeepSeek-R1-Distill-Qwen-1.5B-ONNX", + name: "DeepSeek-R1 Distill 1.5B (reasoning)", + canUseTool: false, + }, + { + id: "onnx-community/Qwen2.5-Coder-1.5B-Instruct", + name: "Qwen2.5 Coder 1.5B", + canUseTool: true, + }, + { + id: "onnx-community/Qwen2.5-0.5B-Instruct", + name: "Qwen2.5 0.5B", + canUseTool: true, + }, + { + id: "onnx-community/LFM2-1.2B-ONNX", + name: "LFM2 1.2B", + canUseTool: false, + }, + { + id: "onnx-community/Phi-3.5-mini-instruct-onnx-web", + name: "Phi 3.5 Mini", + canUseTool: false, + }, + { + id: "onnx-community/SmolLM2-1.7B-Instruct-ONNX", + name: "SmolLM2 1.7B", + canUseTool: false, + }, + { + id: "onnx-community/SmolLM2-360M-Instruct", + name: "SmolLM2 360M (tiny)", + canUseTool: false, + }, +] + +/** Curated WebLLM (MLC) models — WebGPU, non-ONNX. Type any prebuilt model_id too. */ +export const WEBLLM_MODELS = [ + {id: "Qwen2.5-0.5B-Instruct-q4f16_1-MLC", name: "Qwen2.5 0.5B"}, + {id: "Qwen2.5-1.5B-Instruct-q4f16_1-MLC", name: "Qwen2.5 1.5B"}, + {id: "Qwen2.5-3B-Instruct-q4f16_1-MLC", name: "Qwen2.5 3B"}, + {id: "Llama-3.2-1B-Instruct-q4f16_1-MLC", name: "Llama 3.2 1B"}, + {id: "Llama-3.2-3B-Instruct-q4f16_1-MLC", name: "Llama 3.2 3B"}, + {id: "Phi-3.5-mini-instruct-q4f16_1-MLC", name: "Phi 3.5 Mini"}, + {id: "gemma-2-2b-it-q4f16_1-MLC", name: "Gemma 2 2B"}, + {id: "Mistral-7B-Instruct-v0.3-q4f16_1-MLC", name: "Mistral 7B"}, + {id: "SmolLM2-1.7B-Instruct-q4f16_1-MLC", name: "SmolLM2 1.7B"}, + {id: "TinyLlama-1.1B-Chat-v1.0-q4f16_1-MLC", name: "TinyLlama 1.1B"}, +] + +/** Fetch the OpenRouter model catalogue (with capability metadata). */ +export async function fetchOpenRouterModels() { + const resp = await fetch("https://openrouter.ai/api/v1/models") + const data = await resp.json() + return (data.data || []) + .filter((/** @type {any} */ m) => m.id) + .map((/** @type {any} */ m) => ({ + id: m.id, + name: m.name || m.id, + context_length: m.context_length || m.top_provider?.context_length, + max_completion_tokens: m.top_provider?.max_completion_tokens, + supported_parameters: m.supported_parameters || [], + default_parameters: m.default_parameters || null, + input_modalities: m.architecture?.input_modalities || [], + pricing: m.pricing || null, + })) + .sort((/** @type {{name:string}} */ a, /** @type {{name:string}} */ b) => + a.name.localeCompare(b.name) + ) +} + +// A model's `generation_config.json` carries the sampling settings its authors +// actually recommend — the same numbers a model card quotes. transformers.js +// merges that file over its own defaults, so a value equal to a default tells us +// nothing; only fields that differ are a real suggestion. +const GEN_CONFIG_DEFAULTS = { + temperature: 1, + top_p: 1, + top_k: 50, + repetition_penalty: 1, + min_p: 0, + typical_p: 1, + no_repeat_ngram_size: 0, +} + +/** @type {Record} */ +const GEN_CONFIG_KEYS = { + temperature: "temperature", + top_p: "topP", + top_k: "topK", + repetition_penalty: "repetitionPenalty", + min_p: "minP", + typical_p: "typicalP", + no_repeat_ngram_size: "noRepeatNgramSize", + max_new_tokens: "maxTokens", + max_length: "maxTokens", // older files spell the output cap this way +} + +/** + * Pick the author-set sampling params out of a parsed `generation_config.json`, + * as picker-config keys. Returns `{}` when the file is all defaults. + * @param {any} gc + * @returns {Record} + */ +export function suggestedParams(gc) { + /** @type {Record} */ + const out = {} + if (!gc || typeof gc !== "object") return out + // `do_sample: false` means the authors want greedy decoding, whatever else + // the file says — express that as temperature 0, which is how the rest of + // this config represents it. + if (gc.do_sample === false) return {temperature: 0} + for (const [src, dest] of Object.entries(GEN_CONFIG_KEYS)) { + const v = gc[src] + if (typeof v !== "number") continue + if (v === /** @type {Record} */ (GEN_CONFIG_DEFAULTS)[src]) continue + // max_length defaults to 20 in transformers and means prompt+output, not + // output — a value that small is boilerplate, not a recommendation. + if (dest === "maxTokens" && (v <= 20 || out.maxTokens != null)) continue + out[dest] = v + } + return out +} + +/** + * The non-sampling half of a `generation_config.json` — the stop tokens. Not a + * user knob, but the thing to look at when a model won't shut up: no + * `eos_token_id` means nothing ever ends generation but the token cap. + * @param {any} gc + * @returns {{eos: number[], hasEos: boolean, greedy: boolean}} + */ +export function generationStops(gc) { + const raw = gc?.eos_token_id + const eos = raw == null ? [] : Array.isArray(raw) ? raw : [raw] + return {eos, hasEos: eos.length > 0, greedy: gc?.do_sample === false} +} + +// OpenRouter publishes the same idea under `default_parameters` on each model in +// the catalogue (populated for a couple hundred of them; null-filled otherwise). +/** @type {Record} */ +const OR_PARAM_KEYS = { + temperature: "temperature", + top_p: "topP", + top_k: "topK", + min_p: "minP", + repetition_penalty: "repetitionPenalty", + frequency_penalty: "frequencyPenalty", + presence_penalty: "presencePenalty", +} + +/** + * @param {any} dp a model's `default_parameters` + * @returns {Record} + */ +export function suggestedParamsFromOpenRouter(dp) { + /** @type {Record} */ + const out = {} + if (!dp || typeof dp !== "object") return out + for (const [src, dest] of Object.entries(OR_PARAM_KEYS)) { + if (typeof dp[src] === "number") out[dest] = dp[src] + } + return out +} + +/** + * Fetch a HuggingFace model's `generation_config.json`. Null when the repo + * doesn't ship one (or the request fails — this is best-effort decoration). + * @param {string} id + * @returns {Promise} + */ +export async function fetchGenerationConfig(id) { + try { + const res = await fetch( + `https://huggingface.co/${id}/resolve/main/generation_config.json` + ) + return res.ok ? await res.json() : null + } catch { + return null + } +} + +/** + * Same, for a local ONNX folder picked from disk — read the file we already hold. + * @param {{path:string, blob:Blob}[]} files + * @returns {Promise} + */ +export async function generationConfigFromFiles(files) { + const f = files.find((x) => x.path.split("/").pop() === "generation_config.json") + if (!f) return null + try { + return JSON.parse(await f.blob.text()) + } catch { + return null + } +} + +/** + * Probe an Ollama server for installed models. + * @param {string} [url] + * @returns {Promise} + */ +export async function fetchOllamaModels(url) { + const base = (url || DEFAULTS.ollama.url).replace(/\/$/, "") + const resp = await fetch(base + "/api/tags") + const data = await resp.json() + return (data.models || []).map((/** @type {any} */ m) => m.name || m.model) +} + +/** + * Human label for the current selection. + * @param {LLMConfig} cfg + * @param {{openrouterModels?: Array<{id:string,name:string}>}} [opts] + * @returns {string} + */ +export function describeConfig(cfg, {openrouterModels = []} = {}) { + if (cfg.provider === "local") { + const m = LOCAL_MODELS.find((x) => x.id === cfg.local.model) + const name = m ? m.name : cfg.local.model.replace(/^local\//, "") + return "Browser " + name + } + if (cfg.provider === "openrouter") { + const m = openrouterModels.find((x) => x.id === cfg.openrouter.model) + return "OpenRouter " + (m ? m.name : cfg.openrouter.model) + } + if (cfg.provider === "webllm") { + const m = WEBLLM_MODELS.find((x) => x.id === cfg.webllm.model) + return "WebLLM " + (m ? m.name : cfg.webllm.model) + } + if (cfg.provider === "builtin") return "Built-in (Chrome)" + return "Ollama " + cfg.ollama.model +} diff --git a/libraries/llm/globals.d.ts b/libraries/llm/globals.d.ts new file mode 100644 index 0000000..ecea81d --- /dev/null +++ b/libraries/llm/globals.d.ts @@ -0,0 +1,18 @@ +// Build-only ambient declarations. Augments the host globals Patchwork tools +// rely on (set up by the bootloader) so checkJs can resolve them. Not published +// — consumers bring their own DOM lib + Patchwork globals. +import type {Repo, DocHandle} from "@automerge/automerge-repo" + +declare global { + interface Window { + /** The automerge Repo instance, installed by the Patchwork bootloader. */ + repo: Repo + /** The current user's account DocHandle. */ + accountDocHandle: DocHandle + /** Chrome built-in AI entry points (Gemini Nano), when available. */ + ai?: any + LanguageModel?: any + } +} + +export {} diff --git a/libraries/llm/index.js b/libraries/llm/index.js new file mode 100644 index 0000000..1fd091d --- /dev/null +++ b/libraries/llm/index.js @@ -0,0 +1,112 @@ +/** + * @patchwork/llm — LLM toolkit for Patchwork tools. + * + * import { dom, stream, generate } from "@patchwork/llm" + * + * const el = popup(); root.append(el); el.showPopover() // framed model picker + * const cfg = await el.result // null if cancelled + * box.append(dom({source, tools})) // bare embeddable panel + * + * for await (const ev of stream(messages, { topk: 5 })) { + * if (ev.type === "token") out += ev.delta + * if (ev.type === "prediction") showCandidates(ev.candidates) // next-token dist + * if (ev.type === "stats") showStats(ev) // ttft, tok/s, decode + * } + * + * const { text, stats } = await generate(messages, { onToken, onPrediction }) + * + * Provider/model/key/temperature live on the account doc (set via popup()/dom()). + * Telemetry (top-k next-token predictions + decode stats) works for local + * transformers.js AND OpenRouter. + */ + +export { + // config (account doc + patchwork:llm-config provider) + readConfig, + writeConfig, + callConfig, + normalizeConfig, + subscribeConfig, + resolveConfig, + ensureSettingsDoc, + ensureConfig, + // per-tool / per-doc whole-config overrides + scopedRaw, + hasScopeOverride, + readScopedConfig, + writeScopeOverride, + clearScopeOverride, + settingsDocHandle, + applyPrompts, + effectiveSystem, + DEFAULTS, + PARAM_KEYS, + PROVIDER_CAPS, + TOOL_STORAGE_ID, + CONFIG_SELECTOR, + // catalogues / labels + LOCAL_MODELS, + WEBLLM_MODELS, + fetchOpenRouterModels, + fetchOllamaModels, + describeConfig, +} from "./config.js" + +export { + generate, + generateWithTools, + stream, + predict, + scoreTokens, + preload, + abort, + resume, + onStatus, + registerLocalModel, + computeImportance, + computeAttentionWeights, + extractFeatures, + extractCutFeatures, + decodeTokens, + probeAttention, +} from "./client.js" + +export {dom, popup} from "./picker.js" + +export {builtinSupported, builtinAvailability} from "./builtin.js" + +// LLM tools (user-defined tools the model can be given) +export { + createLLMTool, + createToolFile, + LLMToolDatatype, + sanitizeToolName, + resolveTools, + toToolSchemas, + buildToolsSystem, + parseToolCalls, + loadHandler, + runTool, + runHandlerSandboxed, + // saved prompts (system + pre), same doc shape as tools + createPromptDoc, + resolvePromptDocs, + resolvePromptText, + resolveCfgPrompts, + LLMSystemPromptDatatype, + LLMPrePromptDatatype, + // folders + one-time migration + ensureFolderUrl, + addToFolder, + removeFromFolder, + migrateConfig, +} from "./tools.js" + +// Registers on import. +export { + PatchworkLLMConfigProvider, + definePatchworkLLMConfigProvider, +} from "./provider.js" + +// Built-in prompt templates +export {PROMPT_TEMPLATES} from "./templates.js" diff --git a/libraries/llm/package.json b/libraries/llm/package.json new file mode 100644 index 0000000..2b0391a --- /dev/null +++ b/libraries/llm/package.json @@ -0,0 +1,47 @@ +{ + "name": "@chee/patchwork-llm", + "version": "0.2.1", + "description": "LLM toolkit for Patchwork tools: a model picker, a refresh-surviving SharedWorker that runs local (transformers.js) / OpenRouter / Ollama generation, and a streaming API that carries rich telemetry — next-token predictions, temperature, tokens/sec — alongside the text so you can build UIs that show how the model thinks.", + "type": "module", + "main": "index.js", + "types": "./types/index.d.ts", + "exports": { + ".": { + "types": "./types/index.d.ts", + "default": "./index.js" + }, + "./worker.js": { + "types": "./types/worker.d.ts", + "default": "./worker.js" + } + }, + "scripts": { + "build": "pnpm build:types", + "build:types": "tsc" + }, + "files": [ + "types", + "index.js", + "config.js", + "client.js", + "worker.js", + "picker.js", + "tools.js", + "provider.js", + "builtin.js", + "templates.js", + "README.md" + ], + "author": "chee", + "license": "MIT", + "dependencies": { + "@inkandswitch/patchwork-providers": "^0.3.0" + }, + "devDependencies": { + "@automerge/automerge-repo": "^2.5.6", + "typescript": "^6.0.3" + }, + "peerDependencies": { + "@automerge/automerge-repo": "^2.5.6" + } +} diff --git a/libraries/llm/picker.js b/libraries/llm/picker.js new file mode 100644 index 0000000..9c60095 --- /dev/null +++ b/libraries/llm/picker.js @@ -0,0 +1,2264 @@ +/** + * The config picker UI. Two exports: + * - `dom(opts)` — the BARE picker element (no frame); embed + own it. + * - `popup(opts)` — the picker wrapped in a popover frame (title + Cancel/Done). + * Both return synchronously (Suspense-style spinner) and reads/write a config + * source (the account settings doc by default). Framework-free; injects its own + * namespaced styles (`llmp-`), which inherit the host font + overridable + * `--llmp-*` colour vars. + */ + +import { + DEFAULTS, + PARAM_KEYS, + PROVIDER_CAPS, + LOCAL_MODELS, + WEBLLM_MODELS, + ensureSettingsDoc, + settingsDocHandle, + readConfig, + writeConfig, + readScopedConfig, + writeScopeOverride, + clearScopeOverride, + hasScopeOverride, + describeConfig, + fetchOpenRouterModels, + fetchOllamaModels, + fetchGenerationConfig, + generationConfigFromFiles, + suggestedParams, + suggestedParamsFromOpenRouter, + generationStops, + capNote, +} from "./config.js" +import {registerLocalModel, generateWithTools} from "./client.js" +import {builtinSupported, builtinAvailability} from "./builtin.js" +import { + createLLMTool, + createPromptDoc, + resolveTools, + resolvePromptDocs, + ensureFolderUrl, + addToFolder, + removeFromFolder, +} from "./tools.js" +import {PROMPT_TEMPLATES} from "./templates.js" + +const DTYPES = ["q4f16", "q4", "q8", "int8", "fp16", "fp32"] +const SECTIONS = [ + {id: "model", label: "Model"}, + {id: "params", label: "Parameters"}, + {id: "prompts", label: "Prompts"}, + {id: "tools", label: "Tool Calling"}, +] + +const STYLE_ID = "llmp-picker-styles" +// Theme is overridable + plain by default: it inherits the host font, and every +// colour is a `--llmp-*` var a host can set on any ancestor. Neutrals (line, dim, +// soft fills, highlight) are color-mix'd from the text/accent so the picker +// adapts to whatever foreground/accent the surrounding UI uses. +const CSS = ` +.llmp { + /* Follow the active Patchwork theme: accents from --studio-*, fill/line from + --editor-*. An explicit --llmp-* override still wins; the pink/cream values + are the last-resort fallback (e.g. the "Sundae" theme reproduces them). */ + --accent: var(--llmp-accent, var(--studio-primary, #ff4d97)); + --accent-text: var(--llmp-accent-text, var(--editor-fill, #fff)); + --accent2: var(--llmp-accent2, var(--studio-secondary, #58cfb0)); + --paper: var(--llmp-bg, var(--editor-fill-offset-10, #fdfbf7)); + --card: var(--llmp-card, var(--editor-fill, #fff)); + --ink: var(--llmp-fg, var(--editor-line, #34313a)); + --accent-soft: var(--llmp-accent-soft, color-mix(in srgb, var(--accent) 13%, var(--card))); + --line: var(--llmp-line, color-mix(in srgb, var(--ink) 12%, transparent)); + --highlight: var(--llmp-highlight, color-mix(in srgb, var(--accent) 8%, var(--card))); + --dim: var(--llmp-dim, color-mix(in srgb, var(--ink) 45%, transparent)); + --radius: var(--llmp-radius, var(--studio-radius, 14px)); --radius-sm: var(--llmp-radius-sm, var(--studio-radius-sm, 9px)); + --shadow: var(--llmp-shadow, var(--studio-shadow-lg, 0 12px 34px rgba(0,0,0,.18))); --shadow-sm: 0 1px 3px rgba(0,0,0,.08); + width: min(840px, 96vw); height: min(760px, 90vh); margin: auto; + overflow: hidden; + border: 1px solid var(--line); border-radius: var(--radius); padding: 0; + background: var(--paper); color: var(--ink); box-shadow: var(--shadow); + font-family: inherit; font-size: 14px; line-height: 1.5; +} +.llmp:popover-open { display: flex; flex-direction: column; } +.llmp::backdrop { background: rgba(0,0,0,.26); } +/* bare: dom() — a plain in-flow panel the host sizes + owns; blends with its bg */ +.llmp--bare { display: flex; flex-direction: column; width: 100%; height: 100%; max-width: none; max-height: none; margin: 0; border: none; border-radius: 0; box-shadow: none; --paper: var(--llmp-bg, transparent); } +/* inner: popup()'s content region between the header + footer */ +.llmp-inner { flex: 1; min-height: 0; display: flex; flex-direction: column; } +/* scopebody: the per-scope editor region under the scope switcher bar — grows to + fill so the status bar stays pinned at the bottom of the modal */ +.llmp-scopebody { flex: 1; min-height: 0; display: flex; flex-direction: column; } + +.llmp-statusbar { flex: none; display: flex; align-items: center; gap: 8px; padding: 6px 12px; border-top: 1px solid var(--line); background: var(--card); } +.llmp-statusbar-label { font-size: 9px; font-weight: 700; letter-spacing: .4px; text-transform: uppercase; color: var(--dim); } +.llmp-statusbar-url { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font: 10px/1.4 ui-monospace, Menlo, monospace; color: var(--ink); } +.llmp-statusbar-copy { flex: none; cursor: pointer; padding: 2px 5px; font-size: 12px; color: var(--dim); background: none; border: none; border-radius: 6px; } +.llmp-statusbar-copy:hover { color: var(--ink); background: var(--highlight); } +/* Suspense placeholder while config resolves (dom() returns synchronously). */ +.llmp-loading { flex: 1; min-height: 120px; display: flex; align-items: center; justify-content: center; } +.llmp-spinner { width: 28px; height: 28px; border: 3px solid var(--line); border-top-color: var(--accent); border-radius: 50%; animation: llmp-spin .7s linear infinite; } +@keyframes llmp-spin { to { transform: rotate(360deg); } } + +.llmp-header { + flex: none; display: flex; align-items: center; justify-content: space-between; + padding: 14px 18px; font-weight: 700; font-size: 16px; border-bottom: 1px solid var(--line); +} +.llmp-close { background: none; border: none; color: var(--dim); font-size: 22px; line-height: 1; cursor: pointer; } +.llmp-close:hover { color: var(--ink); } + +.llmp-main { flex: 1; min-height: 0; display: flex; align-items: stretch; } +.llmp-side { flex: 0 0 116px; display: flex; flex-direction: column; gap: 4px; padding: 14px 10px; border-right: 1px solid var(--line); } +.llmp-side button { text-align: left; padding: 8px 11px; font: inherit; font-weight: 600; cursor: pointer; color: var(--ink); background: transparent; border: none; border-radius: var(--radius-sm); } +.llmp-side button:hover { background: var(--accent-soft); } +.llmp-side button.active { background: var(--accent); color: var(--accent-text); } +.llmp-content { flex: 1; min-width: 0; overflow: auto; display: flex; flex-direction: column; } + +.llmp-tabs { display: flex; gap: 6px; padding: 14px 16px 0; } +.llmp-tabs button { flex: 1; padding: 7px 10px; font: inherit; font-weight: 600; cursor: pointer; color: var(--ink); background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); } +.llmp-tabs button:hover { background: var(--highlight); } +.llmp-tabs button.active { background: var(--accent); color: var(--accent-text); border-color: transparent; } + +.llmp-recent { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; padding: 12px 16px 0; } +.llmp-recent:empty { display: none; } +.llmp-recent-label { font-size: 10px; font-weight: 700; letter-spacing: .4px; text-transform: uppercase; color: var(--dim); margin-right: 2px; } +.llmp-chip { max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 4px 10px; font: inherit; font-size: 11px; font-weight: 600; cursor: pointer; color: var(--ink); background: var(--card); border: 1px solid var(--line); border-radius: 999px; box-shadow: var(--shadow-sm); } +.llmp-chip:hover { background: var(--highlight); } +.llmp-chip.active { background: var(--accent-soft); border-color: var(--accent); color: var(--accent); } + +.llmp-params-head { display: flex; justify-content: flex-end; } + +.llmp-body { padding: 14px 16px; display: flex; flex-direction: column; gap: 14px; } +.llmp-label { display: flex; flex-direction: column; gap: 6px; font-size: 12px; font-weight: 600; color: var(--dim); } +.llmp-input, .llmp-body select { width: 100%; box-sizing: border-box; padding: 8px 11px; font: inherit; color: var(--ink); background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); } +.llmp-input:focus, .llmp-body select:focus { outline: none; border-color: var(--accent); box-shadow: 0 0 0 3px var(--accent-soft); } +.llmp-textarea { min-height: 80px; resize: vertical; font-family: ui-monospace, Menlo, monospace; line-height: 1.5; } + +/* Select2-style editable combobox */ +.llmp-combo { position: relative; width: 100%; } +.llmp-row .llmp-combo { flex: 1; } +.llmp-combo-input { padding-right: 30px; } +.llmp-combo-caret { position: absolute; top: 0; right: 1px; height: 100%; width: 28px; display: flex; align-items: center; justify-content: center; color: var(--dim); font-size: 11px; cursor: pointer; user-select: none; transition: transform .12s; } +.llmp-combo-caret:hover { color: var(--ink); } +.llmp-combo-input[aria-expanded=true] + .llmp-combo-caret { transform: rotate(180deg); color: var(--accent); } +.llmp-combo-menu { position: fixed; z-index: 10; overflow-y: auto; padding: 5px; background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); box-shadow: var(--shadow); } +.llmp-combo-opt { display: flex; flex-direction: column; gap: 1px; padding: 7px 10px; border-radius: 7px; cursor: pointer; } +.llmp-combo-opt.active { background: var(--accent-soft); } +.llmp-combo-opt.current .llmp-combo-opt-label::after { content: " ✓"; color: var(--accent); } +.llmp-combo-opt-label { font-size: 13px; font-weight: 600; color: var(--ink); } +.llmp-combo-opt-sub { font-size: 11px; color: var(--dim); font-family: ui-monospace, Menlo, monospace; } +.llmp-combo-empty { padding: 9px 10px; font-size: 12px; color: var(--dim); } +.llmp-row { display: flex; gap: 8px; } +.llmp-row .llmp-input, .llmp-row select { flex: 1; } + +.llmp-note { margin: 0; font-size: 11px; line-height: 1.5; color: var(--dim); } +.llmp-explain { font-style: italic; color: #aaa3ae; } +.llmp-disabled { opacity: 0.45; } +.llmp-locked { font-style: italic; } +.llmp-label-row { display: flex; align-items: center; justify-content: space-between; gap: 8px; } +.llmp-reset-one { padding: 1px 7px; font-size: 10px; font-weight: 600; font-variant-numeric: tabular-nums; color: var(--dim); } +.llmp-reset-one:hover { color: var(--ink); } + +/* saved-prompt manager: an inline-editable list */ +.llmp-pp-list { display: flex; flex-direction: column; gap: 6px; } +.llmp-pp-list:empty { display: none; } +.llmp-pp-row { display: flex; align-items: center; gap: 6px; padding: 5px 7px; background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); } +.llmp-pp-row.active { border-color: var(--accent); background: var(--accent-soft); } +.llmp-pp-pick { flex: 0 0 auto; width: 26px; height: 26px; display: flex; align-items: center; justify-content: center; font-size: 13px; cursor: pointer; color: var(--dim); background: none; border: none; border-radius: 999px; } +.llmp-pp-pick:hover { color: var(--ink); } +.llmp-pp-row.active .llmp-pp-pick { color: var(--accent); } +.llmp-pp-name { flex: 1; min-width: 0; padding: 5px 8px; font-weight: 600; border-color: transparent; background: transparent; box-shadow: none; } +.llmp-pp-name:hover { border-color: var(--line); background: var(--paper); } +.llmp-pp-name:focus { border-color: var(--accent); background: var(--card); } +.llmp-iconbtn { flex: 0 0 auto; width: 26px; height: 26px; display: flex; align-items: center; justify-content: center; font-size: 12px; cursor: pointer; color: var(--dim); background: none; border: none; border-radius: var(--radius-sm); } +.llmp-iconbtn:hover { background: var(--highlight); color: var(--ink); } +.llmp-warn { margin: 0; font-size: 11px; font-weight: 600; line-height: 1.45; color: var(--accent); } +.llmp-warn:empty { display: none; } + +.llmp-pills { display: flex; flex-wrap: wrap; gap: 6px; } +.llmp-pills:empty { display: none; } +.llmp-pill { padding: 3px 9px; font-size: 10px; font-weight: 700; letter-spacing: .2px; border-radius: 999px; background: var(--accent-soft); color: var(--accent); } +.llmp-pill.warn { background: var(--accent); color: var(--accent-text); } +.llmp-pill.muted { background: #f1ece3; color: var(--dim); } + +.llmp-temp { display: flex; align-items: center; gap: 10px; } +.llmp-temp input[type=range] { flex: 1; accent-color: var(--accent); } +.llmp-temp b { min-width: 2.6em; text-align: right; font-weight: 700; font-variant-numeric: tabular-nums; color: var(--ink); } + +.llmp-tools { display: flex; flex-direction: column; gap: 10px; } +.llmp-builtins { display: flex; flex-direction: column; gap: 6px; } +.llmp-builtin { padding: 8px 11px; background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); } +.llmp-builtin b { display: block; font: 600 12px/1.4 ui-monospace, Menlo, monospace; color: var(--accent); margin-bottom: 2px; } +.llmp-tool-card { display: flex; flex-direction: column; gap: 8px; padding: 11px; background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); box-shadow: var(--shadow-sm); } +.llmp-handler { display: block; height: 260px; overflow: hidden; background: var(--paper); border: 1px solid var(--line); border-radius: var(--radius-sm); } +.llmp-tryout { margin: 0; max-height: 220px; overflow: auto; white-space: pre-wrap; word-break: break-word; font: 11px/1.5 ui-monospace, Menlo, monospace; color: var(--ink); background: #f7f2ea; border: 1px solid var(--line); border-radius: var(--radius-sm); padding: 9px; } +.llmp-tryout:empty { display: none; } +.llmp-avail { font-size: 12px; font-weight: 600; color: var(--dim); } + +.llmp-custom-rows { display: flex; flex-direction: column; gap: 8px; } +.llmp-custom-rows:empty { display: none; } +.llmp-custom-row { display: flex; flex-direction: column; gap: 6px; padding: 9px 11px; background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); box-shadow: var(--shadow-sm); } + +.llmp-footer { flex: none; display: flex; gap: 8px; justify-content: flex-end; padding: 12px 16px; border-top: 1px solid var(--line); } +.llmp-btn { padding: 8px 15px; font: inherit; font-weight: 600; cursor: pointer; color: var(--ink); background: var(--card); border: 1px solid var(--line); border-radius: var(--radius-sm); box-shadow: var(--shadow-sm); transition: background .12s, box-shadow .12s, transform .08s; } +.llmp-btn:hover { background: var(--highlight); box-shadow: 0 2px 7px rgba(52,49,58,.12); } +.llmp-btn:active { transform: translateY(1px); } +.llmp-btn[disabled] { opacity: .5; cursor: default; } +.llmp-btn.primary { background: var(--accent); color: var(--accent-text); border-color: transparent; } +.llmp-btn.primary:hover { background: #ff63a6; } +/* scope switcher: text-link buttons; the active one is primary with a primary + underline drawn as a real bottom border (so it sits on the bar baseline) */ +.llmp-scopebtn { padding: 4px 2px; font: inherit; font-weight: 600; cursor: pointer; color: var(--dim); background: none; border: none; border-bottom: 2px solid transparent; border-radius: 0; } +.llmp-scopebtn:hover { color: var(--accent); } +.llmp-scopebtn.active { color: var(--accent); border-bottom-color: var(--accent); } +` + +/** + * @typedef {{value: string, label?: string}} ComboOption + * @typedef {Object} ComboOpts + * @property {string} [value] + * @property {string} [placeholder] + * @property {ComboOption[]} [options] + * @property {(v: string) => void} [onChange] + * @property {(v: string) => void} [onCommit] + * + * @typedef {Object} ComboHandle + * @property {HTMLInputElement} input + * @property {HTMLElement} field + * @property {(next: ComboOption[]) => void} setOptions + * @property {(v: string|null|undefined) => void} setValue + * + * @typedef {Object} PickerSource + * @property {() => import("./config.js").LLMConfig} read + * @property {(next: import("./config.js").LLMConfig) => void} write + * @property {string} [url] + * + * @typedef {Object} PickerOpts + * @property {PickerSource} [source] + * @property {string[]} [locked] + * @property {{name?: string, text?: string}} [toolPrompt] + * @property {any[]} [tools] + * @property {Array<{name: string, description?: string}>} [toolTools] host-tool-provided built-in tools + * @property {string} [toolName] label for the host tool's built-in tools + * @property {() => void} [onRequestClose] + * @property {{toolId: string, docId?: string, toolName?: string, docName?: string}} [scope] + * @property {boolean} [scopePanel] show the Default·Tool·Doc scope switcher (default true when `scope` is given; set false to hide it and edit the default config) + * + * @typedef {{commit: () => any, revert: () => void, cancel: () => void}} Ctl the live picker controller + */ + +function injectStyles() { + // Upsert: if an older bundle already injected this stylesheet, refresh its + // contents so CSS changes (new rules, tweaks) actually take effect instead of + // being shadowed by the stale copy in . + let s = /** @type {HTMLStyleElement|null} */ (document.getElementById(STYLE_ID)) + if (!s) { + s = document.createElement("style") + s.id = STYLE_ID + document.head.appendChild(s) + } + if (s.textContent !== CSS) s.textContent = CSS +} + +/** + * @param {string} tag + * @param {Record} [attrs] + * @param {any} [children] + * @returns {any} + */ +function el(tag, attrs = {}, children = []) { + const node = document.createElement(tag) + for (const [k, v] of Object.entries(attrs)) { + if (k === "class") node.className = v + else if (k === "text") node.textContent = v + else if (k.startsWith("on") && typeof v === "function") + node.addEventListener(k.slice(2).toLowerCase(), v) + else if (v != null) node.setAttribute(k, v) + } + for (const c of /** @type {any[]} */ ([].concat(children))) { + if (c == null) continue + node.append(c.nodeType ? c : document.createTextNode(String(c))) + } + return node +} + +// An editable combobox, Select2-style: a free-text input you can type any id +// into, plus a floating dropdown of suggestions. Clicking the caret opens the +// full list (text selected, unfiltered "as if blank"); typing filters it. +// +// The menu is `position: fixed`, appended into the `.llmp` popover — so it joins +// the top layer (paints above everything) yet escapes the content area's +// `overflow: auto` clipping. `onChange(v)` fires live as you type; `onCommit(v)` +// fires when a value is committed (pick / Enter / blur). +/** + * @param {ComboOpts} opts + * @returns {ComboHandle} + */ +function combo({value, placeholder, options = [], onChange, onCommit}) { + let opts = options.slice() + let view = opts // currently shown (filtered) options + let active = -1 // highlighted index in `view` + let dirty = false // has the user typed since opening? (false = show all) + let isOpen = false + let committed = value || "" + + const input = /** @type {HTMLInputElement} */ (el("input", { + class: "llmp-input llmp-combo-input", + placeholder: placeholder || "", + value: value || "", + autocomplete: "off", + autocapitalize: "off", + spellcheck: "false", + role: "combobox", + "aria-expanded": "false", + })) + const caret = el("span", {class: "llmp-combo-caret", text: "▾"}) + const field = el("div", {class: "llmp-combo"}, [input, caret]) + const menu = el("div", {class: "llmp-combo-menu", role: "listbox"}) + + const computeView = () => { + if (!dirty) return opts + const q = input.value.trim().toLowerCase() + if (!q) return opts + return opts.filter((o) => + (o.value + " " + (o.label || "")).toLowerCase().includes(q) + ) + } + const position = () => { + const r = input.getBoundingClientRect() + menu.style.left = r.left + "px" + menu.style.top = r.bottom + 5 + "px" + menu.style.width = r.width + "px" + menu.style.maxHeight = Math.max(140, Math.min(300, window.innerHeight - r.bottom - 16)) + "px" + } + const paintActive = () => { + ;[...menu.children].forEach((c, i) => c.classList.toggle("active", i === active)) + const a = menu.children[active] + a && a.scrollIntoView({block: "nearest"}) + } + const renderMenu = () => { + view = computeView() + menu.replaceChildren() + if (!view.length) { + menu.append( + el("div", {class: "llmp-combo-empty", text: "No matches — Enter keeps what you typed"}) + ) + return + } + view.forEach((o, i) => { + const item = el("div", { + class: + "llmp-combo-opt" + + (i === active ? " active" : "") + + (o.value === committed ? " current" : ""), + role: "option", + }) + item.append(el("span", {class: "llmp-combo-opt-label", text: o.label || o.value})) + if (o.label && o.label !== o.value) + item.append(el("span", {class: "llmp-combo-opt-sub", text: o.value})) + // mousedown (not click) so it runs before the input's blur closes us. + item.addEventListener("mousedown", (/** @type {MouseEvent} */ e) => { + e.preventDefault() + choose(o.value) + }) + item.addEventListener("mousemove", () => { + if (active !== i) { + active = i + paintActive() + } + }) + menu.append(item) + }) + } + const onScroll = () => isOpen && position() + const open = (/** @type {boolean} */ selectAll) => { + if (!isOpen) { + isOpen = true + dirty = false + active = -1 + input.setAttribute("aria-expanded", "true") + ;(input.closest(".llmp") || document.body).append(menu) + position() + renderMenu() + window.addEventListener("scroll", onScroll, true) + window.addEventListener("resize", position) + } + if (selectAll) input.select() + } + const closeMenu = () => { + if (!isOpen) return + isOpen = false + input.setAttribute("aria-expanded", "false") + menu.remove() + window.removeEventListener("scroll", onScroll, true) + window.removeEventListener("resize", position) + } + /** @param {string} [v] */ + const commit = (v) => { + v = (v == null ? input.value : v).trim() + if (v === committed) return + committed = v + onCommit && onCommit(v) + } + const choose = (/** @type {string} */ v) => { + input.value = v + dirty = false + onChange && onChange(v) + commit(v) + closeMenu() + } + + input.addEventListener("focus", () => open(false)) + input.addEventListener("input", () => { + dirty = true + onChange && onChange(input.value.trim()) + active = -1 + if (!isOpen) open(false) + else { + renderMenu() + position() + } + }) + // Let an option's mousedown land before blur tears the menu down. + input.addEventListener("blur", () => setTimeout(() => { + commit() + closeMenu() + }, 0)) + input.addEventListener("keydown", (/** @type {KeyboardEvent} */ e) => { + if (e.key === "ArrowDown") { + e.preventDefault() + if (!isOpen) return open(false) + active = Math.min(view.length - 1, active + 1) + paintActive() + } else if (e.key === "ArrowUp") { + e.preventDefault() + active = Math.max(0, active - 1) + paintActive() + } else if (e.key === "Enter") { + if (isOpen && active >= 0 && view[active]) { + e.preventDefault() + choose(view[active].value) + } else { + commit() + closeMenu() + } + } else if (e.key === "Escape" && isOpen) { + e.preventDefault() + e.stopPropagation() + closeMenu() + } + }) + caret.addEventListener("mousedown", (/** @type {MouseEvent} */ e) => { + e.preventDefault() + if (isOpen) return closeMenu() + input.focus() + open(true) // full list + select the text, "as if blank" + }) + + return { + input, + field, + setOptions(/** @type {ComboOption[]} */ next) { + opts = next.slice() + if (isOpen) renderMenu() + }, + setValue(/** @type {string|null|undefined} */ v) { + input.value = v || "" + committed = v || "" + }, + } +} + +/** + * @param {string} text + * @param {string} [kind] + */ +function pill(text, kind) { + return el("span", {class: "llmp-pill" + (kind ? " " + kind : ""), text}) +} + +/** @param {number} [n] */ +function fmtParams(n) { + if (!n) return null + return n >= 1e9 + ? (n / 1e9).toFixed(n >= 1e10 ? 0 : 1) + "B params" + : Math.round(n / 1e6) + "M params" +} +/** @param {number} n */ +function fmtCtx(n) { + return n >= 1000 ? Math.round(n / 1000) + "K" : String(n) +} + +// generation_config.json parsed out of an uploaded folder, keyed by +// "local/". Session-only, like the registration itself. +/** @type {Map} */ +const localGenConfig = new Map() + +const PARAM_LABELS = { + temperature: "temp", + topP: "top-p", + topK: "top-k", + minP: "min-p", + typicalP: "typical-p", + repetitionPenalty: "rep", + noRepeatNgramSize: "no-repeat n-gram", + frequencyPenalty: "freq", + presencePenalty: "presence", + maxTokens: "max out", +} + +/** @param {{path:string, blob:Blob}[]} files */ +async function hasChatTemplate(files) { + const at = (/** @type {string} */ n) => + files.find((f) => f.path.split("/").pop() === n) + if (at("chat_template.jinja")) return true + const tc = at("tokenizer_config.json") + if (!tc) return false + try { + return !!JSON.parse(await tc.blob.text()).chat_template + } catch { + return false + } +} + +// Ask the HuggingFace API whether a model exists and ships an ONNX export — +// that's the requirement for transformers.js to run it in the browser. +/** + * @param {string} id + * @returns {Promise<{exists?: boolean, error?: boolean, hasOnnx?: boolean, canUseTool?: boolean, params?: number, gated?: boolean}>} + */ +async function fetchModelInfo(id) { + const res = await fetch("https://huggingface.co/api/models/" + id) + if (res.status === 404) return {exists: false} + if (!res.ok) return {error: true} + const data = await res.json() + const hasOnnx = (data.siblings || []).some((/** @type {any} */ s) => + /(^|\/)onnx\/.+\.onnx$|^.+\.onnx$/.test(s.rfilename || "") + ) + const chatTemplate = + data.config?.tokenizer_config?.chat_template || data.config?.chat_template_jinja + const templates = + typeof chatTemplate === "string" + ? [chatTemplate] + : chatTemplate && typeof chatTemplate === "object" + ? Object.values(chatTemplate) + : [] + let canUseTool = false + for (const template of templates) { + if ( + typeof template === "string" && + /\btools\b/.test(template) && + /tool[_ ]?calls?|tool_response|role\s*==\s*["']tool["']/.test(template) + ) { + canUseTool = true + break + } + } + return { + exists: true, + hasOnnx, + canUseTool, + params: data.safetensors?.total, + gated: !!data.gated, + } +} + +/** + * Build the bare picker UI (sidebar + content + status bar) into `host` — no + * outer frame. `dom()` hands you `host` directly; `popup()` wraps it in a + * popover frame with a header + Cancel/Done. The caller has already resolved + * config (settings doc or custom source), so reads are synchronous here. + * Returns `{commit, revert, cancel}` (the wrappers drive close/cancel). + */ +/** + * @param {HTMLElement} host + * @param {PickerOpts} opts + * @returns {Ctl} + */ +function buildPickerInto(host, opts) { + const source = opts.source || { + read: () => readConfig(), + write: (/** @type {any} */ next) => writeConfig(next), + } + + // Snapshot to revert to if the user cancels (pristine — cfg's nested objects + // are copies, and arrays are only ever replaced, never mutated in place). + const before = source.read() + /** @type {any} */ + const baseCfg = { + ...before, // all scalar params (temperature, topP, topK, penalties, seed, maxTokens, …) + local: {...before.local}, + openrouter: {...before.openrouter}, + ollama: {...before.ollama}, + webllm: {...before.webllm}, + builtin: {...before.builtin}, + // tools / prompts are folder URLs (strings|null); systemUrl/preUrl select + // which prompt docs are active. All come in via ...before. + } + + // Autosave: every change writes through to the account doc (debounced so a + // slider drag or typing coalesces into one write). `cfg` is a reactive proxy + // over `baseCfg`; mutating any field — at any depth — schedules a persist. + /** @type {any} */ + let persistTimer = null + function flushPersist() { + if (persistTimer) { + clearTimeout(persistTimer) + persistTimer = null + } + source.write(baseCfg) + } + function schedulePersist() { + clearTimeout(persistTimer) + persistTimer = setTimeout(flushPersist, 300) + } + const reactive = (/** @type {any} */ target) => + new Proxy(target, { + get(t, k, r) { + const v = Reflect.get(t, k, r) + // Wrap nested plain objects so their mutations bubble up; arrays are + // replaced wholesale (never mutated in place), so leave them raw. + return v && typeof v === "object" && !Array.isArray(v) ? reactive(v) : v + }, + set(t, k, v, r) { + const ok = Reflect.set(t, k, v, r) + schedulePersist() + return ok + }, + deleteProperty(t, k) { + const ok = Reflect.deleteProperty(t, k) + schedulePersist() + return ok + }, + }) + /** @type {any} */ + const cfg = reactive(baseCfg) + + /** @type {any[]} */ + let orModels = [] + /** @type {string[]} */ + let ollamaModels = [] + + const body = el("div", {class: "llmp-body"}) + const tabsBar = el("div", {class: "llmp-tabs"}) + const recentBar = el("div", {class: "llmp-recent"}) + + // Persist any pending change + return the live config (Done). Revert restores + // the open-time snapshot (Cancel). The wrappers (dom/popup) call these. + function commit() { + flushPersist() + return source.read() + } + function revert() { + clearTimeout(persistTimer) + source.write(before) + } + // Drop any pending write WITHOUT touching the source (neither commit nor + // revert writes). Used when the caller is about to delete this scope's config + // entirely — a flush/revert would re-create the very override we're removing. + function cancel() { + clearTimeout(persistTimer) + persistTimer = null + } + // Open a folder doc in the host app, then ask the wrapper to close the picker. + function openFolder(/** @type {string|null|undefined} */ url) { + if (!url) return + host.dispatchEvent( + new CustomEvent("patchwork:open-document", { + detail: {url}, // no toolId — a folder doc opens as a folder by default + bubbles: true, + composed: true, + }) + ) + opts.onRequestClose?.() + } + + // --- recent-models history ------------------------------------------------- + function modelForProvider(/** @type {string} */ provider) { + if (provider === "openrouter") return cfg.openrouter.model + if (provider === "ollama") return cfg.ollama.model + if (provider === "webllm") return cfg.webllm.model + if (provider === "builtin") return null // one model, no id + return cfg.local.model + } + const sameRecent = (/** @type {any} */ a, /** @type {any} */ b) => + a.provider === b.provider && (a.model || null) === (b.model || null) + function recordRecent() { + const entry = {provider: cfg.provider, model: modelForProvider(cfg.provider) || null} + if (entry.provider !== "builtin" && !entry.model) return + const prev = Array.isArray(cfg.recentModels) ? cfg.recentModels : [] + cfg.recentModels = [entry, ...prev.filter((/** @type {any} */ r) => !sameRecent(r, entry))].slice(0, 12) + } + function applyRecent(/** @type {any} */ r) { + cfg.provider = r.provider + if (r.provider === "openrouter") cfg.openrouter.model = r.model + else if (r.provider === "ollama") cfg.ollama.model = r.model + else if (r.provider === "webllm") cfg.webllm.model = r.model + else if (r.provider === "local") cfg.local.model = r.model + recordRecent() // re-selecting bumps it to the front + renderRecent() + renderTabs() + renderBody() + } + function renderRecent() { + recentBar.replaceChildren() + const recents = Array.isArray(cfg.recentModels) ? cfg.recentModels : [] + if (!recents.length) return + recentBar.append(el("span", {class: "llmp-recent-label", text: "Recent"})) + const current = { + provider: cfg.provider, + model: modelForProvider(cfg.provider) || null, + } + for (const r of recents.slice(0, 8)) { + recentBar.append( + el("button", { + class: "llmp-chip" + (sameRecent(r, current) ? " active" : ""), + title: r.provider + (r.model ? " · " + r.model : ""), + text: r.provider === "builtin" ? "Chrome built-in" : r.model || "—", + onClick: () => applyRecent(r), + }) + ) + } + } + + function resetParams() { + for (const k of PARAM_KEYS) cfg[k] = /** @type {Record} */ (DEFAULTS)[k] + renderSection() // re-render the Parameters section with the defaults + } + + /** + * The "the people who made this model suggest…" row, shared by the Browser and + * OpenRouter model pickers. `params` is already in picker-config keys; a param + * the current provider can't do is shown struck through and not applied. + * @param {Record} params + * @param {string} source where the numbers came from, for the pill + * @returns {(HTMLElement|null)[]} + */ + function suggestedRow(params, source) { + const usable = Object.fromEntries( + Object.entries(params).filter(([k]) => !capNote(cfg.provider, k)) + ) + const keys = Object.keys(params) + if (!keys.length) return [] + const label = keys + .map((k) => { + const name = /** @type {any} */ (PARAM_LABELS)[k] || k + return k in usable ? `${name} ${params[k]}` : `${name} n/a` + }) + .join(" · ") + const apply = el("button", { + class: "llmp-btn", + text: "Use these", + disabled: Object.keys(usable).length ? null : "", + onClick: () => { + // The Parameters section is a different section and rebuilds when + // navigated to, so don't renderSection() here — it would tear down + // this very row. + Object.assign(cfg, usable) + apply.textContent = "✓ applied" + apply.disabled = true + }, + }) + return [ + pill(`suggested by ${source || "the model"}: ${label}`, "muted"), + apply, + ] + } + + const isBrowser = () => + cfg.provider === "local" || + cfg.provider === "webllm" || + cfg.provider === "builtin" + function renderTabs() { + const tabs = [ + ["local", "Browser"], + ["openrouter", "OpenRouter"], + ["ollama", "Ollama"], + ] + tabsBar.replaceChildren( + ...tabs.map(([id, label]) => + el("button", { + class: (id === "local" ? isBrowser() : cfg.provider === id) ? "active" : "", + text: label, + onClick: () => { + if (id === "local") { + if (!isBrowser()) cfg.provider = "local" + } else cfg.provider = id + renderTabs() + renderBody() + }, + }) + ) + ) + } + + function renderBody() { + body.replaceChildren() + if (cfg.provider === "openrouter") renderOpenRouter() + else if (cfg.provider === "ollama") renderOllama() + else renderLocal() // local (transformers) / webllm / builtin + renderRecent() // keep the active chip in sync with the current provider + } + + // A "↺ default" affordance beside a param that's been moved off DEFAULTS — + // so applying a model's suggestions (or fiddling) is always one click from + // undone, per param, without resetting the whole section. + /** @param {string|undefined} key */ + function resetOne(key) { + if (!key) return null + const def = /** @type {Record} */ (DEFAULTS)[key] + if ((cfg[key] ?? null) === (def ?? null)) return null + return el("button", { + class: "llmp-btn llmp-reset-one", + text: `↺ ${def == null ? "auto" : def}`, + title: `Reset to the default (${def == null ? "auto" : def})`, + onClick: (/** @type {any} */ e) => { + e.preventDefault() + cfg[key] = def + renderSection() + }, + }) + } + + /** + * @param {string} label + * @param {number} value + * @param {{min?: any, max?: any, step?: any, onInput: (v: number) => void, note?: string, disabled?: string|null, key?: string}} cfg2 + */ + function slider(label, value, {min, max, step, onInput, note, disabled, key}) { + const out = el("b", {text: (+value).toFixed(2)}) + const range = el("input", { + type: "range", + min, + max, + step, + value: String(value), + onInput: (/** @type {any} */ e) => { + const v = +e.currentTarget.value + out.textContent = v.toFixed(2) + onInput(v) + }, + }) + if (disabled) range.disabled = true + const wrapper = el("label", {class: "llmp-label" + (disabled ? " llmp-disabled" : "")}, [ + el("span", {class: "llmp-label-row"}, [label, disabled ? null : resetOne(key)]), + el("div", {class: "llmp-temp"}, [range, out]), + note ? el("p", {class: "llmp-note", text: note}) : null, + ]) + if (disabled) wrapper.append(el("p", {class: "llmp-note llmp-locked", text: disabled})) + return wrapper + } + + /** + * @param {string} label + * @param {number|null} value + * @param {{min?: any, step?: any, placeholder?: string, onInput: (v: number|null) => void, note?: string, disabled?: string|null, key?: string}} [cfg2] + */ + function numberField(label, value, cfg2 = /** @type {any} */ ({})) { + const {min, step, placeholder, onInput, note, disabled, key} = cfg2 + const input = el("input", { + class: "llmp-input", + type: "number", + min, + step, + placeholder, + onInput: (/** @type {any} */ e) => { + const raw = e.currentTarget.value + onInput(raw === "" ? null : +raw) + }, + }) + if (value != null) input.value = String(value) + if (disabled) input.disabled = true + const wrapper = el("label", {class: "llmp-label" + (disabled ? " llmp-disabled" : "")}, [ + el("span", {class: "llmp-label-row"}, [label, disabled ? null : resetOne(key)]), + input, + note ? el("p", {class: "llmp-note", text: note}) : null, + ]) + if (disabled) wrapper.append(el("p", {class: "llmp-note llmp-locked", text: disabled})) + return wrapper + } + + // Dedicated "Parameters" section — every sampling/decoding knob. + function renderParamsSection() { + const wrap = el("div", {class: "llmp-body"}) + content.append(wrap) + + const locked = opts.locked || [] + // Greying a control out without saying why is a small cruelty. Two reasons + // exist: the host tool passes this per call (so your value is overwritten), + // or the provider's runtime has nowhere to put it (see CAP_NOTES). + function paramState(/** @type {string} */ key) { + if (locked.includes(key)) + return `Set by the tool you're in — it passes this with every request, so a value here wouldn't reach the model.` + return capNote(cfg.provider, key) + } + + const atDefaults = PARAM_KEYS.every((k) => (cfg[k] ?? null) === (/** @type {Record} */ (DEFAULTS)[k] ?? null)) + wrap.append( + el("div", {class: "llmp-params-head"}, [ + el("button", { + class: "llmp-btn llmp-reset", + text: "Reset to defaults", + disabled: atDefaults ? "" : null, + onClick: resetParams, + }), + ]) + ) + wrap.append( + slider("Temperature", cfg.temperature, { + min: "0", + max: "2", + step: "0.05", + onInput: (v) => (cfg.temperature = v), + note: "Randomness of each pick. 0 = always the top token (deterministic); ~0.7 balanced; past ~1.5 it tips into incoherence.", + disabled: paramState("temperature"), + key: "temperature", + }), + slider("Top-p (nucleus)", cfg.topP, { + min: "0", + max: "1", + step: "0.01", + onInput: (v) => (cfg.topP = v), + note: "Sample from the smallest set of tokens whose probabilities sum to p. 1 = off.", + disabled: paramState("topP"), + key: "topP", + }), + numberField("Top-k", cfg.topK, { + min: "0", + step: "1", + onInput: (v) => (cfg.topK = v || 0), + note: "Sample only from the k most likely tokens. 0 = off.", + disabled: paramState("topK"), + key: "topK", + }), + slider("Min-p", cfg.minP, { + min: "0", + max: "1", + step: "0.01", + onInput: (v) => (cfg.minP = v), + note: "Drop tokens below this fraction of the top token's probability. 0 = off.", + disabled: paramState("minP"), + key: "minP", + }), + slider("Typical-p", cfg.typicalP, { + min: "0", + max: "1", + step: "0.01", + onInput: (v) => (cfg.typicalP = v), + note: "Sample from tokens whose surprise is closest to the distribution's average, rather than the most likely ones. 1 = off. (Ollama)", + disabled: paramState("typicalP"), + key: "typicalP", + }), + slider("Repetition penalty", cfg.repetitionPenalty, { + min: "1", + max: "2", + step: "0.01", + onInput: (v) => (cfg.repetitionPenalty = v), + note: "Penalise tokens already used. 1 = off. (transformers · Ollama · OpenRouter)", + disabled: paramState("repetitionPenalty"), + key: "repetitionPenalty", + }), + numberField("No-repeat n-gram size", cfg.noRepeatNgramSize, { + min: "0", + step: "1", + onInput: (v) => (cfg.noRepeatNgramSize = v || 0), + note: "Hard-ban any sequence of this many tokens that has already appeared. 3 stops verbatim loops; too low mangles ordinary phrasing. 0 = off. (transformers)", + disabled: paramState("noRepeatNgramSize"), + key: "noRepeatNgramSize", + }), + slider("Frequency penalty", cfg.frequencyPenalty, { + min: "-2", + max: "2", + step: "0.05", + onInput: (v) => (cfg.frequencyPenalty = v), + note: "Penalise tokens by how often they've appeared. 0 = off. (OpenRouter · Ollama · WebLLM)", + disabled: paramState("frequencyPenalty"), + key: "frequencyPenalty", + }), + slider("Presence penalty", cfg.presencePenalty, { + min: "-2", + max: "2", + step: "0.05", + onInput: (v) => (cfg.presencePenalty = v), + note: "Penalise tokens that have appeared at all. 0 = off.", + disabled: paramState("presencePenalty"), + key: "presencePenalty", + }), + numberField("Max output tokens", cfg.maxTokens, { + min: "1", + placeholder: "model default", + onInput: (v) => (cfg.maxTokens = v), + disabled: paramState("maxTokens"), + key: "maxTokens", + }), + numberField("Seed", cfg.seed, { + min: "0", + step: "1", + placeholder: "random", + onInput: (v) => (cfg.seed = v), + note: "Fix the seed for reproducible output where supported. Blank = random.", + disabled: paramState("seed"), + key: "seed", + }) + ) + } + + function renderLocal() { + // Engine: transformers.js (ONNX) · WebLLM (MLC) · Chrome built-in. + const engines = [ + ["local", "transformers.js (ONNX)"], + ["webllm", "WebLLM (MLC · WebGPU)"], + ] + if (builtinSupported()) engines.push(["builtin", "Built-in (Chrome) ✨"]) + const engine = el("select", { + onChange: (/** @type {any} */ e) => { + cfg.provider = e.currentTarget.value + // Built-in has no model combo to commit, so the engine pick *is* the + // model selection — record it directly. + if (cfg.provider === "builtin") recordRecent() + renderBody() + }, + }) + for (const [val, label] of engines) { + const o = el("option", {value: val, text: label}) + if (val === cfg.provider) o.selected = true + engine.append(o) + } + body.append(el("label", {class: "llmp-label"}, ["Engine", engine])) + + if (cfg.provider === "webllm") { + if (!Array.isArray(cfg.webllm.custom)) cfg.webllm.custom = [] + const validCustom = () => + cfg.webllm.custom.filter((/** @type {any} */ c) => (c.model_id || "").trim()) + const webllmOptions = () => [ + ...WEBLLM_MODELS.map((m) => ({value: m.id, label: m.name})), + ...validCustom().map((/** @type {any} */ c) => ({value: c.model_id, label: "📦 " + c.model_id})), + ] + const wc = combo({ + value: cfg.webllm.model, + placeholder: "Qwen2.5-1.5B-Instruct-q4f16_1-MLC or any MLC model_id", + options: webllmOptions(), + onChange: (v) => (cfg.webllm.model = v), + onCommit: () => recordRecent(), + }) + + // Self-compiled MLC models as live-editable rows: model_id + model_lib + // (the weights URL is derived from the id). Edits write straight through + // (autosaved); rows stay editable. There's no delete button — clear a + // row's fields and leave it to drop it. + const rows = el("div", {class: "llmp-custom-rows"}) + function commitRows() { + cfg.webllm.custom = [...rows.children] + .map((/** @type {any} */ r) => ({ + model_id: r._id.value.trim(), + model_lib: r._lib.value.trim(), + })) + .filter((c) => c.model_id || c.model_lib) + wc.setOptions(webllmOptions()) + } + /** + * @param {any} model + * @param {boolean} [focus] + */ + function addRow(model, focus) { + const idIn = el("input", { + class: "llmp-input", + placeholder: "model_id — e.g. owner/Model-q4f16_1-MLC", + value: model?.model_id || "", + }) + const libIn = el("input", { + class: "llmp-input", + placeholder: "model_lib URL — compiled .wasm", + value: model?.model_lib || "", + }) + const row = el("div", {class: "llmp-custom-row"}, [idIn, libIn]) + row._id = idIn + row._lib = libIn + idIn.addEventListener("input", commitRows) + libIn.addEventListener("input", commitRows) + // Drop a fully-cleared row once you leave it. + const maybeDrop = () => { + if (!idIn.value.trim() && !libIn.value.trim()) { + row.remove() + commitRows() + } + } + idIn.addEventListener("blur", maybeDrop) + libIn.addEventListener("blur", maybeDrop) + rows.append(row) + if (focus) idIn.focus() + } + for (const c of cfg.webllm.custom) addRow(c, false) + + const addBtn = el("button", { + class: "llmp-btn", + text: "+ Add model", + onClick: () => addRow(null, true), + }) + + body.append( + el("label", {class: "llmp-label"}, ["Model", wc.field]), + el("p", { + class: "llmp-note", + text: "Runs in your browser on WebGPU via MLC WebLLM — a different, non-ONNX engine, often faster. Downloads on first use and exposes next-token probabilities, so the prediction popup works.", + }), + el("label", {class: "llmp-label"}, [ + "Custom models", + el("p", { + class: "llmp-note llmp-explain", + text: "Self-compiled MLC models. Give the model_id (its HuggingFace repo, e.g. owner/Model-q4f16_1-MLC) and the compiled wasm lib URL — the weights URL is derived from the id. Stored in your synced config. Clear a row to remove it.", + }), + rows, + el("div", {class: "llmp-row"}, [addBtn]), + ]) + ) + return + } + if (cfg.provider === "builtin") { + const avail = el("p", {class: "llmp-avail", text: "checking availability…"}) + body.append( + el("p", { + class: "llmp-note", + text: "Chrome's on-device model (Gemini Nano). Nothing to download or manage — but it exposes no next-token probabilities, so the prediction popup is off for built-in.", + }), + avail + ) + builtinAvailability().then((s) => { + avail.textContent = + s === "available" + ? "✓ Ready on this device" + : s === "downloadable" + ? "⤓ Will download on first use" + : s === "downloading" + ? "⤓ Downloading…" + : "⚠ Not available in this browser" + }) + return + } + function localOptions() { + const opts = LOCAL_MODELS.map((m) => ({ + value: m.id, + label: m.name + (m.canUseTool ? " (tool calls)" : ""), + })) + if (cfg.local.model?.startsWith("local/")) + opts.push({value: cfg.local.model, label: "📁 uploaded"}) + return opts + } + const pills = el("div", {class: "llmp-pills"}) + const warn = el("p", {class: "llmp-warn"}) + const suggested = el("div", {class: "llmp-pills"}) + const c = combo({ + value: cfg.local.model, + placeholder: "onnx-community/… or any ONNX HuggingFace id", + options: localOptions(), + onChange: (v) => { + cfg.local.model = v + refreshPills() // light: catalogue pills only, no network + refreshSuggested() + }, + // On a *committed* id (pick / Enter / blur): record it and validate it + // against the HF API — never per-keystroke (that spammed HF with partial + // ids → 401/503/CORS). + onCommit: () => { + recordRecent() + scheduleValidate() + }, + }) + + /** @type {any} */ + let validateTimer = null + // Catalogue/uploaded pills only — no network. + function refreshPills() { + clearTimeout(validateTimer) + pills.replaceChildren() + warn.textContent = "" + const id = cfg.local.model + if (!id) return + if (id.startsWith("local/")) return void pills.append(pill("📁 uploaded", "muted")) + const cat = LOCAL_MODELS.find((m) => m.id === id) + if (cat?.canUseTool) pills.append(pill("supports tool calling")) + } + // The model authors' own sampling settings, from the repo's + // `generation_config.json` (or the uploaded folder's copy). Prefills the + // Parameters section instead of leaving everyone on our generic 0.7/0.9, + // which is far too hot for most small local models. + /** @type {any} */ + let suggestTimer = null + function refreshSuggested() { + clearTimeout(suggestTimer) + suggested.replaceChildren() + const id = cfg.local.model + if (!id || cfg.provider !== "local") return + if (id.startsWith("local/")) return showSuggested(localGenConfig.get(id)) + if (!/^[^/\s]+\/[^/\s]{2,}$/.test(id)) return // must look like org/repo + suggestTimer = setTimeout(() => { + suggested.replaceChildren(pill("⟳ reading generation_config.json…", "muted")) + fetchGenerationConfig(id).then((gc) => { + if (id === cfg.local.model) showSuggested(gc) + }) + }, 500) + } + /** @param {any} gc parsed generation_config.json, or null/undefined */ + function showSuggested(gc) { + const params = suggestedParams(gc) + const {hasEos, greedy} = generationStops(gc) + suggested.replaceChildren(...suggestedRow(params, "generation_config.json")) + if (gc && !hasEos) + suggested.append( + pill("⚠ no eos_token_id — generation only stops at the token cap", "warn") + ) + if (greedy) suggested.append(pill("authors want greedy decoding", "muted")) + } + + function scheduleValidate() { + clearTimeout(validateTimer) + const id = cfg.local.model + if (!id || id.startsWith("local/")) return + if (LOCAL_MODELS.some((m) => m.id === id)) return // catalogue = known good + if (!/^[^/\s]+\/[^/\s]{2,}$/.test(id)) return // must look like org/repo + validateTimer = setTimeout(() => validate(id), 500) + } + async function validate(/** @type {string} */ id) { + refreshPills() + pills.append(pill("checking HuggingFace…", "muted")) + let info + try { + info = await fetchModelInfo(id) + } catch { + refreshPills() // CORS / rate-limited / offline — fail quietly + return + } + if (id !== cfg.local.model) return // changed while we waited + refreshPills() + if (info.error) return + if (!info.exists) { + warn.textContent = "⚠ Not found on HuggingFace — check the id." + return + } + if (info.params) pills.append(pill(/** @type {string} */ (fmtParams(info.params)), "muted")) + if (info.canUseTool) pills.append(pill("supports tool calling")) + if (info.hasOnnx) pills.append(pill("✓ ONNX")) + else { + pills.append(pill("no ONNX", "warn")) + warn.textContent = + "⚠ This repo has no ONNX export, so it can't run in the browser. Use an onnx-community/… model, or load an ONNX folder from disk below." + } + if (info.gated) + warn.textContent = + "⚠ This model is gated on HuggingFace — it likely won't download in the browser." + } + + // ---- load a local ONNX model from disk ---- + const dtypeSel = el("select", {style: "flex:0 0 auto;width:auto"}) + for (const d of DTYPES) { + dtypeSel.append(el("option", {value: d, text: d})) + } + const fileInput = el("input", { + type: "file", + webkitdirectory: "", + directory: "", + multiple: "", + style: "display:none", + }) + const note = el("p", { + class: "llmp-note", + text: "Runs via WebGPU in your browser. Or load your own ONNX model folder (transformers.js layout: config.json, tokenizer.json, onnx/model_.onnx).", + }) + fileInput.addEventListener("change", () => { + const picked = [...(fileInput.files || [])] + if (!picked.length) return + const folder = (picked[0].webkitRelativePath || picked[0].name).split("/")[0] + const files = picked.map((f) => ({ + path: (f.webkitRelativePath || f.name).split("/").slice(1).join("/") || f.name, + blob: f, + })) + const id = "local/" + folder + registerLocalModel(id, files, dtypeSel.value) + cfg.provider = "local" + cfg.local.model = id + c.setValue(id) + c.setOptions(localOptions()) + refreshPills() + note.textContent = `Loaded ${files.length} files as ${id} (dtype ${dtypeSel.value}). It will load on first use.` + generationConfigFromFiles(files).then((gc) => { + localGenConfig.set(id, gc) + if (cfg.local.model === id) showSuggested(gc) + }) + // A model with no chat template can only be prompted as ChatML, which is + // wrong for anything that isn't Qwen-family — say so at upload time + // rather than letting it surface as garbage output. + hasChatTemplate(files).then((ok) => { + if (!ok && cfg.local.model === id) + warn.textContent = + "⚠ No chat template in this folder (tokenizer_config.json / chat_template.jinja) — chat prompts fall back to ChatML, which produces garbage on non-Qwen models." + }) + }) + const loadBtn = el("button", { + class: "llmp-btn", + text: "Load ONNX folder…", + onClick: () => fileInput.click(), + }) + + // Quantization picker for the chosen model id. transformers.js loads + // onnx/model_.onnx, so a repo that only ships, say, q4 needs this + // set explicitly — otherwise the default q4f16 file 404s. "auto" = let the + // worker fall back to the catalogue default / q4f16. + const modelDtypeSel = el("select", {style: "width:auto"}) + for (const d of ["auto", ...DTYPES]) modelDtypeSel.append(el("option", {value: d, text: d})) + modelDtypeSel.value = cfg.local.dtype || "auto" + modelDtypeSel.addEventListener("change", () => { + cfg.local.dtype = modelDtypeSel.value === "auto" ? null : modelDtypeSel.value + }) + + refreshPills() + scheduleValidate() // validate a pre-set custom id once on open + refreshSuggested() + body.append( + el("label", {class: "llmp-label"}, ["Model", c.field]), + pills, + warn, + suggested, + el("label", {class: "llmp-label"}, [ + "Quantization", + el("div", {class: "llmp-row"}, [modelDtypeSel]), + ]), + el("p", {class: "llmp-note", text: "Which ONNX variant to load (onnx/model_.onnx). Auto = the model's default. Set this if a repo only ships a specific one (e.g. q4)."}), + el("label", {class: "llmp-label"}, [ + "Load local ONNX from disk", + el("div", {class: "llmp-row"}, [loadBtn, dtypeSel]), + ]), + fileInput, + note + ) + } + + function renderOpenRouter() { + const keyInput = el("input", { + type: "password", + class: "llmp-input", + placeholder: "sk-or-...", + value: cfg.openrouter.apiKey || "", + onInput: (/** @type {any} */ e) => (cfg.openrouter.apiKey = e.currentTarget.value), + }) + const orOptions = () => orModels.map((m) => ({value: m.id, label: m.name})) + const pills = el("div", {class: "llmp-pills"}) + // OpenRouter's answer to generation_config.json: `default_parameters` on + // each catalogue entry, already in hand from the models fetch. + const suggested = el("div", {class: "llmp-pills"}) + function refreshSuggested() { + const m = orModels.find((x) => x.id === cfg.openrouter.model) + suggested.replaceChildren( + ...suggestedRow(suggestedParamsFromOpenRouter(m?.default_parameters), "OpenRouter") + ) + } + function refreshPills() { + pills.replaceChildren() + refreshSuggested() + const m = orModels.find((x) => x.id === cfg.openrouter.model) + if (!m) return + const sp = m.supported_parameters || [] + const mods = m.input_modalities || [] + if (m.context_length) pills.append(pill(fmtCtx(m.context_length) + " context", "muted")) + if (mods.includes("image")) pills.append(pill("👁 vision")) + if (mods.includes("audio")) pills.append(pill("🔊 audio")) + if (sp.includes("tools")) pills.append(pill("🔧 tool calling")) + if (sp.includes("reasoning") || sp.includes("include_reasoning")) + pills.append(pill("🧠 reasoning")) + if (sp.includes("logprobs")) pills.append(pill("logprobs")) + if (m.max_completion_tokens) + pills.append(pill(fmtCtx(m.max_completion_tokens) + " max out", "muted")) + } + const c = combo({ + value: cfg.openrouter.model, + placeholder: "anthropic/claude-sonnet-4 or any OpenRouter id", + options: orOptions(), + onChange: (v) => { + cfg.openrouter.model = v + const f = orModels.find((m) => m.id === v) + cfg.openrouter.contextLength = f?.context_length ?? null + cfg.openrouter.maxCompletionTokens = f?.max_completion_tokens ?? null + refreshPills() + }, + onCommit: () => recordRecent(), + }) + const refresh = el("button", {class: "llmp-btn", text: "Refresh", onClick: load}) + async function load() { + refresh.textContent = "..." + refresh.disabled = true + try { + orModels = await fetchOpenRouterModels() + } catch (e) { + console.warn("[@patchwork/llm] fetch OpenRouter models:", e) + } + refresh.textContent = "Refresh" + refresh.disabled = false + c.setOptions(orOptions()) + refreshPills() + } + if (!orModels.length) load() + else refreshPills() + body.append( + el("label", {class: "llmp-label"}, [ + "API Key (stored on your account)", + keyInput, + ]), + el("label", {class: "llmp-label"}, [ + "Model", + el("div", {class: "llmp-row"}, [c.field, refresh]), + ]), + pills, + suggested + ) + } + + function renderOllama() { + const urlInput = el("input", { + class: "llmp-input", + placeholder: DEFAULTS.ollama.url, + value: cfg.ollama.url || "", + onInput: (/** @type {any} */ e) => (cfg.ollama.url = e.currentTarget.value), + }) + const select = el("select", { + onChange: (/** @type {any} */ e) => { + cfg.ollama.model = e.currentTarget.value + recordRecent() + }, + }) + const refresh = el("button", {class: "llmp-btn", text: "Refresh", onClick: load}) + function paint() { + select.replaceChildren() + if (!ollamaModels.length) + select.append(el("option", {value: cfg.ollama.model, text: cfg.ollama.model})) + for (const m of ollamaModels) { + const o = el("option", {value: m, text: m}) + if (m === cfg.ollama.model) o.selected = true + select.append(o) + } + select.value = cfg.ollama.model + } + async function load() { + refresh.textContent = "..." + refresh.disabled = true + try { + ollamaModels = await fetchOllamaModels(cfg.ollama.url) + } catch (e) { + console.warn("[@patchwork/llm] probe Ollama:", e) + ollamaModels = [] + } + refresh.textContent = "Refresh" + refresh.disabled = false + paint() + } + paint() + if (!ollamaModels.length) load() + body.append( + el("label", {class: "llmp-label"}, [ + "Ollama URL", + el("div", {class: "llmp-row"}, [urlInput, refresh]), + ]), + el("label", {class: "llmp-label"}, ["Model", select]) + ) + } + + // ---- Prompts section ---- + function renderPromptsSection() { + const wrap = el("div", {class: "llmp-body"}) + content.append(wrap) + renderPromptPicker(wrap, "system") + renderPromptPicker(wrap, "pre") + } + + // A saved-prompt manager (used for both system + pre): a list of named prompts + // you can rename in place, copy, remove, or make active (●); `+ New` creates + // one, and you can import by URL. The active prompt's text editor shows below. + /** + * @param {any} wrap + * @param {"system"|"pre"} kind + */ + function renderPromptPicker(wrap, kind) { + const repo = typeof window !== "undefined" ? window.repo : null + const isPre = kind === "pre" + const promptType = isPre ? "llm:pre-prompt" : "llm:system-prompt" + const urlKey = isPre ? "preUrl" : "systemUrl" + const header = isPre ? "Pre-prompt" : "System prompt" + const note = isPre + ? "“Start with this text.” Literal text glued to the front of your input on every call — used everywhere, including completions and predictions." + : "“Be like this.” Standing instructions sent as a chat system message. ⚠ Not used during raw completions (continuation / keystroke predictions): a completion has no system role." + + const sel = () => cfg[urlKey] + const setSel = (/** @type {string|null|undefined} */ u) => (cfg[urlKey] = u || null) + const newName = isPre ? "Pre-prompt" : "System prompt" + /** @type {any[]} */ + let resolved = [] + /** @type {(() => void)|null} */ + let paintToolCard = null // refreshes the tool-default card's active state + + // Add a prompt doc URL to the prompts folder (creating the folder if needed). + /** + * @param {string} url + * @param {string} [name] + */ + async function addLink(url, name) { + if (!repo || !url) return + cfg.prompts = await ensureFolderUrl(repo, cfg.prompts, "LLM Prompts") + await addToFolder(repo, cfg.prompts, {name: name || "Prompt", type: promptType, url}) + } + // Rename: the name lives on the wrapper doc. (Folder DocLinks resolve their + // name from the doc, so renaming the doc is enough.) + /** + * @param {string} url + * @param {string} [name] + */ + async function rename(url, name) { + if (!repo || !url) return + const h = await repo.find(/** @type {any} */ (url)) + h.change((/** @type {any} */ d) => (d.name = name || "Prompt")) + } + + const list = el("div", {class: "llmp-pp-list"}) + const empty = el("p", {class: "llmp-note llmp-explain", text: "No saved prompts yet — “+ New” to make one."}) + const editorBox = el("div", {class: "llmp-tools"}) + + function paint() { + list.replaceChildren() + for (const p of resolved) { + const active = p.url === sel() + const nameIn = el("input", { + class: "llmp-input llmp-pp-name", + value: p.name || "", + spellcheck: "false", + }) + /** @type {any} */ + let renameTimer = null + nameIn.addEventListener("input", () => { + clearTimeout(renameTimer) + const v = nameIn.value + renameTimer = setTimeout(() => rename(p.url, v.trim()), 350) + }) + const pick = el("button", { + class: "llmp-pp-pick", + title: active ? "Active prompt" : "Make active", + text: active ? "●" : "○", + onClick: () => { + setSel(active ? null : p.url) // click the active one to deactivate + paint() + }, + }) + const copyBtn = el("button", { + class: "llmp-iconbtn", + title: "Copy URL", + text: "⧉", + onClick: () => navigator.clipboard?.writeText(p.url), + }) + const rm = el("button", { + class: "llmp-iconbtn", + title: "Remove", + text: "✕", + onClick: async () => { + if (repo && cfg.prompts) await removeFromFolder(repo, cfg.prompts, p.url) + if (active) setSel(null) + reload() + }, + }) + list.append( + el("div", {class: "llmp-pp-row" + (active ? " active" : "")}, [ + pick, + nameIn, + copyBtn, + rm, + ]) + ) + } + empty.style.display = resolved.length ? "none" : "" + // Editor for the active prompt. + editorBox.replaceChildren() + const cur = resolved.find((p) => p.url === sel()) + if (cur?.promptUrl) { + const view = document.createElement("patchwork-view") + view.setAttribute("doc-url", cur.promptUrl) // the .txt file doc + view.setAttribute("tool-id", "file") + view.className = "llmp-handler" + editorBox.append(view) + } + if (paintToolCard) paintToolCard() + } + async function reload() { + resolved = repo ? await resolvePromptDocs(cfg, kind, repo) : [] + paint() + } + + const newBtn = el("button", { + class: "llmp-btn primary", + text: "+ New", + onClick: async () => { + if (!repo) return + const w = await createPromptDoc(repo, kind, {name: newName}) + await addLink(w.url, newName) + setSel(w.url) + reload() + }, + }) + const templates = PROMPT_TEMPLATES.filter((t) => t.kind === kind) + const templateBtns = templates.map((t) => + el("button", { + class: "llmp-btn", + text: "+ " + t.name, + onClick: async () => { + if (!repo) return + const w = await createPromptDoc(repo, kind, {name: t.name, text: t.text}) + await addLink(w.url, t.name) + setSel(w.url) + reload() + }, + }) + ) + const importIn = el("input", { + class: "llmp-input", + placeholder: "paste a URL to import a shared prompt…", + }) + importIn.addEventListener("change", async () => { + const v = importIn.value.trim() + if (!/^automerge:/i.test(v)) return + importIn.value = "" + await addLink(v) + setSel(v) + reload() + }) + const openBtn = el("button", { + class: "llmp-btn", + text: "Open folder ↗", + onClick: () => openFolder(cfg.prompts), + }) + + // A tool can pass its built-in system prompt via `opts.toolPrompt` + // ({name, text}). Shown read-only with an active dot (active = the tool + // default, i.e. NO saved prompt selected) and a "Fork & edit" button that + // creates an editable copy and selects it — overriding the default. The dot + // refreshes via paintToolCard() whenever the selection changes. System only. + /** @type {any} */ + let toolCard = null + const toolPrompt = !isPre && opts && opts.toolPrompt + if (toolPrompt && toolPrompt.text) { + const preview = el("pre", {class: "llmp-note"}) + preview.textContent = toolPrompt.text + preview.style.cssText = + "max-height:120px;overflow:auto;white-space:pre-wrap;font:11px/1.45 ui-monospace,Menlo,monospace;opacity:0.8;margin:6px 0 0;" + const pick = el("button", { + class: "llmp-pp-pick", + onClick: () => { + setSel(null) // clicking returns to the tool default + paint() + }, + }) + const status = el("span", {class: "llmp-note", style: "margin-left:auto;font-style:italic;white-space:nowrap"}) + const forkBtn = el("button", { + class: "llmp-btn primary", + text: "Fork & edit", + onClick: async () => { + if (!repo) return + const name = (toolPrompt.name || "Tool prompt") + " (fork)" + const w = await createPromptDoc(repo, "system", {name, text: toolPrompt.text}) + await addLink(w.url, name) + setSel(w.url) // the fork becomes active → overrides the tool default + reload() + }, + }) + toolCard = el("div", {class: "llmp-builtin"}, [ + el("div", {class: "llmp-row", style: "align-items:center;gap:8px"}, [ + pick, + el("b", {text: toolPrompt.name || "Provided by the tool"}), + status, + forkBtn, + ]), + el("p", {class: "llmp-note llmp-explain", text: "The tool's built-in default — active when no saved prompt below is selected. Fork it to make an editable copy that overrides it."}), + preview, + ]) + paintToolCard = () => { + const active = !sel() + pick.textContent = active ? "●" : "○" + pick.title = active ? "Active (tool default)" : "Click to use the tool default" + status.textContent = active ? "active — default" : "overridden ↓" + toolCard.classList.toggle("active", active) + } + } + + wrap.append( + el("label", {class: "llmp-label"}, [ + header, + el("p", {class: "llmp-note llmp-explain", text: note}), + ...(toolCard ? [toolCard] : []), + list, + empty, + el("div", {class: "llmp-row"}, [newBtn, ...templateBtns, openBtn]), + importIn, + ]), + editorBox + ) + reload() + } + + // ---- Tools section ---- + function renderToolsSection() { + const repo = typeof window !== "undefined" ? window.repo : null + const wrap = el("div", {class: "llmp-body"}) + content.append(wrap) + + const urlInput = el("input", { + class: "llmp-input", + placeholder: "automerge:… (paste an llm:tool URL)", + }) + const list = el("div", {class: "llmp-tools"}) + const tryBox = el("div") + + /** + * @param {string} url + * @param {string} [name] + */ + async function addLink(url, name) { + if (!repo || !url) return + cfg.tools = await ensureFolderUrl(repo, cfg.tools, "Tool Calls") + await addToFolder(repo, cfg.tools, {name: name || "Tool Call", type: "llm:tool", url}) + reload() + } + const addBtn = el("button", { + class: "llmp-btn", + text: "Add", + onClick: async () => { + const url = urlInput.value.trim() + urlInput.value = "" + await addLink(url) + }, + }) + const newBtn = el("button", { + class: "llmp-btn primary", + text: "New tool call", + onClick: async () => { + if (!repo) return + const h = await createLLMTool(repo) + await addLink(h.url, h.doc()?.name || "New tool call") + }, + }) + const openBtn = el("button", { + class: "llmp-btn", + text: "Open folder ↗", + onClick: () => openFolder(cfg.tools), + }) + + let tools = [] + async function reload() { + tools = repo ? await resolveTools(cfg, repo) : [] + openBtn.disabled = !cfg.tools + list.replaceChildren() + for (const t of tools) { + const card = el("div", {class: "llmp-tool-card"}) + list.append(card) + renderToolCard(card, t.url, repo, reload) + } + renderTry() + } + function renderTry() { + tryBox.replaceChildren() + if (!tools.length) return + const tryInput = el("input", {class: "llmp-input", placeholder: "Ask something that needs a tool call…"}) + const tryOut = el("pre", {class: "llmp-tryout"}) + const runBtn = el("button", { + class: "llmp-btn primary", + text: "Run", + onClick: async () => { + if (!tryInput.value.trim()) return + runBtn.disabled = true + tryOut.textContent = "…" + try { + await generateWithTools(tryInput.value, /** @type {any} */ ({ + config: baseCfg, // raw target — a Proxy can't be postMessage'd to the worker + onToken: (/** @type {any} */ _d, /** @type {any} */ full) => (tryOut.textContent = full), + onToolCall: (/** @type {any} */ c) => { + tryOut.textContent += + `\n\n▶ ${c.name}(${JSON.stringify(c.args)}) → ` + + (c.error ? "⚠ " + c.error : JSON.stringify(c.result)) + + "\n" + }, + })) + } catch (/** @type {any} */ e) { + tryOut.textContent = "Error: " + (e?.message || e) + } + runBtn.disabled = false + }, + }) + tryBox.append( + el("label", {class: "llmp-label"}, [ + "Try it", + el("div", {class: "llmp-row"}, [tryInput, runBtn]), + ]), + tryOut + ) + } + + // Built-in tools the host has already wired in (e.g. duet's fetch / ask_*). + // Read-only — shown so you can see what this model already has. + const builtin = Array.isArray(opts.tools) ? opts.tools : [] + const builtinGroup = builtin.length + ? el("label", {class: "llmp-label"}, [ + "Built-in tool calls", + el("p", { + class: "llmp-note llmp-explain", + text: "Provided by this tool — always available to the model, not editable here.", + }), + el( + "div", + {class: "llmp-builtins"}, + builtin.map((t) => + el("div", {class: "llmp-builtin"}, [ + el("b", {text: t.name + (t.args ? "(" + t.args + ")" : "")}), + el("span", {class: "llmp-note", text: t.description || t.desc || ""}), + ]) + ) + ), + ]) + : null + + // Sandbox toggle: run folder-tool handlers in an isolated Worker (no page + // access). Autosaved to the config (cfg.toolSandbox), so the "Try it" runner + // and every generateWithTools call honour it. + const sandboxCb = el("input", {type: "checkbox"}) + sandboxCb.checked = !!cfg.toolSandbox + sandboxCb.addEventListener("change", () => (cfg.toolSandbox = sandboxCb.checked)) + const sandboxToggle = el( + "label", + { + class: "llmp-label", + style: "flex-direction: row; align-items: center; gap: 8px; color: var(--ink);", + }, + [sandboxCb, el("span", {text: "Run handlers in a sandbox (no page access)"})] + ) + const sandboxNote = el("p", { + class: "llmp-note llmp-explain", + text: "Runs each tool call's JS in an isolated Worker with no access to this page, your repo, or your account — safer for tool calls added by URL. Off = handlers run on the page with full access (needed for tool calls that read/write documents or the DOM).", + }) + + // Tools the host tool itself provides (opts.toolTools: [{name, description}]). + // Read-only — their handlers live in the tool — but each can be toggled off; + // the tool reads cfg.toolToggles[name] to decide what to offer the model. + let providedCard = null + const provided = (opts && opts.toolTools) || [] + if (provided.length) { + if (!cfg.toolToggles) cfg.toolToggles = {} + const rows = provided.map((/** @type {{name: string, description?: string}} */ t) => { + const cb = el("input", {type: "checkbox"}) + cb.checked = cfg.toolToggles[t.name] !== false + cb.addEventListener("change", () => { + cfg.toolToggles = {...(cfg.toolToggles || {}), [t.name]: cb.checked} + }) + return el("label", {class: "llmp-pp-row", style: "display:flex;align-items:flex-start;gap:8px;cursor:pointer"}, [ + cb, + el("div", {}, [ + el("b", {text: t.name}), + t.description + ? el("p", {class: "llmp-note", style: "margin:2px 0 0"}, [t.description]) + : null, + ]), + ]) + }) + providedCard = el("div", {class: "llmp-builtin"}, [ + el("b", {text: (opts.toolName ? opts.toolName + " — " : "") + "Built-in tool calls"}), + el("p", {class: "llmp-note llmp-explain", text: "Tool calls this tool gives the model. Toggle one off to stop offering it."}), + ...rows, + ]) + } + + wrap.append( + ...[ + providedCard, + builtinGroup, + el("p", { + class: "llmp-note llmp-explain", + text: "Tool calls you give the model: a name, a description of how/when to use it + its parameters, and a JS handler (edited with the file tool). They live in a folder you can open and manage.", + }), + sandboxToggle, + sandboxNote, + el("div", {class: "llmp-row"}, [urlInput, addBtn, newBtn, openBtn]), + list, + tryBox, + ].filter(Boolean) + ) + reload() + } + + /** + * @param {any} card + * @param {string} url + * @param {any} repo + * @param {() => void} [reload] + */ + async function renderToolCard(card, url, repo, reload) { + if (!repo) return + let handle + try { + handle = await repo.find(url) + } catch { + card.append(el("p", {class: "llmp-warn", text: "⚠ Couldn't load " + url})) + return + } + const doc = handle.doc() || {} + const nameInput = el("input", { + class: "llmp-input", + placeholder: "tool call name", + onInput: (/** @type {any} */ e) => handle.change((/** @type {any} */ d) => (d.name = e.currentTarget.value)), + }) + nameInput.value = doc.name || "" + const desc = el("textarea", { + class: "llmp-input llmp-textarea", + placeholder: "How/when to use it + its parameters…", + onInput: (/** @type {any} */ e) => handle.change((/** @type {any} */ d) => (d.description = e.currentTarget.value)), + }) + desc.value = doc.description || "" + const copyBtn = el("button", { + class: "llmp-btn", + text: "Copy URL", + onClick: () => { + navigator.clipboard?.writeText(url) + copyBtn.textContent = "Copied!" + setTimeout(() => (copyBtn.textContent = "Copy URL"), 1200) + }, + }) + const removeBtn = el("button", { + class: "llmp-btn", + text: "Remove", + onClick: async () => { + if (cfg.tools) await removeFromFolder(repo, cfg.tools, url) + reload?.() + }, + }) + /** @type {any} */ + let view = null + const editBtn = el("button", { + class: "llmp-btn", + text: "Edit handler", + onClick: () => { + if (view) { + view.remove() + view = null + editBtn.textContent = "Edit handler" + return + } + const d = handle.doc() + const handlerUrl = d?.tool ?? d?.handlerUrl // `tool`, or legacy `handlerUrl` + if (!handlerUrl) return + view = document.createElement("patchwork-view") + view.setAttribute("doc-url", handlerUrl) // the JS handler file doc + view.setAttribute("tool-id", "file") + view.className = "llmp-handler" + card.append(view) + editBtn.textContent = "Hide handler" + }, + }) + card.append( + el("div", {class: "llmp-row"}, [nameInput, copyBtn]), + desc, + el("div", {class: "llmp-row"}, [editBtn, removeBtn]) + ) + } + + // ---- assemble: header + (sidebar | content) + footer ---- + const sideNav = el("div", {class: "llmp-side"}) + const content = el("div", {class: "llmp-content"}) + let section = "model" + function renderNav() { + sideNav.replaceChildren( + ...SECTIONS.map((s) => + el("button", { + class: section === s.id ? "active" : "", + text: s.label, + onClick: () => { + section = s.id + renderNav() + renderSection() + }, + }) + ) + ) + } + function renderSection() { + content.replaceChildren() + if (section === "params") renderParamsSection() + else if (section === "prompts") renderPromptsSection() + else if (section === "tools") renderToolsSection() + else { + content.append(recentBar, tabsBar, body) + renderRecent() + renderTabs() + renderBody() + } + } + + // The status bar (config-doc URL) lives at the very bottom of the modal now, + // built once by buildScopedPicker — not per scope editor. See statusBar(). + host.append(el("div", {class: "llmp-main"}, [sideNav, content])) + + renderNav() + renderSection() + return {commit, revert, cancel} +} + +// The bottom status bar: the URL of the config doc being edited (the account +// settings doc by default, or `source.url` when a host scopes it). Rendered once +// at the bottom of the modal, below every scope editor. +/** @param {PickerOpts} opts */ +function statusBar(opts) { + const configUrl = opts.source?.url || settingsDocHandle()?.url || null + return el("div", {class: "llmp-statusbar"}, [ + el("span", {class: "llmp-statusbar-label", text: "config"}), + el("code", { + class: "llmp-statusbar-url", + text: configUrl || "(unsaved)", + title: configUrl || "", + }), + configUrl + ? el("button", { + class: "llmp-statusbar-copy", + title: "Copy config URL", + text: "⧉", + onClick: () => navigator.clipboard?.writeText(configUrl), + }) + : null, + ]) +} + +const spinner = () => el("div", {class: "llmp-loading"}, [el("div", {class: "llmp-spinner"})]) + +// Build the picker into `mount`, optionally with a whole-scope switcher. When +// `opts.scope = {toolId, docId?, toolName?, docName?}` is given, a Default · This +// tool · This doc bar swaps which config the editor edits: a scope either has its +// own complete override or inherits the default. Returns a getter for the live +// controller (commit/revert target the active scope's editor). No scope → plain. +/** + * @param {HTMLElement} mount + * @param {PickerOpts} opts + * @param {() => void} onRequestClose + * @returns {() => Ctl|null} + */ +function buildScopedPicker(mount, opts, onRequestClose) { + /** @type {Ctl|null} */ + let ctl = null + const getCtl = () => ctl + // The scope switcher is shown only when a tool opts in: it must pass a `scope` + // AND not disable the panel (`scopePanel: false`). Without it we render a plain + // picker that edits the default config (or the caller's `source`). + const showPanel = !!(opts.scope && opts.scope.toolId && opts.scopePanel !== false) + if (!showPanel) { + ctl = buildPickerInto(mount, {...opts, onRequestClose}) + mount.append(statusBar(opts)) + return getCtl + } + const scope = /** @type {NonNullable} */ (opts.scope) + + const scopeFor = (/** @type {string} */ which) => + which === "doc" + ? {toolId: scope.toolId, docId: scope.docId} + : {toolId: scope.toolId} + const sourceFor = (/** @type {string} */ which) => { + if (which === "default") + return opts.source || {read: () => readConfig(), write: (/** @type {any} */ n) => writeConfig(n)} + const sc = scopeFor(which) + return {read: () => readScopedConfig(sc), write: (/** @type {any} */ n) => writeScopeOverride(sc, n)} + } + + // Open on the most-specific scope that already has its own settings — so the + // modal shows what's actually in effect for this tool/doc, not always Default. + const initialActive = () => { + const raw = readConfig() + if (scope.docId && hasScopeOverride(raw, scopeFor("doc"))) return "doc" + if (hasScopeOverride(raw, scopeFor("tool"))) return "tool" + return "default" + } + let active = initialActive() // "default" | "tool" | "doc" + + const bar = el("div", { + class: "llmp-row", + style: "padding:8px 12px;gap:6px;align-items:center;flex-wrap:wrap;border-bottom:1px solid var(--line,rgba(128,128,128,0.25))", + }) + const bodyHost = el("div", {class: "llmp-scopebody"}) + mount.replaceChildren(bar, bodyHost, statusBar(opts)) + + function rebuild() { + // Flush any pending edit of the scope we're leaving before swapping. + try { ctl && ctl.commit && ctl.commit() } catch {} + const mkBtn = (/** @type {string} */ id, /** @type {string} */ label) => + el("button", { + class: "llmp-scopebtn" + (active === id ? " active" : ""), + text: label, + onClick: () => { active = id; rebuild() }, + }) + bar.replaceChildren( + ...[ + el("span", {class: "llmp-note", text: "Settings for"}), + mkBtn("default", "Default"), + mkBtn("tool", scope.toolName || "This tool"), + scope.docId ? mkBtn("doc", scope.docName || "This doc") : null, + ].filter(Boolean) + ) + + bodyHost.replaceChildren() + if (active === "default") { + ctl = buildPickerInto(bodyHost, {...opts, source: sourceFor("default"), onRequestClose}) + return + } + const sc = scopeFor(active) + if (!hasScopeOverride(readConfig(), sc)) { + ctl = null + const what = active === "doc" ? "This document" : "“" + (scope.toolName || "This tool") + "”" + bodyHost.append( + el("div", {class: "llmp-body"}, [ + el("p", {class: "llmp-note llmp-explain", text: what + " inherits the default settings."}), + el("button", { + class: "llmp-btn primary", + text: "Create separate settings", + onClick: () => { writeScopeOverride(sc); rebuild() }, + }), + ]) + ) + return + } + const editorHost = el("div", {class: "llmp-scopebody"}) + const resetRow = el("div", {class: "llmp-row", style: "padding:8px 12px"}, [ + el("button", { + class: "llmp-btn", + text: "Reset to default (remove these settings)", + // Drop the editor's pending write FIRST (cancel, not commit/revert — + // either would re-write the override we're about to delete), then null + // `ctl` so rebuild()'s leading commit can't resurrect it either. + onClick: () => { + try { ctl && ctl.cancel && ctl.cancel() } catch {} + ctl = null + clearScopeOverride(sc) + active = "default" + rebuild() + }, + }), + ]) + bodyHost.append(resetRow, editorHost) + ctl = buildPickerInto(editorHost, {...opts, source: sourceFor(active), onRequestClose}) + } + rebuild() + return getCtl +} + +/** + * The config picker as a BARE inline element (no popover, no header/footer) — for + * a tool to embed and own. Returned synchronously, Suspense-style: it shows a + * spinner and fills in once config resolves (no defaults-flash). + * + * const panel = llm.dom({source, tools}) + * box.append(panel) + * + * Scope which config it edits with `{source:{read,write,url?}}` (url shows in the + * status bar). Pass `{tools:[{name,description}]}` to surface the host's built-in + * tools in the Tools section. The element carries `.result` (resolves on + * `.destroy()`), `.destroy()` (flush + remove), `.revert()` (revert + remove). + * + * @param {PickerOpts} [opts] + * @returns {HTMLElement} + */ +export function dom(opts = {}) { + injectStyles() + const root = el("div", {class: "llmp llmp--bare", role: "group"}) + root.append(spinner()) + /** @type {(v: any) => void} */ + let resolveResult + root.result = new Promise((r) => (resolveResult = r)) + /** @type {Ctl|null} */ + let ctl = null + let done = false + /** @type {() => Ctl|null} */ + let getCtl = () => ctl + const finish = (/** @type {any} */ saved) => { + if (done) return + done = true + resolveResult(saved) + root.remove() + } + root.destroy = () => { const c = getCtl(); finish(c ? c.commit() : null) } + root.revert = () => { + getCtl()?.revert() + finish(null) + } + const start = () => { + root.replaceChildren() + getCtl = buildScopedPicker(root, opts, () => root.destroy()) + } + if (opts.source) start() + else ensureSettingsDoc().then(start) + return root +} + +/** + * The config picker wrapped in an outer popover frame (title + ×, Cancel/Done). + * Returned synchronously; mount it and show it: + * + * const el = llm.popup(); root.append(el); el.showPopover() + * const cfg = await el.result // resolves on close (null if cancelled) + * + * Same options as `dom()`. Changes autosave live; Done keeps them, Cancel reverts + * to the open-time snapshot, light-dismiss keeps them. + * + * @param {PickerOpts} [opts] + * @returns {HTMLElement} + */ +export function popup(opts = {}) { + injectStyles() + const frame = el("div", {class: "llmp", popover: "auto", role: "dialog"}) + frame.append(spinner()) + /** @type {(v: any) => void} */ + let resolveResult + frame.result = new Promise((r) => (resolveResult = r)) + /** @type {Ctl|null} */ + let ctl = null + /** @type {() => Ctl|null} */ + let getCtl = () => ctl + let done = false + let reverting = false + const finalize = () => { + if (done) return + done = true + const c = getCtl() + resolveResult(reverting ? null : c ? c.commit() : null) + frame.remove() + } + const close = () => { + if (frame.matches(":popover-open")) frame.hidePopover() + else finalize() + } + const cancel = () => { + reverting = true + getCtl()?.revert() + close() + } + const onEsc = (/** @type {KeyboardEvent} */ e) => { + if (e.key === "Escape") { e.stopPropagation(); close() } + } + frame.addEventListener("toggle", (/** @type {any} */ e) => { + if (e.newState === "open") window.addEventListener("keydown", onEsc) + else { window.removeEventListener("keydown", onEsc); finalize() } + }) + const start = () => { + const inner = el("div", {class: "llmp-inner"}) + frame.replaceChildren( + el("div", {class: "llmp-header"}, [ + el("span", {text: "Large Language Model"}), + el("button", {class: "llmp-close", text: "×", onClick: close}), + ]), + inner, + el("div", {class: "llmp-footer"}, [ + el("button", {class: "llmp-btn", text: "Cancel", onClick: cancel}), + el("button", {class: "llmp-btn primary", text: "Done", onClick: close}), + ]) + ) + getCtl = buildScopedPicker(inner, opts, close) + } + if (opts.source) start() + else ensureSettingsDoc().then(start) + return frame +} + +export {describeConfig} diff --git a/libraries/llm/pnpm-lock.yaml b/libraries/llm/pnpm-lock.yaml new file mode 100644 index 0000000..e038e14 --- /dev/null +++ b/libraries/llm/pnpm-lock.yaml @@ -0,0 +1,214 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@inkandswitch/patchwork-providers': + specifier: ^0.3.0 + version: 0.3.0(@automerge/automerge-repo@2.5.6) + devDependencies: + '@automerge/automerge-repo': + specifier: ^2.5.6 + version: 2.5.6 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + +packages: + + '@automerge/automerge-repo@2.5.6': + resolution: {integrity: sha512-ZXM6TOAwm192g3+zIxYvlB+Z3O00NP+psErOwvbSype8fFO+dhc8tB/jPwfZJqmn1ULUz5w7gssKEynwcYFRSA==} + + '@automerge/automerge@3.4.0': + resolution: {integrity: sha512-THmghtTNGGt2xsI0pM3o1i3PM8oZKcYFgOj25FOzW7l6e94SQOivNtCwy6xc0I8hVJsQSSotoBNs+yk/9hM2dg==} + + '@cbor-extract/cbor-extract-darwin-arm64@2.2.2': + resolution: {integrity: sha512-ZKZ/F8US7JR92J4DMct6cLW/Y66o2K576+zjlEN/MevH70bFIsB10wkZEQPLzl2oNh2SMGy55xpJ9JoBRl5DOA==} + cpu: [arm64] + os: [darwin] + + '@cbor-extract/cbor-extract-darwin-x64@2.2.2': + resolution: {integrity: sha512-32b1mgc+P61Js+KW9VZv/c+xRw5EfmOcPx990JbCBSkYJFY0l25VinvyyWfl+3KjibQmAcYwmyzKF9J4DyKP/Q==} + cpu: [x64] + os: [darwin] + + '@cbor-extract/cbor-extract-linux-arm64@2.2.2': + resolution: {integrity: sha512-wfqgzqCAy/Vn8i6WVIh7qZd0DdBFaWBjPdB6ma+Wihcjv0gHqD/mw3ouVv7kbbUNrab6dKEx/w3xQZEdeXIlzg==} + cpu: [arm64] + os: [linux] + + '@cbor-extract/cbor-extract-linux-arm@2.2.2': + resolution: {integrity: sha512-tNg0za41TpQfkhWjptD+0gSD2fggMiDCSacuIeELyb2xZhr7PrhPe5h66Jc67B/5dmpIhI2QOUtv4SBsricyYQ==} + cpu: [arm] + os: [linux] + + '@cbor-extract/cbor-extract-linux-x64@2.2.2': + resolution: {integrity: sha512-rpiLnVEsqtPJ+mXTdx1rfz4RtUGYIUg2rUAZgd1KjiC1SehYUSkJN7Yh+aVfSjvCGtVP0/bfkQkXpPXKbmSUaA==} + cpu: [x64] + os: [linux] + + '@cbor-extract/cbor-extract-win32-x64@2.2.2': + resolution: {integrity: sha512-dI+9P7cfWxkTQ+oE+7Aa6onEn92PHgfWXZivjNheCRmTBDBf2fx6RyTi0cmgpYLnD1KLZK9ZYrMxaPZ4oiXhGA==} + cpu: [x64] + os: [win32] + + '@inkandswitch/patchwork-providers@0.3.0': + resolution: {integrity: sha512-CUxWbONfOiz5SCzOpX/7zDTMePWgq7e4778eBJpbjC5Et4lnyL2/UrOZ6YJ4HiCG7r7MbfM86GUi63bqjTeutg==} + peerDependencies: + '@automerge/automerge-repo': '*' + + '@noble/hashes@1.8.0': + resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} + engines: {node: ^14.21.3 || >=16} + + base-x@4.0.1: + resolution: {integrity: sha512-uAZ8x6r6S3aUM9rbHGVOIsR15U/ZSc82b3ymnCPsT45Gk1DDvhDPdIgB5MrhirZWt+5K0EEPQH985kNqZgNPFw==} + + bs58@5.0.0: + resolution: {integrity: sha512-r+ihvQJvahgYT50JD05dyJNKlmmSlMoOGwn1lCcEzanPglg7TxYjioQUYehQ9mAR/+hOSd2jRc/Z2y5UxBymvQ==} + + bs58check@3.0.1: + resolution: {integrity: sha512-hjuuJvoWEybo7Hn/0xOrczQKKEKD63WguEjlhLExYs2wUBcebDC1jDNK17eEAD2lYfw82d5ASC1d7K3SWszjaQ==} + + cbor-extract@2.2.2: + resolution: {integrity: sha512-hlSxxI9XO2yQfe9g6msd3g4xCfDqK5T5P0fRMLuaLHhxn4ViPrm+a+MUfhrvH2W962RGxcBwEGzLQyjbDG1gng==} + hasBin: true + + cbor-x@1.6.5: + resolution: {integrity: sha512-yO64CxnSh6kp+pHNRK9IfwnMvCB+c8HvmUjQY/9l9YRF0/cAPka/tUHLwS64QqUpFCq3/OtbKziVJYXH2EaRig==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + node-gyp-build-optional-packages@5.1.1: + resolution: {integrity: sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw==} + hasBin: true + + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + + xstate@5.32.5: + resolution: {integrity: sha512-ULazi1oe6wGrXl0Frb6otSlkm5HLifbbVTkMk5kkSKqz4TkxJaVpnl6jOJwKeid3ORPxYyZQgNLUSYX9q65SIA==} + +snapshots: + + '@automerge/automerge-repo@2.5.6': + dependencies: + '@automerge/automerge': 3.4.0 + bs58check: 3.0.1 + cbor-x: 1.6.5 + debug: 4.4.3 + eventemitter3: 5.0.4 + fast-sha256: 1.3.0 + uuid: 9.0.1 + xstate: 5.32.5 + transitivePeerDependencies: + - supports-color + + '@automerge/automerge@3.4.0': {} + + '@cbor-extract/cbor-extract-darwin-arm64@2.2.2': + optional: true + + '@cbor-extract/cbor-extract-darwin-x64@2.2.2': + optional: true + + '@cbor-extract/cbor-extract-linux-arm64@2.2.2': + optional: true + + '@cbor-extract/cbor-extract-linux-arm@2.2.2': + optional: true + + '@cbor-extract/cbor-extract-linux-x64@2.2.2': + optional: true + + '@cbor-extract/cbor-extract-win32-x64@2.2.2': + optional: true + + '@inkandswitch/patchwork-providers@0.3.0(@automerge/automerge-repo@2.5.6)': + dependencies: + '@automerge/automerge-repo': 2.5.6 + + '@noble/hashes@1.8.0': {} + + base-x@4.0.1: {} + + bs58@5.0.0: + dependencies: + base-x: 4.0.1 + + bs58check@3.0.1: + dependencies: + '@noble/hashes': 1.8.0 + bs58: 5.0.0 + + cbor-extract@2.2.2: + dependencies: + node-gyp-build-optional-packages: 5.1.1 + optionalDependencies: + '@cbor-extract/cbor-extract-darwin-arm64': 2.2.2 + '@cbor-extract/cbor-extract-darwin-x64': 2.2.2 + '@cbor-extract/cbor-extract-linux-arm': 2.2.2 + '@cbor-extract/cbor-extract-linux-arm64': 2.2.2 + '@cbor-extract/cbor-extract-linux-x64': 2.2.2 + '@cbor-extract/cbor-extract-win32-x64': 2.2.2 + optional: true + + cbor-x@1.6.5: + optionalDependencies: + cbor-extract: 2.2.2 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + detect-libc@2.1.2: + optional: true + + eventemitter3@5.0.4: {} + + fast-sha256@1.3.0: {} + + ms@2.1.3: {} + + node-gyp-build-optional-packages@5.1.1: + dependencies: + detect-libc: 2.1.2 + optional: true + + typescript@6.0.3: {} + + uuid@9.0.1: {} + + xstate@5.32.5: {} diff --git a/libraries/llm/pnpm-workspace.yaml b/libraries/llm/pnpm-workspace.yaml new file mode 100644 index 0000000..118ade9 --- /dev/null +++ b/libraries/llm/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + cbor-extract: true diff --git a/libraries/llm/provider.js b/libraries/llm/provider.js new file mode 100644 index 0000000..d89dcaf --- /dev/null +++ b/libraries/llm/provider.js @@ -0,0 +1,111 @@ +/** + * — scope an LLM config to a DOM subtree. + * + * Any `@patchwork/llm` consumer inside this element (more precisely: inside a + * that this element wraps) resolves its config from here + * instead of the account doc. Lets you give one tool/view a different model: + * + * + * … a tool using @patchwork/llm … + * + * + * Configure it three ways: + * - attributes: `provider`, `model`, `temperature` + * - the `.config` property (a full/partial LLM config object) + * - `el.configure()` — opens the picker scoped to THIS element (writes back + * here, not the account doc), returns a Promise + * + * A bare provider (no config set) does NOT answer — consumers fall through to + * the account doc as usual. + */ + +import {accept} from "@inkandswitch/patchwork-providers" +import {normalizeConfig, CONFIG_SELECTOR} from "./config.js" +import {dom} from "./picker.js" + +const TAG = "patchwork-llm-config-provider" + +export class PatchworkLLMConfigProvider extends HTMLElement { + constructor() { + super() + /** @type {import("./config.js").LLMConfig | null} */ + this._config = null // null = "not configured" → don't answer + this._subs = new Set() // live responders (one per consumer subscription) + this._onSubscribe = this._onSubscribe.bind(this) + } + + static get observedAttributes() { + return ["provider", "model", "temperature"] + } + + connectedCallback() { + this.addEventListener("patchwork:subscribe", this._onSubscribe) + } + disconnectedCallback() { + this.removeEventListener("patchwork:subscribe", this._onSubscribe) + } + attributeChangedCallback() { + this._config = this._fromAttrs(this._config ?? {}) + this._emit() + } + + /** @param {any} base */ + _fromAttrs(base) { + const raw = {...base} + const provider = this.getAttribute("provider") + const model = this.getAttribute("model") + const temp = this.getAttribute("temperature") + if (provider) raw.provider = provider + if (temp != null && temp !== "") raw.temperature = +temp + if (model && provider) raw[provider] = {...(raw[provider] || {}), model} + return normalizeConfig(raw) + } + + /** @param {any} e */ + _onSubscribe(e) { + if (e.detail?.selector?.type !== CONFIG_SELECTOR.type) return + if (!this._config) return // not configured — let it bubble to the account doc + accept(e, (/** @type {(cfg: any) => void} */ respond) => { + this._subs.add(respond) + respond(this._config) + return () => this._subs.delete(respond) + }) + } + _emit() { + if (!this._config) return + for (const r of this._subs) r(this._config) + } + + /** @returns {import("./config.js").LLMConfig | null} */ + get config() { + return this._config + } + set config(c) { + this._config = c ? normalizeConfig(c) : null + this._emit() + } + + /** Open the picker scoped to this provider (writes back here). */ + configure() { + const node = /** @type {any} */ ( + dom({ + source: { + read: () => this._config ?? normalizeConfig({}), + write: (/** @type {any} */ cfg) => (this.config = cfg), + }, + }) + ) + this.appendChild(node) // keep it in this subtree so embedded views get context + node.showPopover() + return node.result + } +} + +export function definePatchworkLLMConfigProvider() { + if (typeof customElements !== "undefined" && !customElements.get(TAG)) { + customElements.define(TAG, PatchworkLLMConfigProvider) + } +} + +// Auto-register on import so `` just works. +definePatchworkLLMConfigProvider() diff --git a/libraries/llm/templates.js b/libraries/llm/templates.js new file mode 100644 index 0000000..ae37780 --- /dev/null +++ b/libraries/llm/templates.js @@ -0,0 +1,179 @@ +/** + * Built-in prompt templates. Each entry has a `name`, a `kind` ("system" or + * "pre"), and a `text` string. The picker offers these under "From template…" + * next to the "+ New" button. + */ + +export const PROMPT_TEMPLATES = [ + { + name: "Patchwork tool builder", + kind: "system", + text: `\ +You help people build Patchwork tools — small web plugins that render into a \ +host app and read/write collaborative automerge documents. + +# Plugin shape + +A tool module exports a \`plugins\` array: + +\`\`\`js +export const plugins = [ + { + type: "patchwork:datatype", + id: "my-thing", + name: "My Thing", + icon: "Sparkles", // lucide icon name + async load() { return MyDatatype }, + }, + { + type: "patchwork:tool", + id: "my-thing", // must match the datatype id + name: "My Thing", + icon: "Sparkles", + supportedDatatypes: ["my-thing"], + async load() { return MyTool }, + }, +] +\`\`\` + +# Datatype contract + +\`\`\`js +const MyDatatype = { + init(doc) { + doc.title = "My Thing" + doc.items = [] + }, + getTitle(doc) { return doc.title || "My Thing" }, + setTitle(doc, title) { doc.title = title }, +} +\`\`\` + +\`init\` seeds a new document inside a change callback. Keep the shape flat and \ +JSON-like (objects, arrays, strings, numbers, booleans, null). \ +You CANNOT assign \`undefined\` — use \`null\` or \`delete d.prop\` inside a change. + +# Tool render contract — (handle, element) => cleanup + +\`\`\`js +function MyTool(handle, element) { + const root = document.createElement("div") + const style = document.createElement("style") + style.textContent = \`.my-tool { /* namespaced CSS */ }\` + element.append(style, root) + + function render() { + const doc = handle.doc() // current snapshot (synchronous) + if (!doc) return // may be undefined initially + root.innerHTML = \`

\${doc.title}

\` + } + render() + handle.on("change", render) // re-render on local + remote edits + + return () => { // cleanup (mandatory) + handle.off("change", render) + root.remove() + style.remove() + } +} +\`\`\` + +# Reading & writing documents + +\`\`\`js +const doc = handle.doc() // synchronous snapshot +handle.change(d => { d.count++ }) // all writes go through change() +handle.on("change", fn) // fires on local + remote edits +handle.off("change", fn) +\`\`\` + +# Globals + +\`\`\`js +window.repo // the automerge Repo +window.accountDocHandle // current user's account DocHandle + +const handle = await repo.find(url) // returns Promise (already ready) +const fresh = await repo.create2(initial) // create a new doc +\`\`\` + +Do NOT use the old pattern \`repo.find(url)\` then \`handle.whenReady()\` — \ +\`repo.find\` already returns a ready handle. \`repo.create()\` is deprecated; use \`repo.create2()\`. + +# Ephemeral messaging (multiplayer) + +\`\`\`js +handle.broadcast({ type: "cursor", x, y }) +handle.on("ephemeral-message", ({ message }) => { /* … */ }) +\`\`\` + +Messages reach only currently-connected peers and are never persisted. + +# Custom DOM events + +\`\`\`js +import { openDocument } from "@inkandswitch/patchwork-elements" +openDocument(element, url, toolId) // navigate to another document +\`\`\` + +# Available imports (bare specifiers via importmap) + +- \`@automerge/automerge\`, \`@automerge/automerge/slim\` +- \`@automerge/automerge-repo\`, \`@automerge/automerge-repo/slim\` +- \`@inkandswitch/patchwork-elements\`, \`-filesystem\`, \`-plugins\`, \`-bootloader\` +- \`@codemirror/state\`, \`@codemirror/view\`, \`@codemirror/language\` +- \`solid-js\`, \`solid-js/web\`, \`solid-js/html\`, \`solid-js/store\`, \`solid-js/h\` + +No CDN URLs needed — use direct imports. + +# Solid (only when you need fine-grained reactivity) + +\`\`\`js +import { render } from "solid-js/web" +import html from "solid-js/html" +import { createSignal } from "solid-js" + +function MyTool(handle, element) { + const [doc, setDoc] = createSignal(handle.doc()) + const onChange = () => setDoc(handle.doc()) + handle.on("change", onChange) + + const dispose = render( + () => html\`\`, + element, + ) + return () => { handle.off("change", onChange); dispose() } +} +\`\`\` + +Use \`solid-js/html\` tagged templates (no JSX, no build step needed). + +# Styling + +Write plain CSS. No Tailwind, no CSS frameworks. Namespace all class names. + +Use CSS variables from the theme (with fallbacks): +- Background/foreground: \`var(--studio-fill, white)\` / \`var(--studio-line, black)\` +- Tinted backgrounds: \`var(--studio-fill-offset-10)\` through \`-50\` +- Muted text: \`var(--studio-line-offset-50)\` +- Accents: \`var(--studio-primary)\`, \`--studio-secondary\`, \`--studio-danger\` +- Fonts: \`var(--studio-family-sans, system-ui, sans-serif)\`, \`var(--studio-family-code, ui-monospace, monospace)\` +- Spacing: \`var(--studio-space-2xs)\` (4px) through \`var(--studio-space-2xl)\` (48px) +- Radius: \`var(--studio-radius-sm, 4px)\` through \`var(--studio-radius-round, 9999px)\` + +Derive local variables in \`:root, :host, [theme] { }\`, then use those in rules. \ +Never use raw hex colors — derive everything from theme vars with \`color-mix()\`. + +# Key rules + +- Plain vanilla JavaScript, no TypeScript +- No shadow DOM — tools render into the light DOM, so namespace your CSS classes +- Never \`stopPropagation()\` on \`click\` events (breaks Solid's event delegation) +- Always return a cleanup function from the render function +- Guard \`handle.doc()\` — it may be undefined before the document loads +- No \`undefined\` in automerge — use \`null\` or \`delete\` +`, + }, +] diff --git a/libraries/llm/tools.js b/libraries/llm/tools.js new file mode 100644 index 0000000..43988b0 --- /dev/null +++ b/libraries/llm/tools.js @@ -0,0 +1,724 @@ +/** + * LLM tools — user-defined tools the model can be given. + * + * Each tool is two automerge docs: + * - a handler: a real `file` doc (UnixFileEntry, `.js`) holding a block of JS, + * editable with the built-in `file` tool (tool-id="file"). + * - a wrapper `llm:tool` doc: { name, description, handlerUrl } whose URL can + * be copied and shared; add a tool to your set by pasting its URL. + * + * The tools folder URL lives in the settings doc at `tools` (see config.js). + */ + +import {ensureSettingsDoc} from "./config.js" + +/** + * @typedef {import("@automerge/automerge-repo").Repo} Repo + * @typedef {import("./config.js").LLMConfig} LLMConfig + * @typedef {import("./config.js").DocHandle} DocHandle + */ + +/** + * @typedef {Object} DocLink + * @property {string} name + * @property {string} type + * @property {string} url + * @property {string} [icon] + * @property {string} [copyOf] + */ + +/** + * @typedef {Object} FolderDoc + * @property {string} [title] + * @property {DocLink[]} [docs] + */ + +/** + * @typedef {Object} ResolvedTool + * @property {string} url + * @property {string} name + * @property {string} description + * @property {string} [handlerUrl] + * @property {any} [parameters] + */ + +/** + * @typedef {Object} ToolCall + * @property {string} name + * @property {Record} args + */ + +/** + * @typedef {Object} PromptKind + * @property {string} type + * @property {string} listKey + * @property {string} urlKey + * @property {string} default + */ + +/** Sanitize a tool name for OpenAI's function-calling format: [a-zA-Z0-9_-]{1,64}. + * @param {string} [name] + */ +export function sanitizeToolName(name) { + return (name || "tool") + .replace(/[^a-zA-Z0-9_-]/g, "_") + .replace(/^[_-]+|[_-]+$/g, "") + .slice(0, 64) || "tool" +} + +const DEFAULT_HANDLER = `// Tool handler. The model calls this tool by name; \`args\` is an object of the +// parameters you describe in the tool's description. Return a string or any +// JSON-serialisable value — it's fed back to the model. +export default async function handle(args) { +\treturn "TODO: implement. got args = " + JSON.stringify(args) +} +` + +const DEFAULT_DESCRIPTION = + "Describe what this tool does, when the model should call it, and the parameters it takes — e.g. { city: string, units?: \"c\" | \"f\" }." + +/** @param {Repo} [repo] @returns {Repo} */ +function theRepo(repo) { + return /** @type {Repo} */ (repo || (typeof window !== "undefined" && window.repo) || null) +} + +/** @param {string} [name] */ +function slug(name) { + return (name || "tool").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "tool" +} + +/** Create the handler file doc (a standard `file` doc the file tool can edit). + * @param {Repo} [repo] @param {string} [name] @param {string} [content] + */ +export async function createToolFile(repo, name = "handler.js", content = DEFAULT_HANDLER) { + const r = theRepo(repo) + return r.create2({ + "@patchwork": {type: "file"}, + name, + extension: "js", + mimeType: "text/javascript", + content, + }) +} + +/** Create a new llm-tool (handler file + wrapper doc). Returns the wrapper handle. + * @param {Repo} [repo] @param {{name?: string, description?: string}} [opts] + */ +export async function createLLMTool(repo, {name = "New tool", description = DEFAULT_DESCRIPTION} = {}) { + const r = theRepo(repo) + const file = await createToolFile(r, slug(name) + ".js") + return r.create2({ + "@patchwork": {type: "llm:tool"}, + name, + description, + tool: file.url, // the handler file (was `handlerUrl`) + }) +} + +// --------------------------------------------------------------------------- +// Folders — tools + prompts each live in a `folder` doc (so they're openable / +// manageable as a normal Patchwork folder). cfg.tools / cfg.prompts are the +// folder URLs; the folder's `.docs` are the DocLinks. +// --------------------------------------------------------------------------- + +/** Read a folder's DocLinks, optionally filtered by `.type`. + * @param {string|null|undefined} folderUrl @param {string} [type] @param {Repo} [repo] + * @returns {Promise} + */ +export async function folderLinks(folderUrl, type, repo) { + if (typeof folderUrl !== "string") return [] + try { + const folder = /** @type {FolderDoc} */ ((await theRepo(repo).find(/** @type {any} */ (folderUrl))).doc()) + const docs = folder?.docs || [] + return type ? docs.filter((l) => l.type === type) : docs + } catch { + return [] + } +} + +/** Ensure a folder URL exists; create an empty `folder` doc if missing. Returns the URL. + * @param {Repo} [repo] @param {string|null|undefined} [url] @param {string} [title] + */ +export async function ensureFolderUrl(repo, url, title = "Folder") { + if (typeof url === "string") return url + return ( + await theRepo(repo).create2({"@patchwork": {type: "folder"}, title, docs: []}) + ).url +} + +/** @param {Repo} repo @param {string} folderUrl @param {DocLink} docLink */ +export async function addToFolder(repo, folderUrl, docLink) { + const h = await theRepo(repo).find(/** @type {any} */ (folderUrl)) + h.change((/** @type {FolderDoc} */ d) => { + if (!d.docs) d.docs = [] + d.docs.push(docLink) + }) +} + +/** @param {Repo} repo @param {string} folderUrl @param {string} docUrl */ +export async function removeFromFolder(repo, folderUrl, docUrl) { + const h = await theRepo(repo).find(/** @type {any} */ (folderUrl)) + h.change((/** @type {FolderDoc} */ d) => { + if (!d.docs) return + const i = d.docs.findIndex((l) => l.url === docUrl) + if (i !== -1) d.docs.splice(i, 1) + }) +} + +/** Resolve the tools folder into [{ url, name, description, handlerUrl }]. + * @param {LLMConfig} [cfg] @param {Repo} [repo] @returns {Promise} + */ +export async function resolveTools(cfg, repo) { + const r = theRepo(repo) + const links = await folderLinks(cfg?.tools, "llm:tool", repo) + /** @type {ResolvedTool[]} */ + const out = [] + for (const link of links) { + try { + const d = /** @type {any} */ ((await r.find(/** @type {any} */ (link.url))).doc()) + if (d) + out.push({ + url: link.url, + name: d.name || link.name || "Tool", + description: d.description || "", + handlerUrl: d.tool ?? d.handlerUrl, // `tool`, or legacy `handlerUrl` + // Folder tools carry no JSON Schema — permissive params; the model + // learns the shape from the description. + parameters: d.parameters || {type: "object", additionalProperties: true}, + }) + } catch { + /* unreachable — skip */ + } + } + return out +} + +/** OpenAI-style tool schemas, for providers with native function calling. + * @param {ResolvedTool[]} [tools] + */ +export function toToolSchemas(tools) { + return (tools || []).map((t) => ({ + type: "function", + function: { + name: sanitizeToolName(t.name), + description: t.description || "", + parameters: t.parameters || {type: "object", properties: {}, additionalProperties: true}, + }, + })) +} + +/** + * System-prompt block for providers WITHOUT native tool calling (local + * transformers, Chrome built-in). Uses the Hermes/Qwen `` XML + * convention — what those models are tuned to emit. + */ +/** @param {ResolvedTool[]} [tools] */ +export function buildToolsSystem(tools) { + if (!tools || !tools.length) return "" + const describe = (/** @type {ResolvedTool} */ t) => { + const props = t.parameters?.properties + const params = + props && Object.keys(props).length + ? " — args: " + + Object.entries(props) + .map(([k, v]) => `${k}${v?.type ? ":" + v.type : ""}`) + .join(", ") + : "" + return `- ${sanitizeToolName(t.name)}: ${t.description || ""}${params}` + } + return [ + "You can call tools. To call one, emit a tool call wrapped in tags containing JSON, exactly:", + '{"name": "", "arguments": { ... }}', + "You'll then be given the tool's result and can call another tool or answer. Call a tool only when it genuinely helps — otherwise just answer in plain prose.", + "", + "Available tools:", + tools.map(describe).join("\n"), + ].join("\n") +} + +/** + * Parse tool calls out of model TEXT (the prompt-convention fallback for + * local/built-in). Handles, in order of preference: + * - {…} (Hermes/Qwen XML) + * - ```json / ```tool_call / ```tool-call fenced JSON + * - a bare {…} object containing a name/tool key + * Each accepts {name|tool, arguments|args}. Returns [{ name, args }]. + */ +/** @param {string} [text] @returns {ToolCall[]} */ +export function parseToolCalls(text) { + if (!text) return [] + /** @type {ToolCall[]} */ + const calls = [] + const push = (/** @type {any} */ obj) => { + const name = obj?.name || obj?.tool + if (!name) return + let args = obj.arguments ?? obj.args ?? {} + if (typeof args === "string") { + try { + args = JSON.parse(args) + } catch { + args = {} + } + } + calls.push({name, args: args || {}}) + } + const thinkingEnd = text.lastIndexOf("") + const lfmRegionStart = thinkingEnd === -1 ? 0 : thinkingEnd + "".length + const lfmMatch = text + .slice(lfmRegionStart) + .search(/\[\s*[a-zA-Z_][a-zA-Z0-9_]*\s*\(/) + const lfmStart = lfmMatch === -1 ? -1 : lfmRegionStart + lfmMatch + if (lfmStart !== -1) { + let depth = 0 + let quote = "" + let escaped = false + let lfmEnd = -1 + for (let i = lfmStart; i < text.length; i++) { + const ch = text[i] + if (escaped) { + escaped = false + continue + } + if (quote) { + if (ch === "\\") escaped = true + else if (ch === quote) quote = "" + continue + } + if (ch === "'" || ch === '"') { + quote = ch + continue + } + if (ch === "[" || ch === "(" || ch === "{") depth++ + else if (ch === "]" || ch === ")" || ch === "}") { + depth-- + if (depth === 0) { + lfmEnd = i + break + } + } + } + if (lfmEnd !== -1) { + const split = (source = "") => { + const parts = [] + let start = 0 + let nested = 0 + let string = "" + let slash = false + for (let i = 0; i < source.length; i++) { + const ch = source[i] + if (slash) { + slash = false + continue + } + if (string) { + if (ch === "\\") slash = true + else if (ch === string) string = "" + continue + } + if (ch === "'" || ch === '"') string = ch + else if (ch === "[" || ch === "(" || ch === "{") nested++ + else if (ch === "]" || ch === ")" || ch === "}") nested-- + else if (ch === "," && nested === 0) { + parts.push(source.slice(start, i).trim()) + start = i + 1 + } + } + const last = source.slice(start).trim() + if (last) parts.push(last) + return parts + } + const literal = (source = "") => { + let json = "" + for (let i = 0; i < source.length; i++) { + const ch = source[i] + if (ch !== "'" && ch !== '"') { + json += ch + continue + } + let value = "" + for (i++; i < source.length; i++) { + const next = source[i] + if (next === ch) break + if (next !== "\\") { + value += next + continue + } + const escaped = source[++i] + value += + escaped === "n" + ? "\n" + : escaped === "r" + ? "\r" + : escaped === "t" + ? "\t" + : escaped + } + json += JSON.stringify(value) + } + try { + return JSON.parse( + json.replace(/\bTrue\b/g, "true").replace(/\bFalse\b/g, "false").replace(/\bNone\b/g, "null") + ) + } catch { + return source + } + } + for (const expression of split(text.slice(lfmStart + 1, lfmEnd))) { + const match = /^([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([\s\S]*)\)$/.exec(expression) + if (!match) continue + const args = Object.fromEntries([]) + for (const arg of split(match[2])) { + const eq = arg.indexOf("=") + if (eq < 1) continue + const name = arg.slice(0, eq).trim() + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) continue + args[name] = literal(arg.slice(eq + 1).trim()) + } + calls.push({name: match[1], args}) + } + if (calls.length) return calls + } + } + let m + const xml = /\s*([\s\S]*?)\s*<\/tool_call>/g + let sawXml = false + while ((m = xml.exec(text))) { + sawXml = true + try { + push(JSON.parse(m[1].trim())) + } catch {} + } + if (sawXml) return calls + const fence = /```(?:json|tool[_-]call)?\s*([\s\S]*?)```/g + while ((m = fence.exec(text))) { + try { + push(JSON.parse(m[1].trim())) + } catch {} + } + if (calls.length) return calls + // Bare JSON objects — brace-depth-aware scan so nested objects (e.g. + // "arguments": { ... }) are captured whole instead of truncated at the + // first inner `}`. + let depth = 0, start = -1, inStr = false, esc = false + for (let i = 0; i < text.length; i++) { + const ch = text[i] + if (esc) { esc = false; continue } + if (ch === '\\' && inStr) { esc = true; continue } + if (ch === '"') { inStr = !inStr; continue } + if (inStr) continue + if (ch === '{') { + if (depth === 0) start = i + depth++ + } else if (ch === '}') { + depth-- + if (depth === 0 && start >= 0) { + const block = text.slice(start, i + 1) + if (/"(?:name|tool)"/.test(block)) { + try { push(JSON.parse(block)) } catch {} + } + start = -1 + } + } + } + return calls +} + +/** Fetch a handler file doc's JS source as a string. + * @param {string} handlerUrl @param {Repo} [repo] + */ +async function loadHandlerCode(handlerUrl, repo) { + const h = await theRepo(repo).find(/** @type {any} */ (handlerUrl)) + const content = /** @type {any} */ (h.doc())?.content + return typeof content === "string" + ? content + : new TextDecoder().decode(content || new Uint8Array()) +} + +/** + * Load a tool's handler as a function. The handler file's JS is imported as an + * ES module (via a blob URL) and runs in the MAIN thread with full page access + * (window.repo, the account doc, the DOM). Use a sandbox (see runTool) for + * untrusted / shared tools that shouldn't have that reach. + * @param {string} handlerUrl @param {Repo} [repo] + */ +export async function loadHandler(handlerUrl, repo) { + const code = await loadHandlerCode(handlerUrl, repo) + const blobUrl = URL.createObjectURL(new Blob([code], {type: "text/javascript"})) + try { + const mod = await import(/* @vite-ignore */ blobUrl) + const fn = mod.default || mod.handle + if (typeof fn !== "function") + throw new Error("tool handler must `export default` a function") + return fn + } finally { + URL.revokeObjectURL(blobUrl) + } +} + +// A module worker that imports the handler code in its OWN realm and runs it on +// just the args we hand in — no window, no DOM, no window.repo, no account doc, +// no network of ours. Only structured-cloneable args/results cross the boundary. +const SANDBOX_BOOTSTRAP = ` +self.onmessage = async (e) => { + const {code, args} = e.data + let url + try { + url = URL.createObjectURL(new Blob([code], {type: "text/javascript"})) + const mod = await import(url) + const fn = mod.default || mod.handle + if (typeof fn !== "function") throw new Error("tool handler must export default a function") + const result = await fn(args || {}) + self.postMessage({ok: true, result}) + } catch (err) { + self.postMessage({ok: false, error: (err && err.message) || String(err)}) + } finally { + if (url) URL.revokeObjectURL(url) + } +} +` + +/** + * Run handler `code` in an isolated Worker — no page access. A runaway handler + * is killed after `timeoutMs` (default 10s). Throws on handler error/timeout. + * @param {string} code @param {any} args @param {{timeoutMs?: number}} [opts] + */ +export async function runHandlerSandboxed(code, args, {timeoutMs = 10000} = {}) { + const bootUrl = URL.createObjectURL(new Blob([SANDBOX_BOOTSTRAP], {type: "text/javascript"})) + const worker = new Worker(bootUrl, {type: "module"}) + try { + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("tool handler timed out")), timeoutMs) + worker.onmessage = (e) => { + clearTimeout(timer) + if (e.data?.ok) resolve(e.data.result) + else reject(new Error(e.data?.error || "tool handler failed")) + } + worker.onerror = (e) => { + clearTimeout(timer) + reject(new Error(e.message || "tool handler worker error")) + } + worker.postMessage({code, args: args || {}}) + }) + } finally { + worker.terminate() + URL.revokeObjectURL(bootUrl) + } +} + +/** + * Run a resolved tool with args. By default loads + calls its handler in the + * MAIN thread (full page access). Pass `{sandbox: true}` to run it in an + * isolated Worker with no page access — for untrusted / shared tools. + * + * @param {ResolvedTool} tool resolved tool ({handlerUrl, …}) + * @param {any} args + * @param {any} [opts] options, or a Repo (back-compat positional repo) + */ +export async function runTool(tool, args, opts = {}) { + // Back-compat: runTool(tool, args, repo) — a Repo has a `.find` method. + const o = opts && typeof opts.find === "function" ? {repo: opts} : opts || {} + if (o.sandbox) { + const code = await loadHandlerCode(/** @type {string} */ (tool.handlerUrl), o.repo) + return runHandlerSandboxed(code, args, {timeoutMs: o.timeoutMs}) + } + const fn = await loadHandler(/** @type {string} */ (tool.handlerUrl), o.repo) + return fn(args || {}) +} + +/** Datatype so an `llm:tool` doc has a title/icon and can be opened. */ +export const LLMToolDatatype = { + /** @param {any} doc */ + init(doc) { + doc["@patchwork"] = {type: "llm:tool"} + doc.name = "New tool" + doc.description = DEFAULT_DESCRIPTION + }, + /** @param {any} doc */ + getTitle(doc) { + return doc.name || "LLM tool" + }, + /** @param {any} doc @param {string} title */ + setTitle(doc, title) { + doc.name = title + }, + /** @param {any} doc */ + markCopy(doc) { + doc.name = "Copy of " + (doc.name || "tool") + }, +} + +// --------------------------------------------------------------------------- +// Saved prompts (system + pre) — same shape as tools: the prompt TEXT lives in a +// real `file` (.txt) doc you edit with the file tool, wrapped in an +// `llm:system-prompt` / `llm:pre-prompt` doc whose URL can be copied/shared. +// Libraries live at `llm.systemPrompts` / `llm.prePrompts`; the chosen one at +// `llm.prompts.systemUrl` / `llm.prompts.preUrl`. +// --------------------------------------------------------------------------- + +/** @type {Record} */ +const PROMPT_KINDS = { + system: {type: "llm:system-prompt", listKey: "systemPrompts", urlKey: "systemUrl", default: "LLMs are a computer program. They should respond like a computer program."}, + pre: {type: "llm:pre-prompt", listKey: "prePrompts", urlKey: "preUrl", default: "Genre: noir detective."}, +} + +/** Create a saved prompt (text file + wrapper). `kind` = "system" | "pre". + * @param {Repo} [repo] @param {string} [kind] @param {{name?: string, text?: string}} [opts] + */ +export async function createPromptDoc(repo, kind, {name, text} = {}) { + const r = theRepo(repo) + const k = PROMPT_KINDS[/** @type {string} */ (kind)] || PROMPT_KINDS.system + const file = await r.create2({ + "@patchwork": {type: "file"}, + name: slug(name || kind) + ".txt", + extension: "txt", + mimeType: "text/plain", + content: text ?? k.default, + }) + return r.create2({ + "@patchwork": {type: k.type}, + name: name || "New prompt", + promptUrl: file.url, + }) +} + +/** Resolve the prompts folder (filtered by `kind`) into [{ url, name, promptUrl }]. + * @param {LLMConfig} [cfg] @param {string} [kind] @param {Repo} [repo] + */ +export async function resolvePromptDocs(cfg, kind, repo) { + const k = PROMPT_KINDS[/** @type {string} */ (kind)] || PROMPT_KINDS.system + const r = theRepo(repo) + const links = await folderLinks(cfg?.prompts, k.type, repo) + /** @type {{url: string, name: string, promptUrl: any}[]} */ + const out = [] + for (const link of links) { + try { + const d = /** @type {any} */ ((await r.find(/** @type {any} */ (link.url))).doc()) + out.push({url: link.url, name: d?.name || link.name || "Prompt", promptUrl: d?.promptUrl}) + } catch { + /* unreachable — skip */ + } + } + return out +} + +/** Read the text of the currently-selected prompt for `kind` (its file content). + * @param {LLMConfig} [cfg] @param {string} [kind] @param {Repo} [repo] + */ +export async function resolvePromptText(cfg, kind, repo) { + const k = PROMPT_KINDS[/** @type {string} */ (kind)] || PROMPT_KINDS.system + const url = /** @type {any} */ (cfg)?.[k.urlKey] // top-level systemUrl / preUrl + if (!url) return "" + const r = theRepo(repo) + try { + const promptUrl = /** @type {any} */ ((await r.find(url)).doc())?.promptUrl + if (!promptUrl) return "" + const content = /** @type {any} */ ((await r.find(promptUrl)).doc())?.content + return typeof content === "string" + ? content + : new TextDecoder().decode(content || new Uint8Array()) + } catch { + return "" + } +} + +/** Resolve a cfg's selected system + pre prompt docs into `cfg.resolved.{system,pre}` text. + * @param {LLMConfig} [cfg] @param {Repo} [repo] + */ +export async function resolveCfgPrompts(cfg, repo) { + const system = await resolvePromptText(cfg, "system", repo) + const pre = await resolvePromptText(cfg, "pre", repo) + return {...cfg, resolved: {system, pre}} +} + +/** + * One-time migration. Converts the legacy `tools` (URL array), + * `systemPrompts`/`prePrompts` (URL arrays) and `prompts` (object) inside the + * settings doc (see `ensureSettingsDoc`) into `folder` docs + the new scalar + * shape. Idempotent. + * @param {Repo} [repo] + */ +export async function migrateConfig(repo) { + const r = theRepo(repo) + const handle = await ensureSettingsDoc() + if (!r || !handle) return + const llm = /** @type {any} */ (handle.doc() ?? {}) + const oldTools = Array.isArray(llm.tools) ? llm.tools : null + const oldPromptsObj = llm.prompts && typeof llm.prompts === "object" ? llm.prompts : null + const oldSys = Array.isArray(llm.systemPrompts) ? llm.systemPrompts : [] + const oldPre = Array.isArray(llm.prePrompts) ? llm.prePrompts : [] + const needsPrompts = + !!oldPromptsObj || oldSys.length || oldPre.length || "systemPrompts" in llm || "prePrompts" in llm + if (!oldTools && !needsPrompts) return // already migrated + + const linkFor = async (/** @type {any} */ url, /** @type {string} */ type, /** @type {string} */ fallback) => { + try { + return {name: /** @type {any} */ ((await r.find(url)).doc())?.name || fallback, type, url} + } catch { + return {name: fallback, type, url} + } + } + /** @type {string|undefined} */ + let toolsFolder + /** @type {string|undefined} */ + let promptsFolder + if (oldTools) { + const links = await Promise.all(oldTools.map((/** @type {any} */ u) => linkFor(u, "llm:tool", "Tool"))) + toolsFolder = ( + await r.create2({"@patchwork": {type: "folder"}, title: "LLM Tools", docs: links}) + ).url + } + if (needsPrompts) { + const sysLinks = await Promise.all(oldSys.map((/** @type {any} */ u) => linkFor(u, "llm:system-prompt", "System prompt"))) + const preLinks = await Promise.all(oldPre.map((/** @type {any} */ u) => linkFor(u, "llm:pre-prompt", "Pre-prompt"))) + promptsFolder = ( + await r.create2({ + "@patchwork": {type: "folder"}, + title: "LLM Prompts", + docs: [...sysLinks, ...preLinks], + }) + ).url + } + handle.change((/** @type {any} */ d) => { + // `d` is the settings-doc body (the config itself). + if (toolsFolder) { + delete d.tools + d.tools = toolsFolder + } + if (needsPrompts) { + if (oldPromptsObj?.systemUrl && !d.systemUrl) d.systemUrl = oldPromptsObj.systemUrl + if (oldPromptsObj?.preUrl && !d.preUrl) d.preUrl = oldPromptsObj.preUrl + delete d.prompts + d.prompts = promptsFolder + } + delete d.systemPrompts + delete d.prePrompts + }) +} + +/** @param {string} kind */ +function promptDatatype(kind) { + const k = PROMPT_KINDS[kind] + return { + /** @param {any} doc */ + init(doc) { + doc["@patchwork"] = {type: k.type} + doc.name = "New prompt" + }, + /** @param {any} doc */ + getTitle(doc) { + return doc.name || "Prompt" + }, + /** @param {any} doc @param {string} title */ + setTitle(doc, title) { + doc.name = title + }, + /** @param {any} doc */ + markCopy(doc) { + doc.name = "Copy of " + (doc.name || "prompt") + }, + } +} + +/** Datatypes so the wrapper docs have a title/icon. */ +export const LLMSystemPromptDatatype = promptDatatype("system") +export const LLMPrePromptDatatype = promptDatatype("pre") diff --git a/libraries/llm/tsconfig.json b/libraries/llm/tsconfig.json new file mode 100644 index 0000000..5dc1f2d --- /dev/null +++ b/libraries/llm/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "declaration": true, + "emitDeclarationOnly": true, + "module": "esnext", + "moduleResolution": "bundler", + "target": "esnext", + "skipLibCheck": true, + "outDir": "types" + }, + "include": [ + "globals.d.ts", + "index.js", + "config.js", + "client.js", + "worker.js", + "picker.js", + "tools.js", + "provider.js", + "builtin.js", + "templates.js" + ] +} diff --git a/libraries/llm/worker.js b/libraries/llm/worker.js new file mode 100644 index 0000000..6f90f1b --- /dev/null +++ b/libraries/llm/worker.js @@ -0,0 +1,2176 @@ +/** + * @patchwork/llm SharedWorker + * + * Runs ALL generation (local transformers.js / OpenRouter / Ollama) off the + * main thread, so a stream survives a page refresh and is shared across tabs + * keyed by an optional `sessionKey`. Merges chat's SharedWorker with rlm's + * teaching telemetry: alongside the text we stream the model's next-token + * distribution ("predictions") and decode stats (TTFT, tokens/sec, the exact + * sampling settings used) — for local AND OpenRouter. + * + * IN: + * { type:"generate", id, sessionKey?, provider, messages, config } + * config: { model, apiKey?, url?, temperature?, topk?, maxNewTokens?, + * contextLength?, maxCompletionTokens? } + * { type:"preload", provider, config } + * { type:"resume", sessionKey } + * { type:"abort", sessionKey } + * { type:"list-local-models" } + * + * OUT (per generation `id` unless noted): + * { type:"token", id, delta, text } + * { type:"prediction", id, step, candidates:[{token,p}] } // next-token top-k + * { type:"stats", id, ...stats } + * { type:"result", id, text } + * { type:"error", id, message } + * { type:"status", message } // broadcast: model loading, etc. + * { type:"ready", ...modelInfo } + * { type:"local-models", models } + * { type:"resumed"|"resume-result", id, text } | { type:"no-active-generation" } + */ + +// v4: the version where TextStreamer + a logits_processor probe (the per-token +// prediction telemetry) are verified working together (rlm uses this combo). + +/** + * @typedef {import("./config.js").CallConfig} CallConfig + * The config the worker actually receives. Shaped like a {@link CallConfig} but + * kept fully loose: the worker reads extra fields (tools, continuation, custom) + * and compares some numeric fields against "" defensively, so a permissive + * record is intentional here. + * @typedef {Record} WorkerConfig + * + * Any worker message payload (request or response) — these are dynamic blobs + * over postMessage, so a permissive shape is intentional. + * @typedef {Record} AnyMsg + * + * A single in-flight generation entry. + * @typedef {Object} Gen + * @property {any} id + * @property {any} port + * @property {string} fullText + * @property {boolean} done + * @property {string} [finalText] + * @property {AbortController} abortController + * + * @typedef {{token:string, p:number}} Candidate + */ + +const CDN = "https://cdn.jsdelivr.net/npm/@huggingface/transformers@4" +/** @type {Set} */ +const ports = new Set() + +/** @param {AnyMsg} msg */ +function broadcast(msg) { + for (const port of ports) { + try { + port.postMessage(msg) + } catch {} + } +} + +// The big model weights are fetched by onnxruntime-web DIRECTLY (an external-data +// blob: a `.onnx_data`/`.bin`/hash-named LFS file), bypassing transformers.js's +// `progress_callback` — which only ever sees the small tokenizer/config/onnx-graph +// files. So those instantly hit "100%" while a 400MB+ weights download silently +// streams for minutes with no feedback. Fix: wrap the worker's global `fetch` and +// stream-count bytes on the large weights download, broadcasting real progress. +// +// We deliberately do NOT track `.onnx` files (transformers.js already reports those) +// or anything under the size threshold (tokenizer/config/API calls) — only the big +// external-data blob transformers.js is blind to. +/** @param {number} b */ +function fmtMB(b) { + return (b / 1048576).toFixed(0) + " MB" +} + +function installWeightsProgress() { + const sg = /** @type {any} */ (self) + if (sg.__llmWeightsFetchPatched) return + sg.__llmWeightsFetchPatched = true + const orig = self.fetch.bind(self) + const THRESHOLD = 50 * 1024 * 1024 // 50MB — well above tokenizer/config/graph files + self.fetch = async (input, init) => { + const res = await orig(input, init) + try { + const cl = +(res.headers.get("content-length") || 0) + const url = res.url || (typeof input === "string" ? input : /** @type {any} */ (input)?.url || "") + const name = (url.split("?")[0].split("/").pop() || "").toLowerCase() + // `.onnx` graph files are already tracked by transformers.js; skip them so + // the two don't fight over the status line. Everything else big is weights. + if (res.ok && res.body && cl > THRESHOLD && !name.endsWith(".onnx")) { + return trackDownload(res, cl) + } + } catch {} + return res + } +} + +/** @param {Response} res @param {number} total */ +function trackDownload(res, total) { + let loaded = 0 + let lastPct = -1 + const reader = /** @type {ReadableStream} */ (res.body).getReader() + const stream = new ReadableStream({ + async pull(controller) { + const {done, value} = await reader.read() + if (done) { + controller.close() + return + } + loaded += value.byteLength + const pct = total ? Math.round((100 * loaded) / total) : 0 + if (pct !== lastPct) { + lastPct = pct + broadcast({ + type: "status", + message: `Downloading model weights… ${pct}% (${fmtMB(loaded)} / ${fmtMB(total)})`, + }) + } + controller.enqueue(value) + }, + cancel(reason) { + return reader.cancel(reason) + }, + }) + return new Response(stream, { + status: res.status, + statusText: res.statusText, + headers: res.headers, + }) +} + +// Forward worker diagnostics to every connected page. A SharedWorker has its own +// (usually hidden) console, so without this its logs are invisible from the tool. +// `client.js` re-prints `{type:"log"}` on the main thread. Each arg is reduced to +// something structured-clonable so postMessage never throws on a live object. +/** @param {...any} args */ +function log(...args) { + try { + console.log("[llm worker]", ...args) + } catch {} + broadcast({ + type: "log", + args: args.map((a) => { + if (a == null || typeof a !== "object") return a + try { + return JSON.parse(JSON.stringify(a)) + } catch { + return String(a) + } + }), + }) +} + +// sessionKey -> { id, port, fullText, done, finalText, abortController } +/** @type {Map} */ +const activeGenerations = new Map() + +self.addEventListener("error", (e) => + broadcast({type: "status", message: "Worker error: " + (/** @type {ErrorEvent} */ (e).message || "")}) +) +self.addEventListener("unhandledrejection", (e) => + broadcast({ + type: "status", + message: "Worker error: " + (/** @type {PromiseRejectionEvent} */ (e).reason?.message || /** @type {PromiseRejectionEvent} */ (e).reason || ""), + }) +) + +// --------------------------------------------------------------------------- +// Local models (transformers.js) +// --------------------------------------------------------------------------- + +const LOCAL_MODELS = [ + {id: "LiquidAI/LFM2.5-2.6B-ONNX", name: "LFM2.5 2.6B", dtype: "q4f16"}, + {id: "onnx-community/Qwen3-4B-ONNX", name: "Qwen3 4B (best)", dtype: "q4f16"}, + {id: "onnx-community/Qwen3-1.7B-ONNX", name: "Qwen3 1.7B", dtype: "q4f16"}, + {id: "onnx-community/Qwen3-0.6B-ONNX", name: "Qwen3 0.6B (fast)", dtype: "q4f16"}, + {id: "onnx-community/Llama-3.2-1B-Instruct-ONNX", name: "Llama 3.2 1B", dtype: "q4f16"}, + {id: "onnx-community/Phi-3.5-mini-instruct-onnx-web", name: "Phi 3.5 Mini", dtype: "q4f16"}, + {id: "onnx-community/SmolLM2-1.7B-Instruct-ONNX", name: "SmolLM2 1.7B", dtype: "q4f16"}, +] +const DEFAULT_MODEL_ID = "onnx-community/Qwen3-0.6B-ONNX" +const PREDICTION_CAP = 256 // cap per-step prediction events so a long gen can't flood + +/** @type {any} */ +let TF = null +/** @type {any} */ +let generator = null +let currentModelId = DEFAULT_MODEL_ID +/** @type {string|null} */ +let currentDtype = null // the dtype the loaded model was compiled with +let loading = false +/** @type {Promise|null} */ +let loadingPromise = null +// The real error from the most recent loadModel() failure. Without this the +// device-fallback loop swallows the cause ("WASM failed") and callers can only +// report a useless "Model not loaded". Surfaced via modelLoadError(). +/** @type {any} */ +let lastLoadError = null + +// Map a raw backend error to something a user can act on. onnxruntime's +// out-of-memory surfaces as `std::bad_alloc` / "Can't create a session" / +// "memory access out of bounds" — none of which mean "your machine is out of +// RAM". The WASM backend is wasm32 (a ~4GB address-space ceiling, independent of +// system RAM) and WebGPU is bound by GPU buffer/VRAM limits (and is disabled in +// incognito). Say that, and point at the escape hatches. +/** @param {any} err */ +function friendlyError(err) { + const raw = String((err && (err.message || err)) || "unknown error") + // A genuine allocation failure. Big models really can exceed the wasm32 + // address space; this message is only right when the allocator actually + // said so. + if (/bad_alloc|out of memory|can'?t create a session/i.test(raw)) { + return ( + "Out of memory while loading this model. Big models can exceed the browser's " + + "WASM memory ceiling (~4GB — independent of your system RAM). Try a smaller model, " + + "or open in a normal (non-incognito) window so WebGPU can run it on the GPU instead. " + + "(raw: " + raw + ")" + ) + } + // A WASM *trap*, not an allocation failure. `memory access out of bounds` / + // `RuntimeError: Aborted` on the CPU backend almost always means an op the + // WASM build can't run — most commonly an fp16 dtype (q4f16/fp16), which is + // a WebGPU dtype. A tiny model hits this without being anywhere near the 4GB + // ceiling, so DON'T call it "out of memory". + if (/memory access out of bounds|RuntimeError: Aborted/i.test(raw)) { + return ( + "The WASM backend crashed running this model — usually an fp16 weight " + + "format (q4f16/fp16) the CPU backend can't execute, not a size problem. " + + "Use an integer quantization (q4/q8) for WASM, or open in a normal " + + "(non-incognito) window so WebGPU is available. (raw: " + raw + ")" + ) + } + if (/unaligned/i.test(raw)) { + return "The WASM backend hit an unaligned-access fault. Try a normal window so WebGPU is available, or a smaller model. (raw: " + raw + ")" + } + if (/tensor shape is too large|failed to call OrtRun|op_kernel\.cc/i.test(raw)) { + return ( + "The model produced a tensor too large for this backend during the forward pass. " + + "That usually means either a very long prompt, or a model that emits extra big outputs " + + "every step — e.g. an attention-export model (…-attn) meant for analysis/overlays, not " + + "chat generation. Use the standard (non-attention) model for chat, shorten the prompt, " + + "or open in a normal window for WebGPU. (raw: " + raw + ")" + ) + } + return raw +} + +// Map a dtype to something the WASM/CPU backend can actually execute. The fp16 +// formats (q4f16, q8f16, fp16) are WebGPU-only and trap on WASM, so swap them +// for their integer-quant / fp32 equivalents. Anything already WASM-safe is +// returned unchanged. +/** @param {string} dtype @returns {string} */ +function wasmSafeDtype(dtype) { + switch (dtype) { + case "q4f16": + return "q4" + case "q8f16": + return "q8" + case "fp16": + return "fp32" + default: + return dtype + } +} + +function modelLoadError() { + return lastLoadError + ? "Model load failed: " + friendlyError(lastLoadError) + : "Model not loaded" +} + +// Dispose the current model's onnxruntime session(s) before dropping the +// reference. GC does NOT free native wasm/GPU sessions, so without an explicit +// dispose the old model's memory lingers and loading another can `std::bad_alloc` +// even though nothing uses the old one. Nulls synchronously (so a caller's +// `if (generator)` guard sees it gone immediately) then disposes in background. +function releaseGenerator() { + const g = generator + const wasModel = currentModelId + generator = null + if (!g) return Promise.resolve() + return (async () => { + try { + if (typeof g.dispose === "function") await g.dispose() + else if (g.model && typeof g.model.dispose === "function") await g.model.dispose() + log("releaseGenerator: disposed previous session", {model: wasModel}) + } catch (/** @type {any} */ e) { + log("releaseGenerator: dispose failed", {model: wasModel, message: e?.message || String(e)}) + } + })() +} +/** @type {Set} */ +const compiledModels = new Set() + +// User-supplied local ONNX models (in transformers.js layout) uploaded from +// disk. We serve their files to transformers.js by patching the worker's fetch +// — the same trick rlm uses for CDN files — so a model id like "local/" +// loads from memory instead of the network. +/** @type {Map, dtype: string}>} */ +const localModelFiles = new Map() // id -> { files: Map, dtype } + +// Drop anything transformers.js already cached under a local model's id, so a +// re-upload of the same folder name can't be served stale bytes from an earlier +// (possibly broken, possibly different) export. +/** @param {string} id */ +async function purgeCachedModel(id) { + try { + const cache = await caches.open("transformers-cache") + const keys = await cache.keys() + let n = 0 + for (const req of keys) { + if (req.url.includes(id + "/")) { + await cache.delete(req) + n++ + } + } + if (n) log("purged cached entries for local model", {id, n}) + } catch (/** @type {any} */ e) { + log("purgeCachedModel failed", {id, message: e?.message || String(e)}) + } +} + +function ensureLocalFetchPatch() { + const sg = /** @type {any} */ (self) + if (sg.__llmFetchPatched) return + sg.__llmFetchPatched = true + const realFetch = self.fetch.bind(self) + self.fetch = (input, init) => { + try { + const url = typeof input === "string" ? input : /** @type {any} */ (input)?.url || "" + for (const [id, entry] of localModelFiles) { + const marker = "/" + id + "/resolve/" + const at = url.indexOf(marker) + if (at === -1) continue + const after = url.slice(at + marker.length) // "/" + const rel = after.substring(after.indexOf("/") + 1) + const file = + entry.files.get(rel) || entry.files.get(rel.split("/").pop() || "") + if (file) + return Promise.resolve( + new Response(file, { + status: 200, + headers: {"content-length": String(file.size)}, + }) + ) + // transformers.js treats several of these as optional and swallows the + // 404 into `{}` — a missing generation_config.json costs you + // eos_token_id, a missing tokenizer_config.json costs you the chat + // template. Both produce garbage rather than an error, so say so. + log("local model: 404", {id, rel, have: [...entry.files.keys()]}) + return Promise.resolve( + new Response("local model file not found: " + rel, {status: 404}) + ) + } + } catch {} + return realFetch(input, init) + } +} + +/** @param {Promise} promise @param {number} ms @param {string} label */ +function withTimeout(promise, ms, label) { + return Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(label + " timed out")), ms) + ), + ]) +} + +/** @param {string} [modelId] @param {string} [dtypeOverride] */ +async function loadModel(modelId, dtypeOverride) { + modelId = modelId || DEFAULT_MODEL_ID + // A dtype change for the same id must force a reload (e.g. switching a custom + // HF model from q4f16 → q4 to match the variant the repo actually ships). + const dtypeChanged = dtypeOverride && currentDtype && dtypeOverride !== currentDtype + if (generator && (currentModelId !== modelId || dtypeChanged)) { + log("loadModel: switching model — releasing previous", {from: currentModelId, to: modelId, dtypeChanged}) + await releaseGenerator() + } + if (generator) return + if (loading && loadingPromise) return loadingPromise + currentModelId = modelId + loading = true + /** @type {() => void} */ + let resolveLoading = () => {} + loadingPromise = new Promise((r) => (resolveLoading = r)) + lastLoadError = null + const reg = localModelFiles.get(modelId) + ensureLocalFetchPatch() + // dtype precedence: explicit override (picker) > registered upload > catalogue + // entry > q4f16. Lets you pick the quantization suffix for any HF id. + const baseDef = reg + ? {dtype: reg.dtype} + : LOCAL_MODELS.find((m) => m.id === modelId) || {dtype: "q4f16"} + const modelDef = {...baseDef, dtype: dtypeOverride || baseDef.dtype} + currentDtype = modelDef.dtype + log("loadModel: start", {modelId, dtype: modelDef.dtype}) + + try { + broadcast({type: "status", message: "Loading transformers.js…"}) + TF = await import(/* @vite-ignore */ CDN) + TF.env.allowLocalModels = false + installWeightsProgress() // byte-level progress for the external-data weights blob + // transformers.js routes ALL of its I/O through `env.fetch`, which it binds + // from `globalThis.fetch` once, at module-eval time. Anything we patch onto + // `self.fetch` after the import is therefore invisible to it. Delegate to + // the live `self.fetch` so patch order stops mattering. + TF.env.fetch = (/** @type {any} */ input, /** @type {any} */ init) => + self.fetch(input, init) + // A local upload is a fake repo: transformers.js caches it under + // `local//` and reads the cache BEFORE calling fetch. Re-upload + // a corrected (or entirely different) folder under the same name and you get + // the first upload's bytes forever — often a tokenizer from one model beside + // weights from another, which reads as fluent gibberish. + TF.env.useBrowserCache = !reg + if (navigator.storage?.persist) await navigator.storage.persist() + } catch (/** @type {any} */ err) { + lastLoadError = err + console.error("[llm worker] failed to load transformers.js from", CDN, err) + broadcast({type: "status", message: "Failed to load transformers.js: " + (err?.message || err)}) + loading = false + loadingPromise = null + resolveLoading() + return + } + + const isFirstCompile = !compiledModels.has(modelId) + /** @param {string} backend */ + function progressCb(backend) { + /** @type {Map} */ + const fileProgress = new Map() + return (/** @type {any} */ p) => { + if (p.status === "progress" && p.progress != null && p.file) { + fileProgress.set(p.file, {loaded: p.loaded || 0, total: p.total || 0}) + const shortName = p.file.split("/").pop() || p.file + let tl = 0, + ts = 0 + for (const f of fileProgress.values()) { + tl += f.loaded + ts += f.total + } + const overall = ts > 0 ? Math.round((100 * tl) / ts) : Math.round(p.progress) + broadcast({ + type: "status", + message: `Downloading ${shortName}… ${Math.round(p.progress)}% (overall ${overall}%)`, + }) + } else if (p.status === "ready") { + broadcast({ + type: "status", + message: isFirstCompile + ? `⚠️ Compiling shaders for ${backend} (first time — might freeze for a bit)…` + : `Compiling shaders for ${backend}…`, + }) + } + } + } + + const hasWebGPU = typeof navigator !== "undefined" && !!navigator.gpu + const attempts = [] + if (hasWebGPU) attempts.push({device: "webgpu", label: "WebGPU"}) + attempts.push({device: undefined, label: "WASM"}) + // Tell the page WHY a backend isn't in the running, so "it went to WASM" is + // never a mystery. navigator.gpu is absent in incognito and in any context + // that doesn't expose WebGPU (e.g. SharedWorkers — but this is a dedicated + // Worker, where it should be present). + if (!hasWebGPU) { + broadcast({type: "status", message: "WebGPU unavailable (navigator.gpu missing — incognito or unsupported context); using WASM"}) + } + log("loadModel: backends to try", {hasWebGPU, order: attempts.map((a) => a.label)}) + + for (const attempt of attempts) { + // The WASM/CPU backend can't execute fp16 — q4f16/fp16 trap with "memory + // access out of bounds". Downgrade to the integer-quant equivalent for the + // WASM attempt only; WebGPU keeps the requested dtype. + const attemptDtype = attempt.device ? modelDef.dtype : wasmSafeDtype(modelDef.dtype) + try { + log("loadModel: trying backend", {backend: attempt.label, dtype: attemptDtype, requested: modelDef.dtype}) + generator = await withTimeout( + TF.pipeline("text-generation", modelId, { + dtype: attemptDtype, + device: attempt.device, + progress_callback: progressCb(attempt.label), + }), + 180000, + attempt.label + " pipeline" + ) + compiledModels.add(modelId) + // Record the dtype that actually loaded (WASM may have downgraded), so + // the reload-on-dtype-change check and model-info report stay accurate. + currentDtype = attemptDtype + // Probe the model for attention output support + try { + const model = generator.model + /** @type {Record} */ + const sessionInfo = {} + const sessions = model?.sessions || {} + for (const [name, session] of Object.entries(sessions)) { + const outNames = session?.outputNames || [] + sessionInfo[name] = outNames + } + broadcast({type: "model-info", model: modelId, sessions: sessionInfo, + config: model?.config ? { + output_attentions: model.config.output_attentions, + num_hidden_layers: model.config.num_hidden_layers, + num_attention_heads: model.config.num_attention_heads, + model_type: model.config.model_type, + } : null, + }) + } catch (/** @type {any} */ e) { + broadcast({type: "model-info", model: modelId, error: e.message}) + } + log("loadModel: ready", {modelId, backend: attempt.label, dtype: modelDef.dtype}) + broadcast({type: "status", message: `Model ready (${attempt.label})`}) + broadcast({type: "ready", model: modelId, device: attempt.label}) + lastLoadError = null + break + } catch (/** @type {any} */ err) { + lastLoadError = err + // Keep the actual cause — the device-fallback loop used to discard it, + // leaving only "WASM failed" with no way to see what actually broke. + log("loadModel: backend failed", {backend: attempt.label, modelId, message: err?.message || String(err)}) + console.error(`[llm worker] ${attempt.label} load failed for ${modelId}:`, err) + broadcast({ + type: "status", + message: `${attempt.label} failed: ${err?.message || err}` + (attempt.device ? " — trying WASM…" : ""), + }) + } + } + // Every device attempt failed — surface the last real error to the main + // thread (callers otherwise report a contentless "Model not loaded"). + if (!generator) { + log("loadModel: all backends failed", {modelId, message: lastLoadError?.message || String(lastLoadError)}) + console.error(`[llm worker] all backends failed to load ${modelId}:`, lastLoadError) + broadcast({type: "status", message: modelLoadError()}) + broadcast({type: "load-error", model: modelId, message: modelLoadError()}) + } + loading = false + loadingPromise = null + resolveLoading() +} + +// Top-k of a logits row as softmax probabilities (two passes + a tiny top-k +// scan — cheap enough to run every decode step). Adapted from rlm. +/** @param {ArrayLike} data @param {number} vocab @param {number} k */ +function topkFromLogits(data, vocab, k) { + let max = -Infinity + for (let i = 0; i < vocab; i++) if (data[i] > max) max = data[i] + let sum = 0 + /** @type {number[]} */ + const idx = [] + /** @type {number[]} */ + const val = [] + for (let i = 0; i < vocab; i++) { + const v = data[i] + sum += Math.exp(v - max) + if (idx.length < k) { + idx.push(i) + val.push(v) + } else { + let mi = 0 + for (let j = 1; j < k; j++) if (val[j] < val[mi]) mi = j + if (v > val[mi]) { + val[mi] = v + idx[mi] = i + } + } + } + return idx + .map((id, j) => ({id, p: Math.exp(val[j] - max) / sum})) + .sort((a, b) => b.p - a.p) +} + +// Optional OpenAI-style sampling params, omitting off/default values so a +// provider that doesn't support one isn't upset. +// Parse a tool-call arguments value (string or already-object) → object. +/** @param {any} v @returns {any} */ +function safeJson(v) { + if (v && typeof v === "object") return v + if (typeof v !== "string") return {} + try { + return JSON.parse(v) + } catch { + return {} + } +} + +/** @param {WorkerConfig} config */ +function samplingExtras(config) { + /** @type {Record} */ + const p = {} + if (config.topK > 0) p.top_k = config.topK + if (config.minP > 0) p.min_p = config.minP + if (config.repetitionPenalty && config.repetitionPenalty !== 1) + p.repetition_penalty = config.repetitionPenalty + if (config.frequencyPenalty) p.frequency_penalty = config.frequencyPenalty + if (config.presencePenalty) p.presence_penalty = config.presencePenalty + if (config.seed != null && config.seed !== "") p.seed = config.seed + return p +} + +// Render a chat conversation to a plain ChatML prompt string. Fallback for +// local models (base / coder checkpoints) whose tokenizer ships no +// `chat_template` — without this, transformers.js throws inside the pipeline. +/** @param {any[]} messages */ +function messagesToPrompt(messages) { + let out = "" + for (const m of messages) { + const content = typeof m.content === "string" ? m.content : "" + out += `<|im_start|>${m.role || "user"}\n${content}<|im_end|>\n` + } + return out + "<|im_start|>assistant\n" +} + +/** @param {Gen} gen @param {any} input @param {WorkerConfig} config */ +async function doGenerateLocal(gen, input, config) { + const tokenizer = generator.tokenizer + const isText = typeof input === "string" + const temperature = config.temperature ?? 0.7 + const topk = config.topk | 0 + const maxNewTokens = config.maxNewTokens ?? 2048 + + // The text-generation pipeline applies the chat template itself when handed a + // messages array. If the model has no template, do it ourselves and pass a + // plain string instead (a string input skips templating entirely). + const usedChatFallback = !isText && !tokenizer.chat_template + let genInput = usedChatFallback ? messagesToPrompt(input) : input + let templateTools = !isText && config.tools?.length ? config.tools : undefined + if (templateTools && tokenizer.chat_template) { + const rendered = tokenizer.apply_chat_template(genInput, { + tools: templateTools, + tokenize: false, + add_generation_prompt: true, + }) + let exposesTools = false + for (const tool of templateTools) { + const name = tool?.function?.name || tool?.name + if (name && rendered.includes(name)) { + exposesTools = true + break + } + } + if (!exposesTools) { + const system = config.toolSystem + if (system) { + genInput = [...genInput] + if (genInput[0]?.role === "system") + genInput[0] = { + ...genInput[0], + content: [system, genInput[0].content].filter(Boolean).join("\n\n"), + } + else genInput.unshift({role: "system", content: system}) + } + templateTools = undefined + } + } + // ChatML is Qwen's format, not a universal one. Handing it to a Llama / Gemma / + // LFM2 checkpoint produces confident nonsense, so this fallback is a last + // resort and must be visible — most often it means tokenizer_config.json (or + // chat_template.jinja) never loaded, which transformers.js swallows into `{}`. + if (usedChatFallback) { + log("doGenerateLocal: NO CHAT TEMPLATE — falling back to ChatML", { + model: currentModelId, + warning: + "output will be garbage unless this model is ChatML-native; check that tokenizer_config.json / chat_template.jinja loaded", + }) + broadcast({ + type: "status", + message: `⚠️ ${currentModelId} has no chat template — using ChatML. Output may be garbage.`, + }) + } + + // On the no-template fallback a base model may not stop at the turn boundary + // and keeps going, role-playing further <|im_start|> turns — which surface as + // repeated text. For such models these aren't special tokens, so they stream + // as literal text and we can cut the output there. + const stopMarkers = usedChatFallback ? ["<|im_end|>", "<|im_start|>"] : [] + + let promptTokens = 0 + try { + const prompt = + typeof genInput === "string" + ? genInput + : tokenizer.apply_chat_template(genInput, { + tokenize: false, + add_generation_prompt: true, + ...(templateTools ? {tools: templateTools} : {}), + }) + promptTokens = tokenizer.encode(prompt).length + } catch (/** @type {any} */ e) { + log("doGenerateLocal: prompt build/encode failed (continuing)", { + model: currentModelId, + hasChatTemplate: !!tokenizer.chat_template, + usedFallback: usedChatFallback, + message: e?.message || String(e), + }) + } + + const tStart = performance.now() + let tFirst = 0 + let full = "" + /** @type {any} */ + let heartbeat = null + + let stopped = false + let posted = 0 + const streamer = new TF.TextStreamer(tokenizer, { + skip_prompt: true, + skip_special_tokens: true, + callback_function: (/** @type {string} */ text) => { + if (stopped) return + if (!tFirst) { + tFirst = performance.now() + if (heartbeat) { + clearInterval(heartbeat) + heartbeat = null + } + log("doGenerateLocal: first token", { + ttftMs: Math.round(tFirst - tStart), + promptTokens, + }) + } + full += text + let cut = -1 + for (const m of stopMarkers) { + const i = full.indexOf(m) + if (i !== -1 && (cut === -1 || i < cut)) cut = i + } + if (cut !== -1) { + full = full.slice(0, cut) + stopped = true + } + gen.fullText = full + const delta = full.slice(posted) + posted = full.length + if (delta) post(gen, {type: "token", delta, text: full}) + }, + }) + + // Teaching probe: a plain function is a valid logits processor; we read the + // next-token distribution and stream the top candidates, returning logits + // unchanged so generation is untouched. + let step = 0 + const logits_processor = + topk > 0 + ? [ + (/** @type {any} */ inputIds, /** @type {any} */ logits) => { + if (step < PREDICTION_CAP) { + try { + const vocab = logits.dims.at(-1) + const data = logits.data + const candidates = topkFromLogits(data, vocab, topk).map( + ({id, p}) => ({token: tokenizer.decode([id]), p: +p.toFixed(4)}) + ) + // Full-distribution entropy (bits) from raw logits + let mx = -Infinity + for (let j = 0; j < vocab; j++) if (data[j] > mx) mx = data[j] + let sm = 0 + for (let j = 0; j < vocab; j++) sm += Math.exp(data[j] - mx) + let ent = 0 + for (let j = 0; j < vocab; j++) { + const p = Math.exp(data[j] - mx) / sm + if (p > 0) ent -= p * Math.log2(p) + } + post(gen, {type: "prediction", step, candidates, entropy: +ent.toFixed(3)}) + } catch {} + } + step++ + return logits + }, + ] + : undefined + + // The await below is the big silent gap: prefill of the whole prompt runs + // before the first token streams. On the WASM backend a long prompt can take + // many seconds, so emit a heartbeat (also resets a caller's inactivity + // watchdog) and log the parameters we're running with. + log("doGenerateLocal: generating", { + model: currentModelId, + dtype: currentDtype, + promptTokens, + maxNewTokens, + temperature, + inputKind: typeof genInput === "string" ? "text" : "messages", + }) + heartbeat = setInterval(() => { + if (tFirst) return + const s = Math.round((performance.now() - tStart) / 1000) + broadcast({type: "status", message: `Generating… ${s}s (prefilling, no token yet)`}) + log("doGenerateLocal: still prefilling", {elapsedS: s}) + }, 2000) + + let output + try { + // Only what transformers.js actually reads: its multinomial sampler consults + // `top_k` alone. `top_p` and `typical_p` are declared on GenerationConfig and + // never looked at; `min_p` isn't even a field. Passing them would just make + // the decode stats below lie about what ran. See PROVIDER_CAPS.local. + output = await generator(genInput, { + ...(templateTools ? {tools: templateTools} : {}), + max_new_tokens: maxNewTokens, + do_sample: temperature > 0, + temperature, + ...(config.topK > 0 ? {top_k: config.topK} : {}), + repetition_penalty: config.repetitionPenalty ?? 1.1, + ...(config.noRepeatNgramSize > 0 + ? {no_repeat_ngram_size: config.noRepeatNgramSize} + : {}), + streamer, + logits_processor, + }) + } finally { + if (heartbeat) { + clearInterval(heartbeat) + heartbeat = null + } + } + + const text = + full || output?.[0]?.generated_text?.at(-1)?.content || "" + + let genTokens = 0 + try { + genTokens = tokenizer.encode(text).length + } catch {} + const now = performance.now() + post(gen, { + type: "stats", + provider: "local", + model: currentModelId, + promptTokens, + genTokens, + ttftMs: tFirst ? Math.round(tFirst - tStart) : null, + totalMs: Math.round(now - tStart), + tokPerSec: tFirst ? +(genTokens / ((now - tFirst) / 1000)).toFixed(1) : null, + decode: { + greedy: !(temperature > 0), + temperature, + top_k: config.topK > 0 ? config.topK : null, // null = the model's own generation_config + repetition_penalty: config.repetitionPenalty ?? 1.1, + no_repeat_ngram_size: config.noRepeatNgramSize || 0, + maxNewTokens, + }, + }) + // Local has no native tool API; the client parses the model's text (XML/JSON) + // when tools were requested via the system prompt. + return {text, toolCalls: null, toolMode: templateTools ? "template" : "text"} +} + +// --------------------------------------------------------------------------- +// OpenRouter (SSE) — with logprobs → predictions, usage → stats +// --------------------------------------------------------------------------- + +// Chat-only providers (OpenRouter, etc.) can't do raw completion — they'd +// *answer* the text. So a raw continuation is framed as a chat turn instructing +// the model to continue and emit ONLY the continuation. +const CONTINUE_SYS = + "You are a text-continuation engine inside a writing tool. Continue the user's text seamlessly from exactly where it ends, matching its voice, tense, and style. Output ONLY the continuation — no preamble, no commentary, no explanation, no quotation marks — and never restate or acknowledge the user's text. If it ends mid-word or mid-sentence, finish it." + +/** @param {Gen} gen @param {any} input @param {WorkerConfig} config */ +async function doGenerateOpenRouter(gen, input, config) { + const isText = typeof input === "string" + const temperature = config.temperature ?? 0.7 + const topk = config.topk | 0 + /** @type {Record} */ + const body = { + model: config.model || "anthropic/claude-sonnet-4", + stream: true, + stream_options: {include_usage: true}, + temperature, + } + if (config.topP != null) body.top_p = config.topP + Object.assign(body, samplingExtras(config)) + // Always chat; a raw string becomes a "continue this" chat turn. + body.messages = isText + ? [{role: "system", content: CONTINUE_SYS}, {role: "user", content: input}] + : input + if (config.maxNewTokens) body.max_tokens = config.maxNewTokens + else if (config.contextLength) { + const inputEstimate = Math.ceil(JSON.stringify(input).length / 4) + const maxOutput = config.maxCompletionTokens || 8192 + body.max_tokens = Math.min( + maxOutput, + Math.max(1024, config.contextLength - inputEstimate - 256) + ) + } + // Native function calling streams just like a normal completion: we emit + // content deltas live AND reassemble the tool_call deltas (each chunk carries + // partial {index, id?, function:{name?, arguments?}}) so the text still streams + // token-by-token instead of arriving all at once. logprobs and tools don't + // combine on OpenRouter, so only request logprobs when no tools are in play. + const hasTools = !!(config.tools && config.tools.length) + if (hasTools) { + body.tools = config.tools + body.tool_choice = "auto" + } else if (topk > 0) { + body.logprobs = true + body.top_logprobs = Math.min(topk, 20) // OpenAI caps top_logprobs at 20 + } + + const res = await fetch("https://openrouter.ai/api/v1/chat/completions", { + method: "POST", + headers: { + Authorization: "Bearer " + config.apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + signal: gen.abortController.signal, + }) + if (!res.ok) throw new Error("OpenRouter: " + (await res.text())) + + const tStart = performance.now() + let tFirst = 0 + let full = "" + let step = 0 + /** @type {any} */ + let usage = null + const reader = /** @type {ReadableStream} */ (res.body).getReader() + const decoder = new TextDecoder() + let buf = "" + // Reassemble streamed tool_calls: each chunk carries partial deltas keyed by + // `index`; `id`/`name` arrive once, `arguments` accrues as a JSON string. + /** @type {Map} */ + const toolAcc = new Map() + + const handleLine = (/** @type {string} */ line) => { + if (!line.startsWith("data: ")) return + const data = line.slice(6).trim() + if (data === "[DONE]") return + let parsed + try { + parsed = JSON.parse(data) + } catch { + return + } + if (parsed.usage) usage = parsed.usage + const choice = parsed.choices?.[0] + const delta = choice?.delta?.content + if (delta) { + if (!tFirst) tFirst = performance.now() + full += delta + gen.fullText = full + post(gen, {type: "token", delta, text: full}) + } + for (const tc of choice?.delta?.tool_calls || []) { + const idx = tc.index ?? 0 + let acc = toolAcc.get(idx) + if (!acc) { + acc = {id: undefined, name: undefined, args: ""} + toolAcc.set(idx, acc) + } + if (tc.id) acc.id = tc.id + if (tc.function?.name) acc.name = tc.function.name + if (tc.function?.arguments) acc.args += tc.function.arguments + } + if (topk > 0) { + for (const item of choice?.logprobs?.content || []) { + if (step >= PREDICTION_CAP) break + const candidates = (item.top_logprobs || []).map((/** @type {any} */ tl) => ({ + token: tl.token, + p: +Math.exp(tl.logprob).toFixed(4), + })) + if (candidates.length) post(gen, {type: "prediction", step, candidates}) + step++ + } + } + } + + while (true) { + const {done, value} = await reader.read() + if (done) break + buf += decoder.decode(value, {stream: true}) + const lines = buf.split("\n") + buf = lines.pop() || "" + for (const line of lines) handleLine(line) + } + if (buf.trim()) for (const line of buf.split("\n")) handleLine(line) + + const now = performance.now() + const genTokens = usage?.completion_tokens ?? null + post(gen, { + type: "stats", + provider: "openrouter", + model: config.model, + promptTokens: usage?.prompt_tokens ?? null, + genTokens, + ttftMs: tFirst ? Math.round(tFirst - tStart) : null, + totalMs: Math.round(now - tStart), + tokPerSec: + tFirst && genTokens + ? +(genTokens / ((now - tFirst) / 1000)).toFixed(1) + : null, + decode: {greedy: temperature === 0, temperature}, + }) + // Build structured tool calls from the accumulated deltas (named ones only). + const toolCalls = toolAcc.size + ? [...toolAcc.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([, acc]) => ({id: acc.id, name: acc.name, args: safeJson(acc.args)})) + .filter((tc) => tc.name) + : [] + return {text: full, toolCalls: toolCalls.length ? toolCalls : null} +} + +// --------------------------------------------------------------------------- +// Ollama (NDJSON) — tokens + basic stats (no logprobs available) +// --------------------------------------------------------------------------- + +/** @param {Gen} gen @param {any} input @param {WorkerConfig} config */ +async function doGenerateOllama(gen, input, config) { + const isText = typeof input === "string" + const baseUrl = (config.url || "http://localhost:11434").replace(/\/$/, "") + /** @type {Record} */ + const body = { + model: config.model || "llama3.2", + stream: true, + options: { + temperature: config.temperature ?? 0.7, + ...(config.topP != null ? {top_p: config.topP} : {}), + ...(config.topK > 0 ? {top_k: config.topK} : {}), + ...(config.minP > 0 ? {min_p: config.minP} : {}), + ...(config.typicalP != null && config.typicalP < 1 + ? {typical_p: config.typicalP} + : {}), + ...(config.repetitionPenalty && config.repetitionPenalty !== 1 + ? {repeat_penalty: config.repetitionPenalty} + : {}), + ...(config.frequencyPenalty ? {frequency_penalty: config.frequencyPenalty} : {}), + ...(config.presencePenalty ? {presence_penalty: config.presencePenalty} : {}), + ...(config.seed != null && config.seed !== "" ? {seed: config.seed} : {}), + }, + } + if (isText) body.prompt = input + else body.messages = input + + // Native tool calling (chat only): non-streaming, parse message.tool_calls. + if (config.tools && config.tools.length && !isText) { + body.tools = config.tools + body.stream = false + const t0 = performance.now() + const r = await fetch(baseUrl + "/api/chat", { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify(body), + signal: gen.abortController.signal, + }) + if (!r.ok) throw new Error("Ollama: " + (await r.text())) + const data = await r.json() + const text = data.message?.content || "" + if (text) post(gen, {type: "token", delta: text, text}) + gen.fullText = text + const toolCalls = (data.message?.tool_calls || []).map((/** @type {any} */ tc, /** @type {number} */ i) => ({ + id: tc.id || "call_" + i, + name: tc.function?.name, + args: safeJson(tc.function?.arguments), // Ollama already gives an object + })) + post(gen, { + type: "stats", + provider: "ollama", + model: config.model, + promptTokens: data.prompt_eval_count ?? null, + genTokens: data.eval_count ?? null, + ttftMs: null, + totalMs: Math.round(performance.now() - t0), + tokPerSec: null, + decode: {greedy: (config.temperature ?? 0.7) === 0, temperature: config.temperature ?? 0.7}, + }) + return {text, toolCalls} + } + + const res = await fetch(baseUrl + (isText ? "/api/generate" : "/api/chat"), { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify(body), + signal: gen.abortController.signal, + }) + if (!res.ok) throw new Error("Ollama: " + (await res.text())) + + const tStart = performance.now() + let tFirst = 0 + let full = "" + /** @type {any} */ + let final = null + const reader = /** @type {ReadableStream} */ (res.body).getReader() + const decoder = new TextDecoder() + let buf = "" + while (true) { + const {done, value} = await reader.read() + if (done) break + buf += decoder.decode(value, {stream: true}) + const lines = buf.split("\n") + buf = lines.pop() || "" + for (const line of lines) { + if (!line.trim()) continue + try { + const parsed = JSON.parse(line) + if (parsed.done) final = parsed + const content = isText ? parsed.response : parsed.message?.content + if (content) { + if (!tFirst) tFirst = performance.now() + full += content + gen.fullText = full + post(gen, {type: "token", delta: content, text: full}) + } + } catch {} + } + } + const now = performance.now() + const genTokens = final?.eval_count ?? null + post(gen, { + type: "stats", + provider: "ollama", + model: config.model, + promptTokens: final?.prompt_eval_count ?? null, + genTokens, + ttftMs: tFirst ? Math.round(tFirst - tStart) : null, + totalMs: Math.round(now - tStart), + tokPerSec: + final?.eval_count && final?.eval_duration + ? +(final.eval_count / (final.eval_duration / 1e9)).toFixed(1) + : null, + decode: {temperature: config.temperature ?? 0.7}, + }) + return {text: full, toolCalls: null} +} + +// --------------------------------------------------------------------------- +// Predict: a single forward pass → the next-token distribution at the cursor. +// Powers "predict as you type" without generating. Local reads the real logits; +// OpenRouter is best-effort via the /completions logprobs (chat-only models +// won't return any — the caller just sees an empty list). +// --------------------------------------------------------------------------- + +/** @param {any} text @param {WorkerConfig} config */ +async function predictLocal(text, config) { + const topk = Math.max(1, config.topk | 0 || 10) + const tokenizer = generator.tokenizer + /** @type {Candidate[]} */ + let candidates = [] + await generator(text || " ", { + max_new_tokens: 1, + do_sample: false, + logits_processor: [ + (/** @type {any} */ inputIds, /** @type {any} */ logits) => { + if (!candidates.length) { + try { + const vocab = logits.dims.at(-1) + candidates = topkFromLogits(logits.data, vocab, topk).map(({id, p}) => ({ + token: tokenizer.decode([id]), + p: +p.toFixed(4), + })) + } catch {} + } + return logits + }, + ], + }) + return candidates +} + +// Score every token position in the input — one forward pass per position, +// extracting exact probability (from full vocab), rank, entropy, and top-k +// alternatives. Powers the attention heatmap: "how surprised was the model +// by what you actually wrote?" +/** @param {Gen} gen @param {any} text @param {WorkerConfig} config */ +async function scoreTokensLocal(gen, text, config) { + const tokenizer = generator.tokenizer + const ids = tokenizer.encode(text) + /** @type {any[]} */ + const scores = [] + + for (let i = 0; i < ids.length; i++) { + if (gen.abortController.signal.aborted) break + + const prefix = i === 0 ? "" : tokenizer.decode(ids.slice(0, i), {skip_special_tokens: true}) + /** @type {any} */ + let result = null + + await generator(prefix || " ", { + max_new_tokens: 1, + do_sample: false, + logits_processor: [ + (/** @type {any} */ inputIds, /** @type {any} */ logits) => { + if (result) return logits // only first call matters + const vocab = logits.dims.at(-1) + const data = logits.data + const actualId = ids[i] + + // Softmax (numerically stable) + let mx = -Infinity + for (let j = 0; j < vocab; j++) if (data[j] > mx) mx = data[j] + let sm = 0 + for (let j = 0; j < vocab; j++) sm += Math.exp(data[j] - mx) + + // Exact probability + rank of actual next token + const actualP = Math.exp(data[actualId] - mx) / sm + let rank = 1 + const actualLogit = data[actualId] + for (let j = 0; j < vocab; j++) { + if (data[j] > actualLogit + 1e-8) rank++ + } + + // Full-distribution entropy + let ent = 0 + for (let j = 0; j < vocab; j++) { + const p = Math.exp(data[j] - mx) / sm + if (p > 0) ent -= p * Math.log2(p) + } + + // Top-k alternatives for context + const topk = topkFromLogits(data, vocab, 10).map(({id, p}) => ({ + token: tokenizer.decode([id]), + p: +p.toFixed(4), + })) + + result = { + token: tokenizer.decode([actualId]), + p: +actualP.toFixed(6), + rank, + entropy: +ent.toFixed(3), + topk, + } + return logits + }, + ], + }) + + if (result) scores.push(result) + post(gen, {type: "score-progress", step: i, total: ids.length}) + } + + // Build character position spans via cumulative decode + const spans = [] + let prevLen = 0 + for (let i = 0; i < ids.length && i < scores.length; i++) { + const partial = tokenizer.decode(ids.slice(0, i + 1), {skip_special_tokens: true}) + const from = prevLen + const to = partial.length + if (to > from) spans.push({from, to, index: i}) + prevLen = partial.length + } + const decoded = tokenizer.decode(ids, {skip_special_tokens: true}) + + post(gen, {type: "token-scores", scores, spans, decoded}) +} + +// Perturbation-based token importance (erasure-based attribution). +// For each token, mask it out and measure how much the model's predictions +// change — tokens whose removal most affects the output are most important. +// N+1 forward passes (one baseline + one per token). Genuine importance +// scores, not a proxy. +/** @param {Gen} gen @param {any} text @param {WorkerConfig} config */ +async function computeImportanceLocal(gen, text, config) { + const model = generator.model + const tokenizer = generator.tokenizer + const inputs = tokenizer(text, {return_tensor: true}) + const ids = Array.from(inputs.input_ids.data) + const seqLen = ids.length + if (seqLen < 2) { post(gen, {type: "importance-scores", decoded: text, spans: []}); return } + + const vocab = model.config?.vocab_size || 151936 + const lastOff = (seqLen - 1) * vocab + + // Helper: softmax of logits at the last position + /** @param {ArrayLike} logitsData */ + function lastProbs(logitsData) { + const p = new Float64Array(vocab) + let mx = -Infinity + for (let j = 0; j < vocab; j++) { const v = logitsData[lastOff + j]; if (v > mx) mx = v } + let sm = 0 + for (let j = 0; j < vocab; j++) { p[j] = Math.exp(logitsData[lastOff + j] - mx); sm += p[j] } + for (let j = 0; j < vocab; j++) p[j] /= sm + return p + } + + // JS divergence between two probability distributions + /** @param {ArrayLike} p @param {ArrayLike} q */ + function jsDiv(p, q) { + let d = 0 + for (let j = 0; j < vocab; j++) { + const m = (p[j] + q[j]) / 2 + if (m > 0) { + if (p[j] > 0) d += p[j] * Math.log(p[j] / m) + if (q[j] > 0) d += q[j] * Math.log(q[j] / m) + } + } + return d / 2 + } + + // 1. Baseline forward pass + broadcast({type: "status", message: "Saliency: baseline…"}) + const baseOut = await model(inputs) + const baseP = lastProbs(baseOut.logits.data) + + // 2. For each token, replace it with unk/pad and measure divergence + const importance = new Float64Array(seqLen) + const idsData = new BigInt64Array(inputs.input_ids.data) + + for (let i = 0; i < seqLen; i++) { + if (gen.abortController.signal.aborted) break + broadcast({type: "status", message: `Saliency: ${i + 1}/${seqLen}…`}) + post(gen, {type: "score-progress", step: i, total: seqLen}) + + // Replace token i with 0 (unk/pad) — semantic erasure + const modIds = new BigInt64Array(idsData) + modIds[i] = 0n + const modInputs = { + input_ids: new TF.Tensor("int64", modIds, inputs.input_ids.dims), + attention_mask: inputs.attention_mask, + } + const modOut = await model(modInputs) + const modP = lastProbs(modOut.logits.data) + importance[i] = jsDiv(baseP, modP) + } + + // Normalize to [0, 1] + let lo = Infinity, hi = -Infinity + for (let i = 0; i < seqLen; i++) { + if (importance[i] < lo) lo = importance[i] + if (importance[i] > hi) hi = importance[i] + } + const range = hi - lo || 1 + + // Build spans via cumulative decode + const spans = [] + let prevLen = 0 + for (let i = 0; i < ids.length; i++) { + const partial = tokenizer.decode(ids.slice(0, i + 1), {skip_special_tokens: true}) + const from = prevLen + const to = partial.length + if (to > from) { + spans.push({from, to, importance: (importance[i] - lo) / range}) + } + prevLen = partial.length + } + const decoded = tokenizer.decode(ids, {skip_special_tokens: true}) + + post(gen, {type: "importance-scores", decoded, spans}) + broadcast({type: "status", message: ""}) +} + +// REAL attention weights (not erasure). For models exported with an `attentions` +// output (see glomper-tuning/onnx_attn.py — shape [batch, layers, heads, seq, +// seq], post-softmax, rows sum to 1 over keys). One forward pass; we reduce the +// big [L,H,S,S] tensor server-side to two per-(layer,head) vectors so the client +// can re-slice layer/head/view instantly without re-running: +// received[l][h][j] = mean over queries i≥j of A[i,j] — how attended-to key j +// is across the whole sequence (causal: only i≥j see it) +// fromLast[l][h][j] = A[S-1, j] — what the final token +// (the next-token prediction position) attends to +// If the model has no `attentions` output, posts {supported:false}. +/** @param {Gen} gen @param {any} text @param {WorkerConfig} config */ +async function computeAttentionWeightsLocal(gen, text, config) { + const model = generator.model + const tokenizer = generator.tokenizer + const inputs = tokenizer(text, {return_tensor: true}) + const ids = Array.from(inputs.input_ids.data) + const seqLen = ids.length + if (seqLen < 2) { + post(gen, {type: "attention-weights", supported: true, dims: {layers: 0, heads: 0, seq: seqLen}, received: new Float32Array(0), fromLast: new Float32Array(0), spans: [], tokens: [], decoded: text}) + return + } + + log("computeAttentionWeights: forward pass", {model: currentModelId, seqLen}) + broadcast({type: "status", message: "Attention: forward pass…"}) + const outputs = await model(inputs) + const att = outputs.attentions + if (!att || !att.dims || att.dims.length !== 5) { + // Report what the forward pass actually produced so the UI can say why + // (wrong model vs. an attentions output that's named/shaped unexpectedly). + const outputKeys = Object.keys(outputs || {}) + log("computeAttentionWeights: no usable attentions output", {model: currentModelId, outputKeys, attnDims: att?.dims || null}) + console.warn(`[llm worker] ${currentModelId} has no usable attentions output. forward outputs:`, outputKeys, att ? `(attentions dims: [${att.dims}])` : "(no `attentions` key)") + post(gen, {type: "attention-weights", supported: false, model: currentModelId, outputKeys, attnDims: att?.dims || null}) + broadcast({type: "status", message: ""}) + return + } + + // dims: [batch, layers, heads, queries, keys] + const [, L, H, Sq, Sk] = att.dims + const data = att.data // Float32Array, batch index 0 + const S = Sk + const strideL = H * Sq * Sk + const strideH = Sq * Sk + const strideQ = Sk + const received = new Float32Array(L * H * S) + const fromLast = new Float32Array(L * H * S) + const lastQ = Sq - 1 + for (let l = 0; l < L; l++) { + for (let h = 0; h < H; h++) { + const base = l * strideL + h * strideH + const out = (l * H + h) * S + const lastRow = base + lastQ * strideQ + for (let j = 0; j < S; j++) fromLast[out + j] = data[lastRow + j] + for (let j = 0; j < S; j++) { + let sum = 0, cnt = 0 + for (let i = j; i < Sq; i++) { sum += data[base + i * strideQ + j]; cnt++ } + received[out + j] = cnt ? sum / cnt : 0 + } + } + } + + // Char-position spans (skip_special_tokens, like the other passes) keyed by + // token index so the client can map a span back to its attention row/col. + const spans = [] + const tokens = [] + let prevLen = 0 + for (let i = 0; i < seqLen; i++) { + const partial = tokenizer.decode(ids.slice(0, i + 1), {skip_special_tokens: true}) + const from = prevLen + const to = partial.length + tokens.push(tokenizer.decode([ids[i]])) + if (to > from) spans.push({from, to, index: i}) + prevLen = partial.length + } + const decoded = tokenizer.decode(ids, {skip_special_tokens: true}) + + post(gen, {type: "attention-weights", supported: true, dims: {layers: L, heads: H, seq: S}, received, fromLast, spans, tokens, decoded}) + broadcast({type: "status", message: ""}) +} + +/** @param {any} text @param {WorkerConfig} config @param {AbortSignal} signal */ +async function predictOpenRouter(text, config, signal) { + const topk = Math.min(Math.max(1, config.topk | 0 || 10), 20) + // Chat with the continuation framing (raw /completions doesn't work for + // chat-only models); the first token's logprobs are the next-token dist. + const res = await fetch("https://openrouter.ai/api/v1/chat/completions", { + method: "POST", + headers: { + Authorization: "Bearer " + config.apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: config.model, + messages: config.continuation + ? [{role: "system", content: CONTINUE_SYS}, {role: "user", content: text || " "}] + : [{role: "user", content: text || " "}], + max_tokens: 1, + temperature: 0, + logprobs: true, + top_logprobs: topk, + }), + signal, + }) + if (!res.ok) throw new Error("OpenRouter predict: " + (await res.text())) + const data = await res.json() + const top = data.choices?.[0]?.logprobs?.content?.[0]?.top_logprobs + if (!top) return [] + return top + .map((/** @type {any} */ tl) => ({token: tl.token, p: +Math.exp(tl.logprob).toFixed(4)})) + .sort((/** @type {any} */ a, /** @type {any} */ b) => b.p - a.p) +} + +/** @param {any} port @param {AnyMsg} data */ +function handlePredict(port, data) { + const {id, text, config = {}} = data + const provider = data.provider + const sessionKey = data.sessionKey || id + // Register an abortable entry so an "abort" message (keyed by sessionKey) can + // cancel the in-flight request — the OpenRouter/WebLLM fetch honours the signal. + const abortController = new AbortController() + activeGenerations.set(sessionKey, {id, port, done: false, fullText: "", abortController}) + const cleanup = () => activeGenerations.delete(sessionKey) + const reply = (/** @type {any} */ candidates) => { + cleanup() + port.postMessage({type: "predictions", id, candidates}) + } + const failPredict = (/** @type {any} */ message) => { + cleanup() + if (!abortController.signal.aborted) port.postMessage({type: "error", id, message}) + } + + if (provider === "openrouter") { + predictOpenRouter(text, config, abortController.signal) + .then(reply) + .catch((e) => failPredict(e?.message || String(e))) + return + } + if (provider === "ollama") { + reply([]) // Ollama's API exposes no logprobs + return + } + if (provider === "webllm") { + predictWebLLM(text, config) + .then(reply) + .catch((e) => failPredict(e?.message || String(e))) + return + } + const requested = config.model || DEFAULT_MODEL_ID + const run = () => + predictLocal(text, config) + .then(reply) + .catch((e) => failPredict(e?.message || String(e))) + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) generator = null + loadModel(requested, config.dtype).then(() => + generator ? run() : failPredict(modelLoadError()) + ) + } else run() +} + +// --------------------------------------------------------------------------- +// WebLLM (MLC) — WebGPU, non-ONNX, loaded from a CDN like transformers.js. We're +// already in a worker, so CreateMLCEngine runs the model right here. logprobs +// give the same per-token predictions as the rest. (Mirrors rlm's WebLLMClient.) +// --------------------------------------------------------------------------- + +/** @type {any} */ +let webllmMod = null +/** @type {any} */ +let webllmEngine = null +/** @type {any} */ +let webllmModel = null + +/** @param {string} [model] @param {any} [custom] */ +async function ensureWebLLM(model, custom) { + model = model || "Qwen2.5-1.5B-Instruct-q4f16_1-MLC" + if (!webllmMod) { + broadcast({type: "status", message: "Loading WebLLM…"}) + // @ts-ignore — remote ESM URL, no type declarations + webllmMod = await import(/* @vite-ignore */ "https://esm.run/@mlc-ai/web-llm") + } + if (webllmEngine && webllmModel === model) return + if (webllmEngine) { + try { + await webllmEngine.unload?.() + } catch {} + webllmEngine = null + } + webllmModel = model + // Merge any self-compiled MLC model records into the prebuilt list so a custom + // model_id resolves to its weights + wasm lib. We only store {model_id, + // model_lib}; the weights URL is the model_id's HuggingFace repo. Stored in the + // config (not localStorage), threaded through as config.custom. + const customList = (Array.isArray(custom) ? custom : []) + .filter((/** @type {any} */ c) => c && c.model_id && c.model_lib) + .map((/** @type {any} */ c) => ({ + ...c, + // weights URL defaults to the HF repo named by model_id (matches rlm — + // WebLLM appends the resolve path itself, so NO /resolve/main/ suffix) + model: c.model || "https://huggingface.co/" + c.model_id, + })) + const appConfig = customList.length + ? { + ...webllmMod.prebuiltAppConfig, + model_list: [...webllmMod.prebuiltAppConfig.model_list, ...customList], + } + : undefined + webllmEngine = await webllmMod.CreateMLCEngine(model, { + appConfig, + initProgressCallback: (/** @type {any} */ r) => + broadcast({ + type: "status", + message: + r.text || + "Loading… " + + (typeof r.progress === "number" ? Math.round(r.progress * 100) + "%" : ""), + }), + }) + broadcast({type: "status", message: "Model ready (WebLLM)"}) + broadcast({type: "ready", model, device: "WebGPU"}) +} + +/** @param {Gen} gen @param {any} input @param {WorkerConfig} config */ +async function doGenerateWebLLM(gen, input, config) { + await ensureWebLLM(config.model, config.custom) + const isText = typeof input === "string" + + // Native tool calling (chat only): non-streaming, parse message.tool_calls. + if (config.tools && config.tools.length && !isText) { + const t0 = performance.now() + const res = await webllmEngine.chat.completions.create({ + messages: input, + tools: config.tools, + tool_choice: "auto", + stream: false, + temperature: config.temperature ?? 0.7, + ...(config.topP != null ? {top_p: config.topP} : {}), + ...(config.maxNewTokens ? {max_tokens: config.maxNewTokens} : {}), + }) + const msg = res.choices?.[0]?.message || {} + const text = msg.content || "" + if (text) post(gen, {type: "token", delta: text, text}) + gen.fullText = text + const toolCalls = (msg.tool_calls || []).map((/** @type {any} */ tc) => ({ + id: tc.id, + name: tc.function?.name, + args: safeJson(tc.function?.arguments), + })) + post(gen, { + type: "stats", + provider: "webllm", + model: config.model, + promptTokens: res.usage?.prompt_tokens ?? null, + genTokens: res.usage?.completion_tokens ?? null, + ttftMs: null, + totalMs: Math.round(performance.now() - t0), + tokPerSec: null, + decode: {temperature: config.temperature ?? 0.7}, + }) + return {text, toolCalls} + } + const temperature = config.temperature ?? 0.7 + const topk = config.topk | 0 + const common = { + stream: true, + stream_options: {include_usage: true}, + temperature, + ...(config.topP != null ? {top_p: config.topP} : {}), + ...(config.frequencyPenalty ? {frequency_penalty: config.frequencyPenalty} : {}), + ...(config.presencePenalty ? {presence_penalty: config.presencePenalty} : {}), + ...(config.seed != null && config.seed !== "" ? {seed: config.seed} : {}), + ...(config.maxNewTokens ? {max_tokens: config.maxNewTokens} : {}), + ...(topk > 0 ? {logprobs: true, top_logprobs: topk} : {}), + } + const tStart = performance.now() + let tFirst = 0 + let full = "" + let usage = null + let step = 0 + const stream = isText + ? await webllmEngine.completions.create({prompt: input, ...common}) + : await webllmEngine.chat.completions.create({messages: input, ...common}) + for await (const chunk of stream) { + if (gen.abortController.signal.aborted) break + if (chunk.usage) usage = chunk.usage + const ch = chunk.choices?.[0] + const delta = isText ? ch?.text : ch?.delta?.content + const lp = ch?.logprobs?.content + if (lp && topk > 0) { + for (const e of lp) { + if (step >= PREDICTION_CAP) break + const candidates = (e.top_logprobs || []).map((/** @type {any} */ c) => ({ + token: c.token, + p: +Math.exp(c.logprob).toFixed(4), + })) + if (candidates.length) post(gen, {type: "prediction", step, candidates}) + step++ + } + } + if (delta) { + if (!tFirst) tFirst = performance.now() + full += delta + gen.fullText = full + post(gen, {type: "token", delta, text: full}) + } + } + const now = performance.now() + const genTokens = usage?.completion_tokens ?? null + post(gen, { + type: "stats", + provider: "webllm", + model: config.model, + promptTokens: usage?.prompt_tokens ?? null, + genTokens, + ttftMs: tFirst ? Math.round(tFirst - tStart) : null, + totalMs: Math.round(now - tStart), + tokPerSec: + tFirst && genTokens ? +(genTokens / ((now - tFirst) / 1000)).toFixed(1) : null, + decode: {temperature, top_p: config.topP}, + }) + return {text: full, toolCalls: null} +} + +/** @param {any} text @param {WorkerConfig} config */ +async function predictWebLLM(text, config) { + await ensureWebLLM(config.model, config.custom) + const topk = Math.max(1, config.topk | 0 || 10) + const res = await webllmEngine.completions.create({ + prompt: text || " ", + max_tokens: 1, + temperature: 0, + logprobs: true, + top_logprobs: topk, + stream: false, + }) + const lp = res.choices?.[0]?.logprobs?.content?.[0]?.top_logprobs + if (!lp) return [] + return lp + .map((/** @type {any} */ c) => ({token: c.token, p: +Math.exp(c.logprob).toFixed(4)})) + .sort((/** @type {any} */ a, /** @type {any} */ b) => b.p - a.p) +} + +// --------------------------------------------------------------------------- +// Dispatch + lifecycle +// --------------------------------------------------------------------------- + +/** @param {Gen} gen @param {AnyMsg} msg */ +function post(gen, msg) { + try { + gen.port.postMessage({...msg, id: gen.id}) + } catch {} +} + +/** @param {any} sessionKey @param {Gen} gen @param {string} text @param {any} [toolCalls] @param {string} [toolMode] */ +function finalize(sessionKey, gen, text, toolCalls, toolMode) { + gen.done = true + gen.finalText = text + try { + gen.port.postMessage({type: "result", id: gen.id, text, toolCalls: toolCalls || null, toolMode}) + } catch {} + broadcast({type: "status", message: ""}) + if (sessionKey) setTimeout(() => activeGenerations.delete(sessionKey), 5000) +} + +/** @param {any} sessionKey @param {Gen} gen @param {string} message */ +function fail(sessionKey, gen, message) { + try { + gen.port.postMessage({type: "error", id: gen.id, message}) + } catch {} + broadcast({type: "status", message: ""}) + if (sessionKey) activeGenerations.delete(sessionKey) +} + +// `input` is either chat messages (array → chat-templated) or a raw string +// (→ plain continuation, what the loom editor wants). +/** @param {any} sessionKey @param {Gen} gen @param {string} provider @param {any} input @param {WorkerConfig} config */ +async function runGeneration(sessionKey, gen, provider, input, config) { + try { + log("runGeneration: start", { + provider, + model: config.model || currentModelId, + inputKind: typeof input === "string" ? "text" : "messages", + messages: Array.isArray(input) ? input.length : undefined, + }) + broadcast({type: "status", message: "Thinking…"}) + let out + if (provider === "openrouter") out = await doGenerateOpenRouter(gen, input, config) + else if (provider === "ollama") out = await doGenerateOllama(gen, input, config) + else if (provider === "webllm") out = await doGenerateWebLLM(gen, input, config) + else out = await doGenerateLocal(gen, input, config) + log("runGeneration: done", {provider, chars: out.text?.length || 0}) + finalize( + sessionKey, + gen, + out.text, + out.toolCalls, + "toolMode" in out && typeof out.toolMode === "string" ? out.toolMode : undefined + ) + } catch (/** @type {any} */ err) { + if (gen.abortController.signal.aborted) { + log("runGeneration: aborted", {provider, chars: gen.fullText?.length || 0}) + return + } + log("runGeneration: error", {provider, model: currentModelId, message: err?.message || String(err)}) + if (gen.fullText) finalize(sessionKey, gen, gen.fullText) + else fail(sessionKey, gen, friendlyError(err)) + } +} + +// Extract per-position features for training a LoRA adapter ON the head: ONE +// forward pass → the final hidden state h and the base logits at EVERY position, +// plus token ids. Requires a model exported with a `last_hidden_state` output +// (glomper-tuning/onnx_hidden.py); otherwise reports {supported:false}. +/** @param {Gen} gen @param {any} text */ +async function extractFeaturesLocal(gen, text) { + const model = generator.model + const tokenizer = generator.tokenizer + const inputs = tokenizer(text || " ", {return_tensor: true}) + const ids = Array.from(inputs.input_ids.data, Number) + const seq = ids.length + + broadcast({type: "status", message: "Extracting features (forward pass)…"}) + const outputs = await model(inputs) + const hs = outputs.last_hidden_state + const lg = outputs.logits + if (!hs || !lg) { + post(gen, { + type: "features", + supported: false, + outputKeys: Object.keys(outputs || {}), + message: `${currentModelId} has no last_hidden_state output — needs a hidden-state export (onnx_hidden.py).`, + }) + broadcast({type: "status", message: ""}) + return + } + const H = hs.dims.at(-1) + const V = lg.dims.at(-1) + const hidden = Float32Array.from(hs.data) // [seq*H], batch 0 + const logits = Float32Array.from(lg.data) // [seq*V], batch 0 + + // char-position spans per token (skip_special_tokens), like the other passes + const spans = [] + let prevLen = 0 + for (let i = 0; i < seq; i++) { + const partial = tokenizer.decode(ids.slice(0, i + 1), {skip_special_tokens: true}) + if (partial.length > prevLen) spans.push({from: prevLen, to: partial.length, index: i}) + prevLen = partial.length + } + const decoded = tokenizer.decode(ids, {skip_special_tokens: true}) + const tokens = ids.map((/** @type {any} */ tid) => tokenizer.decode([tid])) + + post(gen, {type: "features", supported: true, seq, H, V, ids, tokens, spans, decoded, hidden, logits}) + broadcast({type: "status", message: ""}) +} + +// Like extractFeaturesLocal but reads the `cut_hidden` output (the residual just +// before the last block's MLP) — for rung 2 (LoRA on the last block's MLP). +// Requires a model exported with onnx_block.py. +/** @param {Gen} gen @param {any} text */ +async function extractCutFeaturesLocal(gen, text) { + const model = generator.model + const tokenizer = generator.tokenizer + const inputs = tokenizer(text || " ", {return_tensor: true}) + const ids = Array.from(inputs.input_ids.data, Number) + const seq = ids.length + broadcast({type: "status", message: "Extracting cut features (forward pass)…"}) + const outputs = await model(inputs) + const cut = outputs.cut_hidden + if (!cut) { + post(gen, { + type: "cut-features", + supported: false, + outputKeys: Object.keys(outputs || {}), + message: `${currentModelId} has no cut_hidden output — needs a cut-point export (onnx_block.py).`, + }) + broadcast({type: "status", message: ""}) + return + } + const d = cut.dims.at(-1) + const hidden = Float32Array.from(cut.data) // [seq*d], batch 0 + const spans = [] + let prevLen = 0 + for (let i = 0; i < seq; i++) { + const partial = tokenizer.decode(ids.slice(0, i + 1), {skip_special_tokens: true}) + if (partial.length > prevLen) spans.push({from: prevLen, to: partial.length, index: i}) + prevLen = partial.length + } + const decoded = tokenizer.decode(ids, {skip_special_tokens: true}) + const tokens = ids.map((/** @type {any} */ tid) => tokenizer.decode([tid])) + post(gen, {type: "cut-features", supported: true, seq, d, ids, tokens, spans, decoded, hidden}) + broadcast({type: "status", message: ""}) +} + +/** @param {any} port @param {AnyMsg} data */ +function handleMessage(port, data) { + const {type, id} = data + const sessionKey = data.sessionKey || id + + // If a local request asks for a dtype different from the loaded model's, drop + // the cached generator so every handler's `!generator` guard forces a reload. + // (Same model id, different quantization — e.g. q4f16 → q4 — still needs a + // fresh compile.) Handled centrally so individual handlers stay simple. + const reqDtype = data.config?.dtype + if (generator && !loading && reqDtype && currentDtype && reqDtype !== currentDtype) { + generator = null + } + + if (type === "list-local-models") { + port.postMessage({type: "local-models", models: LOCAL_MODELS}) + return + } + if (type === "predict") { + handlePredict(port, data) + return + } + if (type === "score-tokens") { + const {provider, config = {}} = data + if (provider !== "local") { + port.postMessage({type: "token-scores", id, scores: []}) + return + } + const gen = {id, port, fullText: "", done: false, finalText: "", abortController: new AbortController()} + activeGenerations.set(sessionKey, gen) + const requested = config.model || DEFAULT_MODEL_ID + const run = () => + scoreTokensLocal(gen, data.text, config) + .then(() => { gen.done = true; activeGenerations.delete(sessionKey) }) + .catch((e) => { + if (!gen.abortController.signal.aborted) + port.postMessage({type: "error", id, message: e?.message || String(e)}) + activeGenerations.delete(sessionKey) + }) + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, config.dtype).then(() => { + if (!generator) { port.postMessage({type: "error", id, message: modelLoadError()}); return } + run() + }) + } else run() + return + } + if (type === "compute-importance") { + const {provider, config = {}} = data + if (provider !== "local") { + port.postMessage({type: "importance-scores", id, scores: []}) + return + } + const gen = {id, port, fullText: "", done: false, finalText: "", abortController: new AbortController()} + activeGenerations.set(sessionKey, gen) + const requested = config.model || DEFAULT_MODEL_ID + const run = () => + computeImportanceLocal(gen, data.text, config) + .then(() => { gen.done = true; activeGenerations.delete(sessionKey) }) + .catch((e) => { + if (!gen.abortController.signal.aborted) + port.postMessage({type: "error", id, message: e?.message || String(e)}) + activeGenerations.delete(sessionKey) + }) + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, config.dtype).then(() => { + if (!generator) { port.postMessage({type: "error", id, message: modelLoadError()}); return } + run() + }) + } else run() + return + } + if (type === "compute-attention-weights") { + const {provider, config = {}} = data + if (provider !== "local") { + port.postMessage({type: "attention-weights", id, supported: false}) + return + } + const gen = {id, port, fullText: "", done: false, finalText: "", abortController: new AbortController()} + activeGenerations.set(sessionKey, gen) + const requested = config.model || DEFAULT_MODEL_ID + const run = () => { + log("compute-attention-weights: run", {model: requested}) + return computeAttentionWeightsLocal(gen, data.text, config) + .then(() => { gen.done = true; activeGenerations.delete(sessionKey) }) + .catch((e) => { + const aborted = gen.abortController.signal.aborted + // Always log the real cause — even on abort, where the client only + // ever sees a generic AbortError and the true error would be lost. + log("compute-attention-weights: error", {model: requested, aborted, message: e?.message || String(e)}) + if (!aborted) + port.postMessage({type: "error", id, message: friendlyError(e)}) + activeGenerations.delete(sessionKey) + }) + } + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, config.dtype) + .then(() => { + if (!generator) { port.postMessage({type: "error", id, message: modelLoadError()}); return } + run() + }) + .catch((e) => { + log("compute-attention-weights: loadModel threw", {model: requested, message: e?.message || String(e)}) + port.postMessage({type: "error", id, message: friendlyError(e)}) + }) + } else run() + return + } + if (type === "extract-features") { + const {provider, config = {}} = data + if (provider !== "local") { + port.postMessage({type: "features", id, supported: false, message: "features require a local model"}) + return + } + const gen = {id, port, fullText: "", done: false, finalText: "", abortController: new AbortController()} + activeGenerations.set(sessionKey, gen) + const requested = config.model || DEFAULT_MODEL_ID + const run = () => { + log("extract-features: run", {model: requested}) + return extractFeaturesLocal(gen, data.text) + .then(() => { gen.done = true; activeGenerations.delete(sessionKey) }) + .catch((e) => { + const aborted = gen.abortController.signal.aborted + log("extract-features: error", {model: requested, aborted, message: e?.message || String(e)}) + if (!aborted) + port.postMessage({type: "error", id, message: friendlyError(e)}) + activeGenerations.delete(sessionKey) + }) + } + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, config.dtype) + .then(() => { + if (!generator) { port.postMessage({type: "error", id, message: modelLoadError()}); return } + run() + }) + .catch((e) => { + log("extract-features: loadModel threw", {model: requested, message: e?.message || String(e)}) + port.postMessage({type: "error", id, message: friendlyError(e)}) + }) + } else run() + return + } + if (type === "extract-cut-features") { + const {provider, config = {}} = data + if (provider !== "local") { + port.postMessage({type: "cut-features", id, supported: false, message: "features require a local model"}) + return + } + const gen = {id, port, fullText: "", done: false, finalText: "", abortController: new AbortController()} + activeGenerations.set(sessionKey, gen) + const requested = config.model || DEFAULT_MODEL_ID + const run = () => + extractCutFeaturesLocal(gen, data.text) + .then(() => { gen.done = true; activeGenerations.delete(sessionKey) }) + .catch((e) => { + if (!gen.abortController.signal.aborted) + port.postMessage({type: "error", id, message: e?.message || String(e)}) + activeGenerations.delete(sessionKey) + }) + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, config.dtype).then(() => { + if (!generator) { port.postMessage({type: "error", id, message: modelLoadError()}); return } + run() + }) + } else run() + return + } + if (type === "decode-tokens") { + // Decode a list of vocab ids to their token strings (for labelling the + // LoRA panel's next-token bars). Uses the loaded model's tokenizer. + const run = () => { + try { + const tokenizer = generator.tokenizer + const strings = (data.ids || []).map((/** @type {any} */ tid) => tokenizer.decode([tid])) + port.postMessage({type: "decoded-tokens", id, strings}) + } catch (/** @type {any} */ e) { + port.postMessage({type: "error", id, message: e?.message || String(e)}) + } + } + const requested = data.config?.model || DEFAULT_MODEL_ID + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, data.config?.dtype).then(() => generator ? run() : port.postMessage({type: "error", id, message: modelLoadError()})) + } else run() + return + } + if (type === "probe-attention") { + // Diagnostic: try to get attention weights from the model directly. + const run = async () => { + const model = generator.model + const tokenizer = generator.tokenizer + const text = data.text || "Hello world" + const inputs = tokenizer(text, {return_tensor: true}) + /** @type {Record} */ + const report = {type: "probe-attention-result", id} + + // What sessions does the ONNX model have? + /** @type {Record} */ + const sessions = {} + for (const [name, session] of Object.entries(model?.sessions || {})) { + sessions[name] = { + inputNames: /** @type {any} */ (session)?.inputNames || [], + outputNames: /** @type {any} */ (session)?.outputNames || [], + } + } + report.sessions = sessions + + // Try a direct forward pass and see what the output object contains + try { + const outputs = await model(inputs) + report.outputKeys = Object.keys(outputs || {}) + report.hasAttentions = "attentions" in (outputs || {}) + // Check for attention-like keys + report.attentionKeys = Object.keys(outputs || {}).filter(k => + /attention|attn/i.test(k) + ) + } catch (/** @type {any} */ e) { + report.forwardError = e.message + } + + // Try calling generate with output_attentions + try { + const genOut = await model.generate({ + ...inputs, + max_new_tokens: 1, + output_attentions: true, + return_dict_in_generate: true, + }) + report.generateKeys = Object.keys(genOut || {}) + report.generateHasAttentions = "attentions" in (genOut || {}) + } catch (/** @type {any} */ e) { + report.generateError = e.message + } + + port.postMessage(report) + } + const requested = data.config?.model || DEFAULT_MODEL_ID + if (!generator || currentModelId !== requested) { + loadModel(requested, data.config?.dtype).then(() => generator ? run() : port.postMessage({type: "probe-attention-result", id, error: modelLoadError()})) + } else run() + return + } + if (type === "register-local-model") { + /** @type {Map} */ + const files = new Map((data.files || []).map((/** @type {any} */ f) => [f.path, f.blob])) + localModelFiles.set(data.id, {files, dtype: data.dtype || "q4f16"}) + ensureLocalFetchPatch() + if (currentModelId === data.id) releaseGenerator() // force a reload (dispose old session) + purgeCachedModel(data.id).then(() => + port.postMessage({type: "local-model-registered", id: data.id, count: files.size}) + ) + return + } + if (type === "preload") { + if (data.provider === "local") { + if (!generator && !loading) loadModel(data.config?.model, data.config?.dtype) + if (generator) port.postMessage({type: "ready"}) + } else port.postMessage({type: "ready"}) + return + } + if (type === "resume") { + const key = data.sessionKey + const gen = activeGenerations.get(key) + if (gen && !gen.done) { + gen.port = port // re-point the live stream at the reconnecting tab + port.postMessage({type: "resumed", id: gen.id, sessionKey: key, text: gen.fullText}) + } else if (gen && gen.done) { + port.postMessage({type: "resume-result", id: gen.id, sessionKey: key, text: gen.finalText}) + activeGenerations.delete(key) + } else port.postMessage({type: "no-active-generation", sessionKey: key}) + return + } + if (type === "abort") { + const gen = activeGenerations.get(data.sessionKey) + if (gen && !gen.done) { + gen.abortController.abort() + activeGenerations.delete(data.sessionKey) + } + return + } + if (type === "generate") { + const {provider, config = {}} = data + const input = data.text != null ? data.text : data.messages + const gen = { + id, + port, + fullText: "", + done: false, + finalText: "", + abortController: new AbortController(), + } + activeGenerations.set(sessionKey, gen) + if (provider === "local") { + const requested = config.model || DEFAULT_MODEL_ID + if (!generator || currentModelId !== requested) { + if (currentModelId !== requested) releaseGenerator() + loadModel(requested, config.dtype) + .then(() => { + if (!generator) + return fail(sessionKey, gen, `Model "${requested}" failed to load (still not ready after load). Try a different local model, or check the device has enough memory.`) + runGeneration(sessionKey, gen, provider, input, config) + }) + .catch((e) => + fail(sessionKey, gen, `Couldn't load local model "${requested}": ${e?.message || e}`), + ) + } else runGeneration(sessionKey, gen, provider, input, config) + } else { + runGeneration(sessionKey, gen, provider, input, config) + } + } +} + +// SharedWorker entry — one port per connecting tab. +/** @type {any} */ (self).onconnect = (/** @type {any} */ e) => { + const port = e.ports[0] + ports.add(port) + port.onmessage = (/** @type {any} */ ev) => handleMessage(port, ev.data) + if (generator) port.postMessage({type: "ready"}) + port.start() +} + +// Dedicated-Worker fallback (browsers without module SharedWorker, e.g. Safari): +// treat `self` as the single port. `self.postMessage` reaches the main thread. +if (typeof (/** @type {any} */ (self).SharedWorkerGlobalScope) === "undefined") { + ports.add(self) + self.onmessage = (/** @type {any} */ ev) => handleMessage(self, ev.data) +}