From 9e69fb14f69c0907c19400e57e2fe35f4aec26cb Mon Sep 17 00:00:00 2001 From: Negrescu Alexandru Date: Sat, 31 Oct 2026 19:45:51 +0200 Subject: [PATCH 1/2] test(smoke): drive real omp turns through a scripted model The smoke ran omp with --no-tools and failed on any agent_start, so the pinned reminder and the pstackpolicy refusal never ran in a real omp. omp-port/scripted-model.mjs is a loopback OpenAI-completions provider that answers `tool: ` prompts with that tool call and keeps every request body. Each smoke leg now runs in a scratch HOME with an env -i style environment and a logging gh shim first on PATH. It checks that an unpinned system prompt lacks skill://poteto-mode and a pinned one has it, that bash reaches the gh shim, and that a bash `gh pr merge` comes back to the model as `pstackpolicy: refused` with no gh call. The checks run twice, once with -e and --plugin-dir and once after `omp plugin link plugins/pstack` with no -e. --- omp-port/scripted-model.mjs | 51 ++++++ omp-port/smoke-plugin.mjs | 300 ++++++++++++++++++++++-------------- 2 files changed, 235 insertions(+), 116 deletions(-) create mode 100644 omp-port/scripted-model.mjs diff --git a/omp-port/scripted-model.mjs b/omp-port/scripted-model.mjs new file mode 100644 index 0000000..365dd04 --- /dev/null +++ b/omp-port/scripted-model.mjs @@ -0,0 +1,51 @@ +// A deterministic OpenAI-completions provider on loopback, so a smoke can drive real omp turns +// without a model. Every request body lands in `requests`. The last user message decides the reply: +// a line `tool: ` one tool call with those arguments +// a tool result as the last message the text `scripted-model: tool result received` +// anything else the text `scripted-model: no script for this prompt` + +export const text = (content) => + typeof content === "string" + ? content + : (content ?? []).map((part) => (part.type === "text" ? part.text : "")).join("\n"); + +function decide(messages) { + if (messages.at(-1)?.role === "tool") return { text: "scripted-model: tool result received" }; + const user = messages.findLast((m) => m.role === "user"); + const call = /^tool:(\w+) (\{.*\})\s*$/m.exec(text(user?.content)); + if (call) return { tool: call[1], args: call[2] }; + return { text: "scripted-model: no script for this prompt" }; +} + +const chunk = (delta, finish, extra = {}) => + `data: ${JSON.stringify({ + id: "scripted", + object: "chat.completion.chunk", + created: 0, + model: "scripted", + choices: [{ index: 0, delta, finish_reason: finish }], + ...extra, + })}\n\n`; + +function reply(plan) { + const call = plan.tool && { id: "call_scripted", type: "function", function: { name: plan.tool, arguments: plan.args } }; + const delta = call ? { role: "assistant", tool_calls: [{ index: 0, ...call }] } : { role: "assistant", content: plan.text }; + const usage = { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }; + const body = chunk(delta, null) + chunk({}, call ? "tool_calls" : "stop", { usage }) + "data: [DONE]\n\n"; + return new Response(body, { headers: { "content-type": "text/event-stream" } }); +} + +export function scriptedModel() { + const requests = []; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + if (!new URL(request.url).pathname.endsWith("/chat/completions")) return new Response("not found", { status: 404 }); + const body = await request.json(); + requests.push(body); + return reply(decide(body.messages)); + }, + }); + return { baseUrl: `http://127.0.0.1:${server.port}/v1`, requests, stop: () => server.stop(true) }; +} diff --git a/omp-port/smoke-plugin.mjs b/omp-port/smoke-plugin.mjs index 050ef8e..632a67d 100755 --- a/omp-port/smoke-plugin.mjs +++ b/omp-port/smoke-plugin.mjs @@ -1,22 +1,25 @@ #!/usr/bin/env bun -// Loads the plugin tree into a real omp and checks what only omp can answer: that every skill -// directory registers as a command, and that the pin extension records and reloads its state. +// Loads the plugin tree into a real omp once per install shape and checks what only omp can answer: +// every skill directory registers as a command, the pin extension records and reloads its state, a +// pinned turn's system prompt carries the poteto-mode reminder, and pstackpolicy refuses a bash +// `gh pr merge` before gh runs. Turns go to a scripted model on loopback, so no credential is used. // // bun omp-port/smoke-plugin.mjs [plugin-dir] plugin-dir defaults to plugins/pstack // // No omp on PATH prints SKIPPED and exits 0, unless OMP_SMOKE_REQUIRED=1 makes that a failure. import assert from "node:assert"; import { spawn, spawnSync } from "node:child_process"; -import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; +import { scriptedModel, text } from "./scripted-model.mjs"; const plugin = resolve(process.argv[2] ?? fileURLToPath(new URL("../plugins/pstack", import.meta.url))); -const version = spawnSync("omp", ["--version"], { encoding: "utf8" }); -if (version.error?.code === "ENOENT") { +const omp = Bun.which("omp"); +if (!omp) { if (process.env.OMP_SMOKE_REQUIRED === "1") { console.error("smoke-plugin: FAIL omp is not on PATH and OMP_SMOKE_REQUIRED=1"); process.exit(1); @@ -24,6 +27,7 @@ if (version.error?.code === "ENOENT") { console.log("smoke-plugin: SKIPPED omp is not on PATH; set OMP_SMOKE_REQUIRED=1 to fail instead"); process.exit(0); } +const version = spawnSync(omp, ["--version"], { encoding: "utf8" }); if (version.status !== 0) { console.error(`smoke-plugin: FAIL omp --version exited ${version.status}: ${version.stderr}`); process.exit(1); @@ -34,125 +38,189 @@ const skills = (await readdir(join(plugin, "skills"), { withFileTypes: true })) .map((entry) => entry.name) .sort(); -// A fresh HOME keeps the operator's plugins, extensions and credentials out. The variables stripped -// below would relocate omp's roots past it. -const home = await mkdtemp(join(tmpdir(), "pstack-smoke-")); -await mkdir(join(home, ".omp", "agent"), { recursive: true }); -// omp refuses to start without a model. A local provider on the discard port satisfies that with -// no credentials, and a turn that did start fails on a refused connection instead of reaching one. -await writeFile( - join(home, ".omp", "agent", "models.yml"), - [ - "providers:", - " smoke:", - " baseUrl: http://127.0.0.1:9/v1", - " auth: none", - " api: openai-completions", - " models:", - " - id: no-inference", - "", - ].join("\n"), -); -const env = { ...process.env, HOME: home, USERPROFILE: home }; -for (const key of Object.keys(env)) { - if (/^(?:OMP_|PI_|XDG_(?:CONFIG|DATA|STATE|CACHE)_HOME$|CLAUDE_CONFIG_DIR$)|API_KEY|TOKEN|SECRET|PASSWORD/i.test(key)) { - delete env[key]; - } -} +const MERGE = "gh pr merge 1 --repo pstack-verify/no-such-repo --squash"; // --plugin-dir adds the tree to skill discovery but does not load package.json omp.extensions, so -// the same directory also goes in as -e, which does. -const child = spawn( - "omp", - [ - "--mode", "rpc", "--no-session", "--no-title", "--no-lsp", "--no-tools", "--no-rules", - "--no-extensions", "--plugin-dir", plugin, "-e", plugin, "--model", "smoke/no-inference", - ], - { cwd: home, env, stdio: ["pipe", "pipe", "pipe"] }, -); -let stderr = ""; -child.stderr.on("data", (chunk) => { stderr += chunk; }); -let spawnError; -child.on("error", (error) => { spawnError = error; }); -const exited = new Promise((done) => child.once("close", (code) => done(code))); -const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); -const timer = setTimeout(() => child.kill(), 60_000); - -const notices = []; -async function frame() { - const { value, done } = await lines.next(); - assert(!done, spawnError?.message ?? "omp exited before the smoke finished"); - const message = JSON.parse(value); - assert(message.type !== "agent_start", "a smoke step started a model turn"); - assert(message.type !== "extension_error", `extension error: ${value}`); - if (message.type === "extension_ui_request" && message.method === "notify") notices.push(message.message); - return message; -} +// the -e shape passes the directory twice. --no-extensions also drops a linked plugin's skills and +// extensions, so the link shape leaves discovery on. +const shapes = [ + { name: "-e", args: ["--no-extensions", "--plugin-dir", plugin, "-e", plugin] }, + { name: "omp plugin link", link: true, args: [] }, +]; -let nextId = 0; -async function request(type, fields = {}) { - const id = String(++nextId); - child.stdin.write(`${JSON.stringify({ id, type, ...fields })}\n`); - for (;;) { - const message = await frame(); - if (message.id !== id) continue; - if (message.type === "response") { - assert(message.success === true, `${type} failed: ${message.error}`); - if (type !== "prompt" || message.data?.agentInvoked === false) return message.data; +const systemPrompt = (body) => + body.messages.filter((m) => m.role === "system").map((m) => text(m.content)).join("\n"); + +async function leg(shape) { + const home = await mkdtemp(join(tmpdir(), "pstack-smoke-")); + const bin = join(home, "bin"); + const ghCalls = join(home, "gh-calls.log"); + const model = scriptedModel(); + let child; + let exited; + let timer; + let stderr = ""; + try { + await mkdir(join(home, ".omp", "agent"), { recursive: true }); + await mkdir(bin); + await writeFile( + join(home, ".omp", "agent", "models.yml"), + [ + "providers:", + " smoke:", + ` baseUrl: ${model.baseUrl}`, + " auth: none", + " api: openai-completions", + " models:", + " - id: scripted", + "", + ].join("\n"), + ); + await writeFile(join(bin, "gh"), `#!/bin/sh\nprintf '%s\\n' "$*" >>'${ghCalls}'\necho "gh-shim: $*"\n`, { + mode: 0o755, + }); + // As under env -i, nothing from the operator's environment reaches omp. That keeps out every + // credential, every variable that moves omp's roots past the scratch HOME, and the real gh. + const env = { HOME: home, PATH: `${bin}:/usr/local/bin:/usr/bin:/bin` }; + + if (shape.link) { + const linked = spawnSync(omp, ["plugin", "link", plugin], { cwd: home, env, encoding: "utf8" }); + assert(linked.status === 0, `omp plugin link exited ${linked.status}: ${linked.stderr}`); } - if (message.type === "prompt_result") { - assert(message.agentInvoked === false, `${fields.message} reached the agent`); - assert(message.status === "completed", `${fields.message} ended ${message.status}`); + + // Under yolo only pstackpolicy's tool_call veto stands between bash and gh. That is its case. + child = spawn( + omp, + [ + "--mode", "rpc", "--no-session", "--no-title", "--no-lsp", "--no-rules", "--tools", "bash", + "--approval-mode", "yolo", "--model", "smoke/scripted", ...shape.args, + ], + { cwd: home, env, stdio: ["pipe", "pipe", "pipe"] }, + ); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + let spawnError; + child.on("error", (error) => { spawnError = error; }); + exited = new Promise((done) => child.once("close", (code) => done(code))); + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + timer = setTimeout(() => child.kill(), 60_000); + + const notices = []; + async function frame() { + const { value, done } = await lines.next(); + assert(!done, spawnError?.message ?? "omp exited before the smoke finished"); + const message = JSON.parse(value); + assert(message.type !== "extension_error", `extension error: ${value}`); + if (message.type === "extension_ui_request" && message.method === "notify") notices.push(message.message); return message; } + + let nextId = 0; + async function request(type, fields = {}) { + const id = String(++nextId); + child.stdin.write(`${JSON.stringify({ id, type, ...fields })}\n`); + for (;;) { + const message = await frame(); + if (message.id !== id) continue; + if (message.type === "response") { + assert(message.success === true, `${type} failed: ${message.error}`); + if (type !== "prompt" || message.data?.agentInvoked === false) return message.data; + } + if (message.type === "prompt_result") { + assert(message.status === "completed", `${fields.message} ended ${message.status}: ${JSON.stringify(message.error)}`); + return message; + } + } + } + + const command = async (message) => + assert((await request("prompt", { message })).agentInvoked === false, `${message} reached the agent`); + + async function turn(message) { + assert((await request("prompt", { message })).agentInvoked === true, `${message} started no model turn`); + return model.requests.filter((body) => + text(body.messages.findLast((m) => m.role === "user")?.content).includes(message), + ); + } + + async function bash(commandLine) { + const requests = await turn(`tool:bash ${JSON.stringify({ command: commandLine })}`); + const result = requests.map((body) => body.messages.at(-1)).find((m) => m.role === "tool"); + assert(result, `the model never got a tool result for ${commandLine}`); + return text(result.content); + } + + const pinEntry = async () => + (await request("get_entries")).entries + .filter((entry) => entry.type === "custom" && entry.customType === "poteto-mode") + .at(-1)?.data?.pinned; + + while ((await frame()).type !== "ready"); + + const { commands } = await request("get_available_commands"); + const registered = new Set(commands.filter((c) => c.source === "skill").map((c) => c.name)); + const missing = skills.filter((name) => !registered.has(`skill:${name}`)); + assert( + missing.length === 0, + `${skills.length - missing.length} of ${skills.length} skill directories registered as skill:; ` + + `missing ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ` and ${missing.length - 5} more` : ""}`, + ); + assert( + commands.some((c) => c.name === "poteto-mode" && c.source === "extension"), + "the potetomode extension did not register /poteto-mode", + ); + + const [unpinned] = await turn("smoke: unpinned turn"); + assert(unpinned, "the unpinned turn sent no request to the model"); + assert( + !systemPrompt(unpinned).includes("skill://poteto-mode"), + "an unpinned system prompt already names skill://poteto-mode, so the pinned check could not fail", + ); + + await command("/poteto-mode on"); + assert((await pinEntry()) === true, "/poteto-mode on did not record pinned: true"); + const [pinned] = await turn("smoke: pinned turn"); + assert(pinned, "the pinned turn sent no request to the model"); + assert(systemPrompt(pinned).includes("skill://poteto-mode"), "the pinned turn's system prompt lacks the poteto-mode reminder"); + + assert((await bash("gh --version")).includes("gh-shim: --version"), "bash did not resolve gh to the logging shim"); + const merged = await bash(MERGE); + assert(merged.includes("pstackpolicy: refused"), `the model got ${JSON.stringify(merged.slice(0, 200))} back for ${MERGE}`); + const calls = (await readFile(ghCalls, "utf8")).trim().split("\n"); + assert.deepStrictEqual(calls, ["--version"], `gh ran for: ${calls.join(" | ")}`); + + await command("/poteto-mode off"); + assert((await pinEntry()) === false, "/poteto-mode off did not record pinned: false"); + + // /new arrives as session_switch; the pin from the session it left must not follow it. + await command("/poteto-mode on"); + await request("new_session"); + await command("/poteto-mode status"); + assert(notices.at(-1) === "poteto-mode: not pinned", `the pin survived new_session (${notices.at(-1)})`); + + child.stdin.end(); + assert((await exited) === 0, "omp did not exit cleanly"); + } catch (error) { + if (stderr.trim()) error.message += `\nomp stderr:\n${stderr.trim()}`; + throw error; + } finally { + clearTimeout(timer); + if (child?.exitCode === null) child.kill(); + await exited; + model.stop(); + await rm(home, { recursive: true, force: true }); } } -const pinEntry = async () => - (await request("get_entries")).entries - .filter((entry) => entry.type === "custom" && entry.customType === "poteto-mode") - .at(-1)?.data?.pinned; - -try { - while ((await frame()).type !== "ready"); - - const { commands } = await request("get_available_commands"); - const registered = new Set(commands.filter((c) => c.source === "skill").map((c) => c.name)); - const missing = skills.filter((name) => !registered.has(`skill:${name}`)); - assert( - missing.length === 0, - `${skills.length - missing.length} of ${skills.length} skill directories registered as skill:; ` + - `missing ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ` and ${missing.length - 5} more` : ""}`, - ); - assert( - commands.some((c) => c.name === "poteto-mode" && c.source === "extension"), - "the potetomode extension did not register /poteto-mode", - ); - - for (const [argument, pinned] of [["on", true], ["off", false]]) { - await request("prompt", { message: `/poteto-mode ${argument}` }); - assert((await pinEntry()) === pinned, `/poteto-mode ${argument} did not record pinned: ${pinned}`); +for (const shape of shapes) { + try { + await leg(shape); + console.log( + `smoke-plugin: PASS ${version.stdout.trim()} via ${shape.name}: ${skills.length}/${skills.length} skills ` + + "registered; /poteto-mode on/off recorded; pinned system prompt names skill://poteto-mode; " + + "pstackpolicy refused the merge and gh never ran; pin reloaded on new_session", + ); + } catch (error) { + console.error(`smoke-plugin: FAIL ${version.stdout.trim()} via ${shape.name}: ${error.message}`); + process.exitCode = 1; } - - // /new arrives as session_switch; the pin from the session it left must not follow it. - await request("prompt", { message: "/poteto-mode on" }); - await request("new_session"); - await request("prompt", { message: "/poteto-mode status" }); - assert(notices.at(-1) === "poteto-mode: not pinned", `the pin survived new_session (${notices.at(-1)})`); - - child.stdin.end(); - assert((await exited) === 0, "omp did not exit cleanly"); - console.log( - `smoke-plugin: PASS ${version.stdout.trim()}: ${skills.length}/${skills.length} skills registered; ` + - "/poteto-mode on/off recorded; pin reloaded on new_session; no model turn", - ); -} catch (error) { - console.error(`smoke-plugin: FAIL ${error.message}`); - if (stderr.trim()) console.error(`omp stderr:\n${stderr.trim()}`); - process.exitCode = 1; -} finally { - clearTimeout(timer); - if (child.exitCode === null) child.kill(); - await exited; - await rm(home, { recursive: true, force: true }); } From 2794c9d341735e24c94f9530045ebb8af7562115 Mon Sep 17 00:00:00 2001 From: Negrescu Alexandru Date: Sat, 31 Oct 2026 20:54:39 +0200 Subject: [PATCH 2/2] test(smoke): install through the marketplace from the checkout The smoke covered -e and omp plugin link but not the README's user install, /marketplace add then /marketplace install pstack@pstack-omp. A third leg runs the CLI equivalents in the scratch HOME against the checkout root. omp 18.14.2 classifies an absolute path as a local marketplace source and copies the plugin from the working tree at install time, so the leg tests this tree and never reaches GitHub. It then runs the same assertions as the other legs. Each shape now carries its setup commands as data, and the optional argument is the checkout root rather than the plugin directory, since the catalog lives at the root. A temp copy whose catalog source names ./plugins/pstack-missing passes -e and link and fails the marketplace leg with omp's "Plugin source directory does not exist". A temp copy with an extra uncommitted skill reports 61/61 on all three. --- omp-port/smoke-plugin.mjs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/omp-port/smoke-plugin.mjs b/omp-port/smoke-plugin.mjs index 632a67d..83c35fd 100755 --- a/omp-port/smoke-plugin.mjs +++ b/omp-port/smoke-plugin.mjs @@ -4,7 +4,7 @@ // pinned turn's system prompt carries the poteto-mode reminder, and pstackpolicy refuses a bash // `gh pr merge` before gh runs. Turns go to a scripted model on loopback, so no credential is used. // -// bun omp-port/smoke-plugin.mjs [plugin-dir] plugin-dir defaults to plugins/pstack +// bun omp-port/smoke-plugin.mjs [checkout] checkout defaults to this repository // // No omp on PATH prints SKIPPED and exits 0, unless OMP_SMOKE_REQUIRED=1 makes that a failure. import assert from "node:assert"; @@ -16,7 +16,8 @@ import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; import { scriptedModel, text } from "./scripted-model.mjs"; -const plugin = resolve(process.argv[2] ?? fileURLToPath(new URL("../plugins/pstack", import.meta.url))); +const checkout = resolve(process.argv[2] ?? fileURLToPath(new URL("..", import.meta.url))); +const plugin = join(checkout, "plugins", "pstack"); const omp = Bun.which("omp"); if (!omp) { @@ -42,10 +43,16 @@ const MERGE = "gh pr merge 1 --repo pstack-verify/no-such-repo --squash"; // --plugin-dir adds the tree to skill discovery but does not load package.json omp.extensions, so // the -e shape passes the directory twice. --no-extensions also drops a linked plugin's skills and -// extensions, so the link shape leaves discovery on. +// extensions, so the installed shapes leave discovery on. The marketplace shape runs the README's +// add and install against the checkout, which omp copies from the working tree, not from GitHub. const shapes = [ - { name: "-e", args: ["--no-extensions", "--plugin-dir", plugin, "-e", plugin] }, - { name: "omp plugin link", link: true, args: [] }, + { name: "-e", setup: [], args: ["--no-extensions", "--plugin-dir", plugin, "-e", plugin] }, + { name: "omp plugin link", setup: [["plugin", "link", plugin]], args: [] }, + { + name: "marketplace install", + setup: [["plugin", "marketplace", "add", checkout], ["plugin", "install", "pstack@pstack-omp"]], + args: [], + }, ]; const systemPrompt = (body) => @@ -83,9 +90,9 @@ async function leg(shape) { // credential, every variable that moves omp's roots past the scratch HOME, and the real gh. const env = { HOME: home, PATH: `${bin}:/usr/local/bin:/usr/bin:/bin` }; - if (shape.link) { - const linked = spawnSync(omp, ["plugin", "link", plugin], { cwd: home, env, encoding: "utf8" }); - assert(linked.status === 0, `omp plugin link exited ${linked.status}: ${linked.stderr}`); + for (const argv of shape.setup) { + const ran = spawnSync(omp, argv, { cwd: home, env, encoding: "utf8" }); + assert(ran.status === 0, `omp ${argv.join(" ")} exited ${ran.status}: ${ran.stderr.trim()}`); } // Under yolo only pstackpolicy's tool_call veto stands between bash and gh. That is its case.