From 4781c3c72df1ba32b99e2c5ee29213fa6b8472c7 Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Thu, 8 Oct 2026 23:48:00 +0800 Subject: [PATCH 1/3] fix(plugin): let skill resonance see tool sub-step evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #2460 ## Summary - include `summary` and each `toolCalls` entry's `name` + `input` in the per-trace resonance text inside `computeResonance` (`core/skill/verifier.ts`), in addition to the existing `userText`/`agentText`/`reflection` - add regression coverage: tool sub-step evidence with a faithful draft now verifies; summary-only traces count as hits; drafts sharing no tokens with tool-only evidence are still rejected ## Why Evidence traces produced by tool sub-steps carry `userText: ""` and `agentText: ""` — `core/capture/step-extractor.ts` leaves them empty on purpose (the viewer's `flattenChat` must not re-render the user bubble per sub-step); the turn's real payload lives in `toolCalls` and `summary`. `computeResonance` only read the two text fields plus `reflection`, so in tool-heavy sessions most evidence traces contributed empty strings: with the default `minResonance` of 0.5 and a ≥2-token overlap per trace, drafts describing exactly what happened were rejected en masse as `resonance-low` (often `resonance=0`). The fix is strictly additive on the text side: the structured tool coverage check (set containment over `draft.tools` vs evidence `toolCalls` names) is untouched. `tc.input` is stringified defensively (same local `safeStringify` pattern as `core/capture/embedder.ts`); outputs are deliberately excluded (unbounded, and outside the reported scope). ## Validation - `npx vitest run tests/unit/skill/verifier.test.ts` - red on unfixed `main` @ `a7367d0`: the issue's minimal repro (two tool sub-step traces, `userText`/`agentText` empty, inputs `git -C /repo status` / `git -C /repo log`, faithful draft) → `resonance=0`, `skill.verify.fail reason="resonance-low"`, `reason="resonance=0.00<0.5"` — 2 new tests failed - green after the fix: 11/11 (8 pre-existing + 3 new) - `npx vitest run tests/unit/skill/` — 11 files, 78/78 - `npm run test:unit` — 181 files, 1592 passed, 2 skipped - `npm run lint` (`tsc -p tsconfig.json --noEmit`) — clean Follow-up (not in this PR): attaching `policyId` to the `skill.verification.failed` event and adding a cooldown, per the issue's part 2 — it touches the event schema and the persistence consumer, so it should be its own change. --- .../memos-local-plugin/core/skill/verifier.ts | 25 ++++++- .../tests/unit/skill/verifier.test.ts | 74 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/skill/verifier.ts b/apps/memos-local-plugin/core/skill/verifier.ts index 276d21815..33888a2a1 100644 --- a/apps/memos-local-plugin/core/skill/verifier.ts +++ b/apps/memos-local-plugin/core/skill/verifier.ts @@ -118,7 +118,20 @@ function computeResonance( if (draftTokens.size === 0) return 0; let hit = 0; for (const t of evidence) { - const txt = `${t.userText}\n${t.agentText}\n${t.reflection ?? ""}`.toLowerCase(); + // Tool sub-steps leave `userText`/`agentText` empty on purpose (the + // viewer's flattenChat contract — see core/capture/step-extractor.ts), + // so their real payload lives in `summary` and `toolCalls`. Include + // those in the resonance text or tool-heavy sessions contribute empty + // strings and every draft fails with resonance-low. Names + inputs + // only — outputs are unbounded. Inputs are capped at the same 300-char + // horizon the embedder uses (core/capture/embedder.ts), so an oversized + // tool input can neither dominate the token budget nor force spurious + // matches on the reward-tick path. + const toolTxt = (t.toolCalls ?? []) + .map((tc) => `${tc.name} ${safeStringify(tc.input).slice(0, 300)}`) + .join(" "); + const txt = + `${t.userText}\n${t.agentText}\n${t.summary ?? ""}\n${t.reflection ?? ""}\n${toolTxt}`.toLowerCase(); const toks = tokensOf(txt); let overlap = 0; for (const tok of draftTokens) if (toks.has(tok)) overlap += 1; @@ -127,6 +140,16 @@ function computeResonance( return hit / evidence.length; } +function safeStringify(v: unknown): string { + if (v === undefined || v === null) return ""; + if (typeof v === "string") return v; + try { + return JSON.stringify(v); + } catch { + return String(v); + } +} + function tokensOf(s: string): Set { const out = new Set(); const asciiMatches = s.match(/[a-z0-9_][a-z0-9_./-]{3,}/g) ?? []; diff --git a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts index 930c95aff..47d1d5288 100644 --- a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts @@ -11,6 +11,7 @@ function trace( userText: string, agentText: string, toolCalls: Partial[] = [], + summary?: string | null, ): TraceRow { return { id: id as TraceRow["id"], @@ -27,6 +28,7 @@ function trace( endedAt: 0 as ToolCallDTO["endedAt"], })), reflection: null, + summary: summary ?? null, value: 0.5, alpha: 0.5 as TraceRow["alpha"], rHuman: null, @@ -181,3 +183,75 @@ describe("skill/verifier", () => { expect(r.ok).toBe(false); }); }); + +describe("skill/verifier — resonance over tool sub-step evidence (#2460)", () => { + // Minimal repro from the issue: evidence traces produced by tool + // sub-steps carry `userText: ""` / `agentText: ""` (step-extractor leaves + // them empty on purpose for the viewer's flattenChat); the turn's real + // payload lives in `toolCalls` and `summary`. A draft that faithfully + // describes those tool calls used to be rejected with resonance = 0. + it("counts toolCalls name/input toward resonance for tool sub-step traces", () => { + const draft = makeDraft({ + summary: "run git commands with an explicit target directory, locally or over ssh", + tools: ["shell"], + steps: [ + { title: "check status", body: "git -C /repo status" }, + { title: "review history", body: "git -C /repo log" }, + ], + }); + const evidence = [ + trace("tr_t1", "", "", [{ name: "shell", input: "git -C /repo status" }]), + trace("tr_t2", "", "", [{ name: "shell", input: "git -C /repo log" }]), + ].map((t) => ({ ...t, reflection: "success" as const })); + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); + expect(r.resonance).toBeGreaterThanOrEqual(0.5); + expect(r.ok).toBe(true); + }); + + it("counts a summary-only trace (no user/agent text) as a resonance hit", () => { + const draft = makeDraft({ + summary: "run git commands with an explicit target directory, locally or over ssh", + tools: ["shell"], + steps: [ + { title: "check status", body: "git -C /repo status" }, + ], + }); + const evidence = [ + trace( + "tr_s1", + "", + "", + [{ name: "shell", input: "git -C /repo status" }], + "agent inspected /repo status before answering", + ), + ]; + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); + expect(r.resonance).toBe(1); + expect(r.ok).toBe(true); + }); + + it("still rejects drafts that share no tokens with tool-only evidence", () => { + // Guard: widening the resonance text must not become a blanket pass. + // Coverage passes (the tool name matches), but nothing in the tool + // inputs overlaps the draft's vocabulary. + const draft = makeDraft({ + summary: "rotate aws credentials through the secrets manager", + tools: ["shell"], + steps: [ + { title: "rotate", body: "call secrets rotate with the new key" }, + ], + }); + const evidence = [ + trace("tr_g1", "", "", [{ name: "shell", input: "tail -n 200 /var/log/syslog" }]), + ]; + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); + expect(r.ok).toBe(false); + expect(r.reason).toContain("resonance="); + }); +}); From 5af69df7323d7d8fb4ec35c619916871b81ed6fb Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Fri, 9 Oct 2026 01:27:18 +0800 Subject: [PATCH 2/3] fix(skill): truncate verifier tool names and harden safeStringify OCR review follow-ups on #2466: - computeResonance: cap tc.name at 60 chars in the resonance text, mirroring the existing 300-char cap on tc.input, so a pathological tool name cannot bypass the token-budget guard. - safeStringify: if String(v) itself throws (custom toString), return a "[unstringifiable]" sentinel instead of letting verification throw. Both covered by new red-first unit assertions in verifier.test.ts. --- .../memos-local-plugin/core/skill/verifier.ts | 17 +++++-- .../tests/unit/skill/verifier.test.ts | 49 +++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/verifier.ts b/apps/memos-local-plugin/core/skill/verifier.ts index 33888a2a1..eef0d828b 100644 --- a/apps/memos-local-plugin/core/skill/verifier.ts +++ b/apps/memos-local-plugin/core/skill/verifier.ts @@ -126,9 +126,14 @@ function computeResonance( // only — outputs are unbounded. Inputs are capped at the same 300-char // horizon the embedder uses (core/capture/embedder.ts), so an oversized // tool input can neither dominate the token budget nor force spurious - // matches on the reward-tick path. + // matches on the reward-tick path. Names get a tighter 60-char cap for + // the same reason: a malformed/pathological name is still just a + // `string` and must not bypass the budget guard. const toolTxt = (t.toolCalls ?? []) - .map((tc) => `${tc.name} ${safeStringify(tc.input).slice(0, 300)}`) + .map( + (tc) => + `${tc.name.slice(0, 60)} ${safeStringify(tc.input).slice(0, 300)}`, + ) .join(" "); const txt = `${t.userText}\n${t.agentText}\n${t.summary ?? ""}\n${t.reflection ?? ""}\n${toolTxt}`.toLowerCase(); @@ -146,7 +151,13 @@ function safeStringify(v: unknown): string { try { return JSON.stringify(v); } catch { - return String(v); + try { + return String(v); + } catch { + // A custom .toString() may throw too — degrade to a sentinel rather + // than let verification blow up on a hostile payload. + return "[unstringifiable]"; + } } } diff --git a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts index 47d1d5288..ed84fc5ba 100644 --- a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts @@ -234,6 +234,55 @@ describe("skill/verifier — resonance over tool sub-step evidence (#2460)", () expect(r.ok).toBe(true); }); + it("caps tool-name contribution at 60 chars in the resonance text", () => { + // A pathologically long tool name must not smuggle tokens past the + // 60-char horizon — only the name's head may contribute to resonance, + // mirroring the 300-char cap already applied to tc.input. + const longName = `${"x".repeat(60)} zwqxalpha zwqxbeta`; + const draft = makeDraft({ + summary: "combine zwqxalpha with zwqxbeta", + steps: [{ title: "combine", body: "zwqxalpha plus zwqxbeta" }], + tools: [longName], + }); + const evidence = [ + trace("tr_n1", "", "", [{ name: longName, input: { hello: "world" } }]), + ]; + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); // coverage still sees the full name + expect(r.resonance).toBe(0); // tail tokens past the cap contribute nothing + expect(r.ok).toBe(false); + expect(r.reason).toContain("resonance="); + }); + + it("does not throw when a tool input defeats both stringify and String()", () => { + // JSON.stringify fails (cyclic) and the custom toString() throws too — + // safeStringify must degrade to a sentinel instead of blowing up + // verification. + const evil: Record = {}; + evil.self = evil; + evil.toString = () => { + throw new Error("boom"); + }; + + const draft = makeDraft({ + summary: "check the repository status with git", + tools: ["shell"], + steps: [{ title: "git status", body: "git -C /repo status" }], + }); + const evidence = [ + trace("tr_e1", "check repo", "git output follows", [ + { name: "shell", input: evil }, + ]), + ]; + + let r: ReturnType; + expect(() => { + r = verifyDraft({ draft, evidence }, { log }); + }).not.toThrow(); + expect(r!.coverage).toBe(1); + }); + it("still rejects drafts that share no tokens with tool-only evidence", () => { // Guard: widening the resonance text must not become a blanket pass. // Coverage passes (the tool name matches), but nothing in the tool From b784e66937ff002f0e686ffedc8ed7e0cc4c61d6 Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Fri, 9 Oct 2026 05:03:46 +0800 Subject: [PATCH 3/3] fix(skill): align ToolCallDTO name type with usage --- apps/memos-local-plugin/core/skill/verifier.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/verifier.ts b/apps/memos-local-plugin/core/skill/verifier.ts index eef0d828b..8fccc8cc0 100644 --- a/apps/memos-local-plugin/core/skill/verifier.ts +++ b/apps/memos-local-plugin/core/skill/verifier.ts @@ -127,12 +127,14 @@ function computeResonance( // horizon the embedder uses (core/capture/embedder.ts), so an oversized // tool input can neither dominate the token budget nor force spurious // matches on the reward-tick path. Names get a tighter 60-char cap for - // the same reason: a malformed/pathological name is still just a - // `string` and must not bypass the budget guard. + // the same reason. `ToolCallDTO.name` is typed `string`, but rows + // replayed from persistence or older schema versions may carry + // null/undefined at runtime, so go through `safeStringify` like the + // input field instead of trusting the annotation. const toolTxt = (t.toolCalls ?? []) .map( (tc) => - `${tc.name.slice(0, 60)} ${safeStringify(tc.input).slice(0, 300)}`, + `${safeStringify(tc.name).slice(0, 60)} ${safeStringify(tc.input).slice(0, 300)}`, ) .join(" "); const txt =