diff --git a/apps/memos-local-plugin/core/skill/verifier.ts b/apps/memos-local-plugin/core/skill/verifier.ts index 276d21815..8fccc8cc0 100644 --- a/apps/memos-local-plugin/core/skill/verifier.ts +++ b/apps/memos-local-plugin/core/skill/verifier.ts @@ -118,7 +118,27 @@ function computeResonance( if (draftTokens.size === 0) return 0; let hit = 0; for (const t of evidence) { - const txt = `${t.userText}\n${t.agentText}\n${t.reflection ?? ""}`.toLowerCase(); + // Tool sub-steps leave `userText`/`agentText` empty on purpose (the + // viewer's flattenChat contract — see core/capture/step-extractor.ts), + // so their real payload lives in `summary` and `toolCalls`. Include + // those in the resonance text or tool-heavy sessions contribute empty + // strings and every draft fails with resonance-low. Names + inputs + // only — outputs are unbounded. Inputs are capped at the same 300-char + // horizon the embedder uses (core/capture/embedder.ts), so an oversized + // tool input can neither dominate the token budget nor force spurious + // matches on the reward-tick path. Names get a tighter 60-char cap for + // the same reason. `ToolCallDTO.name` is typed `string`, but rows + // replayed from persistence or older schema versions may carry + // null/undefined at runtime, so go through `safeStringify` like the + // input field instead of trusting the annotation. + const toolTxt = (t.toolCalls ?? []) + .map( + (tc) => + `${safeStringify(tc.name).slice(0, 60)} ${safeStringify(tc.input).slice(0, 300)}`, + ) + .join(" "); + const txt = + `${t.userText}\n${t.agentText}\n${t.summary ?? ""}\n${t.reflection ?? ""}\n${toolTxt}`.toLowerCase(); const toks = tokensOf(txt); let overlap = 0; for (const tok of draftTokens) if (toks.has(tok)) overlap += 1; @@ -127,6 +147,22 @@ function computeResonance( return hit / evidence.length; } +function safeStringify(v: unknown): string { + if (v === undefined || v === null) return ""; + if (typeof v === "string") return v; + try { + return JSON.stringify(v); + } catch { + try { + return String(v); + } catch { + // A custom .toString() may throw too — degrade to a sentinel rather + // than let verification blow up on a hostile payload. + return "[unstringifiable]"; + } + } +} + function tokensOf(s: string): Set { const out = new Set(); const asciiMatches = s.match(/[a-z0-9_][a-z0-9_./-]{3,}/g) ?? []; diff --git a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts index 930c95aff..ed84fc5ba 100644 --- a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts @@ -11,6 +11,7 @@ function trace( userText: string, agentText: string, toolCalls: Partial[] = [], + summary?: string | null, ): TraceRow { return { id: id as TraceRow["id"], @@ -27,6 +28,7 @@ function trace( endedAt: 0 as ToolCallDTO["endedAt"], })), reflection: null, + summary: summary ?? null, value: 0.5, alpha: 0.5 as TraceRow["alpha"], rHuman: null, @@ -181,3 +183,124 @@ describe("skill/verifier", () => { expect(r.ok).toBe(false); }); }); + +describe("skill/verifier — resonance over tool sub-step evidence (#2460)", () => { + // Minimal repro from the issue: evidence traces produced by tool + // sub-steps carry `userText: ""` / `agentText: ""` (step-extractor leaves + // them empty on purpose for the viewer's flattenChat); the turn's real + // payload lives in `toolCalls` and `summary`. A draft that faithfully + // describes those tool calls used to be rejected with resonance = 0. + it("counts toolCalls name/input toward resonance for tool sub-step traces", () => { + const draft = makeDraft({ + summary: "run git commands with an explicit target directory, locally or over ssh", + tools: ["shell"], + steps: [ + { title: "check status", body: "git -C /repo status" }, + { title: "review history", body: "git -C /repo log" }, + ], + }); + const evidence = [ + trace("tr_t1", "", "", [{ name: "shell", input: "git -C /repo status" }]), + trace("tr_t2", "", "", [{ name: "shell", input: "git -C /repo log" }]), + ].map((t) => ({ ...t, reflection: "success" as const })); + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); + expect(r.resonance).toBeGreaterThanOrEqual(0.5); + expect(r.ok).toBe(true); + }); + + it("counts a summary-only trace (no user/agent text) as a resonance hit", () => { + const draft = makeDraft({ + summary: "run git commands with an explicit target directory, locally or over ssh", + tools: ["shell"], + steps: [ + { title: "check status", body: "git -C /repo status" }, + ], + }); + const evidence = [ + trace( + "tr_s1", + "", + "", + [{ name: "shell", input: "git -C /repo status" }], + "agent inspected /repo status before answering", + ), + ]; + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); + expect(r.resonance).toBe(1); + expect(r.ok).toBe(true); + }); + + it("caps tool-name contribution at 60 chars in the resonance text", () => { + // A pathologically long tool name must not smuggle tokens past the + // 60-char horizon — only the name's head may contribute to resonance, + // mirroring the 300-char cap already applied to tc.input. + const longName = `${"x".repeat(60)} zwqxalpha zwqxbeta`; + const draft = makeDraft({ + summary: "combine zwqxalpha with zwqxbeta", + steps: [{ title: "combine", body: "zwqxalpha plus zwqxbeta" }], + tools: [longName], + }); + const evidence = [ + trace("tr_n1", "", "", [{ name: longName, input: { hello: "world" } }]), + ]; + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); // coverage still sees the full name + expect(r.resonance).toBe(0); // tail tokens past the cap contribute nothing + expect(r.ok).toBe(false); + expect(r.reason).toContain("resonance="); + }); + + it("does not throw when a tool input defeats both stringify and String()", () => { + // JSON.stringify fails (cyclic) and the custom toString() throws too — + // safeStringify must degrade to a sentinel instead of blowing up + // verification. + const evil: Record = {}; + evil.self = evil; + evil.toString = () => { + throw new Error("boom"); + }; + + const draft = makeDraft({ + summary: "check the repository status with git", + tools: ["shell"], + steps: [{ title: "git status", body: "git -C /repo status" }], + }); + const evidence = [ + trace("tr_e1", "check repo", "git output follows", [ + { name: "shell", input: evil }, + ]), + ]; + + let r: ReturnType; + expect(() => { + r = verifyDraft({ draft, evidence }, { log }); + }).not.toThrow(); + expect(r!.coverage).toBe(1); + }); + + it("still rejects drafts that share no tokens with tool-only evidence", () => { + // Guard: widening the resonance text must not become a blanket pass. + // Coverage passes (the tool name matches), but nothing in the tool + // inputs overlaps the draft's vocabulary. + const draft = makeDraft({ + summary: "rotate aws credentials through the secrets manager", + tools: ["shell"], + steps: [ + { title: "rotate", body: "call secrets rotate with the new key" }, + ], + }); + const evidence = [ + trace("tr_g1", "", "", [{ name: "shell", input: "tail -n 200 /var/log/syslog" }]), + ]; + + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.coverage).toBe(1); + expect(r.ok).toBe(false); + expect(r.reason).toContain("resonance="); + }); +});