Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion apps/memos-local-plugin/core/skill/verifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,27 @@ function computeResonance(
if (draftTokens.size === 0) return 0;
let hit = 0;
for (const t of evidence) {
const txt = `${t.userText}\n${t.agentText}\n${t.reflection ?? ""}`.toLowerCase();
// Tool sub-steps leave `userText`/`agentText` empty on purpose (the
// viewer's flattenChat contract — see core/capture/step-extractor.ts),
// so their real payload lives in `summary` and `toolCalls`. Include
// those in the resonance text or tool-heavy sessions contribute empty
// strings and every draft fails with resonance-low. Names + inputs
// only — outputs are unbounded. Inputs are capped at the same 300-char
// horizon the embedder uses (core/capture/embedder.ts), so an oversized
// tool input can neither dominate the token budget nor force spurious
// matches on the reward-tick path. Names get a tighter 60-char cap for
// the same reason. `ToolCallDTO.name` is typed `string`, but rows
// replayed from persistence or older schema versions may carry
// null/undefined at runtime, so go through `safeStringify` like the
// input field instead of trusting the annotation.
const toolTxt = (t.toolCalls ?? [])
.map(
(tc) =>
`${safeStringify(tc.name).slice(0, 60)} ${safeStringify(tc.input).slice(0, 300)}`,
)
.join(" ");
const txt =
`${t.userText}\n${t.agentText}\n${t.summary ?? ""}\n${t.reflection ?? ""}\n${toolTxt}`.toLowerCase();
const toks = tokensOf(txt);
let overlap = 0;
for (const tok of draftTokens) if (toks.has(tok)) overlap += 1;
Expand All @@ -127,6 +147,22 @@ function computeResonance(
return hit / evidence.length;
}

function safeStringify(v: unknown): string {
if (v === undefined || v === null) return "";
if (typeof v === "string") return v;
try {
return JSON.stringify(v);
} catch {
try {
return String(v);
} catch {
// A custom .toString() may throw too — degrade to a sentinel rather
// than let verification blow up on a hostile payload.
return "[unstringifiable]";
}
}
}

function tokensOf(s: string): Set<string> {
const out = new Set<string>();
const asciiMatches = s.match(/[a-z0-9_][a-z0-9_./-]{3,}/g) ?? [];
Expand Down
123 changes: 123 additions & 0 deletions apps/memos-local-plugin/tests/unit/skill/verifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ function trace(
userText: string,
agentText: string,
toolCalls: Partial<ToolCallDTO>[] = [],
summary?: string | null,
): TraceRow {
return {
id: id as TraceRow["id"],
Expand All @@ -27,6 +28,7 @@ function trace(
endedAt: 0 as ToolCallDTO["endedAt"],
})),
reflection: null,
summary: summary ?? null,
value: 0.5,
alpha: 0.5 as TraceRow["alpha"],
rHuman: null,
Expand Down Expand Up @@ -181,3 +183,124 @@ describe("skill/verifier", () => {
expect(r.ok).toBe(false);
});
});

describe("skill/verifier — resonance over tool sub-step evidence (#2460)", () => {
// Minimal repro from the issue: evidence traces produced by tool
// sub-steps carry `userText: ""` / `agentText: ""` (step-extractor leaves
// them empty on purpose for the viewer's flattenChat); the turn's real
// payload lives in `toolCalls` and `summary`. A draft that faithfully
// describes those tool calls used to be rejected with resonance = 0.
it("counts toolCalls name/input toward resonance for tool sub-step traces", () => {
const draft = makeDraft({
summary: "run git commands with an explicit target directory, locally or over ssh",
tools: ["shell"],
steps: [
{ title: "check status", body: "git -C /repo status" },
{ title: "review history", body: "git -C /repo log" },
],
});
const evidence = [
trace("tr_t1", "", "", [{ name: "shell", input: "git -C /repo status" }]),
trace("tr_t2", "", "", [{ name: "shell", input: "git -C /repo log" }]),
].map((t) => ({ ...t, reflection: "success" as const }));

const r = verifyDraft({ draft, evidence }, { log });
expect(r.coverage).toBe(1);
expect(r.resonance).toBeGreaterThanOrEqual(0.5);
expect(r.ok).toBe(true);
});

it("counts a summary-only trace (no user/agent text) as a resonance hit", () => {
const draft = makeDraft({
summary: "run git commands with an explicit target directory, locally or over ssh",
tools: ["shell"],
steps: [
{ title: "check status", body: "git -C /repo status" },
],
});
const evidence = [
trace(
"tr_s1",
"",
"",
[{ name: "shell", input: "git -C /repo status" }],
"agent inspected /repo status before answering",
),
];

const r = verifyDraft({ draft, evidence }, { log });
expect(r.coverage).toBe(1);
expect(r.resonance).toBe(1);
expect(r.ok).toBe(true);
});

it("caps tool-name contribution at 60 chars in the resonance text", () => {
// A pathologically long tool name must not smuggle tokens past the
// 60-char horizon — only the name's head may contribute to resonance,
// mirroring the 300-char cap already applied to tc.input.
const longName = `${"x".repeat(60)} zwqxalpha zwqxbeta`;
const draft = makeDraft({
summary: "combine zwqxalpha with zwqxbeta",
steps: [{ title: "combine", body: "zwqxalpha plus zwqxbeta" }],
tools: [longName],
});
const evidence = [
trace("tr_n1", "", "", [{ name: longName, input: { hello: "world" } }]),
];

const r = verifyDraft({ draft, evidence }, { log });
expect(r.coverage).toBe(1); // coverage still sees the full name
expect(r.resonance).toBe(0); // tail tokens past the cap contribute nothing
expect(r.ok).toBe(false);
expect(r.reason).toContain("resonance=");
});

it("does not throw when a tool input defeats both stringify and String()", () => {
// JSON.stringify fails (cyclic) and the custom toString() throws too —
// safeStringify must degrade to a sentinel instead of blowing up
// verification.
const evil: Record<string, unknown> = {};
evil.self = evil;
evil.toString = () => {
throw new Error("boom");
};

const draft = makeDraft({
summary: "check the repository status with git",
tools: ["shell"],
steps: [{ title: "git status", body: "git -C /repo status" }],
});
const evidence = [
trace("tr_e1", "check repo", "git output follows", [
{ name: "shell", input: evil },
]),
];

let r: ReturnType<typeof verifyDraft>;
expect(() => {
r = verifyDraft({ draft, evidence }, { log });
}).not.toThrow();
expect(r!.coverage).toBe(1);
});

it("still rejects drafts that share no tokens with tool-only evidence", () => {
// Guard: widening the resonance text must not become a blanket pass.
// Coverage passes (the tool name matches), but nothing in the tool
// inputs overlaps the draft's vocabulary.
const draft = makeDraft({
summary: "rotate aws credentials through the secrets manager",
tools: ["shell"],
steps: [
{ title: "rotate", body: "call secrets rotate with the new key" },
],
});
const evidence = [
trace("tr_g1", "", "", [{ name: "shell", input: "tail -n 200 /var/log/syslog" }]),
];

const r = verifyDraft({ draft, evidence }, { log });
expect(r.coverage).toBe(1);
expect(r.ok).toBe(false);
expect(r.reason).toContain("resonance=");
});
});