feat: track and display LLM token usage across CLI, reports, and extension

AGENTS.md

@@ -195,6 +195,7 @@ npm test                          # vitest in core/
 | `core/src/execute/runAgentLoop.ts`                             | Thin wrapper: `runAgentAttack(...)` = `runAttack(new AgentAttackDriver(...))`. Shared by the Node (`evaluatorLoop`) and browser (`runAllBrowser`) loops.                                                                                                                                                                                                                   |
 | `core/src/execute/baselineScanner.ts`                          | MCP-only pre-flight scans run before evaluator attacks (tool-poisoning, resource PII/secret leakage, etc.).                                                                                                                                                                                                                                                                |
 | `core/src/execute/runListener.ts`                              | `RunListener` observer SPI — run-level (`onRunStart/Finish/Error`) + per-attack progress hooks. CLI attaches `ConsoleProgressListener` + `JsonlEventListener` (NDJSON via `--events`).                                                                                                                                                                                     |
+| `core/src/execute/tokenTracker.ts`                             | `TokenTracker` class — lightweight accumulator for LLM token usage. Created per-run in `runAll`/`runAllBrowser`, per-evaluator child trackers aggregate into the parent. Auto-records from `withRetry` and bare `generateText` results.                                                                                                                                    |
 | `core/src/execute/aggregate.ts`                                | Folds `AttackResult`s into `EvaluatorResult` / `UnifiedRunReport` (`toEvaluatorResult`, `buildUnifiedReport`, `summarizeVerdicts`).                                                                                                                                                                                                                                        |
 | `core/src/execute/runAllBrowser.ts`                            | Browser-safe variant: takes preloaded evaluators + a pre-built `AgentTarget`, no Node-only imports                                                                                                                                                                                                                                                                         |
 | `core/src/generate/generateAttacks.ts`                         | Generates `AttackSpec[]` for one evaluator — agent-prompt or MCP tool-call shape                                                                                                                                                                                                                                                                                           |
@@ -278,6 +279,8 @@ There is no longer a separate `generate` step. `opfor run --config <file>` does
 
 **Cancellation.** `RunAllOptions` accepts an optional `signal?: AbortSignal`. When aborted, the evaluator loop finishes the in-flight attack, skips remaining evaluators/attacks, and returns a partial report with `stopReason: "user-interrupted"`. The CLI wires this to SIGINT (first Ctrl+C = graceful stop, second = force kill). The SDK can reuse the same mechanism for programmatic cancellation.
 
+**Token usage tracking.** `runAll` creates a `TokenTracker` (see `core/src/execute/tokenTracker.ts`) and threads it through the evaluator loop → attack drivers → `withRetry` / `generateText` / `chatCompletionJsonContent` call sites. Every LLM call auto-records its `usage` (input/output tokens). Per-evaluator child trackers aggregate into evaluator-level totals (`EvaluatorResult.tokenUsage`); the parent accumulates run-level totals (`UnifiedRunReport.summary.tokenUsage`). The CLI prints a summary line, the HTML report shows a stat card, and the JSON report includes the data for CI. `runAllBrowser` follows the same pattern so the extension popup can show a token count.
+
 `runAllBrowser` is the same loop in browser-safe form: takes preloaded `EvaluatorSpec[]` + a pre-built `AgentTarget` (e.g. `DomTarget`), skips disk reads.
 
 ---

core/src/evaluators/judge.ts

@@ -10,6 +10,7 @@ import { formatUpstreamSessions } from "../lib/summarizeSessionContext.js";
 import { log } from "../lib/logger.js";
 import { JUDGE_AGENT_SYSTEM } from "../prompts/judge-agent.js";
 import { withRetry, isStopError } from "../lib/llmRetry.js";
+import type { TokenTracker } from "../execute/tokenTracker.js";
 import { errorJudge, type JudgeResult, type Verdict } from "../lib/judgeTypes.js";
 import { verdictParser } from "./verdictParser.js";
 
@@ -118,7 +119,8 @@ export async function judgeResponse(
   observability?: JudgeObservabilityContext,
   conversationHistory?: ConversationTurn[],
   attackContext?: AttackContext,
-  upstreamSessions?: SessionContext[]
+  upstreamSessions?: SessionContext[],
+  tokenTracker?: TokenTracker
 ): Promise<JudgeResult> {
   const obsLines: string[] = [];
   if (observability?.propagatedTraceId?.trim()) {
@@ -204,7 +206,7 @@ export async function judgeResponse(
   try {
     const result = await withRetry(
       () => generateText({ model, system: JUDGE_SYSTEM, prompt: judgePrompt }),
-      { context: "Judge", maxRetries: 3 }
+      { context: "Judge", maxRetries: 3, tokenTracker }
     );
     return parseJudgeOutput(result.text);
   } catch (err) {

core/src/execute/agentAttackDriver.ts

@@ -19,6 +19,7 @@ import type { TelemetryConfig } from "../config/types.js";
 import type { UnifiedTargetConfig } from "./types.js";
 import { ConversationHistory } from "./conversationHistory.js";
 import type { AttackDriver } from "./attackRunner.js";
+import type { TokenTracker } from "./tokenTracker.js";
 
 export interface AgentAttackContext {
   targetConfig?: UnifiedTargetConfig;
@@ -32,6 +33,7 @@ export interface AgentAttackContext {
    * id, capture whatever the target returns).
    */
   initialSessionId?: string;
+  tokenTracker?: TokenTracker;
 }
 
 /**
@@ -136,6 +138,7 @@ export class AgentAttackDriver implements AttackDriver<string, string> {
       traceContext: this.attack.traceContext,
       previousTechnique: this.previousTechnique,
       upstreamSessions: this.attack.upstreamSessions,
+      tokenTracker: this.context?.tokenTracker,
     });
     this.previousTechnique = result.technique;
     log.dim(
@@ -218,7 +221,8 @@ export class AgentAttackDriver implements AttackDriver<string, string> {
               ),
               this.history.size > 2 ? this.history.messages : undefined,
               { patternName: this.attack.patternName, judgeHint: this.attack.judgeHint },
-              this.attack.upstreamSessions
+              this.attack.upstreamSessions,
+              this.context?.tokenTracker
             );
 
     return {

core/src/execute/evaluatorLoop.ts

@@ -16,6 +16,7 @@ import { errorJudge } from "../lib/judgeTypes.js";
 import { verdictIcon } from "../lib/verdictIcon.js";
 import { TurnPlan } from "./turnPlan.js";
 import { isStopError, getStopReason } from "../lib/llmRetry.js";
+import type { TokenTracker } from "./tokenTracker.js";
 import { log } from "../lib/logger.js";
 import type {
   RunConfig,
@@ -43,6 +44,8 @@ export interface EvaluatorLoopContext {
   notify: (event: ProgressEvent) => void;
   /** Cancellation signal — when aborted, the loop finishes the in-flight attack then stops. */
   signal?: AbortSignal;
+  /** Token usage accumulator. Per-evaluator children are auto-created in the loop. */
+  tokenTracker?: TokenTracker;
 }
 
 /**
@@ -63,6 +66,7 @@ export async function runEvaluatorAttacks(
     traceContext,
     notify,
     signal,
+    tokenTracker,
   } = ctx;
   const sessionMap = new Map<string, SessionContext>();
   const evaluatorResults: EvaluatorResult[] = [];
@@ -97,6 +101,7 @@ export async function runEvaluatorAttacks(
     }
 
     const { turnMode, effectiveTurns } = TurnPlan.from(config);
+    const evalTracker = tokenTracker?.child();
 
     let attacks: AttackSpec[];
     try {
@@ -113,6 +118,7 @@ export async function runEvaluatorAttacks(
           upstreamSessions,
           attackObjective: config.attackObjective,
           businessUseCase: config.businessUseCase,
+          tokenTracker: evalTracker,
         },
       });
     } catch (err) {
@@ -172,15 +178,19 @@ export async function runEvaluatorAttacks(
       try {
         result =
           attack.kind === "mcp"
-            ? await runMcpAttack(attack, mcpTarget!, attackModel, judgeLlmConfig)
+            ? await runMcpAttack(attack, mcpTarget!, attackModel, judgeLlmConfig, evalTracker)
             : await runAgentAttack(
                 attack,
                 attackModel,
                 judgeModel,
                 attack.id,
                 evaluator.patterns,
                 ctx.agentTarget ?? createAgentTarget(config.target as AgentTargetConfig),
-                { targetConfig: config.target, telemetry: config.telemetry }
+                {
+                  targetConfig: config.target,
+                  telemetry: config.telemetry,
+                  tokenTracker: evalTracker,
+                }
               );
       } catch (err) {
         const makeFailedResult = (reason: string): AttackResult =>
@@ -230,7 +240,9 @@ export async function runEvaluatorAttacks(
     const { passed, failed, errors } = summarizeVerdicts(attackResults);
     notify({ type: "evaluator_done", evaluatorId: evaluator.id, passed, failed, errors });
 
-    evaluatorResults.push(toEvaluatorResult(evaluatorMeta, attackResults));
+    const evResult = toEvaluatorResult(evaluatorMeta, attackResults);
+    if (evalTracker) evResult.tokenUsage = evalTracker.totals;
+    evaluatorResults.push(evResult);
     sessionMap.set(evaluator.id, captureSessionContext(evaluator, attackResults));
   }
 

core/src/execute/mcpAttackDriver.ts

@@ -11,6 +11,7 @@ import type { LlmConfig } from "../config/types.js";
 import type { McpTarget, McpToolCallResult } from "../targets/mcpTarget.js";
 import type { McpAttackSpec, McpTurnRecord, AttackResult } from "./types.js";
 import { runAttack, type AttackDriver } from "./attackRunner.js";
+import type { TokenTracker } from "./tokenTracker.js";
 
 /**
  * Drives one MCP attack: call a tool (seed args on turn 1, else adaptively
@@ -35,7 +36,8 @@ export class McpAttackDriver implements AttackDriver<Record<string, unknown>, Mc
     private readonly target: McpTarget,
     private readonly toolName: string,
     private readonly attackModel: LanguageModel,
-    private readonly judgeLlm: LlmConfig
+    private readonly judgeLlm: LlmConfig,
+    private readonly tokenTracker?: TokenTracker
   ) {
     this.judgeHint = attack.judgeHint;
     this.totalTurns = attack.turns;
@@ -50,7 +52,8 @@ export class McpAttackDriver implements AttackDriver<Record<string, unknown>, Mc
       `${this.attack.patternName} — ${this.attack.evaluatorName}`,
       this.toolName,
       this.attack.toolArguments ?? {},
-      this.attackModel
+      this.attackModel,
+      this.tokenTracker
     );
     if (next.judgeHint) this.judgeHint = next.judgeHint;
     return next.args;
@@ -146,6 +149,7 @@ export class McpAttackDriver implements AttackDriver<Record<string, unknown>, Mc
       toolError,
       judgeHint: this.judgeHint,
       priorTurns: this.mcpHistory.length > 1 ? this.mcpHistory.slice(0, -1) : undefined,
+      tokenTracker: this.tokenTracker,
     });
     return sanitizeJudgeResult(result, {
       attackSummary: this.attack.patternName,
@@ -164,7 +168,8 @@ export async function runMcpAttack(
   attack: McpAttackSpec,
   target: McpTarget,
   attackModel: LanguageModel,
-  judgeLlm: LlmConfig
+  judgeLlm: LlmConfig,
+  tokenTracker?: TokenTracker
 ): Promise<AttackResult> {
   if (!attack.toolName) {
     return {
@@ -179,5 +184,7 @@ export async function runMcpAttack(
       judge: mcpErrorJudge("no toolName in attack spec"),
     };
   }
-  return runAttack(new McpAttackDriver(attack, target, attack.toolName, attackModel, judgeLlm));
+  return runAttack(
+    new McpAttackDriver(attack, target, attack.toolName, attackModel, judgeLlm, tokenTracker)
+  );
 }

core/src/execute/runAll.ts

@@ -16,6 +16,7 @@ import { createModel } from "../providers/factory.js";
 import type { LlmConfig } from "../config/types.js";
 import { getAdapter } from "../telemetry/adapter.js";
 import { runSetupTraceCuration } from "../telemetry/curation.js";
+import { TokenTracker } from "./tokenTracker.js";
 import { log } from "../lib/logger.js";
 
 export interface RunAllOptions {
@@ -62,6 +63,7 @@ export async function runAll(
   // if listTools() throws after connect). Everything else — model + evaluator
   // resolution included — runs inside the try so any failure reaches onRunError.
   let mcpTarget: Awaited<ReturnType<typeof createMcpTarget>> | null = null;
+  const tokenTracker = new TokenTracker();
 
   try {
     const attackModel = resolveModel(config.attackerLlm);
@@ -130,12 +132,17 @@ export async function runAll(
       agentTarget: options?.agentTarget,
       notify,
       signal: options?.signal,
+      tokenTracker,
     });
     evaluatorResults.push(...loop.evaluatorResults);
     const stopReason = loop.stopReason;
 
-    // Build report (partial or complete) with stop reason if applicable.
+    // Build report (partial or complete) with stop reason and token usage.
     const report = buildReport(config, evaluatorResults);
+    const usage = tokenTracker.totals;
+    if (usage.totalTokens > 0) {
+      report.summary.tokenUsage = usage;
+    }
     if (stopReason) {
       (report as UnifiedRunReport & { stopReason?: string }).stopReason = stopReason;
     }

core/src/execute/runAllBrowser.ts

@@ -19,6 +19,7 @@ import {
   buildUnifiedReport,
   modelLabel,
 } from "./aggregate.js";
+import { TokenTracker } from "./tokenTracker.js";
 import type {
   AgentAttackSpec,
   AttackResult,
@@ -72,13 +73,15 @@ export async function runAllBrowser(
   const attackModel = createModel(config.attackerLlm);
   const judgeModel = createModel(config.judgeLlm ?? config.attackerLlm);
   const evaluatorResults: EvaluatorResult[] = [];
+  const tokenTracker = new TokenTracker();
   let stopReason: string | undefined;
 
   evaluatorLoop: for (const evaluator of evaluators) {
     notify({ type: "evaluator_start", evaluatorId: evaluator.id, evaluatorName: evaluator.name });
     log.info(`\n▶ ${evaluator.name} (${evaluator.id})`);
 
     const { turnMode, effectiveTurns } = TurnPlan.from(config);
+    const evalTracker = tokenTracker.child();
 
     let generated;
     try {
@@ -97,6 +100,7 @@ export async function runAllBrowser(
         options: {
           attackObjective: config.attackObjective,
           businessUseCase: config.businessUseCase,
+          tokenTracker: evalTracker,
         },
       });
     } catch (err) {
@@ -145,7 +149,10 @@ export async function runAllBrowser(
           attack.id,
           evaluator.patterns,
           agentTarget,
-          options?.initialHistory ? { initialHistory: options.initialHistory } : undefined
+          {
+            ...(options?.initialHistory ? { initialHistory: options.initialHistory } : {}),
+            tokenTracker: evalTracker,
+          }
         );
       } catch (err) {
         // Handle LLM stop errors (attacker/judge)
@@ -222,20 +229,25 @@ export async function runAllBrowser(
     const { passed, failed, errors } = summarizeVerdicts(attackResults);
     notify({ type: "evaluator_done", evaluatorId: evaluator.id, passed, failed, errors });
 
-    evaluatorResults.push(
-      toEvaluatorResult(
-        {
-          evaluatorId: evaluator.id,
-          evaluatorName: evaluator.name,
-          standards: evaluator.standards,
-          severity: evaluator.severity,
-        },
-        attackResults
-      )
+    const evResult = toEvaluatorResult(
+      {
+        evaluatorId: evaluator.id,
+        evaluatorName: evaluator.name,
+        standards: evaluator.standards,
+        severity: evaluator.severity,
+      },
+      attackResults
     );
+    evResult.tokenUsage = evalTracker.totals;
+    evaluatorResults.push(evResult);
   }
 
-  return buildBrowserReport(config, evaluatorResults, stopReason);
+  const report = buildBrowserReport(config, evaluatorResults, stopReason);
+  const usage = tokenTracker.totals;
+  if (usage.totalTokens > 0) {
+    report.summary.tokenUsage = usage;
+  }
+  return report;
 }
 
 function buildBrowserReport(

core/src/execute/tokenTracker.ts

@@ -0,0 +1,65 @@
+/**
+ * Lightweight accumulator for LLM token usage across a run.
+ *
+ * Created per-run and threaded through RunAllOptions → EvaluatorLoopContext →
+ * attack drivers. Each `generateText` / `generateObject` call site records its
+ * usage after the call completes (including retries). Aggregated totals are
+ * surfaced in the CLI summary, HTML/JSON report, and extension popup.
+ */
+
+export interface TokenUsage {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+}
+
+export const ZERO_USAGE: TokenUsage = Object.freeze({
+  inputTokens: 0,
+  outputTokens: 0,
+  totalTokens: 0,
+});
+
+export class TokenTracker {
+  private input = 0;
+  private output = 0;
+
+  /** Record usage from a single LLM call. Safe to call with undefined/partial usage. */
+  record(usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }): void {
+    if (!usage) return;
+    this.input += usage.inputTokens ?? 0;
+    this.output += usage.outputTokens ?? 0;
+  }
+
+  get totals(): TokenUsage {
+    return {
+      inputTokens: this.input,
+      outputTokens: this.output,
+      totalTokens: this.input + this.output,
+    };
+  }
+
+  /** Create a child tracker whose totals can be read independently. */
+  child(): TokenTracker {
+    return new ChildTracker(this);
+  }
+}
+
+/**
+ * A child tracker that records to itself AND to its parent. Used per-evaluator
+ * so the evaluator's own usage is available while the parent accumulates the
+ * run-level total.
+ */
+class ChildTracker extends TokenTracker {
+  constructor(private readonly parent: TokenTracker) {
+    super();
+  }
+
+  override record(usage?: {
+    inputTokens?: number;
+    outputTokens?: number;
+    totalTokens?: number;
+  }): void {
+    super.record(usage);
+    this.parent.record(usage);
+  }
+}