feat: surface quality scores, duration, token usage, and case outcomes in PR reports

src/action.ts

@@ -169,6 +169,8 @@ export async function runAction(): Promise<void> {
       report.code = 'vally-execution-failed';
       report.category = 'execution';
       delete report.executed;
+      delete report.durationMs;
+      delete report.tokens;
       if (executionInputs.command === 'run') {
         report.selection = sha256(JSON.stringify([inputs.suite ?? '', inputs.evalPaths, inputs.tags]));
         if (inputs.suite) report.suite = inputs.suite;

src/native.ts

@@ -4,7 +4,7 @@ import { open, readFile, stat } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
 import type { Inputs } from './inputs.js';
 import { inertEnvironment, object } from './policy.js';
-import type { Report } from './reporting.js';
+import type { Report, ReportTokens } from './reporting.js';
 import { sha256, type EngineIdentity } from './tooling.js';
 
 export class NativeError extends Error {
@@ -20,6 +20,8 @@ export interface NativeResult {
   executed?: number;
   passed?: number;
   failed?: number;
+  durationMs?: number;
+  tokens?: ReportTokens;
   nativeRunId?: string;
   groups?: Report['groups'];
 }
@@ -132,6 +134,33 @@ function count(value: unknown): number {
   return value;
 }
 
+function safeNonNegativeInteger(value: unknown): number | undefined {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
+}
+
+interface BucketAccumulator {
+  trials: number;
+  stimuli: Set<string>;
+  skipped: number;
+  name?: string;
+  durationMs: number;
+  durationTrials: number;
+  totalTokens: number;
+  tokenTrials: number;
+  inputTokens: number;
+  outputTokens: number;
+  ioTokenTrials: number;
+}
+
+function emptyBucket(): BucketAccumulator {
+  return {
+    trials: 0, stimuli: new Set<string>(), skipped: 0,
+    durationMs: 0, durationTrials: 0,
+    totalTokens: 0, tokenTrials: 0,
+    inputTokens: 0, outputTokens: 0, ioTokenTrials: 0,
+  };
+}
+
 export function parseNativeEvidence(text: string, runId: string, exitCode: number | null): NativeResult {
   const rows: Record<string, unknown>[] = [];
   try {
@@ -144,12 +173,12 @@ export function parseNativeEvidence(text: string, runId: string, exitCode: numbe
     throw new NativeError('native-run-summary-missing', 'evidence');
   }
   if (rows.some((row) => row.runId !== runId) || typeof summary.passed !== 'boolean' ||
-      typeof summary.hadExecutionErrors !== 'boolean' || !Array.isArray(summary.evals)) {
+      typeof summary.hadExecutionErrors !== 'boolean' || !Array.isArray(summary.evals) || summary.evals.length > 1000) {
     throw new NativeError('native-evidence-identity-or-schema-mismatch', 'evidence');
   }
   const trials = rows.slice(0, -1);
   const itemIds = new Set<string>();
-  const buckets = new Map<string, { trials: number; stimuli: Set<string>; skipped: number; name?: string }>();
+  const buckets = new Map<string, BucketAccumulator>();
   let executionErrors = summary.hadExecutionErrors;
   for (const trial of trials) {
     if (trial.type !== 'trial-result' || typeof trial.itemId !== 'string' || itemIds.has(trial.itemId) ||
@@ -160,7 +189,7 @@ export function parseNativeEvidence(text: string, runId: string, exitCode: numbe
     }
     itemIds.add(trial.itemId);
     const key = sha256(JSON.stringify([trial.evalFilePath, trial.variant, trial.model ?? '']));
-    const bucket = buckets.get(key) ?? { trials: 0, stimuli: new Set<string>(), skipped: 0 };
+    const bucket = buckets.get(key) ?? emptyBucket();
     if (trial.evalName !== undefined) {
       if (typeof trial.evalName !== 'string' || !trial.evalName || trial.evalName.length > 256 ||
           /[\r\n]/.test(trial.evalName) || (bucket.name !== undefined && bucket.name !== trial.evalName)) {
@@ -172,6 +201,11 @@ export function parseNativeEvidence(text: string, runId: string, exitCode: numbe
     else {
       bucket.trials++;
       bucket.stimuli.add(trial.stimulus);
+      const trialDurationMs = safeNonNegativeInteger(trial.durationMs);
+      if (trialDurationMs !== undefined) {
+        bucket.durationMs += trialDurationMs;
+        bucket.durationTrials++;
+      }
       if (trial.status === 'error') executionErrors = true;
       if (trial.status === 'success' && (trial.gradeResult === undefined || trial.trajectory === null || trial.trajectory === undefined)) {
         throw new NativeError('native-trial-evidence-missing', 'evidence');
@@ -180,6 +214,26 @@ export function parseNativeEvidence(text: string, runId: string, exitCode: numbe
         try {
           const trajectory = object(trial.trajectory);
           if (typeof trajectory.id !== 'string' || typeof trajectory.output !== 'string') throw new Error('Invalid native trajectory.');
+          if (trajectory.metrics !== undefined && trajectory.metrics !== null &&
+              typeof trajectory.metrics === 'object' && !Array.isArray(trajectory.metrics)) {
+            const metrics = trajectory.metrics as Record<string, unknown>;
+            if (metrics.tokenUsage !== undefined && metrics.tokenUsage !== null &&
+                typeof metrics.tokenUsage === 'object' && !Array.isArray(metrics.tokenUsage)) {
+              const usage = metrics.tokenUsage as Record<string, unknown>;
+              const totalTokens = safeNonNegativeInteger(usage.totalTokens);
+              const inputTokens = safeNonNegativeInteger(usage.inputTokens);
+              const outputTokens = safeNonNegativeInteger(usage.outputTokens);
+              if (totalTokens !== undefined) {
+                bucket.totalTokens += totalTokens;
+                bucket.tokenTrials++;
+                if (inputTokens !== undefined && outputTokens !== undefined && inputTokens + outputTokens <= totalTokens) {
+                  bucket.inputTokens += inputTokens;
+                  bucket.outputTokens += outputTokens;
+                  bucket.ioTokenTrials++;
+                }
+              }
+            }
+          }
           // Null is the native contract for an ungraded trial. Never invent a grader requirement.
           const pending: unknown[] = trial.gradeResult === null ? [] : [trial.gradeResult];
           while (pending.length) {
@@ -208,13 +262,14 @@ export function parseNativeEvidence(text: string, runId: string, exitCode: numbe
         typeof group.passed !== 'boolean' || typeof group.scoringApplied !== 'boolean' ||
         (group.model !== undefined && typeof group.model !== 'string')) throw new NativeError('invalid-native-eval-summary', 'evidence');
     if (group.scoringApplied && (typeof group.overallScore !== 'number' || !Number.isFinite(group.overallScore) ||
+        group.overallScore < 0 || group.overallScore > 1 ||
         typeof group.threshold !== 'number' || !Number.isFinite(group.threshold) || group.threshold < 0 || group.threshold > 1)) {
       throw new NativeError('invalid-native-scoring-evidence', 'evidence');
     }
     const key = sha256(JSON.stringify([group.evalFilePath, group.variant, group.model ?? '']));
     if (seen.has(key)) throw new NativeError('duplicate-native-eval-summary', 'evidence');
     seen.add(key);
-    const bucket = buckets.get(key) ?? { trials: 0, stimuli: new Set<string>(), skipped: 0 };
+    const bucket = buckets.get(key) ?? emptyBucket();
     const stimuliRun = count(group.stimuliRun);
     const stimuliTotal = count(group.stimuliTotal);
     const skipped = count(group.stimuliSkipped ?? 0);
@@ -224,13 +279,55 @@ export function parseNativeEvidence(text: string, runId: string, exitCode: numbe
     if (group.error !== undefined) executionErrors = true;
     const name = bucket.name ?? basename(group.evalFilePath);
     if (!name || name.length > 256 || /[\r\n]/.test(name)) throw new NativeError('invalid-native-eval-name', 'evidence');
-    groups.push({ identity: key, name, passed: group.passed, trials: bucket.trials, stimuliRun, stimuliTotal, skipped });
+    const groupDurationMs = bucket.trials > 0 && bucket.durationTrials === bucket.trials ? bucket.durationMs : undefined;
+    const groupTokens: ReportTokens | undefined = bucket.trials > 0 && bucket.tokenTrials === bucket.trials
+      ? {
+          ...(bucket.ioTokenTrials === bucket.trials
+            ? { input: bucket.inputTokens, output: bucket.outputTokens }
+            : {}),
+          total: bucket.totalTokens,
+        }
+      : undefined;
+    groups.push({
+      identity: key,
+      name,
+      passed: group.passed,
+      trials: bucket.trials,
+      stimuliRun,
+      stimuliTotal,
+      skipped,
+      ...(bucket.trials > 0 && group.scoringApplied ? { overallScore: group.overallScore as number, threshold: group.threshold as number } : {}),
+      ...(groupDurationMs !== undefined ? { durationMs: groupDurationMs } : {}),
+      ...(groupTokens !== undefined ? { tokens: groupTokens } : {}),
+    });
   }
   if ([...buckets.keys()].some((key) => !seen.has(key)) || (summary.passed && groups.some((group) => !group.passed))) {
     throw new NativeError('inconsistent-native-verdict', 'evidence');
   }
   const executed = groups.reduce((total, group) => total + group.trials, 0);
-  const base = { completeness: 'complete' as const, executed, nativeRunId: runId, groups };
+  const executedGroups = groups.filter((group) => group.trials > 0);
+  const durationMs = executedGroups.length > 0 && executedGroups.every((group) => group.durationMs !== undefined)
+    ? executedGroups.reduce((total, group) => total + group.durationMs!, 0)
+    : undefined;
+  const tokens: ReportTokens | undefined = executedGroups.length > 0 && executedGroups.every((group) => group.tokens !== undefined)
+    ? {
+        ...(executedGroups.every((group) => group.tokens!.input !== undefined && group.tokens!.output !== undefined)
+          ? {
+              input: executedGroups.reduce((sum, group) => sum + group.tokens!.input!, 0),
+              output: executedGroups.reduce((sum, group) => sum + group.tokens!.output!, 0),
+            }
+          : {}),
+        total: executedGroups.reduce((sum, group) => sum + group.tokens!.total, 0),
+      }
+    : undefined;
+  const base: Omit<NativeResult, 'outcome' | 'category' | 'code'> = {
+    completeness: 'complete' as const,
+    executed,
+    nativeRunId: runId,
+    groups,
+    ...(durationMs !== undefined ? { durationMs } : {}),
+    ...(tokens !== undefined ? { tokens } : {}),
+  };
   if (executionErrors) return { ...base, outcome: 'error', category: 'execution', code: 'native-execution-error' };
   // Never use text matching to reinterpret a nonzero native exit as an empty successful run.
   if (executed === 0 && exitCode === 0 && summary.passed) return { ...base, outcome: 'not-run', category: 'none', code: 'native-empty-selection' };

src/reporter.ts

@@ -1,7 +1,7 @@
 import { parseDocument } from 'yaml';
 import { commentMarker, orderedBody, selectComment, type Comment, type CommentIdentity, type ReportOrder } from './comments.js';
 import { object } from './policy.js';
-import { escapeMarkdown, type Report } from './reporting.js';
+import { escapeMarkdown, hasQualityDetails, renderQualitySection, type Report } from './reporting.js';
 import { ReportingError, type GitHubClient } from './github.js';
 
 export const commentConcurrencyGroup = 'evals-${{ github.workflow_ref }}';
@@ -149,15 +149,15 @@ export async function postReport(
   return decision.action === 'create' ? 'created' : 'updated';
 }
 
-function renderComment(
+export function renderComment(
   report: Report, artifactId: number | undefined, companion: { report: Report; artifactId: number } | undefined, run: string,
 ): string {
   const lint = companion?.report ?? (report.command === 'lint' ? report : undefined);
   const evaluation = report.command === 'run' ? report : undefined;
   const status = evaluation?.outcome ?? lint?.outcome ?? report.outcome;
   const rows = evaluation?.groups ?? [];
   const suite = evaluation?.suite;
-  return `# Evals: ${status}\n\n` +
+  const base = `# Evals: ${status}\n\n` +
     (suite ? `**Suite:** ${escapeMarkdown(suite)}\n\n` : evaluation ? '**Selection:** Native eval paths or tags\n\n' : '') +
     (lint ? `**Lint:** ${lint.outcome}${lint.outcome === 'validated' ? ' (no model execution)' : ` - ${escapeMarkdown(lint.code)}`}` +
       ((companion?.artifactId ?? artifactId) !== undefined ? ` - [Evidence](${run}/artifacts/${companion?.artifactId ?? artifactId})` : '') + '\n\n' : '') +
@@ -170,4 +170,8 @@ function renderComment(
           `| ${escapeMarkdown(group.name ?? group.identity.slice(0, 12))} | ${group.trials} | ${group.trials ? group.passed ? 'passed' : 'failed' : 'not-run'} |`,
         ).join('\n') + '\n\n' +
         (rows.length > 100 ? `${rows.length - 100} further groups are in the structured artifact.\n\n` : '') : '') : '');
+  if (!evaluation || !rows.length || !hasQualityDetails(rows)) return base;
+  const maxQualityBytes = Math.min(16_000, Math.max(0, 57_500 - Buffer.byteLength(base, 'utf8')));
+  const quality = renderQualitySection('### Quality & token summary', rows, 25, maxQualityBytes);
+  return quality ? `${base}${quality}\n\n` : base;
 }