diff --git a/evalboard/app/_components/__tests__/harness-badge.test.tsx b/evalboard/app/_components/__tests__/harness-badge.test.tsx
new file mode 100644
index 00000000..32b384c8
--- /dev/null
+++ b/evalboard/app/_components/__tests__/harness-badge.test.tsx
@@ -0,0 +1,45 @@
+import { describe, expect, test } from "vitest";
+import { render, screen } from "@testing-library/react";
+import { KNOWN_HARNESSES } from "@/lib/harness";
+import { HarnessBadge, harnessShortLabel } from "../harness-badge";
+
+describe("harnessShortLabel", () => {
+ test("every known harness has a human label", () => {
+ // A missing entry falls through to the raw id, which would put
+ // "delegate-sdk" in a legend next to "Codex" and "Antigravity".
+ for (const h of KNOWN_HARNESSES) {
+ expect(harnessShortLabel(h)).not.toBe(h);
+ }
+ });
+
+ test("the UiPath harness reads as Delegate", () => {
+ // The run data's id stays `delegate-sdk` (it's the registered
+ // `agent.type`); only the label people read is the short one.
+ expect(harnessShortLabel("delegate-sdk")).toBe("Delegate");
+ });
+
+ test("an unknown harness falls back to its id rather than a wrong name", () => {
+ expect(harnessShortLabel("some-new-agent")).toBe("some-new-agent");
+ });
+});
+
+describe("HarnessBadge", () => {
+ test("names the vendor in the alt text, not just the product", () => {
+ render();
+ expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument();
+ });
+
+ test("renders the id as text when there is no logo for it", () => {
+ // Better a raw id than another vendor's mark on someone else's run.
+ render();
+ expect(screen.getByText("some-new-agent")).toBeInTheDocument();
+ });
+
+ test("takes a size so the chart legend can sit inside 11px text", () => {
+ render();
+ expect(screen.getByAltText("Codex · OpenAI")).toHaveAttribute(
+ "width",
+ "14",
+ );
+ });
+});
diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx
index da16b704..02a7bbaa 100644
--- a/evalboard/app/_components/harness-badge.tsx
+++ b/evalboard/app/_components/harness-badge.tsx
@@ -22,6 +22,16 @@ const HARNESS_LOGO: Record
);
diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx
index 7a814f0e..9e7acc0a 100644
--- a/evalboard/app/_components/harness-selector.tsx
+++ b/evalboard/app/_components/harness-selector.tsx
@@ -3,44 +3,96 @@
import { usePathname, useRouter, useSearchParams } from "next/navigation";
import { HarnessBadge, harnessShortLabel } from "./harness-badge";
-// Segmented control for the trends page's harness scope. Sets `?h=`
-// while preserving the active q/tag params, mirroring WindowSelector. Each
-// segment shows the vendor logo + short label so it reads like the per-run
-// harness badge on the runs tables.
+// Segmented control for a page's harness scope. Sets `?h=` while
+// preserving the active q/tag params, mirroring WindowSelector. Each segment
+// shows the vendor logo + short label so it reads like the per-run harness badge
+// on the runs tables.
+//
+// `current` is null only on pages that support the all-harness view (the
+// overview, whose charts draw a line per harness and whose tiles and run list
+// then cover every harness). Pages that need exactly one harness — trends
+// collapses per-task history across runs, which only means something inside one
+// harness — pass a concrete harness and leave `includeAll` off.
export function HarnessSelector({
current,
harnesses,
+ includeAll = false,
}: {
- current: string;
+ current: string | null;
harnesses: readonly string[];
+ includeAll?: boolean;
}) {
const router = useRouter();
const pathname = usePathname();
const searchParams = useSearchParams();
- const set = (h: string) => {
+ const set = (h: string | null) => {
const p = new URLSearchParams(searchParams.toString());
- p.set("h", h);
- router.replace(`${pathname}?${p.toString()}`, { scroll: false });
+ if (h == null) {
+ // The unscoped view is the default, so it's the absence of the param
+ // rather than `h=all` — keeps the canonical URL clean.
+ p.delete("h");
+ } else {
+ p.set("h", h);
+ }
+ // Changing scope changes which runs are in play, so the paged-out row
+ // counts no longer describe the new set; drop them back to page one.
+ p.delete("limit");
+ p.delete("alimit");
+ const qs = p.toString();
+ router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false });
};
// Always show the active harness, even if it has aged out of the recent
// window (so a deep-linked `?h=` still reads as selected rather than absent).
- const opts = harnesses.includes(current)
- ? harnesses
- : [current, ...harnesses];
+ const opts =
+ current == null || harnesses.includes(current)
+ ? harnesses
+ : [current, ...harnesses];
+ // shrink-0 + whitespace-nowrap: a segment must keep its label on one line
+ // and never compress, so on a narrow screen the control scrolls (see the
+ // wrapper) instead of wrapping "Claude Code" onto two lines and stretching
+ // the whole page past the viewport.
+ const segment = (active: boolean) =>
+ `flex shrink-0 items-center gap-1.5 whitespace-nowrap px-2.5 py-1 sm:px-3 ${
+ active
+ ? "bg-studio-blue text-white"
+ : "bg-white text-gray-700 hover:bg-gray-50"
+ }`;
+ // Below `sm` the vendor logo carries the identity on its own (each button
+ // keeps the full name as its accessible label and tooltip) — five spelled-out
+ // segments don't fit on a phone, and the chart legend below names every line.
+ const label = (text: string) => (
+ {text}
+ );
return (
-
- {opts.map((h) => {
+
+ {includeAll && (
+
+ )}
+ {opts.map((h, i) => {
const active = h === current;
+ const name = harnessShortLabel(h);
return (
);
})}
diff --git a/evalboard/app/_components/window-selector.tsx b/evalboard/app/_components/window-selector.tsx
deleted file mode 100644
index 3394eeef..00000000
--- a/evalboard/app/_components/window-selector.tsx
+++ /dev/null
@@ -1,33 +0,0 @@
-"use client";
-
-import { usePathname, useRouter, useSearchParams } from "next/navigation";
-import { WINDOWS, type Window } from "@/lib/reviews-types";
-
-export function WindowSelector({ current }: { current: Window }) {
- const router = useRouter();
- const pathname = usePathname();
- const searchParams = useSearchParams();
- const set = (w: Window) => {
- const p = new URLSearchParams(searchParams.toString());
- p.set("window", w);
- router.replace(`${pathname}?${p.toString()}`, { scroll: false });
- };
- return (
-
- {WINDOWS.map((w) => {
- const active = w === current;
- return (
-
- );
- })}
-
- );
-}
diff --git a/evalboard/app/_overview/__tests__/harness-legend.test.tsx b/evalboard/app/_overview/__tests__/harness-legend.test.tsx
new file mode 100644
index 00000000..f55698f4
--- /dev/null
+++ b/evalboard/app/_overview/__tests__/harness-legend.test.tsx
@@ -0,0 +1,130 @@
+import { describe, expect, test } from "vitest";
+import { render, screen } from "@testing-library/react";
+import { harnessColor } from "@/lib/harness";
+import { HarnessLegend, HarnessTooltip } from "../harness-legend";
+import { seriesKey } from "../harness-series";
+
+function series(...harnesses: string[]) {
+ return harnesses.map((harness, i) => ({
+ harness,
+ dataKey: seriesKey(harness, i),
+ color: harnessColor(harness),
+ }));
+}
+
+describe("HarnessLegend", () => {
+ test("names every series, so identity is never color-alone", () => {
+ render();
+ expect(screen.getByText("Claude Code")).toBeInTheDocument();
+ expect(screen.getByText("Codex")).toBeInTheDocument();
+ });
+
+ test("renders nothing for a single series", () => {
+ // A one-line chart is already named by its own heading; a legend box
+ // there is noise.
+ const { container } = render(
+ ,
+ );
+ expect(container).toBeEmptyDOMElement();
+ });
+
+ test("renders nothing when there is no data at all", () => {
+ const { container } = render();
+ expect(container).toBeEmptyDOMElement();
+ });
+});
+
+describe("HarnessTooltip", () => {
+ const s = series("claude-code", "codex");
+
+ test("shows only the harnesses that actually ran at the hovered x", () => {
+ // Recharts hands over every series, including the ones with no value
+ // here. Listing those would invent runs that never happened.
+ render(
+ ,
+ );
+ expect(screen.getByText("Claude Code")).toBeInTheDocument();
+ expect(screen.getByText("92.5% success")).toBeInTheDocument();
+ expect(screen.queryByText("Codex")).not.toBeInTheDocument();
+ });
+
+ test("keeps a real zero instead of filtering it out as falsy", () => {
+ render(
+ ,
+ );
+ expect(screen.getByText("0.0% success")).toBeInTheDocument();
+ });
+
+ test("falls back to the empty text when no series has a value", () => {
+ render(
+ ,
+ );
+ expect(
+ screen.getByText("no tasks with a turn budget"),
+ ).toBeInTheDocument();
+ });
+
+ test("ignores payload entries for unknown series keys", () => {
+ render(
+ ,
+ );
+ expect(screen.getByText("no tasks")).toBeInTheDocument();
+ });
+
+ test("renders nothing while inactive", () => {
+ const { container } = render(
+ ,
+ );
+ expect(container).toBeEmptyDOMElement();
+ });
+
+ test("renders the UTC timestamp header from the x label", () => {
+ render(
+ ,
+ );
+ expect(screen.getByText("2026-07-30 04:27 UTC")).toBeInTheDocument();
+ });
+});
diff --git a/evalboard/app/_overview/__tests__/harness-series.test.ts b/evalboard/app/_overview/__tests__/harness-series.test.ts
new file mode 100644
index 00000000..399bbd02
--- /dev/null
+++ b/evalboard/app/_overview/__tests__/harness-series.test.ts
@@ -0,0 +1,148 @@
+import { describe, expect, test } from "vitest";
+import { harnessColor } from "@/lib/harness";
+import type { RunPoint } from "@/lib/overview";
+import { pivotByHarness, seriesKey } from "../harness-series";
+
+// pivotByHarness is the only thing standing between "one line per harness" and
+// the zigzag the charts used to draw across incomparable harnesses. Its output
+// is consumed by recharts, so a wrong key or a dropped row shows up as a missing
+// line rather than an error — nothing else would fail.
+
+function point(
+ harness: string,
+ timestamp: number,
+ successRate: number | null,
+ turnBudgetRate: number | null = null,
+): RunPoint {
+ return {
+ runId: `run-${timestamp}`,
+ timestamp,
+ harness,
+ successRate,
+ turnBudgetRate,
+ };
+}
+
+describe("seriesKey", () => {
+ test("prefixes and sanitizes so recharts can't read it as a path", () => {
+ // A dataKey containing "." is a nested-object path in recharts, which
+ // would resolve to undefined and silently draw no line.
+ expect(seriesKey("gpt-5.5", 0)).not.toContain(".");
+ expect(seriesKey("claude-code", 0)).toBe("h0_claude_code");
+ });
+
+ test("is injective even when two ids sanitize alike", () => {
+ // Without the index prefix "a.b" and "a-b" both become "h_a_b", and one
+ // harness's line would silently take over the other's column.
+ const ids = ["claude-code", "codex", "gpt-5.5", "gpt-5-5"];
+ const keys = ids.map((id, i) => seriesKey(id, i));
+ expect(new Set(keys).size).toBe(ids.length);
+ });
+});
+
+describe("pivotByHarness", () => {
+ test("gives each harness its own column, keyed by run timestamp", () => {
+ const { rows, series } = pivotByHarness(
+ [point("claude-code", 100, 90), point("codex", 200, 70)],
+ ["claude-code", "codex"],
+ "successRate",
+ );
+ expect(series.map((s) => s.harness)).toEqual(["claude-code", "codex"]);
+ const [cc, cx] = series.map((s) => s.dataKey);
+ expect(rows).toEqual([
+ { timestamp: 100, [cc]: 90 },
+ { timestamp: 200, [cx]: 70 },
+ ]);
+ // The absent key on each row is what connectNulls bridges — that is how
+ // interleaved runs become one continuous line per harness.
+ expect(rows[0]).not.toHaveProperty(cx);
+ });
+
+ test("colors come from the harness, not the series index", () => {
+ const { series } = pivotByHarness(
+ [point("codex", 100, 70)],
+ ["codex"],
+ "successRate",
+ );
+ // codex is slot 2, but it is the ONLY series here. If color were assigned
+ // positionally it would come back as slot 1's blue.
+ expect(series[0].color).toBe(harnessColor("codex"));
+ expect(series[0].color).not.toBe(harnessColor("claude-code"));
+ });
+
+ test("merges two harnesses that share a timestamp into one row", () => {
+ const { rows, series } = pivotByHarness(
+ [point("claude-code", 100, 90), point("codex", 100, 70)],
+ ["claude-code", "codex"],
+ "successRate",
+ );
+ const [cc, cx] = series.map((s) => s.dataKey);
+ expect(rows).toEqual([{ timestamp: 100, [cc]: 90, [cx]: 70 }]);
+ });
+
+ test("sorts rows by timestamp regardless of input order", () => {
+ const { rows } = pivotByHarness(
+ [point("codex", 300, 1), point("codex", 100, 2)],
+ ["codex"],
+ "successRate",
+ );
+ expect(rows.map((r) => r.timestamp)).toEqual([100, 300]);
+ });
+
+ test("reads the requested metric, not the other one", () => {
+ // Both charts share this pivot and the same RunPoint[], differing only
+ // by the metric key — reading the wrong one would put the success rate
+ // on the turn-budget chart with no visible error.
+ const pts = [point("codex", 100, 90, 40)];
+ const key = seriesKey("codex", 0);
+ expect(pivotByHarness(pts, ["codex"], "successRate").rows[0]).toEqual({
+ timestamp: 100,
+ [key]: 90,
+ });
+ expect(pivotByHarness(pts, ["codex"], "turnBudgetRate").rows[0]).toEqual(
+ { timestamp: 100, [key]: 40 },
+ );
+ });
+
+ test("omits a null metric rather than plotting it as zero", () => {
+ // The turn-budget metric is null for a run with no budgeted task. A 0%
+ // there would read as "every task blew its budget", which is the
+ // opposite of "there was nothing to measure".
+ const { rows } = pivotByHarness(
+ [point("codex", 100, 90, null)],
+ ["codex"],
+ "turnBudgetRate",
+ );
+ expect(rows).toEqual([]);
+ });
+
+ test("drops series with no plotted point so the legend stays honest", () => {
+ // antigravity is a known harness with no run in this window; an empty
+ // line would still put it in the legend.
+ const { series } = pivotByHarness(
+ [point("codex", 100, 70)],
+ ["codex", "antigravity"],
+ "successRate",
+ );
+ expect(series.map((s) => s.harness)).toEqual(["codex"]);
+ });
+
+ test("keeps a harness whose only value is a real zero", () => {
+ // 0% is a legitimate pass rate (every task failed) and must not be
+ // confused with "no data" by a truthiness check.
+ const { rows, series } = pivotByHarness(
+ [point("codex", 100, 0)],
+ ["codex"],
+ "successRate",
+ );
+ expect(series).toHaveLength(1);
+ expect(rows[0][series[0].dataKey]).toBe(0);
+ });
+
+ test("handles an empty window", () => {
+ expect(pivotByHarness([], ["codex"], "successRate")).toEqual({
+ rows: [],
+ series: [],
+ });
+ });
+});
diff --git a/evalboard/app/_overview/daily-chart.tsx b/evalboard/app/_overview/daily-chart.tsx
index 721dd655..51309e7a 100644
--- a/evalboard/app/_overview/daily-chart.tsx
+++ b/evalboard/app/_overview/daily-chart.tsx
@@ -10,6 +10,8 @@ import {
YAxis,
} from "recharts";
import type { RunPoint } from "@/lib/overview";
+import { pivotByHarness } from "./harness-series";
+import { HarnessLegend, HarnessTooltip } from "./harness-legend";
// MM-DD tick label on the axis; full date+time appears in the tooltip.
function shortLabel(ms: number): string {
@@ -19,94 +21,86 @@ function shortLabel(ms: number): string {
return `${m}-${day}`;
}
-function fullLabel(ms: number): string {
- const d = new Date(ms);
- const y = d.getUTCFullYear();
- const m = String(d.getUTCMonth() + 1).padStart(2, "0");
- const day = String(d.getUTCDate()).padStart(2, "0");
- const h = String(d.getUTCHours()).padStart(2, "0");
- const min = String(d.getUTCMinutes()).padStart(2, "0");
- return `${y}-${m}-${day} ${h}:${min} UTC`;
-}
-
-function CustomTooltip({
- active,
- payload,
-}: {
- active?: boolean;
- payload?: Array<{ payload: RunPoint }>;
-}) {
- if (!active || !payload?.length) return null;
- const point = payload[0].payload;
- return (
-
- );
-}
-
+// One line per harness present in the window. Unscoped (the page default) that
+// means every harness on one chart, which is the comparison the page is for;
+// scoped to one harness it degenerates to the single line this chart used to be.
export function DailySuccessChart({
data,
+ harnesses,
windowStart,
windowEnd,
}: {
data: RunPoint[];
+ harnesses: string[];
windowStart: number;
windowEnd: number;
}) {
+ const { rows, series } = pivotByHarness(data, harnesses, "successRate");
return (
-
);
}
diff --git a/evalboard/app/_overview/harness-legend.tsx b/evalboard/app/_overview/harness-legend.tsx
new file mode 100644
index 00000000..750c600e
--- /dev/null
+++ b/evalboard/app/_overview/harness-legend.tsx
@@ -0,0 +1,111 @@
+"use client";
+
+// Legend + tooltip shared by the two overview charts. Both are multi-series
+// whenever the page is unscoped, and a multi-series chart must never identify a
+// line by color alone — so the legend pairs each swatch with the harness's
+// vendor badge and short label, and the tooltip repeats that pairing per row.
+
+import { harnessShortLabel, HarnessBadge } from "@/app/_components/harness-badge";
+import type { HarnessSeries } from "./harness-series";
+
+export function HarnessLegend({ series }: { series: HarnessSeries[] }) {
+ // One series needs no legend box — the chart's own heading names it.
+ if (series.length < 2) return null;
+ return (
+
+ );
+}
+
+// Recharts hands every series to the tooltip, including the ones with no value
+// at the hovered x. Only the harnesses that actually ran at this timestamp are
+// worth a row.
+export interface TooltipEntry {
+ dataKey?: string | number;
+ value?: number | string | null;
+ color?: string;
+}
+
+function fullLabel(ms: number): string {
+ const d = new Date(ms);
+ const y = d.getUTCFullYear();
+ const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+ const day = String(d.getUTCDate()).padStart(2, "0");
+ const h = String(d.getUTCHours()).padStart(2, "0");
+ const min = String(d.getUTCMinutes()).padStart(2, "0");
+ return `${y}-${m}-${day} ${h}:${min} UTC`;
+}
+
+export function HarnessTooltip({
+ active,
+ payload,
+ label,
+ series,
+ suffix,
+ emptyText,
+}: {
+ active?: boolean;
+ payload?: TooltipEntry[];
+ // The x value under the cursor; recharts passes the XAxis dataKey's value.
+ label?: number | string;
+ series: HarnessSeries[];
+ // Trailing prose after the number, e.g. "success".
+ suffix: string;
+ // Shown when the hovered run produced no value for this metric at all.
+ emptyText: string;
+}) {
+ if (!active || !payload?.length) return null;
+ const byKey = new Map(series.map((s) => [s.dataKey, s]));
+ const rows = payload.filter(
+ (e) => typeof e.value === "number" && byKey.has(String(e.dataKey)),
+ );
+ const ms = typeof label === "number" ? label : Number(label);
+ return (
+
+ );
+}
diff --git a/evalboard/app/_overview/harness-series.ts b/evalboard/app/_overview/harness-series.ts
new file mode 100644
index 00000000..1b50f111
--- /dev/null
+++ b/evalboard/app/_overview/harness-series.ts
@@ -0,0 +1,75 @@
+// Pivot the flat RunPoint[] into the wide, one-column-per-harness shape recharts
+// needs to draw several lines on one chart.
+//
+// Each run belongs to exactly one harness, so a row carries a value for that
+// harness's column and leaves the others undefined. Every then uses
+// connectNulls to bridge its own gaps, which is what turns sparse interleaved
+// runs (claude-code nightly, codex twice a week) into one continuous line each
+// rather than a single zigzag across incomparable harnesses.
+
+import { harnessColor } from "@/lib/harness";
+import type { RunPoint } from "@/lib/overview";
+
+// Which per-run rate to plot. Both metrics live on the same RunPoint, so the
+// two overview charts share this module and differ only by this key.
+export type HarnessMetric = "successRate" | "turnBudgetRate";
+
+export interface HarnessSeries {
+ harness: string;
+ // Column name in the pivoted rows. See seriesKey.
+ dataKey: string;
+ color: string;
+}
+
+export interface HarnessChartData {
+ rows: Array>;
+ series: HarnessSeries[];
+}
+
+// Sanitized because recharts reads a dataKey containing "." as a nested object
+// path, which would resolve to undefined and silently draw no line. Prefixed
+// with the series index because sanitizing alone is not injective — `a.b` and
+// `a-b` would collide onto one column and one harness's data would vanish
+// under the other's.
+export function seriesKey(harness: string, index: number): string {
+ return `h${index}_${harness.replace(/[^\w]/g, "_")}`;
+}
+
+export function pivotByHarness(
+ points: RunPoint[],
+ harnesses: string[],
+ metric: HarnessMetric,
+): HarnessChartData {
+ const series: HarnessSeries[] = harnesses.map((harness, i) => ({
+ harness,
+ dataKey: seriesKey(harness, i),
+ color: harnessColor(harness),
+ }));
+ const keyByHarness = new Map(series.map((s) => [s.harness, s.dataKey]));
+
+ // Merge on timestamp so two harnesses that happen to start a run in the same
+ // second share one row instead of producing two points recharts would render
+ // stacked on the same x.
+ const byTimestamp = new Map>();
+ for (const p of points) {
+ if (p[metric] == null) continue;
+ // A point whose harness isn't in the requested series list has no column
+ // to land in; dropping it keeps a stale id out of the chart.
+ const key = keyByHarness.get(p.harness);
+ if (!key) continue;
+ let row = byTimestamp.get(p.timestamp);
+ if (!row) {
+ row = { timestamp: p.timestamp };
+ byTimestamp.set(p.timestamp, row);
+ }
+ row[key] = p[metric] as number;
+ }
+
+ const rows = [...byTimestamp.values()].sort(
+ (a, b) => a.timestamp - b.timestamp,
+ );
+ // Drop series with no plotted point in this window — an empty line would put
+ // an entry in the legend for a harness the chart never draws.
+ const present = series.filter((s) => rows.some((r) => s.dataKey in r));
+ return { rows, series: present };
+}
diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx
index 2bf1f9be..97b0ef09 100644
--- a/evalboard/app/_overview/tag-rail.tsx
+++ b/evalboard/app/_overview/tag-rail.tsx
@@ -1,7 +1,5 @@
import Link from "next/link";
import type { TagCount } from "@/lib/overview";
-import { DEFAULT_HARNESS } from "@/lib/harness";
-import type { Window } from "@/lib/reviews-types";
type Variant = "neutral" | "rose" | "indigo";
@@ -34,18 +32,17 @@ const STYLES: Record<
function hrefForTag(
basePath: string,
tag: string | null,
- window: Window | null,
q: string | null,
harness: string | null,
): string {
const params = new URLSearchParams();
- if (window) params.set("window", window);
if (tag) params.set("tag", tag);
if (q) params.set("q", q);
- // Preserve the active harness scope across tag clicks (omit the default to
- // keep URLs clean). Without this, filtering by a tag would silently reset a
- // codex/antigravity view back to claude-code.
- if (harness && harness !== DEFAULT_HARNESS) params.set("h", harness);
+ // Preserve the active harness scope across tag clicks. null is the
+ // all-harness default and is expressed as the param's absence, so there is
+ // nothing to carry. Without this, filtering by a tag would silently widen a
+ // codex/antigravity view back out to every harness.
+ if (harness) params.set("h", harness);
const qs = params.toString();
return qs ? `${basePath}?${qs}` : basePath;
}
@@ -56,7 +53,6 @@ function TagChip({
variant,
active,
basePath,
- window,
q,
harness,
}: {
@@ -65,14 +61,13 @@ function TagChip({
variant: Variant;
active: boolean;
basePath: string;
- window: Window | null;
q: string | null;
harness: string | null;
}) {
const s = STYLES[variant];
return (
@@ -117,7 +112,6 @@ export function MergedTagRail({
reviewTags,
activeTag,
basePath = "/",
- window = null,
q = null,
harness = null,
limit = 24,
@@ -129,8 +123,6 @@ export function MergedTagRail({
// Path the chip links point at — "/" for the overview, "/trends" for the
// trends page. Query string is preserved.
basePath?: string;
- // Null on pages that don't expose a window selector (e.g. /trends).
- window?: Window | null;
q?: string | null;
// Active harness scope to preserve in chip links (null = not harness-scoped).
harness?: string | null;
@@ -155,7 +147,6 @@ export function MergedTagRail({
variant="indigo"
active={tc.tag === activeTag}
basePath={basePath}
- window={window}
q={q}
harness={harness}
/>
@@ -168,7 +159,6 @@ export function MergedTagRail({
variant="rose"
active={tc.tag === activeTag}
basePath={basePath}
- window={window}
q={q}
harness={harness}
/>
@@ -181,7 +171,6 @@ export function MergedTagRail({
variant="neutral"
active={tc.tag === activeTag}
basePath={basePath}
- window={window}
q={q}
harness={harness}
/>
diff --git a/evalboard/app/_overview/turn-budget-chart.tsx b/evalboard/app/_overview/turn-budget-chart.tsx
index f274f32e..19fb95d7 100644
--- a/evalboard/app/_overview/turn-budget-chart.tsx
+++ b/evalboard/app/_overview/turn-budget-chart.tsx
@@ -10,6 +10,8 @@ import {
YAxis,
} from "recharts";
import type { RunPoint } from "@/lib/overview";
+import { pivotByHarness } from "./harness-series";
+import { HarnessLegend, HarnessTooltip } from "./harness-legend";
// MM-DD tick label on the axis; full date+time appears in the tooltip.
function shortLabel(ms: number): string {
@@ -19,99 +21,90 @@ function shortLabel(ms: number): string {
return `${m}-${day}`;
}
-function fullLabel(ms: number): string {
- const d = new Date(ms);
- const y = d.getUTCFullYear();
- const m = String(d.getUTCMonth() + 1).padStart(2, "0");
- const day = String(d.getUTCDate()).padStart(2, "0");
- const h = String(d.getUTCHours()).padStart(2, "0");
- const min = String(d.getUTCMinutes()).padStart(2, "0");
- return `${y}-${m}-${day} ${h}:${min} UTC`;
-}
-
-function CustomTooltip({
- active,
- payload,
-}: {
- active?: boolean;
- payload?: Array<{ payload: RunPoint }>;
-}) {
- if (!active || !payload?.length) return null;
- const point = payload[0].payload;
- return (
-
-
- {fullLabel(point.timestamp)}
-
-
- {point.turnBudgetRate != null
- ? `${point.turnBudgetRate.toFixed(1)}% within turn budget (budgeted failures count as over)`
- : "no tasks with a turn budget"}
-
-
- );
-}
-
-// Sibling of DailySuccessChart: same axes/styling, plotting the per-run
-// "within expected turns" rate instead of the success rate. Driven by the
-// same windowed RunPoint[] so the shared window selector controls both.
+// Sibling of DailySuccessChart: same axes/styling and the same per-harness
+// series split, plotting the per-run "within expected turns" rate instead of the
+// success rate. Driven by the same windowed RunPoint[] so the shared window and
+// harness selectors control both.
export function TurnBudgetChart({
data,
+ harnesses,
windowStart,
windowEnd,
}: {
data: RunPoint[];
+ harnesses: string[];
windowStart: number;
windowEnd: number;
}) {
+ const { rows, series } = pivotByHarness(data, harnesses, "turnBudgetRate");
return (
-
);
}
diff --git a/evalboard/app/_overview/window-summary.tsx b/evalboard/app/_overview/window-summary.tsx
index cd3f5629..29c06245 100644
--- a/evalboard/app/_overview/window-summary.tsx
+++ b/evalboard/app/_overview/window-summary.tsx
@@ -1,4 +1,5 @@
-import { fmtDuration, fmtUsd, passClass } from "@/lib/format";
+import { fmtDuration, fmtUsd } from "@/lib/format";
+import { passClass } from "@/lib/pass-rate";
import type { RunListingTotals } from "@/lib/overview";
import type { Window } from "@/lib/reviews-types";
@@ -33,10 +34,14 @@ function Tile({
}
// Front-page window rollup: total spend + shape of the runs in scope. Every
-// tile is summed over the same set — `runCount` is the matched-run count
-// (`matchedCount`), which is exactly what `totals` folds in, so the Runs tile
-// can never disagree with the Cost/Tasks/Pass/Compute tiles. The totals are
-// scoped to matching tasks whenever a filter is active.
+// tile is summed over the same set — `runCount` and `totals` both come out of
+// getOverview's single pass, so the Runs tile can never disagree with the
+// Cost/Tasks/Pass/Compute tiles or with the charts. The totals are scoped to
+// matching tasks whenever a filter is active.
+//
+// This describes the window the charts plot, NOT however far the run table below
+// is paged out (the table reads back past this window), so every sub-label names
+// the window even when scoped — otherwise a narrowed view reads as all-time.
export function WindowSummary({
totals,
window,
@@ -52,7 +57,9 @@ export function WindowSummary({
totals.tasksRun > 0
? (totals.tasksSucceeded / totals.tasksRun) * 100
: null;
- const scope = isFiltered ? "matching runs" : `runs · last ${window}`;
+ const scope = isFiltered
+ ? `matching · last ${window}`
+ : `runs · last ${window}`;
return (
@@ -64,7 +71,7 @@ export function WindowSummary({
totals.costPartial
? "some runs missing cost"
: isFiltered
- ? "matching tasks"
+ ? `matching tasks · ${window}`
: `across ${window}`
}
/>
@@ -81,7 +88,7 @@ export function WindowSummary({
0)}
+ valueClass={passClass(pct)}
/>
;
}) {
const params = await searchParams;
- const window = parseWindow(params.window);
const activeTag = parseTag(params.tag);
const q = parseQ(params.q);
- const harness = parseHarnessParam(params.h);
- const limit = parseLimit(params.limit);
- const adhocLimit = parseAdhocLimit(params.alimit);
+ // null = every harness, and that is the default: the page opens on the
+ // cross-harness comparison and narrows from there.
+ const harness = parseHarnessScope(params.h);
+ const limit = parsePagedLimit(params.limit, DEFAULT_LIMIT);
+ const adhocLimit = parsePagedLimit(params.alimit, ADHOC_LIMIT);
+ // Tasks WITHIN a run are narrowed: drives the "Matching tasks" column header
+ // and the clear-filters link.
const isFiltered = activeTag != null || q != null;
+ // The set of RUNS is narrowed, harness scope included: drives every "n of N"
+ // count, since N only means "everything" when nothing is scoped.
+ const isNarrowed = isFiltered || harness != null;
- // The analytics block (chart + rails) is scoped to one harness so the
- // success line stops zigzagging across incomparable harnesses. The run
- // LIST stays all-harness — seeing every recent run is the page's job, and
- // the Harness column already disambiguates each row.
+ // One harness scope drives everything on the page: the charts split into a
+ // line per harness in the window, and the summary tiles + run list are
+ // filtered to the same set. Picking a harness therefore recomputes the
+ // tiles and the charts and narrows the table, instead of re-scoping the
+ // analytics block while the table silently kept showing every harness.
const [overview, listing, adhoc, harnesses] = await Promise.all([
- getOverview(window, activeTag, q, harness),
- getRunListing(window, activeTag, q, limit),
+ getOverview(WINDOW, activeTag, q, harness),
+ getRunListing(activeTag, q, limit, harness),
getAdhocRunListing(adhocLimit),
listRecentHarnesses(),
]);
@@ -131,10 +137,14 @@ export default async function Page({
const reviewTags = filterTagsByQuery(overview.reviewTags, q);
const shownCount = listing.rows.length;
- const matchedCount = listing.matchedCount;
- const totalInWindow = listing.totalInWindow;
- const tableTotalLabel = isFiltered ? matchedCount : totalInWindow;
- const hasMore = limit != null && shownCount < tableTotalLabel;
+ // Another page exists AND the page cap still has room for it. Under a
+ // filter there is no total to count against: proving how many older runs
+ // match would mean reading all of them, so the table reports what it has
+ // and offers another page while one is there.
+ const hasMore = listing.hasMore && shownCount < MAX_LIMIT;
+ const tableCountLabel = isNarrowed
+ ? `${shownCount} matching`
+ : `${shownCount} of ${listing.totalCandidates}`;
// Current URL params, normalized to scalars. Spread as the base of every
// href so toggling one section (main `limit` or ad-hoc `alimit`) carries
@@ -143,11 +153,10 @@ export default async function Page({
const rawAlimit = Array.isArray(params.alimit)
? params.alimit[0]
: params.alimit;
- // Omit the default harness from URLs to keep them clean; carry a non-default
+ // Omit the all-harness scope from URLs to keep them clean; carry a narrowed
// scope through every self-link so it isn't reset by pagination/clear.
- const hParam = harness === DEFAULT_HARNESS ? undefined : harness;
+ const hParam = harness ?? undefined;
const base = {
- window,
tag: activeTag,
q,
h: hParam,
@@ -155,19 +164,25 @@ export default async function Page({
alimit: rawAlimit,
};
+ // Both tables grow one page at a time, up to MAX_LIMIT.
+ const nextPageSize = Math.min(DEFAULT_LIMIT, MAX_LIMIT - shownCount);
const showMoreHref = buildHref({
...base,
- limit: Math.min(tableTotalLabel, shownCount + DEFAULT_LIMIT),
+ limit: shownCount + nextPageSize,
});
- const showAllHref = buildHref({ ...base, limit: "all" });
- const clearAllHref = buildHref({ window, h: hParam });
+ const clearAllHref = buildHref({ h: hParam });
// Ad-hoc section disclosure: rows are filtered (by `q`) then capped to
- // adhocLimit; offer "Show all" while more match than are shown, and a
- // collapse back to the default once expanded past it.
- const adhocExpandable = adhoc.rows.length < adhoc.total;
- const adhocExpanded = adhocLimit == null && adhoc.total > ADHOC_LIMIT;
- const adhocShowAllHref = buildHref({ ...base, alimit: "all" });
+ // adhocLimit; page in another ADHOC_LIMIT while more match than are shown,
+ // and offer a collapse back to the default once expanded past it.
+ const adhocRemaining = Math.min(adhoc.total, MAX_LIMIT) - adhoc.rows.length;
+ const adhocExpandable = adhocRemaining > 0;
+ const adhocNextPageSize = Math.min(ADHOC_LIMIT, adhocRemaining);
+ const adhocExpanded = adhoc.rows.length > ADHOC_LIMIT;
+ const adhocShowMoreHref = buildHref({
+ ...base,
+ alimit: adhoc.rows.length + adhocNextPageSize,
+ });
const adhocShowLessHref = buildHref({ ...base, alimit: undefined });
return (
@@ -183,14 +198,14 @@ export default async function Page({
{/* The analytics block — daily success / turn-budget charts, the
- window selector, and the colored skill/review/tag rail — is an
+ harness selector, and the colored skill/review/tag rail — is an
internal-only surface (see lib/edition.ts). The public OSS
edition drops it so the front page is just the run list. */}
{isInternal && (
@@ -221,12 +236,12 @@ export default async function Page({
)}{" "}
- over the last {window} ·{" "}
+ over the last {WINDOW} ·{" "}
{overview.runs.length} run
{overview.runs.length === 1 ? "" : "s"}
{" · "}
@@ -236,23 +251,30 @@ export default async function Page({
) : (
<>
Success rate per{" "}
- {harnessShortLabel(harness)} run across the
- last {window} · {overview.runs.length} run
+ {harness
+ ? `${harnessShortLabel(harness)} run`
+ : "run, one line per harness,"}{" "}
+ across the last {WINDOW} ·{" "}
+ {overview.runs.length} run
{overview.runs.length === 1 ? "" : "s"}
)}
-
+ {/* min-w-0 + flex-wrap: the two segmented controls are wider
+ than a phone together, so they stack rather than pushing
+ the whole page past the viewport. */}
+
-
@@ -263,11 +285,13 @@ export default async function Page({
% of budgeted tasks that stayed within 1.5× their
expected turns (a budgeted task that failed counts as
- over budget)
+ over budget) · runs with no budgeted task are omitted
+ rather than plotted at 0
{activeTag || q ? " · scoped to the active filter" : ""}
@@ -280,7 +304,6 @@ export default async function Page({
taskTags={taskTags}
reviewTags={reviewTags}
activeTag={activeTag}
- window={window}
q={q}
harness={harness}
limit={24}
@@ -305,9 +328,7 @@ export default async function Page({
Runs
- {isFiltered
- ? `${shownCount} of ${matchedCount} matching · ${totalInWindow} in ${window}`
- : `${shownCount} of ${totalInWindow} in ${window}`}
+ {tableCountLabel}
{isFiltered && (
- Show{" "}
- {Math.min(
- DEFAULT_LIMIT,
- tableTotalLabel - shownCount,
- )}{" "}
- more
-
- ·
-
- Show all ({tableTotalLabel})
+ Show {nextPageSize} more
+
+ {tableCountLabel}
+
+ {/* nowrap: the table already scrolls inside
+ its own container, so a narrow screen
+ should scroll it rather than break the
+ timestamp across three lines. */}
+
{adhocExpandable && (
- Show all ({adhoc.total})
+ Show {adhocNextPageSize} more
)}
+
+ {adhoc.rows.length} of {adhoc.total}
+
{adhocExpanded && (
0,
- )}`}
+ className={`font-medium ${passClass(pct)}`}
>
{pct != null
? `${pct.toFixed(0)}%`
diff --git a/evalboard/app/path-to-ga/page.tsx b/evalboard/app/path-to-ga/page.tsx
index 57b8514b..1432ddb5 100644
--- a/evalboard/app/path-to-ga/page.tsx
+++ b/evalboard/app/path-to-ga/page.tsx
@@ -4,12 +4,12 @@ import {
getTagTaskBreakdown,
listRecentHarnesses,
} from "@/lib/overview";
-import { parseHarnessParam } from "@/lib/harness";
+import { parseHarnessScope } from "@/lib/harness";
import { humanizeTaskId } from "@/lib/format";
-import { WindowSelector } from "../_components/window-selector";
+import { passClass } from "@/lib/pass-rate";
import { HarnessSelector } from "../_components/harness-selector";
import { harnessShortLabel } from "../_components/harness-badge";
-import { WINDOWS, type Window } from "@/lib/reviews-types";
+import { type Window } from "@/lib/reviews-types";
import { DailySuccessChart } from "../_overview/daily-chart";
import { TableScroll } from "../_components/scroll-table";
import { StatusPill } from "@/lib/pills";
@@ -18,32 +18,26 @@ export const dynamic = "force-dynamic";
const TAG = "path-to-ga";
-function parseWindow(raw: string | string[] | undefined): Window {
- const v = Array.isArray(raw) ? raw[0] : raw;
- return WINDOWS.includes(v as Window) ? (v as Window) : "30d";
-}
-
-function passClass(pct: number | null): string {
- if (pct == null) return "text-gray-500";
- if (pct >= 80) return "text-green-700";
- if (pct >= 50) return "text-gray-700";
- return "text-red-700";
-}
+// Fixed 30-day window, matching the front page: no window control, since the
+// readiness question is "where is this task now, and how did it get here" rather
+// than "how did it look in a 1-day slice".
+const WINDOW: Window = "30d";
export default async function PathToGaPage({
searchParams,
}: {
- searchParams: Promise<{ window?: string; h?: string }>;
+ searchParams: Promise<{ h?: string }>;
}) {
const params = await searchParams;
- const window = parseWindow(params.window);
- const harness = parseHarnessParam(params.h);
+ // null = every harness, and that is the default. The chart draws a line per
+ // harness, so the readiness of a task on each is comparable side by side
+ // rather than requiring four page loads. Picking a harness narrows the
+ // chart, the headline numbers, and the task table together.
+ const harness = parseHarnessScope(params.h);
- // Scope to one harness — readiness of a task is per-harness, and a blended
- // chart/pass-rate mixes incomparable regimes.
const [overview, taskRows, harnesses] = await Promise.all([
- getOverview(window, TAG, null, harness),
- getTagTaskBreakdown(window, TAG, harness),
+ getOverview(WINDOW, TAG, null, harness),
+ getTagTaskBreakdown(WINDOW, TAG, harness),
listRecentHarnesses(),
]);
@@ -63,13 +57,18 @@ export default async function PathToGaPage({
Score for every task tagged{" "}
- {TAG} on{" "}
- {harnessShortLabel(harness)}.
+ {TAG}
+ {harness
+ ? ` on ${harnessShortLabel(harness)}.`
+ : ", across every harness."}
- No {TAG} runs in the last {window}.
+ No {TAG} runs in the last {WINDOW}.
)}
-
Tasks
+
+
+ Tasks
+
+ {/* Unscoped, a task's appearances span harnesses, so its rate
+ pools regimes that aren't strictly comparable. Say so
+ rather than let the number read as one harness's. */}
+ {!harness && (
+
+ pooled across harnesses · pick one above to separate
+ them
+
+ )}
+
@@ -192,7 +205,7 @@ export default async function PathToGaPage({
className="py-6 px-4 text-center text-sm text-gray-500"
>
No tasks tagged {TAG} in the last{" "}
- {window}.
+ {WINDOW}.
)}
diff --git a/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx b/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx
new file mode 100644
index 00000000..a2951def
--- /dev/null
+++ b/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx
@@ -0,0 +1,76 @@
+import { describe, expect, test } from "vitest";
+import { render, screen } from "@testing-library/react";
+import { RunIdentity } from "../run-identity";
+
+// The run page's numbers are only comparable across runs on the same harness and
+// model, so getting this strip wrong (or silently omitting it) is what sends
+// someone comparing a codex run's pass rate against a claude-code one.
+describe("RunIdentity", () => {
+ test("names the harness and the model", () => {
+ render(
+ ,
+ );
+ expect(screen.getByText("Codex")).toBeInTheDocument();
+ expect(screen.getByText("gpt-5.2-codex")).toBeInTheDocument();
+ });
+
+ test("uses the vendor logo, not just the raw harness id", () => {
+ render(
+ ,
+ );
+ // The UiPath mark stands in for the Delegate harness, which has no file
+ // of its own under /harness. The id stays `delegate-sdk` (that is what
+ // run.json carries); only the label is shortened.
+ expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument();
+ expect(screen.getByText("Delegate")).toBeInTheDocument();
+ });
+
+ test("flags a multi-model run instead of claiming the dominant one", () => {
+ render(
+ ,
+ );
+ expect(screen.getByText("+2 more")).toBeInTheDocument();
+ });
+
+ test("says nothing extra on a single-model run", () => {
+ render(
+ ,
+ );
+ expect(screen.queryByText(/more/)).not.toBeInTheDocument();
+ });
+
+ test("renders each half independently when the other is missing", () => {
+ const { rerender } = render(
+ ,
+ );
+ expect(screen.getByText("Codex")).toBeInTheDocument();
+
+ rerender(
+ ,
+ );
+ expect(screen.getByText("some-model")).toBeInTheDocument();
+ });
+
+ test("renders nothing for a legacy run that identifies neither", () => {
+ const { container } = render(
+ ,
+ );
+ expect(container).toBeEmptyDOMElement();
+ });
+});
diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx
index 7d0f58f2..72682ccf 100644
--- a/evalboard/app/runs/[id]/page.tsx
+++ b/evalboard/app/runs/[id]/page.tsx
@@ -13,6 +13,7 @@ import { fmtRunTime } from "@/lib/format";
import { AnalysisPanel } from "./analysis-panel";
import { RefreshButton } from "./refresh-button";
import { RunView } from "./run-view";
+import { RunIdentity } from "./run-identity";
import { VersionList } from "@/app/_components/version-list";
import { isInternal } from "@/lib/edition";
@@ -102,6 +103,13 @@ export default async function RunPage({
{meta.description}
)}
+ {/* Harness + model, so the page says what produced these
+ numbers without a trip back to the runs table. */}
+
{/* Component SHAs (cli / agent / sdk / drawer …) point at
internal tooling; internal-only. See lib/edition.ts. */}
{isInternal && (
diff --git a/evalboard/app/runs/[id]/run-identity.tsx b/evalboard/app/runs/[id]/run-identity.tsx
new file mode 100644
index 00000000..a13f24b5
--- /dev/null
+++ b/evalboard/app/runs/[id]/run-identity.tsx
@@ -0,0 +1,55 @@
+// "What am I looking at" strip for the run header. The run page used to identify
+// a run only by its timestamped id, so telling a codex run from a claude-code one
+// meant going back to the runs table (which does carry a Harness column) or
+// opening a task. Harness and model are the two facts that decide whether a
+// number on this page is comparable to a number on another run's page, so they
+// belong in the header.
+
+import {
+ HarnessBadge,
+ harnessShortLabel,
+} from "@/app/_components/harness-badge";
+
+export function RunIdentity({
+ harness,
+ model,
+ modelCount,
+}: {
+ harness: string | null;
+ model: string | null;
+ modelCount: number;
+}) {
+ // Legacy runs identify neither; render nothing rather than an empty frame.
+ if (!harness && !model) return null;
+ return (
+
+ {harness && (
+
+
+ {harnessShortLabel(harness)}
+
+ )}
+ {model && (
+ 1
+ ? `${modelCount} models across this run's tasks; showing the most common`
+ : "Model this run's tasks ran on"
+ }
+ >
+ model
+ {model}
+ {modelCount > 1 && (
+
+ +{modelCount - 1} more
+
+ )}
+
+ )}
+
+ );
+}
diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx
index 160e1303..1aac1316 100644
--- a/evalboard/app/runs/[id]/run-view.tsx
+++ b/evalboard/app/runs/[id]/run-view.tsx
@@ -5,6 +5,7 @@ import { useCallback, useMemo, useState } from "react";
import type { ActivationScore, TaskResultSummary } from "@/lib/runs";
import type { ReviewIndexEntry } from "@/lib/reviews-types";
import { fmtDuration, humanizeTaskId } from "@/lib/format";
+import { passBarClass, passClass } from "@/lib/pass-rate";
import { perTaskPassCounts, statusCategory } from "@/lib/status";
import { ChipLegend } from "@/app/_overview/tag-rail";
import { CollapsibleRail } from "@/app/_components/collapsible-rail";
@@ -404,10 +405,15 @@ export function RunView({
const totalN = hasRepeats
? metrics.taskTotal
: metrics.total;
+ // null when nothing ran, so an empty run reads neutral
+ // rather than as a measured 0%.
+ const tone = totalN > 0 ? pct : null;
return (
<>
)}
+ {/* The meter was unconditionally green, so a 77%
+ run still read as healthy at a glance. It now
+ carries the same traffic-light cutoffs as the
+ number beside it. */}
diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx
index 67654303..0eacba8f 100644
--- a/evalboard/app/trends/trends-view.tsx
+++ b/evalboard/app/trends/trends-view.tsx
@@ -8,6 +8,7 @@ import {
} from "@/lib/trends";
import type { TagCount } from "@/lib/overview";
import { fmtRunTime, fmtDuration, humanizeTaskId } from "@/lib/format";
+import { passClassRatio } from "@/lib/pass-rate";
import {
displayedTurns,
fmtTurnsCount,
@@ -45,12 +46,6 @@ function fmtCount(n: number | null): string {
return n.toFixed(0);
}
-function passRateClass(rate: number, hasRuns: boolean): string {
- if (!hasRuns) return "text-gray-500";
- if (rate >= 1.0) return "text-green-700";
- return "text-red-700";
-}
-
type SortKey =
| "task"
| "runs"
@@ -379,7 +374,11 @@ function TaskRow({
pending: boolean;
onToggle: () => void;
}) {
- const rateClass = passRateClass(t.passRate, t.totalRuns > 0);
+ // Same traffic-light cutoffs as every other pass rate on the site
+ // (lib/pass-rate.ts). This used to be green only at a perfect 100%, which
+ // painted a task that passed 9 of its last 10 runs the same red as one that
+ // never passed.
+ const rateClass = passClassRatio(t.totalRuns > 0 ? t.passRate : null);
return (
<>
{r.skill}
-
+
{pct(r.passRate)}
diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts
index 8dcb064c..c8a07a99 100644
--- a/evalboard/lib/__tests__/harness.test.ts
+++ b/evalboard/lib/__tests__/harness.test.ts
@@ -1,5 +1,13 @@
import { describe, expect, test } from "vitest";
-import { DEFAULT_HARNESS, parseHarnessParam } from "../harness";
+import {
+ ALL_HARNESSES,
+ DEFAULT_HARNESS,
+ KNOWN_HARNESSES,
+ harnessColor,
+ orderHarnesses,
+ parseHarnessParam,
+ parseHarnessScope,
+} from "../harness";
// parseHarnessParam gates the untrusted `?h=` query param at every
// harness-scoped route, and its result decides which runs are counted toward
@@ -47,3 +55,92 @@ describe("parseHarnessParam", () => {
expect(parseHarnessParam("a".repeat(65))).toBe(DEFAULT_HARNESS);
});
});
+
+// parseHarnessScope is the overview's variant: the same untrusted `?h=` param,
+// but null-defaulting so the front page opens on every harness instead of
+// silently scoping to claude-code. Anything that mis-maps here mis-scopes the
+// summary tiles, both charts, and the run table at once, since all four now read
+// this one value.
+describe("parseHarnessScope", () => {
+ test("absent / empty input means all harnesses", () => {
+ expect(parseHarnessScope(undefined)).toBeNull();
+ expect(parseHarnessScope([])).toBeNull();
+ expect(parseHarnessScope("")).toBeNull();
+ expect(parseHarnessScope(" ")).toBeNull();
+ });
+
+ test("the explicit all sentinel means all harnesses", () => {
+ expect(parseHarnessScope(ALL_HARNESSES)).toBeNull();
+ expect(parseHarnessScope([ALL_HARNESSES])).toBeNull();
+ });
+
+ test("passes through any syntactically-valid id", () => {
+ for (const id of [...KNOWN_HARNESSES, "gpt-5.5", "some_harness"]) {
+ expect(parseHarnessScope(id)).toBe(id);
+ }
+ });
+
+ test("trims before validating, and picks the first array element", () => {
+ expect(parseHarnessScope(" codex ")).toBe("codex");
+ expect(parseHarnessScope([" antigravity ", "codex"])).toBe(
+ "antigravity",
+ );
+ });
+
+ test("malformed ids widen to all rather than to a wrong harness", () => {
+ // Contrast with parseHarnessParam, which falls back to claude-code. A
+ // bad param here must not silently produce claude-code-only numbers.
+ for (const bad of ["has space", "a/b", "bad;rm", "a".repeat(65)]) {
+ expect(parseHarnessScope(bad)).toBeNull();
+ }
+ });
+
+ test("differs from parseHarnessParam exactly on the empty case", () => {
+ expect(parseHarnessParam(undefined)).toBe(DEFAULT_HARNESS);
+ expect(parseHarnessScope(undefined)).toBeNull();
+ });
+});
+
+// Color is bound to the harness, never to its index in the series list — a
+// filter that removes one line must not repaint the survivors.
+describe("harnessColor", () => {
+ test("every known harness has its own reserved color", () => {
+ const colors = KNOWN_HARNESSES.map(harnessColor);
+ expect(new Set(colors).size).toBe(KNOWN_HARNESSES.length);
+ });
+
+ test("is stable per harness regardless of what else is on screen", () => {
+ // The regression this guards: assigning by position, so dropping
+ // claude-code would slide codex onto claude-code's blue.
+ expect(harnessColor("codex")).toBe(harnessColor("codex"));
+ expect(harnessColor("codex")).not.toBe(harnessColor("claude-code"));
+ });
+
+ test("unknown harnesses share one neutral, not a generated hue", () => {
+ const a = harnessColor("brand-new-harness");
+ const b = harnessColor("another-newcomer");
+ expect(a).toBe(b);
+ for (const known of KNOWN_HARNESSES) {
+ expect(a).not.toBe(harnessColor(known));
+ }
+ });
+});
+
+describe("orderHarnesses", () => {
+ test("known harnesses come first, in KNOWN_HARNESSES order", () => {
+ expect(
+ orderHarnesses(["antigravity", "claude-code", "codex"]),
+ ).toEqual(["claude-code", "codex", "antigravity"]);
+ });
+
+ test("newcomers follow, alphabetically", () => {
+ expect(
+ orderHarnesses(["zeta", "codex", "alpha-harness"]),
+ ).toEqual(["codex", "alpha-harness", "zeta"]);
+ });
+
+ test("dedupes and drops nothing else", () => {
+ expect(orderHarnesses(["codex", "codex"])).toEqual(["codex"]);
+ expect(orderHarnesses([])).toEqual([]);
+ });
+});
diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts
index eaa935d1..2f8b9440 100644
--- a/evalboard/lib/__tests__/overview.test.ts
+++ b/evalboard/lib/__tests__/overview.test.ts
@@ -2,6 +2,8 @@ import { describe, expect, test, vi } from "vitest";
import {
buildAdhocRows,
collectPipelineRuns,
+ projectRunRow,
+ scopeRunTasks,
summarizeListing,
turnBudgetRateForTasks,
type PerRun,
@@ -297,6 +299,20 @@ describe("collectPipelineRuns", () => {
warn.mockRestore();
});
+ test("probeAll reaches a match sitting past the scan cap", async () => {
+ // The paged run table derives "Show more" from finding one extra row, so
+ // a scan that gave up at the cap would report "no more" and strand every
+ // older matching run. Only r1 matches, 30 ids deep — well past
+ // limit x HARNESS_SCAN_FACTOR.
+ const ids = Array.from({ length: 30 }, (_, i) => `r${30 - i}`);
+ const load = vi.fn(async (id: string) => run(id));
+ const isMatch = (r: PerRun) => r.id === "r1";
+
+ expect(await collectPipelineRuns(ids, 1, load, isMatch)).toEqual([]);
+ const out = await collectPipelineRuns(ids, 1, load, isMatch, true);
+ expect(out.map((r) => r.id)).toEqual(["r1"]);
+ });
+
test("limit=0 and empty ids short-circuit without loading", async () => {
const load = vi.fn(async (id: string) => run(id));
expect(await collectPipelineRuns(["r1"], 0, load)).toEqual([]);
@@ -474,3 +490,153 @@ describe("buildAdhocRows", () => {
expect(row.tasksRun).toBe(2);
});
});
+
+// projectRunRow is the single definition of "does this run count, and with which
+// tasks" — the summary tiles (getWindowRollup) and the paged run table
+// (getRunListing) both go through it. They used to be one loop; if they ever
+// disagreed, the tiles would describe a different set of runs than the table
+// below them with nothing failing.
+describe("projectRunRow", () => {
+ function run(id: string, tasks: RunOverviewTask[], extra?: Partial): PerRun {
+ return {
+ id,
+ overview: {
+ id,
+ tasks,
+ totalCostUsd: 10,
+ taskDurationSeconds: 100,
+ componentShas: [],
+ },
+ reviewTagCounts: {},
+ reviewTagsByTask: {},
+ adhoc: false,
+ title: null,
+ ...extra,
+ };
+ }
+
+ test("unfiltered, reports whole-run totals", () => {
+ const row = projectRunRow(
+ run("r", [
+ task({ taskId: "a", status: "SUCCESS" }),
+ task({ taskId: "b", status: "FAILURE" }),
+ ]),
+ null,
+ null,
+ );
+ expect(row).toMatchObject({
+ id: "r",
+ tasksSucceeded: 1,
+ tasksRun: 2,
+ totalCostUsd: 10,
+ taskDurationSeconds: 100,
+ });
+ });
+
+ test("drops a run with no task matching the tag", () => {
+ const row = projectRunRow(
+ run("r", [task({ taskId: "a", tags: ["smoke"] })]),
+ "path-to-ga",
+ null,
+ );
+ expect(row).toBeNull();
+ });
+
+ test("scopes cost and duration to the matching tasks", () => {
+ // The row's numbers have to describe the filtered slice, not the run —
+ // otherwise filtering to one tag still shows the whole run's spend.
+ const row = projectRunRow(
+ run("r", [
+ task({
+ taskId: "a",
+ tags: ["keep"],
+ totalCostUsd: 3,
+ durationSeconds: 30,
+ }),
+ task({ taskId: "b", totalCostUsd: 99, durationSeconds: 999 }),
+ ]),
+ "keep",
+ null,
+ );
+ expect(row).toMatchObject({
+ tasksRun: 1,
+ totalCostUsd: 3,
+ taskDurationSeconds: 30,
+ });
+ });
+
+ test("nulls a scoped duration when any matching task lacks one", () => {
+ // A partial sum understates the slice, which reads as a fast run rather
+ // than an unmeasured one.
+ const row = projectRunRow(
+ run("r", [
+ task({ taskId: "a", tags: ["keep"], durationSeconds: 30 }),
+ task({ taskId: "b", tags: ["keep"], durationSeconds: null }),
+ ]),
+ "keep",
+ null,
+ );
+ expect(row?.taskDurationSeconds).toBeNull();
+ });
+
+ test("a query matching only the run id keeps the whole-run slice", () => {
+ // Pinning a run by pasting a date fragment must not show 0 tasks just
+ // because no task name contains the date.
+ const row = projectRunRow(
+ run("2026-07-30_04-38-11", [task({ taskId: "a" })]),
+ null,
+ "07-30",
+ );
+ expect(row).toMatchObject({ tasksRun: 1 });
+ });
+
+ test("returns null for a run whose run.json could not be read", () => {
+ const r = run("r", []);
+ expect(projectRunRow({ ...r, overview: null }, null, null)).toBeNull();
+ });
+
+ test("carries the harness through for the table's badge column", () => {
+ const r = run("r", [task({ taskId: "a" })]);
+ const withHarness = {
+ ...r,
+ overview: { ...r.overview!, harness: "codex" },
+ };
+ expect(projectRunRow(withHarness, null, null)?.harness).toBe("codex");
+ });
+
+ // getOverview reads the task LIST out of the same call (for the turn-budget
+ // rate), while the table only ever sees the counts. If the two disagreed
+ // about which tasks are in scope, the charts and the tiles would describe
+ // different runs with nothing on the page to show it.
+ describe("scopeRunTasks, the seam both share", () => {
+ test("hands back the matching tasks themselves, not just a count", () => {
+ const scoped = scopeRunTasks(
+ run("r", [
+ task({ taskId: "keep-me", tags: ["keep"] }),
+ task({ taskId: "drop-me" }),
+ ]),
+ "keep",
+ null,
+ );
+ expect(scoped?.tasks.map((t) => t.taskId)).toEqual(["keep-me"]);
+ });
+
+ test("agrees with projectRunRow on which runs are in scope", () => {
+ const r = run("r", [task({ taskId: "a", tags: ["other"] })]);
+ expect(scopeRunTasks(r, "keep", null)).toBeNull();
+ expect(projectRunRow(r, "keep", null)).toBeNull();
+ });
+
+ test("the run-id fallback widens back to every task", () => {
+ const scoped = scopeRunTasks(
+ run("2026-07-30_04-38-11", [
+ task({ taskId: "a" }),
+ task({ taskId: "b" }),
+ ]),
+ null,
+ "07-30",
+ );
+ expect(scoped?.tasks).toHaveLength(2);
+ });
+ });
+});
diff --git a/evalboard/lib/__tests__/pass-rate.test.ts b/evalboard/lib/__tests__/pass-rate.test.ts
new file mode 100644
index 00000000..5d50a696
--- /dev/null
+++ b/evalboard/lib/__tests__/pass-rate.test.ts
@@ -0,0 +1,75 @@
+import { describe, expect, test } from "vitest";
+import {
+ PASS_GREEN_PCT,
+ PASS_RED_PCT,
+ passBarClass,
+ passClass,
+ passClassRatio,
+ passTone,
+} from "../pass-rate";
+
+// These cutoffs are shared with the nightly Slack rollup's traffic-light dots.
+// A run that pings the channel red and then reads green on the page it links to
+// is the failure this module exists to prevent, so the boundaries are pinned
+// here rather than left to whatever each call site felt like.
+describe("passTone", () => {
+ test("mirrors the Slack rollup's cutoffs", () => {
+ expect(PASS_GREEN_PCT).toBe(95);
+ expect(PASS_RED_PCT).toBe(85);
+ });
+
+ test("is inclusive at green and exclusive at red", () => {
+ expect(passTone(95)).toBe("good");
+ expect(passTone(94.9)).toBe("warn");
+ expect(passTone(85)).toBe("warn");
+ expect(passTone(84.9)).toBe("bad");
+ });
+
+ test("treats a measured 0% as bad, not as missing data", () => {
+ // Every task failed. A falsy check here would paint the worst possible
+ // run the same neutral gray as a run that hasn't started.
+ expect(passTone(0)).toBe("bad");
+ expect(passClass(0)).toBe("text-red-700");
+ });
+
+ test("distinguishes no data from a low rate", () => {
+ expect(passTone(null)).toBe("none");
+ expect(passTone(undefined)).toBe("none");
+ expect(passClass(null)).toBe("text-gray-500");
+ });
+
+ test("does not paint a tone from a non-finite rate", () => {
+ // 0/0 in a caller's own division arrives here as NaN.
+ expect(passTone(Number.NaN)).toBe("none");
+ });
+
+ test("clears 100% and anything above it", () => {
+ expect(passTone(100)).toBe("good");
+ });
+});
+
+describe("class helpers", () => {
+ test("bars are more saturated than text at the same tone", () => {
+ // Text at the 500 step is too light to read; a 2px-tall bar at the 700
+ // step is too dark to distinguish from its track.
+ expect(passClass(99)).toBe("text-green-700");
+ expect(passBarClass(99)).toBe("bg-green-500");
+ expect(passBarClass(90)).toBe("bg-amber-500");
+ expect(passBarClass(10)).toBe("bg-red-500");
+ });
+
+ test("every tone maps to a distinct class", () => {
+ const classes = [passClass(99), passClass(90), passClass(10), passClass(null)];
+ expect(new Set(classes).size).toBe(4);
+ });
+
+ test("the ratio variant scales 0-1 onto the same cutoffs", () => {
+ // trends/watchlist carry rates as fractions; feeding one in unscaled
+ // would put every task in the red band.
+ expect(passClassRatio(0.96)).toBe(passClass(96));
+ expect(passClassRatio(0.9)).toBe(passClass(90));
+ expect(passClassRatio(0.5)).toBe(passClass(50));
+ expect(passClassRatio(1)).toBe("text-green-700");
+ expect(passClassRatio(null)).toBe("text-gray-500");
+ });
+});
diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts
index 5fd0c719..91c8341a 100644
--- a/evalboard/lib/__tests__/runs.test.ts
+++ b/evalboard/lib/__tests__/runs.test.ts
@@ -15,6 +15,7 @@ import {
type ArtifactRef,
clearRunCacheDir,
extractComponentShas,
+ tallyModels,
extractRunConfig,
findMatureSourceRuns,
isExcludedArtifact,
@@ -345,6 +346,89 @@ describe("extractComponentShas", () => {
});
expect(out.map((c) => c.name)).toEqual(["cli", "maestro-tool"]);
});
+
+ // The framework is pinned and consumed by released version (env_info
+ // `coder_eval`), not by checkout — so the chip must be that version and link
+ // to its release, rather than a git SHA pointing at a tree.
+ test("coder_eval reports its released version, linked to the release", () => {
+ const out = extractComponentShas({
+ ...baseEnv,
+ coder_eval: "0.9.1",
+ });
+ const framework = out.find((c) => c.name === "coder_eval");
+ expect(framework?.sha).toBe("0.9.1");
+ expect(framework?.url).toBe(
+ "https://github.com/UiPath/coder_eval/releases/tag/v0.9.1",
+ );
+ });
+
+ test("the version wins over a git_commit present in the same run", () => {
+ // Both keys are captured on every run; the SHA must not shadow the
+ // version just because it comes first in env_info.
+ const out = extractComponentShas({
+ git_commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
+ coder_eval: "0.10.0",
+ });
+ expect(out.find((c) => c.name === "coder_eval")?.sha).toBe("0.10.0");
+ });
+
+ test("falls back to the git SHA when no version was captured", () => {
+ // Legacy runs, and editable checkouts between releases, have only the
+ // SHA — better a tree link than no chip at all.
+ const out = extractComponentShas(baseEnv);
+ const framework = out.find((c) => c.name === "coder_eval");
+ expect(framework?.sha).toBe(baseEnv.git_commit);
+ expect(framework?.url).toBe(
+ `https://github.com/UiPath/coder_eval/tree/${baseEnv.git_commit}`,
+ );
+ });
+
+ test("an 'unknown' version still falls through to the SHA", () => {
+ const out = extractComponentShas({
+ ...baseEnv,
+ coder_eval: "unknown",
+ });
+ expect(out.find((c) => c.name === "coder_eval")?.sha).toBe(
+ baseEnv.git_commit,
+ );
+ });
+});
+
+// The run header claims a single harness and model for the whole run; tallyModels
+// is what keeps that claim honest when an A/B experiment fans variants across
+// models inside one run.
+describe("tallyModels", () => {
+ const row = (model: string | null) => ({
+ task_id: "t",
+ model_used: model,
+ });
+
+ test("returns the most common model and the distinct count", () => {
+ const out = tallyModels([
+ row("claude-sonnet-5"),
+ row("claude-sonnet-5"),
+ row("claude-opus-5"),
+ ]);
+ expect(out).toEqual({ dominant: "claude-sonnet-5", distinct: 2 });
+ });
+
+ test("a single-model run reports distinct 1", () => {
+ expect(tallyModels([row("claude-sonnet-5"), row("claude-sonnet-5")]))
+ .toEqual({ dominant: "claude-sonnet-5", distinct: 1 });
+ });
+
+ test("ignores rows with no model rather than counting them as one", () => {
+ const out = tallyModels([row(null), row("codex-mini"), row(null)]);
+ expect(out).toEqual({ dominant: "codex-mini", distinct: 1 });
+ });
+
+ test("an empty or model-less run reports nothing to display", () => {
+ expect(tallyModels([])).toEqual({ dominant: null, distinct: 0 });
+ expect(tallyModels([row(null)])).toEqual({
+ dominant: null,
+ distinct: 0,
+ });
+ });
});
describe("extractRunConfig", () => {
diff --git a/evalboard/lib/format.ts b/evalboard/lib/format.ts
index 833c5a96..d801c526 100644
--- a/evalboard/lib/format.ts
+++ b/evalboard/lib/format.ts
@@ -49,17 +49,6 @@ export function fmtUsd(n: number | null | undefined): string {
return `$${n.toFixed(digits)}`;
}
-// Pass-rate → Tailwind text color. Shared by the front-page run table
-// (page.tsx) and the window summary tile so the >=80 green / >=50 gray / else
-// red thresholds live in one place. `hasTasks` distinguishes "0%" (red) from
-// "no tasks yet" (neutral gray); pct is null exactly when hasTasks is false.
-export function passClass(pct: number | null, hasTasks: boolean): string {
- if (!hasTasks || pct == null) return "text-gray-500";
- if (pct >= 80) return "text-green-700";
- if (pct >= 50) return "text-gray-700";
- return "text-red-700";
-}
-
export function fmtDuration(s: number | null): string {
if (s == null) return "—";
// Round once on the total to avoid `1m 60s` from rounding the remainder.
diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts
index bf36cb28..c52c992c 100644
--- a/evalboard/lib/harness.ts
+++ b/evalboard/lib/harness.ts
@@ -5,9 +5,15 @@
// The harnesses the nightly rotates through, in preferred display order. This
// is only a display-order hint / default source — the switcher is data-driven
-// (see listRecentHarnesses), so a new harness like "delegate" surfaces
-// automatically without editing this list.
-export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const;
+// (see listRecentHarnesses), so a new harness surfaces automatically without
+// editing this list. Membership here does buy two things: a stable position in
+// the switcher, and a reserved series color (see HARNESS_COLORS).
+export const KNOWN_HARNESSES = [
+ "claude-code",
+ "codex",
+ "antigravity",
+ "delegate-sdk",
+] as const;
export type KnownHarness = (typeof KNOWN_HARNESSES)[number];
// A run with no RunConfig harness predates the stamp; every such run was
@@ -29,3 +35,61 @@ export function parseHarnessParam(raw: string | string[] | undefined): string {
const trimmed = v.trim();
return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS;
}
+
+// URL value for the unscoped view. Reserved: a real harness id could never
+// collide with it, since AgentKind ids are hyphenated vendor names.
+export const ALL_HARNESSES = "all";
+
+// Harness scope for pages that can show every harness at once (the overview).
+// null = all harnesses, which is also the DEFAULT — the front page opens on the
+// comparison view and narrows from there, so `?h=` absent means "everything"
+// rather than "claude-code". Pages that genuinely need a single harness (trends
+// collapses per-task history across runs, which is only meaningful within one
+// harness) keep using parseHarnessParam.
+export function parseHarnessScope(
+ raw: string | string[] | undefined,
+): string | null {
+ const v = Array.isArray(raw) ? raw[0] : raw;
+ if (typeof v !== "string") return null;
+ const trimmed = v.trim();
+ if (!trimmed || trimmed === ALL_HARNESSES) return null;
+ return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : null;
+}
+
+// Series color per harness, for the multi-harness overview charts. Color is
+// bound to the HARNESS, not to its position in the series list — a filter that
+// drops one harness must not repaint the others.
+//
+// Values are slots 1/2/3(dark step)/7 of the validated categorical palette. As
+// an ordered set on a light surface they clear every hard gate: lightness band,
+// chroma floor, adjacent-pair CVD separation (worst ΔE 8.4 protan), the
+// normal-vision floor (worst ΔE 27.1), and 3:1 contrast. Re-run the palette
+// validator before adding a fifth entry rather than eyeballing a new hue — the
+// obvious next slots (yellow, magenta) both fail contrast on white.
+const HARNESS_COLORS: Record = {
+ "claude-code": "#2a78d6", // blue
+ codex: "#eb6834", // orange
+ antigravity: "#199e70", // aqua
+ "delegate-sdk": "#4a3aa7", // violet
+};
+
+// Neutral for any harness with no reserved slot. Deliberately NOT a generated
+// hue: unknown harnesses fold into one "other" color, which stays honest under
+// CVD, and the legend still names each line.
+const HARNESS_COLOR_FALLBACK = "#6b7280";
+
+export function harnessColor(harness: string): string {
+ return HARNESS_COLORS[harness] ?? HARNESS_COLOR_FALLBACK;
+}
+
+// Stable series order for charts and legends: known harnesses in display order,
+// then any newcomers alphabetically. Same rule as listRecentHarnesses, applied
+// to whichever subset actually has data in the current window.
+export function orderHarnesses(harnesses: Iterable): string[] {
+ const seen = new Set(harnesses);
+ const known = KNOWN_HARNESSES.filter((h) => seen.has(h));
+ const extras = [...seen]
+ .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h))
+ .sort();
+ return [...known, ...extras];
+}
diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts
index 4e6cfa6b..e02215d6 100644
--- a/evalboard/lib/overview.ts
+++ b/evalboard/lib/overview.ts
@@ -15,12 +15,16 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review
import { withinTurnBudget } from "./turns";
import { humanizeTaskId } from "./format";
import { mapWithConcurrency } from "./concurrency";
-import { DEFAULT_HARNESS, KNOWN_HARNESSES, normalizeHarness } from "./harness";
+import { DEFAULT_HARNESS, normalizeHarness, orderHarnesses } from "./harness";
import type { Window } from "./reviews-types";
export interface RunPoint {
runId: string;
timestamp: number; // ms since epoch (UTC); used as the chart x-coordinate
+ // The harness this run ran on, normalized (legacy runs fold to claude-code).
+ // Every run belongs to exactly one harness, so the charts split the points
+ // into one series per harness and color each by identity.
+ harness: string;
successRate: number | null;
// % of budgeted tasks whose visible turns stayed within 1.5× their
// expected_turns budget. Only tasks carrying a positive expected_turns
@@ -80,8 +84,17 @@ export interface TagCount {
export interface OverviewData {
runs: RunPoint[]; // one point per run, no daily aggregation
+ // The harnesses actually present in `runs`, in stable display order. Drives
+ // the chart's series list and legend, so a harness with no runs in the
+ // window contributes no empty line.
+ harnesses: string[];
windowStart: number; // ms — chart x-domain start
windowEnd: number; // ms — chart x-domain end
+ // The summary tiles' rollup, folded over the same runs `runs` plots so the
+ // two can't disagree. Independent of the run table's pagination, which reads
+ // back past this window (see getRunListing).
+ totals: RunListingTotals;
+ runCount: number; // matched pipeline runs in the window
skills: TagCount[];
taskTags: TagCount[];
reviewTags: TagCount[];
@@ -150,11 +163,14 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
export interface RunListing {
rows: RunListingRow[]; // after filter + limit, newest first
- totalInWindow: number;
- matchedCount: number; // post-filter, pre-limit
- // Rollup across all matched runs (pre-limit) — powers the front-page
- // window summary; independent of the `rows` limit.
- totals: RunListingTotals;
+ // Every pipeline run in the store, not just the ones loaded for this page.
+ // Free to compute (it's the id count), so the table can say "20 of 94"
+ // without reading 94 run.json files.
+ totalCandidates: number;
+ // Another page exists behind `rows`. Derived from over-fetching by one
+ // rather than from a total match count: counting matches would mean loading
+ // every run in history on every render.
+ hasMore: boolean;
}
const FETCH_CONCURRENCY = 16;
@@ -294,13 +310,7 @@ async function listRecentHarnessesInner(): Promise {
for (const r of perRun) {
if (r.overview) seen.add(normalizeHarness(r.overview.harness));
}
- // Known harnesses first (stable display order), then any newcomers
- // (alphabetical) so the list is deterministic but self-extending.
- const known = KNOWN_HARNESSES.filter((h) => seen.has(h));
- const extras = [...seen]
- .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h))
- .sort();
- const ordered = [...known, ...extras];
+ const ordered = orderHarnesses(seen);
// Never hand back an empty list — the default must always be selectable.
return ordered.length > 0 ? ordered : [DEFAULT_HARNESS];
}
@@ -357,9 +367,16 @@ export async function collectPipelineRuns(
// Optional extra predicate (AND-ed with usability) — e.g. a harness filter.
// When set, the scan reaches further back since matches are sparser.
isMatch?: (r: PerRun) => boolean,
+ // Probe every candidate instead of stopping at a multiple of `limit`. A
+ // capped scan can't tell "no older match exists" from "stopped looking", so
+ // the paged run table — whose "Show more" link comes from finding one extra
+ // row — has to probe fully or it silently hides older matching runs.
+ probeAll = false,
): Promise {
const scanFactor = isMatch ? HARNESS_SCAN_FACTOR : RECENT_SCAN_FACTOR;
- const maxScan = Math.min(ids.length, limit * scanFactor);
+ const maxScan = probeAll
+ ? ids.length
+ : Math.min(ids.length, limit * scanFactor);
const usable = isMatch
? (r: PerRun) => isUsablePipelineRun(r) && isMatch(r)
: isUsablePipelineRun;
@@ -549,39 +566,34 @@ export async function getOverview(
);
const needle = q?.trim().toLowerCase() || null;
- // ---- Per-run chart points ----
- // One point per run plotted at its own timestamp, no daily averaging.
- // When tag or q is active, scope each run's rate to only matching tasks.
+ // ---- Per-run chart points + the window rollup ----
+ // One chart point per run plotted at its own timestamp, no daily averaging,
+ // and one rollup row per run for the summary tiles. Both come out of the
+ // same scopeRunTasks call in the same loop, so the tiles can never describe
+ // a different set of runs than the charts plot.
const runPoints: RunPoint[] = [];
+ const rows: RunListingRow[] = [];
+ const seenHarnesses = new Set();
- for (const { id, overview, reviewTagsByTask } of perRun) {
+ for (const r of perRun) {
+ const { id, overview } = r;
if (!overview || overview.tasks.length === 0) continue;
const date = parseRunIdDate(id);
if (!date) continue;
+ const scoped = scopeRunTasks(r, tag, needle);
+ if (!scoped || scoped.tasks.length === 0) continue;
- let matching = overview.tasks;
- if (tag) {
- matching = matching.filter((t) =>
- taskMatchesTag(t, reviewTagsByTask, tag),
- );
- }
- if (needle) {
- matching = matching.filter((t) =>
- taskMatchesQuery(t, reviewTagsByTask, needle),
- );
- }
- if (matching.length === 0) continue;
-
- const succeeded = matching.filter(
- (t) => t.status === "SUCCESS",
- ).length;
- const rate = (succeeded / matching.length) * 100;
+ const row = rowFromScoped(id, scoped, overview.harness);
+ rows.push(row);
+ const runHarness = normalizeHarness(overview.harness);
+ seenHarnesses.add(runHarness);
runPoints.push({
runId: id,
timestamp: date.getTime(),
- successRate: rate,
- turnBudgetRate: turnBudgetRateForTasks(matching),
+ harness: runHarness,
+ successRate: (row.tasksSucceeded / row.tasksRun) * 100,
+ turnBudgetRate: turnBudgetRateForTasks(scoped.tasks),
});
}
runPoints.sort((a, b) => a.timestamp - b.timestamp);
@@ -597,8 +609,11 @@ export async function getOverview(
return {
runs: runPoints,
+ harnesses: orderHarnesses(seenHarnesses),
windowStart,
windowEnd,
+ totals: summarizeListing(rows),
+ runCount: rows.length,
skills,
taskTags,
reviewTags,
@@ -679,92 +694,148 @@ export async function getTagTaskBreakdown(
return rows.sort((a, b) => a.taskId.localeCompare(b.taskId));
}
+// The slice of a run that the active tag/q filter selects: which tasks count,
+// and the cost/duration summed over exactly those. null means the run has
+// nothing matching and drops out entirely.
+//
+// This is the ONE definition of "does this run count, and over which tasks",
+// shared by the chart points, the summary tiles, and the run table. Each used to
+// carry its own copy, and a disagreement between them would have been invisible
+// on the page.
+export interface ScopedRun {
+ tasks: RunOverviewTask[];
+ totalCostUsd: number | null;
+ taskDurationSeconds: number | null;
+}
+
+export function scopeRunTasks(
+ { id, overview, reviewTagsByTask }: PerRun,
+ tag: string | null,
+ needle: string | null,
+): ScopedRun | null {
+ if (!overview) return null;
+ const wholeRun: ScopedRun = {
+ tasks: overview.tasks,
+ totalCostUsd: overview.totalCostUsd,
+ taskDurationSeconds: overview.taskDurationSeconds,
+ };
+ if (tag == null && needle == null) return wholeRun;
+
+ const matching = overview.tasks.filter((t) => {
+ const passesTag =
+ tag == null || taskMatchesTag(t, reviewTagsByTask, tag);
+ const passesQ =
+ needle == null || taskMatchesQuery(t, reviewTagsByTask, needle);
+ return passesTag && passesQ;
+ });
+ if (matching.length === 0) {
+ // A `q` that matches only the run ID is the date-fragment "pin a run"
+ // case: keep the whole-run slice so the row shows real totals rather
+ // than 0/—/—. Anything else has nothing to show.
+ const idMatchesQ = needle != null && id.toLowerCase().includes(needle);
+ return idMatchesQ ? wholeRun : null;
+ }
+
+ // Cost sums any matching task that recorded one; duration is only meaningful
+ // when every matching task has one (otherwise the partial sum would
+ // understate the slice — mirrors readRunOverview's whole-run rule).
+ let costSum = 0;
+ let costHasAny = false;
+ let durSum = 0;
+ let durAllPresent = true;
+ for (const t of matching) {
+ if (t.totalCostUsd != null) {
+ costSum += t.totalCostUsd;
+ costHasAny = true;
+ }
+ if (t.durationSeconds != null) {
+ durSum += t.durationSeconds;
+ } else {
+ durAllPresent = false;
+ }
+ }
+ return {
+ tasks: matching,
+ totalCostUsd: costHasAny ? costSum : null,
+ taskDurationSeconds: durAllPresent ? durSum : null,
+ };
+}
+
+function rowFromScoped(
+ id: string,
+ scoped: ScopedRun,
+ harness: string | null | undefined,
+): RunListingRow {
+ return {
+ id,
+ tasksSucceeded: scoped.tasks.filter((t) => t.status === "SUCCESS")
+ .length,
+ tasksRun: scoped.tasks.length,
+ totalCostUsd: scoped.totalCostUsd,
+ taskDurationSeconds: scoped.taskDurationSeconds,
+ harness: harness ?? null,
+ };
+}
+
+// One loaded run as a table row, or null when the filter excludes it. Exported
+// for unit testing.
+export function projectRunRow(
+ run: PerRun,
+ tag: string | null,
+ needle: string | null,
+): RunListingRow | null {
+ const scoped = scopeRunTasks(run, tag, needle);
+ return scoped ? rowFromScoped(run.id, scoped, run.overview?.harness) : null;
+}
+
+// The run table. Unlike getOverview's window rollup, this is NOT date-bounded:
+// it pages back through the whole store, a screenful at a time, so history older
+// than the charts' window is still reachable without a time-window control.
+//
+// Rows are loaded lazily because a run.json is multi-MB — reading all of history
+// to count matches would cost hundreds of MB of parsing per render. So the depth
+// of the read is the depth of the page: `hasMore` comes from over-fetching a
+// single row rather than from a total, and the only free count (every pipeline
+// candidate in the store) is reported separately as `totalCandidates`.
export async function getRunListing(
- window: Window,
tag: string | null,
q: string | null,
- limit: number | null, // null = unlimited
+ limit: number,
+ // When set, scope the listing to one harness — applied at the same seam as
+ // the chart's scope in getOverview, so the tiles, the charts, and the table
+ // always describe the same set of runs. null = all harnesses.
+ harness: string | null = null,
): Promise {
- // Exclude ad-hoc runs from the main listing — they appear in their own
- // section (getAdhocRunListing). totalInWindow therefore counts only
- // pipeline runs, matching the chart above it.
- const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc);
- // Run IDs are timestamped — newest first by lexical compare.
- const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id));
- const totalInWindow = sorted.length;
-
+ // Ad-hoc ids aren't date-shaped, so this drops them (they have their own
+ // section) without loading anything. Newest-first: ids are timestamps.
+ const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) != null);
const needle = q?.trim().toLowerCase() || null;
- const needsFilter = tag != null || needle != null;
-
- const matched: RunListingRow[] = [];
- for (const { id, overview, reviewTagsByTask } of sorted) {
- if (!overview) continue;
-
- // Default to the whole-run slice; narrow to matching tasks if a
- // filter is active AND any task matches. When `q` matches only the
- // run ID (date-fragment "pin a run" use case), we keep the whole-run
- // slice so the row shows real totals rather than 0/—/—.
- let scopedTasks = overview.tasks;
- let scopedCost = overview.totalCostUsd;
- let scopedDur = overview.taskDurationSeconds;
-
- if (needsFilter) {
- const matching = overview.tasks.filter((t) => {
- const passesTag =
- tag == null || taskMatchesTag(t, reviewTagsByTask, tag);
- const passesQ =
- needle == null ||
- taskMatchesQuery(t, reviewTagsByTask, needle);
- return passesTag && passesQ;
- });
- const idMatchesQ =
- needle != null && id.toLowerCase().includes(needle);
- if (matching.length === 0 && !idMatchesQ) continue;
-
- if (matching.length > 0) {
- // Scope cost/duration to matching tasks. Cost sums any task
- // with a recorded value; duration is only meaningful when
- // every matching task has a duration recorded (otherwise the
- // partial sum would understate the slice — mirrors
- // readRunOverview's whole-run rule).
- let costSum = 0;
- let costHasAny = false;
- let durSum = 0;
- let durAllPresent = true;
- for (const t of matching) {
- if (t.totalCostUsd != null) {
- costSum += t.totalCostUsd;
- costHasAny = true;
- }
- if (t.durationSeconds != null) {
- durSum += t.durationSeconds;
- } else {
- durAllPresent = false;
- }
- }
- scopedTasks = matching;
- scopedCost = costHasAny ? costSum : null;
- scopedDur = durAllPresent ? durSum : null;
- }
- }
+ const hasFilter = tag != null || needle != null || harness != null;
+ const isMatch = hasFilter
+ ? (r: PerRun) =>
+ (harness == null ||
+ normalizeHarness(r.overview?.harness) === harness) &&
+ projectRunRow(r, tag, needle) != null
+ : undefined;
- matched.push({
- id,
- tasksSucceeded: scopedTasks.filter((t) => t.status === "SUCCESS")
- .length,
- tasksRun: scopedTasks.length,
- totalCostUsd: scopedCost,
- taskDurationSeconds: scopedDur,
- harness: overview.harness ?? null,
- });
- }
+ // Over-fetch by one: if the extra row materializes, another page exists.
+ // probeAll, because a scan that gave up at a cap would report `hasMore:
+ // false` and strand every older matching run behind a link that vanished.
+ const loaded = await collectPipelineRuns(
+ ids,
+ limit + 1,
+ cachedLoadPerRun,
+ isMatch,
+ true,
+ );
+ const rows = loaded
+ .map((r) => projectRunRow(r, tag, needle))
+ .filter((row): row is RunListingRow => row != null);
- const rows = limit == null ? matched : matched.slice(0, limit);
return {
- rows,
- totalInWindow,
- matchedCount: matched.length,
- totals: summarizeListing(matched),
+ rows: rows.slice(0, limit),
+ totalCandidates: ids.length,
+ hasMore: rows.length > limit,
};
}
diff --git a/evalboard/lib/pass-rate.ts b/evalboard/lib/pass-rate.ts
new file mode 100644
index 00000000..d1ff242b
--- /dev/null
+++ b/evalboard/lib/pass-rate.ts
@@ -0,0 +1,54 @@
+// Traffic-light thresholds for a pass rate.
+//
+// These mirror the nightly Slack rollup's dots
+// (coder_eval_uipath/eval_runner/scripts/ci/slack_summary.py, GREEN_PCT /
+// RED_PCT) so a number that reads green in the channel reads green here. The
+// channel is where a regression is first noticed and the evalboard is where it
+// gets opened, so the two disagreeing about which numbers are healthy is worse
+// than either bar being slightly off. Change these only alongside that script.
+export const PASS_GREEN_PCT = 95;
+export const PASS_RED_PCT = 85;
+
+// "none" is the absence of a measurement, which callers signal by passing null.
+// That is not the same as a measured 0% — that one is genuinely bad and must
+// read red.
+export type PassTone = "none" | "good" | "warn" | "bad";
+
+export function passTone(pct: number | null | undefined): PassTone {
+ if (pct == null || !Number.isFinite(pct)) return "none";
+ if (pct >= PASS_GREEN_PCT) return "good";
+ if (pct < PASS_RED_PCT) return "bad";
+ return "warn";
+}
+
+const TEXT_CLASS: Record = {
+ none: "text-gray-500",
+ good: "text-green-700",
+ warn: "text-amber-700",
+ bad: "text-red-700",
+};
+
+// Fills (progress bars, meters) need more saturation than text to read at all,
+// so they take the 500 step rather than 700.
+const BAR_CLASS: Record = {
+ none: "bg-gray-300",
+ good: "bg-green-500",
+ warn: "bg-amber-500",
+ bad: "bg-red-500",
+};
+
+// Pass rate as a percent (0-100) -> Tailwind text color. Pass null when there is
+// nothing to measure so it reads neutral rather than red.
+export function passClass(pct: number | null | undefined): string {
+ return TEXT_CLASS[passTone(pct)];
+}
+
+export function passBarClass(pct: number | null | undefined): string {
+ return BAR_CLASS[passTone(pct)];
+}
+
+// Same, for the tables that carry a rate as a 0-1 fraction (trends, watchlist)
+// rather than a percent.
+export function passClassRatio(rate: number | null | undefined): string {
+ return passClass(rate == null ? null : rate * 100);
+}
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index 45de6f98..e4f98842 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -56,6 +56,16 @@ export interface RunSummary {
tasksError: number;
totalCostUsd: number | null;
componentShas: ComponentSha[];
+ // What actually produced this run, for the run header. `harness` is the
+ // coder-eval AgentKind from the RunConfig stamp (falling back to the per-task
+ // agent_config vote); null only on legacy runs that identify neither.
+ harness: string | null;
+ // Dominant per-task `model_used`, with how many distinct models the run
+ // touched. Normally 1 — a run is one model — but an A/B experiment fans
+ // variants across models in a single run, and a header claiming one model
+ // would be wrong there, so the count is carried and surfaced.
+ model: string | null;
+ modelCount: number;
}
export interface TaskResultSummary {
@@ -471,13 +481,31 @@ const COMPONENTS: {
display: string;
repo: string | null;
nonShaUrl: string | null;
+ // Link target for a semver-shaped value, given the value itself. Takes
+ // precedence over `nonShaUrl` when it returns a URL. This is how a released
+ // component links at its own version rather than at a moving list page.
+ versionUrl?: (version: string) => string | null;
keys: string[];
}[] = [
{
display: "coder_eval",
repo: "UiPath/coder_eval",
nonShaUrl: null,
- keys: ["git_commit"],
+ // Releases are tagged `v` by the Release workflow, so the
+ // version resolves to an exact release page. Runs from before tagging
+ // began (v0.8.2) recorded a hardcoded `0.1.0` that was never released;
+ // render those as plain text rather than a link that 404s.
+ versionUrl: (v) =>
+ v === "0.1.0"
+ ? null
+ : `https://github.com/UiPath/coder_eval/releases/tag/v${v}`,
+ // The framework identifies itself by the released package version
+ // (env_info `coder_eval`, e.g. "0.9.1"), which is what the suite
+ // actually pins — so that is the chip, and it links to the release.
+ // `git_commit` is a fallback for two cases the version can't cover: an
+ // editable checkout between releases, and legacy runs written before the
+ // version was captured. It is a SHA, so it links to the tree.
+ keys: ["coder_eval", "git_commit"],
},
{
display: "skills",
@@ -516,8 +544,10 @@ export function extractComponentShas(
let url: string | null = null;
if (comp.repo && SHA_RE.test(value)) {
url = `https://github.com/${comp.repo}/tree/${value}`;
- } else if (comp.nonShaUrl) {
- url = comp.nonShaUrl;
+ } else {
+ // Not SHA-shaped, so it's a version label. Prefer an exact
+ // per-version link; fall back to the component's list page.
+ url = comp.versionUrl?.(value) ?? comp.nonShaUrl;
}
out.push({ name: comp.display, sha: value, url });
}
@@ -665,6 +695,7 @@ export async function readRunSummary(id: string): Promise {
const taskDurationSeconds = allHaveDuration
? taskDurationSum
: (data.total_duration_seconds ?? null);
+ const models = tallyModels(taskResults);
return {
id,
startTime: data.start_time ?? null,
@@ -676,9 +707,36 @@ export async function readRunSummary(id: string): Promise {
tasksError: data.tasks_error ?? 0,
totalCostUsd: taskResults.length ? totalCost : null,
componentShas: extractComponentShas(data.environment_info),
+ harness: extractRunConfig(data).harness,
+ model: models.dominant,
+ modelCount: models.distinct,
};
}
+// Dominant `model_used` across a run's rows, plus how many distinct models
+// appeared. Mirrors mostCommonAgentType's "the thing these tasks ran on" vote,
+// but also reports the spread so the run header can say when there was more than
+// one instead of silently picking a winner.
+export function tallyModels(rows: RawTaskResult[]): {
+ dominant: string | null;
+ distinct: number;
+} {
+ const counts = new Map();
+ for (const r of rows) {
+ const m = r.model_used;
+ if (typeof m === "string" && m) counts.set(m, (counts.get(m) ?? 0) + 1);
+ }
+ let dominant: string | null = null;
+ let bestN = 0;
+ for (const [model, n] of counts) {
+ if (n > bestN) {
+ dominant = model;
+ bestN = n;
+ }
+ }
+ return { dominant, distinct: counts.size };
+}
+
export async function readRunTasks(
id: string,
): Promise {