From fb59c6f845302413b42eac6b7b2dea062801fd5e Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 30 Jul 2026 14:31:22 -0700 Subject: [PATCH 1/5] feat(evalboard): compare every harness on the overview, and scope the whole page to one The overview opened scoped to claude-code and re-scoped only its analytics block: the summary tiles and the run list kept covering every harness, so picking codex changed the chart while the tiles above it still reported all-harness cost and pass rate. Nothing on the page said which set a number described. Now one scope drives the page, and its default is every harness: - Both charts draw one line per harness in the window instead of a single line zigzagging across incomparable harnesses. RunPoint carries its run's harness and getOverview reports which harnesses are present, so the series list is data-driven and a harness with no runs contributes no empty line. - Line color is bound to the harness, not to its index, so narrowing the view never repaints the survivors. The four reserved colors are slots from the validated categorical palette and clear every hard gate as an ordered set on a light surface (worst adjacent CVD dE 8.4, normal-vision dE 27.1, all above 3:1 contrast); unknown harnesses share one neutral rather than getting a generated hue. Each chart carries a legend pairing the swatch with the vendor badge and label, so identity is never color-alone. - getRunListing takes the same scope, so the tiles (which read its pre-limit rollup) and the table narrow with the charts. - delegate-sdk joins KNOWN_HARNESSES and renders the UiPath mark, the vendor logo it had been missing. Both tables now page one screen at a time and no longer offer "show all": every extra row is another run.json read behind the window load, so an unbounded jump was the one click that could stall the page. A hand-typed ?limit= clamps at 500 rather than being honored. Trends and path-to-ga still require a single harness (collapsing per-task history across runs only means something inside one), so the All segment is opt-in via includeAll. --- evalboard/app/_components/harness-badge.tsx | 22 ++- .../app/_components/harness-selector.tsx | 66 ++++++-- .../__tests__/harness-legend.test.tsx | 130 ++++++++++++++ .../__tests__/harness-series.test.ts | 144 ++++++++++++++++ evalboard/app/_overview/daily-chart.tsx | 150 ++++++++--------- evalboard/app/_overview/harness-legend.tsx | 111 ++++++++++++ evalboard/app/_overview/harness-series.ts | 67 ++++++++ evalboard/app/_overview/tag-rail.tsx | 10 +- evalboard/app/_overview/turn-budget-chart.tsx | 159 +++++++++--------- evalboard/app/page.tsx | 118 +++++++------ evalboard/app/path-to-ga/page.tsx | 1 + evalboard/lib/__tests__/harness.test.ts | 99 ++++++++++- evalboard/lib/harness.ts | 70 +++++++- evalboard/lib/overview.ts | 35 +++- 14 files changed, 946 insertions(+), 236 deletions(-) create mode 100644 evalboard/app/_overview/__tests__/harness-legend.test.tsx create mode 100644 evalboard/app/_overview/__tests__/harness-series.test.ts create mode 100644 evalboard/app/_overview/harness-legend.tsx create mode 100644 evalboard/app/_overview/harness-series.ts diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index da16b704..5a1d12ab 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -22,6 +22,14 @@ const HARNESS_LOGO: Record ); diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index 7a814f0e..d82a9070 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -1,34 +1,72 @@ "use client"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { ALL_HARNESSES } from "@/lib/harness"; import { HarnessBadge, harnessShortLabel } from "./harness-badge"; -// Segmented control for the trends page's harness scope. Sets `?h=` -// while preserving the active q/tag params, mirroring WindowSelector. Each -// segment shows the vendor logo + short label so it reads like the per-run -// harness badge on the runs tables. +// Segmented control for a page's harness scope. Sets `?h=` while +// preserving the active q/tag params, mirroring WindowSelector. Each segment +// shows the vendor logo + short label so it reads like the per-run harness badge +// on the runs tables. +// +// `current` is null only on pages that support the all-harness view (the +// overview, whose charts draw a line per harness and whose tiles and run list +// then cover every harness). Pages that need exactly one harness — trends +// collapses per-task history across runs, which only means something inside one +// harness — pass a concrete harness and leave `includeAll` off. export function HarnessSelector({ current, harnesses, + includeAll = false, }: { - current: string; + current: string | null; harnesses: readonly string[]; + includeAll?: boolean; }) { const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - const set = (h: string) => { + const set = (h: string | null) => { const p = new URLSearchParams(searchParams.toString()); - p.set("h", h); - router.replace(`${pathname}?${p.toString()}`, { scroll: false }); + if (h == null) { + // The unscoped view is the default, so it's the absence of the param + // rather than `h=all` — keeps the canonical URL clean. + p.delete("h"); + } else { + p.set("h", h); + } + // Changing scope changes which runs are in play, so the paged-out row + // counts no longer describe the new set; drop them back to page one. + p.delete("limit"); + p.delete("alimit"); + const qs = p.toString(); + router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); }; // Always show the active harness, even if it has aged out of the recent // window (so a deep-linked `?h=` still reads as selected rather than absent). - const opts = harnesses.includes(current) - ? harnesses - : [current, ...harnesses]; + const opts = + current == null || harnesses.includes(current) + ? harnesses + : [current, ...harnesses]; + const segment = (active: boolean) => + `flex items-center gap-1.5 px-3 py-1 ${ + active + ? "bg-studio-blue text-white" + : "bg-white text-gray-700 hover:bg-gray-50" + }`; return (
+ {includeAll && ( + + )} {opts.map((h) => { const active = h === current; return ( @@ -37,7 +75,7 @@ export function HarnessSelector({ type="button" onClick={() => set(h)} aria-pressed={active} - className={`flex items-center gap-1.5 px-3 py-1 ${active ? "bg-studio-blue text-white" : "bg-white text-gray-700 hover:bg-gray-50"}`} + className={segment(active)} > {harnessShortLabel(h)} @@ -47,3 +85,7 @@ export function HarnessSelector({
); } + +// Re-exported so a caller can build an `?h=all` link without importing the leaf +// constants module directly. +export { ALL_HARNESSES }; diff --git a/evalboard/app/_overview/__tests__/harness-legend.test.tsx b/evalboard/app/_overview/__tests__/harness-legend.test.tsx new file mode 100644 index 00000000..361d1291 --- /dev/null +++ b/evalboard/app/_overview/__tests__/harness-legend.test.tsx @@ -0,0 +1,130 @@ +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { harnessColor } from "@/lib/harness"; +import { HarnessLegend, HarnessTooltip } from "../harness-legend"; +import { seriesKey } from "../harness-series"; + +function series(...harnesses: string[]) { + return harnesses.map((harness) => ({ + harness, + dataKey: seriesKey(harness), + color: harnessColor(harness), + })); +} + +describe("HarnessLegend", () => { + test("names every series, so identity is never color-alone", () => { + render(); + expect(screen.getByText("Claude Code")).toBeInTheDocument(); + expect(screen.getByText("Codex")).toBeInTheDocument(); + }); + + test("renders nothing for a single series", () => { + // A one-line chart is already named by its own heading; a legend box + // there is noise. + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); + + test("renders nothing when there is no data at all", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); +}); + +describe("HarnessTooltip", () => { + const s = series("claude-code", "codex"); + + test("shows only the harnesses that actually ran at the hovered x", () => { + // Recharts hands over every series, including the ones with no value + // here. Listing those would invent runs that never happened. + render( + , + ); + expect(screen.getByText("Claude Code")).toBeInTheDocument(); + expect(screen.getByText("92.5% success")).toBeInTheDocument(); + expect(screen.queryByText("Codex")).not.toBeInTheDocument(); + }); + + test("keeps a real zero instead of filtering it out as falsy", () => { + render( + , + ); + expect(screen.getByText("0.0% success")).toBeInTheDocument(); + }); + + test("falls back to the empty text when no series has a value", () => { + render( + , + ); + expect( + screen.getByText("no tasks with a turn budget"), + ).toBeInTheDocument(); + }); + + test("ignores payload entries for unknown series keys", () => { + render( + , + ); + expect(screen.getByText("no tasks")).toBeInTheDocument(); + }); + + test("renders nothing while inactive", () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); + + test("renders the UTC timestamp header from the x label", () => { + render( + , + ); + expect(screen.getByText("2026-07-30 04:27 UTC")).toBeInTheDocument(); + }); +}); diff --git a/evalboard/app/_overview/__tests__/harness-series.test.ts b/evalboard/app/_overview/__tests__/harness-series.test.ts new file mode 100644 index 00000000..2930d6ac --- /dev/null +++ b/evalboard/app/_overview/__tests__/harness-series.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, test } from "vitest"; +import { harnessColor } from "@/lib/harness"; +import type { RunPoint } from "@/lib/overview"; +import { pivotByHarness, seriesKey } from "../harness-series"; + +// pivotByHarness is the only thing standing between "one line per harness" and +// the zigzag the charts used to draw across incomparable harnesses. Its output +// is consumed by recharts, so a wrong key or a dropped row shows up as a missing +// line rather than an error — nothing else would fail. + +function point( + harness: string, + timestamp: number, + successRate: number | null, + turnBudgetRate: number | null = null, +): RunPoint { + return { + runId: `run-${timestamp}`, + timestamp, + harness, + successRate, + turnBudgetRate, + }; +} + +describe("seriesKey", () => { + test("prefixes and sanitizes so recharts can't read it as a path", () => { + // A dataKey containing "." is a nested-object path in recharts, which + // would resolve to undefined and silently draw no line. + expect(seriesKey("gpt-5.5")).not.toContain("."); + expect(seriesKey("claude-code")).toBe("h_claude_code"); + }); + + test("is injective across the known harness ids", () => { + const ids = ["claude-code", "codex", "antigravity", "delegate-sdk"]; + expect(new Set(ids.map(seriesKey)).size).toBe(ids.length); + }); +}); + +describe("pivotByHarness", () => { + test("gives each harness its own column, keyed by run timestamp", () => { + const { rows, series } = pivotByHarness( + [point("claude-code", 100, 90), point("codex", 200, 70)], + ["claude-code", "codex"], + "successRate", + ); + expect(series.map((s) => s.harness)).toEqual(["claude-code", "codex"]); + expect(rows).toEqual([ + { timestamp: 100, h_claude_code: 90 }, + { timestamp: 200, h_codex: 70 }, + ]); + // The absent key on each row is what connectNulls bridges — that is how + // interleaved runs become one continuous line per harness. + expect(rows[0]).not.toHaveProperty("h_codex"); + }); + + test("colors come from the harness, not the series index", () => { + const { series } = pivotByHarness( + [point("codex", 100, 70)], + ["codex"], + "successRate", + ); + // codex is slot 2, but it is the ONLY series here. If color were assigned + // positionally it would come back as slot 1's blue. + expect(series[0].color).toBe(harnessColor("codex")); + expect(series[0].color).not.toBe(harnessColor("claude-code")); + }); + + test("merges two harnesses that share a timestamp into one row", () => { + const { rows } = pivotByHarness( + [point("claude-code", 100, 90), point("codex", 100, 70)], + ["claude-code", "codex"], + "successRate", + ); + expect(rows).toEqual([ + { timestamp: 100, h_claude_code: 90, h_codex: 70 }, + ]); + }); + + test("sorts rows by timestamp regardless of input order", () => { + const { rows } = pivotByHarness( + [point("codex", 300, 1), point("codex", 100, 2)], + ["codex"], + "successRate", + ); + expect(rows.map((r) => r.timestamp)).toEqual([100, 300]); + }); + + test("reads the requested metric, not the other one", () => { + // Both charts share this pivot and the same RunPoint[], differing only + // by the metric key — reading the wrong one would put the success rate + // on the turn-budget chart with no visible error. + const pts = [point("codex", 100, 90, 40)]; + expect(pivotByHarness(pts, ["codex"], "successRate").rows[0]).toEqual({ + timestamp: 100, + h_codex: 90, + }); + expect(pivotByHarness(pts, ["codex"], "turnBudgetRate").rows[0]).toEqual( + { timestamp: 100, h_codex: 40 }, + ); + }); + + test("omits a null metric rather than plotting it as zero", () => { + // The turn-budget metric is null for a run with no budgeted task. A 0% + // there would read as "every task blew its budget", which is the + // opposite of "there was nothing to measure". + const { rows } = pivotByHarness( + [point("codex", 100, 90, null)], + ["codex"], + "turnBudgetRate", + ); + expect(rows).toEqual([]); + }); + + test("drops series with no plotted point so the legend stays honest", () => { + // antigravity is a known harness with no run in this window; an empty + // line would still put it in the legend. + const { series } = pivotByHarness( + [point("codex", 100, 70)], + ["codex", "antigravity"], + "successRate", + ); + expect(series.map((s) => s.harness)).toEqual(["codex"]); + }); + + test("keeps a harness whose only value is a real zero", () => { + // 0% is a legitimate pass rate (every task failed) and must not be + // confused with "no data" by a truthiness check. + const { rows, series } = pivotByHarness( + [point("codex", 100, 0)], + ["codex"], + "successRate", + ); + expect(series).toHaveLength(1); + expect(rows[0].h_codex).toBe(0); + }); + + test("handles an empty window", () => { + expect(pivotByHarness([], ["codex"], "successRate")).toEqual({ + rows: [], + series: [], + }); + }); +}); diff --git a/evalboard/app/_overview/daily-chart.tsx b/evalboard/app/_overview/daily-chart.tsx index 721dd655..51309e7a 100644 --- a/evalboard/app/_overview/daily-chart.tsx +++ b/evalboard/app/_overview/daily-chart.tsx @@ -10,6 +10,8 @@ import { YAxis, } from "recharts"; import type { RunPoint } from "@/lib/overview"; +import { pivotByHarness } from "./harness-series"; +import { HarnessLegend, HarnessTooltip } from "./harness-legend"; // MM-DD tick label on the axis; full date+time appears in the tooltip. function shortLabel(ms: number): string { @@ -19,94 +21,86 @@ function shortLabel(ms: number): string { return `${m}-${day}`; } -function fullLabel(ms: number): string { - const d = new Date(ms); - const y = d.getUTCFullYear(); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - const h = String(d.getUTCHours()).padStart(2, "0"); - const min = String(d.getUTCMinutes()).padStart(2, "0"); - return `${y}-${m}-${day} ${h}:${min} UTC`; -} - -function CustomTooltip({ - active, - payload, -}: { - active?: boolean; - payload?: Array<{ payload: RunPoint }>; -}) { - if (!active || !payload?.length) return null; - const point = payload[0].payload; - return ( -
-
- {fullLabel(point.timestamp)} -
-
- {point.successRate != null - ? `${point.successRate.toFixed(1)}% success` - : "no tasks"} -
-
- ); -} - +// One line per harness present in the window. Unscoped (the page default) that +// means every harness on one chart, which is the comparison the page is for; +// scoped to one harness it degenerates to the single line this chart used to be. export function DailySuccessChart({ data, + harnesses, windowStart, windowEnd, }: { data: RunPoint[]; + harnesses: string[]; windowStart: number; windowEnd: number; }) { + const { rows, series } = pivotByHarness(data, harnesses, "successRate"); return ( -
- - UTC - - - - - - - } - cursor={{ stroke: "#e5e7eb", strokeDasharray: "3 3" }} - /> - - - +
+
+ + UTC + + + + + + + + } + cursor={{ + stroke: "#e5e7eb", + strokeDasharray: "3 3", + }} + /> + {series.map((s) => ( + + ))} + + +
+
); } diff --git a/evalboard/app/_overview/harness-legend.tsx b/evalboard/app/_overview/harness-legend.tsx new file mode 100644 index 00000000..750c600e --- /dev/null +++ b/evalboard/app/_overview/harness-legend.tsx @@ -0,0 +1,111 @@ +"use client"; + +// Legend + tooltip shared by the two overview charts. Both are multi-series +// whenever the page is unscoped, and a multi-series chart must never identify a +// line by color alone — so the legend pairs each swatch with the harness's +// vendor badge and short label, and the tooltip repeats that pairing per row. + +import { harnessShortLabel, HarnessBadge } from "@/app/_components/harness-badge"; +import type { HarnessSeries } from "./harness-series"; + +export function HarnessLegend({ series }: { series: HarnessSeries[] }) { + // One series needs no legend box — the chart's own heading names it. + if (series.length < 2) return null; + return ( +
+ {series.map((s) => ( + + + + {harnessShortLabel(s.harness)} + + ))} +
+ ); +} + +// Recharts hands every series to the tooltip, including the ones with no value +// at the hovered x. Only the harnesses that actually ran at this timestamp are +// worth a row. +export interface TooltipEntry { + dataKey?: string | number; + value?: number | string | null; + color?: string; +} + +function fullLabel(ms: number): string { + const d = new Date(ms); + const y = d.getUTCFullYear(); + const m = String(d.getUTCMonth() + 1).padStart(2, "0"); + const day = String(d.getUTCDate()).padStart(2, "0"); + const h = String(d.getUTCHours()).padStart(2, "0"); + const min = String(d.getUTCMinutes()).padStart(2, "0"); + return `${y}-${m}-${day} ${h}:${min} UTC`; +} + +export function HarnessTooltip({ + active, + payload, + label, + series, + suffix, + emptyText, +}: { + active?: boolean; + payload?: TooltipEntry[]; + // The x value under the cursor; recharts passes the XAxis dataKey's value. + label?: number | string; + series: HarnessSeries[]; + // Trailing prose after the number, e.g. "success". + suffix: string; + // Shown when the hovered run produced no value for this metric at all. + emptyText: string; +}) { + if (!active || !payload?.length) return null; + const byKey = new Map(series.map((s) => [s.dataKey, s])); + const rows = payload.filter( + (e) => typeof e.value === "number" && byKey.has(String(e.dataKey)), + ); + const ms = typeof label === "number" ? label : Number(label); + return ( +
+ {Number.isFinite(ms) && ( +
+ {fullLabel(ms)} +
+ )} + {rows.length === 0 ? ( +
{emptyText}
+ ) : ( + rows.map((e) => { + const s = byKey.get(String(e.dataKey))!; + return ( +
+ + + {harnessShortLabel(s.harness)} + + + {(e.value as number).toFixed(1)}% {suffix} + +
+ ); + }) + )} +
+ ); +} diff --git a/evalboard/app/_overview/harness-series.ts b/evalboard/app/_overview/harness-series.ts new file mode 100644 index 00000000..369cbcd7 --- /dev/null +++ b/evalboard/app/_overview/harness-series.ts @@ -0,0 +1,67 @@ +// Pivot the flat RunPoint[] into the wide, one-column-per-harness shape recharts +// needs to draw several lines on one chart. +// +// Each run belongs to exactly one harness, so a row carries a value for that +// harness's column and leaves the others undefined. Every then uses +// connectNulls to bridge its own gaps, which is what turns sparse interleaved +// runs (claude-code nightly, codex twice a week) into one continuous line each +// rather than a single zigzag across incomparable harnesses. + +import { harnessColor } from "@/lib/harness"; +import type { RunPoint } from "@/lib/overview"; + +// Which per-run rate to plot. Both metrics live on the same RunPoint, so the +// two overview charts share this module and differ only by this key. +export type HarnessMetric = "successRate" | "turnBudgetRate"; + +export interface HarnessSeries { + harness: string; + // Column name in the pivoted rows. Prefixed and sanitized because recharts + // reads a dataKey containing "." as a nested object path, which would + // silently resolve to undefined for a harness id with a dot in it. + dataKey: string; + color: string; +} + +export interface HarnessChartData { + rows: Array>; + series: HarnessSeries[]; +} + +export function seriesKey(harness: string): string { + return `h_${harness.replace(/[^\w]/g, "_")}`; +} + +export function pivotByHarness( + points: RunPoint[], + harnesses: string[], + metric: HarnessMetric, +): HarnessChartData { + const series: HarnessSeries[] = harnesses.map((harness) => ({ + harness, + dataKey: seriesKey(harness), + color: harnessColor(harness), + })); + + // Merge on timestamp so two harnesses that happen to start a run in the same + // second share one row instead of producing two points recharts would render + // stacked on the same x. + const byTimestamp = new Map>(); + for (const p of points) { + if (p[metric] == null) continue; + let row = byTimestamp.get(p.timestamp); + if (!row) { + row = { timestamp: p.timestamp }; + byTimestamp.set(p.timestamp, row); + } + row[seriesKey(p.harness)] = p[metric] as number; + } + + const rows = [...byTimestamp.values()].sort( + (a, b) => a.timestamp - b.timestamp, + ); + // Drop series with no plotted point in this window — an empty line would put + // an entry in the legend for a harness the chart never draws. + const present = series.filter((s) => rows.some((r) => s.dataKey in r)); + return { rows, series: present }; +} diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx index 2bf1f9be..b3674418 100644 --- a/evalboard/app/_overview/tag-rail.tsx +++ b/evalboard/app/_overview/tag-rail.tsx @@ -1,6 +1,5 @@ import Link from "next/link"; import type { TagCount } from "@/lib/overview"; -import { DEFAULT_HARNESS } from "@/lib/harness"; import type { Window } from "@/lib/reviews-types"; type Variant = "neutral" | "rose" | "indigo"; @@ -42,10 +41,11 @@ function hrefForTag( if (window) params.set("window", window); if (tag) params.set("tag", tag); if (q) params.set("q", q); - // Preserve the active harness scope across tag clicks (omit the default to - // keep URLs clean). Without this, filtering by a tag would silently reset a - // codex/antigravity view back to claude-code. - if (harness && harness !== DEFAULT_HARNESS) params.set("h", harness); + // Preserve the active harness scope across tag clicks. null is the + // all-harness default and is expressed as the param's absence, so there is + // nothing to carry. Without this, filtering by a tag would silently widen a + // codex/antigravity view back out to every harness. + if (harness) params.set("h", harness); const qs = params.toString(); return qs ? `${basePath}?${qs}` : basePath; } diff --git a/evalboard/app/_overview/turn-budget-chart.tsx b/evalboard/app/_overview/turn-budget-chart.tsx index f274f32e..19fb95d7 100644 --- a/evalboard/app/_overview/turn-budget-chart.tsx +++ b/evalboard/app/_overview/turn-budget-chart.tsx @@ -10,6 +10,8 @@ import { YAxis, } from "recharts"; import type { RunPoint } from "@/lib/overview"; +import { pivotByHarness } from "./harness-series"; +import { HarnessLegend, HarnessTooltip } from "./harness-legend"; // MM-DD tick label on the axis; full date+time appears in the tooltip. function shortLabel(ms: number): string { @@ -19,99 +21,90 @@ function shortLabel(ms: number): string { return `${m}-${day}`; } -function fullLabel(ms: number): string { - const d = new Date(ms); - const y = d.getUTCFullYear(); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - const h = String(d.getUTCHours()).padStart(2, "0"); - const min = String(d.getUTCMinutes()).padStart(2, "0"); - return `${y}-${m}-${day} ${h}:${min} UTC`; -} - -function CustomTooltip({ - active, - payload, -}: { - active?: boolean; - payload?: Array<{ payload: RunPoint }>; -}) { - if (!active || !payload?.length) return null; - const point = payload[0].payload; - return ( -
-
- {fullLabel(point.timestamp)} -
-
- {point.turnBudgetRate != null - ? `${point.turnBudgetRate.toFixed(1)}% within turn budget (budgeted failures count as over)` - : "no tasks with a turn budget"} -
-
- ); -} - -// Sibling of DailySuccessChart: same axes/styling, plotting the per-run -// "within expected turns" rate instead of the success rate. Driven by the -// same windowed RunPoint[] so the shared window selector controls both. +// Sibling of DailySuccessChart: same axes/styling and the same per-harness +// series split, plotting the per-run "within expected turns" rate instead of the +// success rate. Driven by the same windowed RunPoint[] so the shared window and +// harness selectors control both. export function TurnBudgetChart({ data, + harnesses, windowStart, windowEnd, }: { data: RunPoint[]; + harnesses: string[]; windowStart: number; windowEnd: number; }) { + const { rows, series } = pivotByHarness(data, harnesses, "turnBudgetRate"); return ( -
- - UTC - - - - - - - } - cursor={{ stroke: "#e5e7eb", strokeDasharray: "3 3" }} - /> - - - +
+
+ + UTC + + + + + + + + } + cursor={{ + stroke: "#e5e7eb", + strokeDasharray: "3 3", + }} + /> + {series.map((s) => ( + + ))} + + +
+
); } diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index d8326a0a..bf410cbd 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -6,7 +6,7 @@ import { listRecentHarnesses, type TagCount, } from "@/lib/overview"; -import { parseHarnessParam, DEFAULT_HARNESS } from "@/lib/harness"; +import { parseHarnessScope } from "@/lib/harness"; import { fmtDuration, fmtRunTime, fmtTimestamp, passClass } from "@/lib/format"; import { WindowSelector } from "./_components/window-selector"; import { WINDOWS, type Window } from "@/lib/reviews-types"; @@ -48,24 +48,32 @@ function parseQ(raw: string | string[] | undefined): string | null { return trimmed ? trimmed.slice(0, 200) : null; } -function parseLimit(raw: string | string[] | undefined): number | null { +// Hard ceiling on how far the tables can be paged out. Both sections grow a +// page at a time and never expose a "show all" jump: every extra row is another +// run.json read behind the window load, and an unbounded expansion on a busy +// window is the one interaction that can stall the page. A hand-typed +// `?limit=` above this clamps here rather than being honored. +const MAX_LIMIT = 500; + +function parsePagedLimit( + raw: string | string[] | undefined, + fallback: number, +): number { const v = Array.isArray(raw) ? raw[0] : raw; - if (v === "all") return null; - if (!v) return DEFAULT_LIMIT; + if (!v) return fallback; const n = parseInt(v, 10); - if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT; - return Math.min(n, 10000); + if (!Number.isFinite(n) || n <= 0) return fallback; + return Math.min(n, MAX_LIMIT); +} + +function parseLimit(raw: string | string[] | undefined): number { + return parsePagedLimit(raw, DEFAULT_LIMIT); } // Separate from the main table's `limit` so expanding one section doesn't -// reset the other's pagination. null = show all matching ad-hoc runs. -function parseAdhocLimit(raw: string | string[] | undefined): number | null { - const v = Array.isArray(raw) ? raw[0] : raw; - if (v === "all") return null; - if (!v) return ADHOC_LIMIT; - const n = parseInt(v, 10); - if (!Number.isFinite(n) || n <= 0) return ADHOC_LIMIT; - return Math.min(n, 10000); +// reset the other's pagination. +function parseAdhocLimit(raw: string | string[] | undefined): number { + return parsePagedLimit(raw, ADHOC_LIMIT); } function fmtCost(c: number | null): string { @@ -110,18 +118,21 @@ export default async function Page({ const window = parseWindow(params.window); const activeTag = parseTag(params.tag); const q = parseQ(params.q); - const harness = parseHarnessParam(params.h); + // null = every harness, and that is the default: the page opens on the + // cross-harness comparison and narrows from there. + const harness = parseHarnessScope(params.h); const limit = parseLimit(params.limit); const adhocLimit = parseAdhocLimit(params.alimit); const isFiltered = activeTag != null || q != null; - // The analytics block (chart + rails) is scoped to one harness so the - // success line stops zigzagging across incomparable harnesses. The run - // LIST stays all-harness — seeing every recent run is the page's job, and - // the Harness column already disambiguates each row. + // One harness scope drives everything on the page: the charts split into a + // line per harness in the window, and the summary tiles + run list are + // filtered to the same set. Picking a harness therefore recomputes the + // tiles and the charts and narrows the table, instead of re-scoping the + // analytics block while the table silently kept showing every harness. const [overview, listing, adhoc, harnesses] = await Promise.all([ getOverview(window, activeTag, q, harness), - getRunListing(window, activeTag, q, limit), + getRunListing(window, activeTag, q, limit, harness), getAdhocRunListing(adhocLimit), listRecentHarnesses(), ]); @@ -134,7 +145,8 @@ export default async function Page({ const matchedCount = listing.matchedCount; const totalInWindow = listing.totalInWindow; const tableTotalLabel = isFiltered ? matchedCount : totalInWindow; - const hasMore = limit != null && shownCount < tableTotalLabel; + // More rows exist AND the page cap still has room for them. + const hasMore = shownCount < Math.min(tableTotalLabel, MAX_LIMIT); // Current URL params, normalized to scalars. Spread as the base of every // href so toggling one section (main `limit` or ad-hoc `alimit`) carries @@ -143,9 +155,9 @@ export default async function Page({ const rawAlimit = Array.isArray(params.alimit) ? params.alimit[0] : params.alimit; - // Omit the default harness from URLs to keep them clean; carry a non-default + // Omit the all-harness scope from URLs to keep them clean; carry a narrowed // scope through every self-link so it isn't reset by pagination/clear. - const hParam = harness === DEFAULT_HARNESS ? undefined : harness; + const hParam = harness ?? undefined; const base = { window, tag: activeTag, @@ -155,19 +167,30 @@ export default async function Page({ alimit: rawAlimit, }; + // Both tables grow one page at a time. There is deliberately no "show all" + // link: each row is backed by a run.json read, so an unbounded jump is the + // one click that can hang the page on a busy window. + const nextPageSize = Math.min( + DEFAULT_LIMIT, + Math.min(tableTotalLabel, MAX_LIMIT) - shownCount, + ); const showMoreHref = buildHref({ ...base, - limit: Math.min(tableTotalLabel, shownCount + DEFAULT_LIMIT), + limit: shownCount + nextPageSize, }); - const showAllHref = buildHref({ ...base, limit: "all" }); const clearAllHref = buildHref({ window, h: hParam }); // Ad-hoc section disclosure: rows are filtered (by `q`) then capped to - // adhocLimit; offer "Show all" while more match than are shown, and a - // collapse back to the default once expanded past it. - const adhocExpandable = adhoc.rows.length < adhoc.total; - const adhocExpanded = adhocLimit == null && adhoc.total > ADHOC_LIMIT; - const adhocShowAllHref = buildHref({ ...base, alimit: "all" }); + // adhocLimit; page in another ADHOC_LIMIT while more match than are shown, + // and offer a collapse back to the default once expanded past it. + const adhocRemaining = Math.min(adhoc.total, MAX_LIMIT) - adhoc.rows.length; + const adhocExpandable = adhocRemaining > 0; + const adhocNextPageSize = Math.min(ADHOC_LIMIT, adhocRemaining); + const adhocExpanded = adhoc.rows.length > ADHOC_LIMIT; + const adhocShowMoreHref = buildHref({ + ...base, + alimit: adhoc.rows.length + adhocNextPageSize, + }); const adhocShowLessHref = buildHref({ ...base, alimit: undefined }); return ( @@ -236,8 +259,11 @@ export default async function Page({ ) : ( <> Success rate per{" "} - {harnessShortLabel(harness)} run across the - last {window} · {overview.runs.length} run + {harness + ? `${harnessShortLabel(harness)} run` + : "run, one line per harness,"}{" "} + across the last {window} ·{" "} + {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} )} @@ -247,12 +273,14 @@ export default async function Page({
@@ -268,6 +296,7 @@ export default async function Page({

@@ -330,21 +359,11 @@ export default async function Page({ scroll={false} className="text-studio-blue hover:underline" > - Show{" "} - {Math.min( - DEFAULT_LIMIT, - tableTotalLabel - shownCount, - )}{" "} - more - - · - - Show all ({tableTotalLabel}) + Show {nextPageSize} more + + {shownCount} of {tableTotalLabel} + ) : undefined } @@ -463,13 +482,16 @@ export default async function Page({
{adhocExpandable && ( - Show all ({adhoc.total}) + Show {adhocNextPageSize} more )} + + {adhoc.rows.length} of {adhoc.total} + {adhocExpanded && ( 0 ? ( diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts index 8dcb064c..c8a07a99 100644 --- a/evalboard/lib/__tests__/harness.test.ts +++ b/evalboard/lib/__tests__/harness.test.ts @@ -1,5 +1,13 @@ import { describe, expect, test } from "vitest"; -import { DEFAULT_HARNESS, parseHarnessParam } from "../harness"; +import { + ALL_HARNESSES, + DEFAULT_HARNESS, + KNOWN_HARNESSES, + harnessColor, + orderHarnesses, + parseHarnessParam, + parseHarnessScope, +} from "../harness"; // parseHarnessParam gates the untrusted `?h=` query param at every // harness-scoped route, and its result decides which runs are counted toward @@ -47,3 +55,92 @@ describe("parseHarnessParam", () => { expect(parseHarnessParam("a".repeat(65))).toBe(DEFAULT_HARNESS); }); }); + +// parseHarnessScope is the overview's variant: the same untrusted `?h=` param, +// but null-defaulting so the front page opens on every harness instead of +// silently scoping to claude-code. Anything that mis-maps here mis-scopes the +// summary tiles, both charts, and the run table at once, since all four now read +// this one value. +describe("parseHarnessScope", () => { + test("absent / empty input means all harnesses", () => { + expect(parseHarnessScope(undefined)).toBeNull(); + expect(parseHarnessScope([])).toBeNull(); + expect(parseHarnessScope("")).toBeNull(); + expect(parseHarnessScope(" ")).toBeNull(); + }); + + test("the explicit all sentinel means all harnesses", () => { + expect(parseHarnessScope(ALL_HARNESSES)).toBeNull(); + expect(parseHarnessScope([ALL_HARNESSES])).toBeNull(); + }); + + test("passes through any syntactically-valid id", () => { + for (const id of [...KNOWN_HARNESSES, "gpt-5.5", "some_harness"]) { + expect(parseHarnessScope(id)).toBe(id); + } + }); + + test("trims before validating, and picks the first array element", () => { + expect(parseHarnessScope(" codex ")).toBe("codex"); + expect(parseHarnessScope([" antigravity ", "codex"])).toBe( + "antigravity", + ); + }); + + test("malformed ids widen to all rather than to a wrong harness", () => { + // Contrast with parseHarnessParam, which falls back to claude-code. A + // bad param here must not silently produce claude-code-only numbers. + for (const bad of ["has space", "a/b", "bad;rm", "a".repeat(65)]) { + expect(parseHarnessScope(bad)).toBeNull(); + } + }); + + test("differs from parseHarnessParam exactly on the empty case", () => { + expect(parseHarnessParam(undefined)).toBe(DEFAULT_HARNESS); + expect(parseHarnessScope(undefined)).toBeNull(); + }); +}); + +// Color is bound to the harness, never to its index in the series list — a +// filter that removes one line must not repaint the survivors. +describe("harnessColor", () => { + test("every known harness has its own reserved color", () => { + const colors = KNOWN_HARNESSES.map(harnessColor); + expect(new Set(colors).size).toBe(KNOWN_HARNESSES.length); + }); + + test("is stable per harness regardless of what else is on screen", () => { + // The regression this guards: assigning by position, so dropping + // claude-code would slide codex onto claude-code's blue. + expect(harnessColor("codex")).toBe(harnessColor("codex")); + expect(harnessColor("codex")).not.toBe(harnessColor("claude-code")); + }); + + test("unknown harnesses share one neutral, not a generated hue", () => { + const a = harnessColor("brand-new-harness"); + const b = harnessColor("another-newcomer"); + expect(a).toBe(b); + for (const known of KNOWN_HARNESSES) { + expect(a).not.toBe(harnessColor(known)); + } + }); +}); + +describe("orderHarnesses", () => { + test("known harnesses come first, in KNOWN_HARNESSES order", () => { + expect( + orderHarnesses(["antigravity", "claude-code", "codex"]), + ).toEqual(["claude-code", "codex", "antigravity"]); + }); + + test("newcomers follow, alphabetically", () => { + expect( + orderHarnesses(["zeta", "codex", "alpha-harness"]), + ).toEqual(["codex", "alpha-harness", "zeta"]); + }); + + test("dedupes and drops nothing else", () => { + expect(orderHarnesses(["codex", "codex"])).toEqual(["codex"]); + expect(orderHarnesses([])).toEqual([]); + }); +}); diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts index bf36cb28..c52c992c 100644 --- a/evalboard/lib/harness.ts +++ b/evalboard/lib/harness.ts @@ -5,9 +5,15 @@ // The harnesses the nightly rotates through, in preferred display order. This // is only a display-order hint / default source — the switcher is data-driven -// (see listRecentHarnesses), so a new harness like "delegate" surfaces -// automatically without editing this list. -export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; +// (see listRecentHarnesses), so a new harness surfaces automatically without +// editing this list. Membership here does buy two things: a stable position in +// the switcher, and a reserved series color (see HARNESS_COLORS). +export const KNOWN_HARNESSES = [ + "claude-code", + "codex", + "antigravity", + "delegate-sdk", +] as const; export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; // A run with no RunConfig harness predates the stamp; every such run was @@ -29,3 +35,61 @@ export function parseHarnessParam(raw: string | string[] | undefined): string { const trimmed = v.trim(); return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS; } + +// URL value for the unscoped view. Reserved: a real harness id could never +// collide with it, since AgentKind ids are hyphenated vendor names. +export const ALL_HARNESSES = "all"; + +// Harness scope for pages that can show every harness at once (the overview). +// null = all harnesses, which is also the DEFAULT — the front page opens on the +// comparison view and narrows from there, so `?h=` absent means "everything" +// rather than "claude-code". Pages that genuinely need a single harness (trends +// collapses per-task history across runs, which is only meaningful within one +// harness) keep using parseHarnessParam. +export function parseHarnessScope( + raw: string | string[] | undefined, +): string | null { + const v = Array.isArray(raw) ? raw[0] : raw; + if (typeof v !== "string") return null; + const trimmed = v.trim(); + if (!trimmed || trimmed === ALL_HARNESSES) return null; + return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : null; +} + +// Series color per harness, for the multi-harness overview charts. Color is +// bound to the HARNESS, not to its position in the series list — a filter that +// drops one harness must not repaint the others. +// +// Values are slots 1/2/3(dark step)/7 of the validated categorical palette. As +// an ordered set on a light surface they clear every hard gate: lightness band, +// chroma floor, adjacent-pair CVD separation (worst ΔE 8.4 protan), the +// normal-vision floor (worst ΔE 27.1), and 3:1 contrast. Re-run the palette +// validator before adding a fifth entry rather than eyeballing a new hue — the +// obvious next slots (yellow, magenta) both fail contrast on white. +const HARNESS_COLORS: Record = { + "claude-code": "#2a78d6", // blue + codex: "#eb6834", // orange + antigravity: "#199e70", // aqua + "delegate-sdk": "#4a3aa7", // violet +}; + +// Neutral for any harness with no reserved slot. Deliberately NOT a generated +// hue: unknown harnesses fold into one "other" color, which stays honest under +// CVD, and the legend still names each line. +const HARNESS_COLOR_FALLBACK = "#6b7280"; + +export function harnessColor(harness: string): string { + return HARNESS_COLORS[harness] ?? HARNESS_COLOR_FALLBACK; +} + +// Stable series order for charts and legends: known harnesses in display order, +// then any newcomers alphabetically. Same rule as listRecentHarnesses, applied +// to whichever subset actually has data in the current window. +export function orderHarnesses(harnesses: Iterable): string[] { + const seen = new Set(harnesses); + const known = KNOWN_HARNESSES.filter((h) => seen.has(h)); + const extras = [...seen] + .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h)) + .sort(); + return [...known, ...extras]; +} diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 4e6cfa6b..08d748bf 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,12 +15,21 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; -import { DEFAULT_HARNESS, KNOWN_HARNESSES, normalizeHarness } from "./harness"; +import { + DEFAULT_HARNESS, + KNOWN_HARNESSES, + normalizeHarness, + orderHarnesses, +} from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { runId: string; timestamp: number; // ms since epoch (UTC); used as the chart x-coordinate + // The harness this run ran on, normalized (legacy runs fold to claude-code). + // Every run belongs to exactly one harness, so the charts split the points + // into one series per harness and color each by identity. + harness: string; successRate: number | null; // % of budgeted tasks whose visible turns stayed within 1.5× their // expected_turns budget. Only tasks carrying a positive expected_turns @@ -80,6 +89,10 @@ export interface TagCount { export interface OverviewData { runs: RunPoint[]; // one point per run, no daily aggregation + // The harnesses actually present in `runs`, in stable display order. Drives + // the chart's series list and legend, so a harness with no runs in the + // window contributes no empty line. + harnesses: string[]; windowStart: number; // ms — chart x-domain start windowEnd: number; // ms — chart x-domain end skills: TagCount[]; @@ -554,6 +567,8 @@ export async function getOverview( // When tag or q is active, scope each run's rate to only matching tasks. const runPoints: RunPoint[] = []; + const seenHarnesses = new Set(); + for (const { id, overview, reviewTagsByTask } of perRun) { if (!overview || overview.tasks.length === 0) continue; const date = parseRunIdDate(id); @@ -576,10 +591,13 @@ export async function getOverview( (t) => t.status === "SUCCESS", ).length; const rate = (succeeded / matching.length) * 100; + const runHarness = normalizeHarness(overview.harness); + seenHarnesses.add(runHarness); runPoints.push({ runId: id, timestamp: date.getTime(), + harness: runHarness, successRate: rate, turnBudgetRate: turnBudgetRateForTasks(matching), }); @@ -597,6 +615,7 @@ export async function getOverview( return { runs: runPoints, + harnesses: orderHarnesses(seenHarnesses), windowStart, windowEnd, skills, @@ -684,11 +703,21 @@ export async function getRunListing( tag: string | null, q: string | null, limit: number | null, // null = unlimited + // When set, scope the listing (and therefore the window rollup that feeds + // the summary tiles) to one harness. null = all harnesses. Applied at the + // same seam as the chart's scope in getOverview, so the tiles, the charts, + // and the table always describe the same set of runs. + harness: string | null = null, ): Promise { // Exclude ad-hoc runs from the main listing — they appear in their own // section (getAdhocRunListing). totalInWindow therefore counts only - // pipeline runs, matching the chart above it. - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + // pipeline runs in scope, matching the chart above it. + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); // Run IDs are timestamped — newest first by lexical compare. const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); const totalInWindow = sorted.length; From 431d162e43edf943a6425de49db0818b5f7c2784 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 30 Jul 2026 14:31:33 -0700 Subject: [PATCH 2/5] feat(evalboard): say which harness, model, and framework version a run used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run page identified a run only by its timestamped id, so telling a codex run from a claude-code one meant going back to the runs table or opening a task — even though harness and model are exactly what decide whether this page's numbers are comparable to another run's. RunSummary now carries both (harness from the RunConfig stamp, model as the dominant per-task model_used) and the header renders them as a badge strip. An A/B run that fans variants across models reports the spread instead of silently claiming the winner. The coder_eval chip also showed a git SHA linking to a tree, but the framework is pinned and consumed by released version, and env_info records that version alongside the SHA. The chip now reads the version and links to its release. The SHA survives as a fallback for the two cases a version can't cover: an editable checkout between releases, and legacy runs written before the version was captured. This also gives in-container runs a chip at all — their git SHA resolves to "unknown" and was dropped, while the version resolves fine. --- .../runs/[id]/__tests__/run-identity.test.tsx | 77 +++++++++++++++++ evalboard/app/runs/[id]/page.tsx | 8 ++ evalboard/app/runs/[id]/run-identity.tsx | 55 ++++++++++++ evalboard/lib/__tests__/runs.test.ts | 84 +++++++++++++++++++ evalboard/lib/runs.ts | 63 +++++++++++++- 5 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 evalboard/app/runs/[id]/__tests__/run-identity.test.tsx create mode 100644 evalboard/app/runs/[id]/run-identity.tsx diff --git a/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx b/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx new file mode 100644 index 00000000..e3d633bf --- /dev/null +++ b/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx @@ -0,0 +1,77 @@ +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { RunIdentity } from "../run-identity"; + +// The run page's numbers are only comparable across runs on the same harness and +// model, so getting this strip wrong (or silently omitting it) is what sends +// someone comparing a codex run's pass rate against a claude-code one. +describe("RunIdentity", () => { + test("names the harness and the model", () => { + render( + , + ); + expect(screen.getByText("Codex")).toBeInTheDocument(); + expect(screen.getByText("gpt-5.2-codex")).toBeInTheDocument(); + }); + + test("uses the vendor logo, not just the raw harness id", () => { + render( + , + ); + // The UiPath mark stands in for the Delegate SDK harness, which has no + // file of its own under /harness. + expect( + screen.getByAltText("Delegate SDK · UiPath"), + ).toBeInTheDocument(); + expect(screen.getByText("Delegate SDK")).toBeInTheDocument(); + }); + + test("flags a multi-model run instead of claiming the dominant one", () => { + render( + , + ); + expect(screen.getByText("+2 more")).toBeInTheDocument(); + }); + + test("says nothing extra on a single-model run", () => { + render( + , + ); + expect(screen.queryByText(/more/)).not.toBeInTheDocument(); + }); + + test("renders each half independently when the other is missing", () => { + const { rerender } = render( + , + ); + expect(screen.getByText("Codex")).toBeInTheDocument(); + + rerender( + , + ); + expect(screen.getByText("some-model")).toBeInTheDocument(); + }); + + test("renders nothing for a legacy run that identifies neither", () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 7d0f58f2..72682ccf 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -13,6 +13,7 @@ import { fmtRunTime } from "@/lib/format"; import { AnalysisPanel } from "./analysis-panel"; import { RefreshButton } from "./refresh-button"; import { RunView } from "./run-view"; +import { RunIdentity } from "./run-identity"; import { VersionList } from "@/app/_components/version-list"; import { isInternal } from "@/lib/edition"; @@ -102,6 +103,13 @@ export default async function RunPage({ {meta.description}

)} + {/* Harness + model, so the page says what produced these + numbers without a trip back to the runs table. */} + {/* Component SHAs (cli / agent / sdk / drawer …) point at internal tooling; internal-only. See lib/edition.ts. */} {isInternal && ( diff --git a/evalboard/app/runs/[id]/run-identity.tsx b/evalboard/app/runs/[id]/run-identity.tsx new file mode 100644 index 00000000..a13f24b5 --- /dev/null +++ b/evalboard/app/runs/[id]/run-identity.tsx @@ -0,0 +1,55 @@ +// "What am I looking at" strip for the run header. The run page used to identify +// a run only by its timestamped id, so telling a codex run from a claude-code one +// meant going back to the runs table (which does carry a Harness column) or +// opening a task. Harness and model are the two facts that decide whether a +// number on this page is comparable to a number on another run's page, so they +// belong in the header. + +import { + HarnessBadge, + harnessShortLabel, +} from "@/app/_components/harness-badge"; + +export function RunIdentity({ + harness, + model, + modelCount, +}: { + harness: string | null; + model: string | null; + modelCount: number; +}) { + // Legacy runs identify neither; render nothing rather than an empty frame. + if (!harness && !model) return null; + return ( +
+ {harness && ( + + + {harnessShortLabel(harness)} + + )} + {model && ( + 1 + ? `${modelCount} models across this run's tasks; showing the most common` + : "Model this run's tasks ran on" + } + > + model + {model} + {modelCount > 1 && ( + + +{modelCount - 1} more + + )} + + )} +
+ ); +} diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 5fd0c719..91c8341a 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -15,6 +15,7 @@ import { type ArtifactRef, clearRunCacheDir, extractComponentShas, + tallyModels, extractRunConfig, findMatureSourceRuns, isExcludedArtifact, @@ -345,6 +346,89 @@ describe("extractComponentShas", () => { }); expect(out.map((c) => c.name)).toEqual(["cli", "maestro-tool"]); }); + + // The framework is pinned and consumed by released version (env_info + // `coder_eval`), not by checkout — so the chip must be that version and link + // to its release, rather than a git SHA pointing at a tree. + test("coder_eval reports its released version, linked to the release", () => { + const out = extractComponentShas({ + ...baseEnv, + coder_eval: "0.9.1", + }); + const framework = out.find((c) => c.name === "coder_eval"); + expect(framework?.sha).toBe("0.9.1"); + expect(framework?.url).toBe( + "https://github.com/UiPath/coder_eval/releases/tag/v0.9.1", + ); + }); + + test("the version wins over a git_commit present in the same run", () => { + // Both keys are captured on every run; the SHA must not shadow the + // version just because it comes first in env_info. + const out = extractComponentShas({ + git_commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + coder_eval: "0.10.0", + }); + expect(out.find((c) => c.name === "coder_eval")?.sha).toBe("0.10.0"); + }); + + test("falls back to the git SHA when no version was captured", () => { + // Legacy runs, and editable checkouts between releases, have only the + // SHA — better a tree link than no chip at all. + const out = extractComponentShas(baseEnv); + const framework = out.find((c) => c.name === "coder_eval"); + expect(framework?.sha).toBe(baseEnv.git_commit); + expect(framework?.url).toBe( + `https://github.com/UiPath/coder_eval/tree/${baseEnv.git_commit}`, + ); + }); + + test("an 'unknown' version still falls through to the SHA", () => { + const out = extractComponentShas({ + ...baseEnv, + coder_eval: "unknown", + }); + expect(out.find((c) => c.name === "coder_eval")?.sha).toBe( + baseEnv.git_commit, + ); + }); +}); + +// The run header claims a single harness and model for the whole run; tallyModels +// is what keeps that claim honest when an A/B experiment fans variants across +// models inside one run. +describe("tallyModels", () => { + const row = (model: string | null) => ({ + task_id: "t", + model_used: model, + }); + + test("returns the most common model and the distinct count", () => { + const out = tallyModels([ + row("claude-sonnet-5"), + row("claude-sonnet-5"), + row("claude-opus-5"), + ]); + expect(out).toEqual({ dominant: "claude-sonnet-5", distinct: 2 }); + }); + + test("a single-model run reports distinct 1", () => { + expect(tallyModels([row("claude-sonnet-5"), row("claude-sonnet-5")])) + .toEqual({ dominant: "claude-sonnet-5", distinct: 1 }); + }); + + test("ignores rows with no model rather than counting them as one", () => { + const out = tallyModels([row(null), row("codex-mini"), row(null)]); + expect(out).toEqual({ dominant: "codex-mini", distinct: 1 }); + }); + + test("an empty or model-less run reports nothing to display", () => { + expect(tallyModels([])).toEqual({ dominant: null, distinct: 0 }); + expect(tallyModels([row(null)])).toEqual({ + dominant: null, + distinct: 0, + }); + }); }); describe("extractRunConfig", () => { diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 45de6f98..0c09d9ed 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -56,6 +56,16 @@ export interface RunSummary { tasksError: number; totalCostUsd: number | null; componentShas: ComponentSha[]; + // What actually produced this run, for the run header. `harness` is the + // coder-eval AgentKind from the RunConfig stamp (falling back to the per-task + // agent_config vote); null only on legacy runs that identify neither. + harness: string | null; + // Dominant per-task `model_used`, with how many distinct models the run + // touched. Normally 1 — a run is one model — but an A/B experiment fans + // variants across models in a single run, and a header claiming one model + // would be wrong there, so the count is carried and surfaced. + model: string | null; + modelCount: number; } export interface TaskResultSummary { @@ -471,13 +481,30 @@ const COMPONENTS: { display: string; repo: string | null; nonShaUrl: string | null; + // Link target for a semver-shaped value, given the value itself. Takes + // precedence over `nonShaUrl` when it returns a URL. This is how a released + // component links at its own version rather than at a moving list page. + versionUrl?: (version: string) => string | null; keys: string[]; }[] = [ { display: "coder_eval", repo: "UiPath/coder_eval", nonShaUrl: null, - keys: ["git_commit"], + // Releases are tagged `v` by the Release workflow, so the + // version resolves to an exact release page. Runs from before tagging + // began (v0.8.2) recorded a placeholder `0.1.0` and will 404 — the honest + // answer, since no such release exists, and not worth a version floor + // that would rot. + versionUrl: (v) => + `https://github.com/UiPath/coder_eval/releases/tag/v${v}`, + // The framework identifies itself by the released package version + // (env_info `coder_eval`, e.g. "0.9.1"), which is what the suite + // actually pins — so that is the chip, and it links to the release. + // `git_commit` is a fallback for two cases the version can't cover: an + // editable checkout between releases, and legacy runs written before the + // version was captured. It is a SHA, so it links to the tree. + keys: ["coder_eval", "git_commit"], }, { display: "skills", @@ -516,8 +543,10 @@ export function extractComponentShas( let url: string | null = null; if (comp.repo && SHA_RE.test(value)) { url = `https://github.com/${comp.repo}/tree/${value}`; - } else if (comp.nonShaUrl) { - url = comp.nonShaUrl; + } else { + // Not SHA-shaped, so it's a version label. Prefer an exact + // per-version link; fall back to the component's list page. + url = comp.versionUrl?.(value) ?? comp.nonShaUrl; } out.push({ name: comp.display, sha: value, url }); } @@ -665,6 +694,7 @@ export async function readRunSummary(id: string): Promise { const taskDurationSeconds = allHaveDuration ? taskDurationSum : (data.total_duration_seconds ?? null); + const models = tallyModels(taskResults); return { id, startTime: data.start_time ?? null, @@ -676,9 +706,36 @@ export async function readRunSummary(id: string): Promise { tasksError: data.tasks_error ?? 0, totalCostUsd: taskResults.length ? totalCost : null, componentShas: extractComponentShas(data.environment_info), + harness: extractRunConfig(data).harness, + model: models.dominant, + modelCount: models.distinct, }; } +// Dominant `model_used` across a run's rows, plus how many distinct models +// appeared. Mirrors mostCommonAgentType's "the thing these tasks ran on" vote, +// but also reports the spread so the run header can say when there was more than +// one instead of silently picking a winner. +export function tallyModels(rows: RawTaskResult[]): { + dominant: string | null; + distinct: number; +} { + const counts = new Map(); + for (const r of rows) { + const m = r.model_used; + if (typeof m === "string" && m) counts.set(m, (counts.get(m) ?? 0) + 1); + } + let dominant: string | null = null; + let bestN = 0; + for (const [model, n] of counts) { + if (n > bestN) { + dominant = model; + bestN = n; + } + } + return { dominant, distinct: counts.size }; +} + export async function readRunTasks( id: string, ): Promise { From 7c925a5cf5e98dee5955e71b01e101368c26517b Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 30 Jul 2026 15:12:18 -0700 Subject: [PATCH 3/5] refactor(evalboard): call the UiPath harness Delegate Shortens the label from "Delegate SDK" to "Delegate" everywhere it is read: the runs-table badge, the chart legend, the harness selector, and the run page's identity strip. The registered `agent.type` stays `delegate-sdk` (that is what run.json carries and what `?h=` accepts), so only the display name changes. Adds coverage for the badge, which had none: every known harness has a human label, an unknown one falls back to its raw id rather than borrowing another vendor's mark, and the size prop the chart legend depends on is honored. Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/harness-badge.test.tsx | 45 +++++++++++++++++++ evalboard/app/_components/harness-badge.tsx | 12 ++--- .../runs/[id]/__tests__/run-identity.test.tsx | 13 +++--- 3 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 evalboard/app/_components/__tests__/harness-badge.test.tsx diff --git a/evalboard/app/_components/__tests__/harness-badge.test.tsx b/evalboard/app/_components/__tests__/harness-badge.test.tsx new file mode 100644 index 00000000..32b384c8 --- /dev/null +++ b/evalboard/app/_components/__tests__/harness-badge.test.tsx @@ -0,0 +1,45 @@ +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { KNOWN_HARNESSES } from "@/lib/harness"; +import { HarnessBadge, harnessShortLabel } from "../harness-badge"; + +describe("harnessShortLabel", () => { + test("every known harness has a human label", () => { + // A missing entry falls through to the raw id, which would put + // "delegate-sdk" in a legend next to "Codex" and "Antigravity". + for (const h of KNOWN_HARNESSES) { + expect(harnessShortLabel(h)).not.toBe(h); + } + }); + + test("the UiPath harness reads as Delegate", () => { + // The run data's id stays `delegate-sdk` (it's the registered + // `agent.type`); only the label people read is the short one. + expect(harnessShortLabel("delegate-sdk")).toBe("Delegate"); + }); + + test("an unknown harness falls back to its id rather than a wrong name", () => { + expect(harnessShortLabel("some-new-agent")).toBe("some-new-agent"); + }); +}); + +describe("HarnessBadge", () => { + test("names the vendor in the alt text, not just the product", () => { + render(); + expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument(); + }); + + test("renders the id as text when there is no logo for it", () => { + // Better a raw id than another vendor's mark on someone else's run. + render(); + expect(screen.getByText("some-new-agent")).toBeInTheDocument(); + }); + + test("takes a size so the chart legend can sit inside 11px text", () => { + render(); + expect(screen.getByAltText("Codex · OpenAI")).toHaveAttribute( + "width", + "14", + ); + }); +}); diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index 5a1d12ab..02a7bbaa 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -22,13 +22,15 @@ const HARNESS_LOGO: Record { render( , ); - // The UiPath mark stands in for the Delegate SDK harness, which has no - // file of its own under /harness. - expect( - screen.getByAltText("Delegate SDK · UiPath"), - ).toBeInTheDocument(); - expect(screen.getByText("Delegate SDK")).toBeInTheDocument(); + // The UiPath mark stands in for the Delegate harness, which has no file + // of its own under /harness. The id stays `delegate-sdk` (that is what + // run.json carries); only the label is shortened. + expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument(); + expect(screen.getByText("Delegate")).toBeInTheDocument(); }); test("flags a multi-model run instead of claiming the dominant one", () => { From b49f07ee9fb2dbac036bc025a10299b407160d54 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 30 Jul 2026 15:12:33 -0700 Subject: [PATCH 4/5] feat(evalboard): one set of pass-rate cutoffs, and a run table that pages through all history Three changes that all pull in the same direction: make a number mean the same thing wherever it appears, and stop making people choose a time window to find a run. Pass-rate colors now come from one module (`lib/pass-rate.ts`) carrying the same cutoffs as the nightly Slack rollup's traffic-light dots (>=95% green, <85% red, amber between). Four call sites had drifted apart: the front-page table and tiles were green at >=80, Trends was green only at a perfect 100% (a task that passed 9 of its last 10 runs looked as broken as one that never passed), the watchlist had a third set, and the run page's meter was unconditionally green, so a 77% run still read as healthy at a glance. A run that pings the channel red and then reads green on the page it links to is the failure this consolidates away. The time-window control (1d/7d/14d/30d) is gone. Charts and summary tiles cover a fixed 30 days; the run table is no longer bounded by that window at all and pages back through the whole store 20 rows at a time, which is what a shorter window was being used for. Rows load lazily, since a run.json is multi-MB and counting every match in history on each render would not be affordable: `hasMore` comes from over-fetching a single row rather than from a total, and the one free count (every pipeline run in the store) is reported alongside it. `getWindowRollup` splits the tiles' 30-day rollup away from the table's paging, and `projectRunRow` becomes the single definition of "does this run count, and with which tasks" that both go through, so the tiles can no longer describe a different set of runs than the table under them. Path to GA defaults to every harness, like the overview: a line per harness in one chart instead of four page loads. Its task table pools appearances across harnesses when unscoped, which mixes regimes that are not strictly comparable, so it says so and points at the selector. Also fixes the overview at phone width, where the two segmented controls could not fit on one row and pushed the page past the viewport: the control row wraps, harness segments keep their labels on one line and drop to logo-only below `sm` (each keeps its name as tooltip and accessible label, and the chart legend names every line), and the run timestamp stops breaking across three lines. Co-Authored-By: Claude Opus 5 (1M context) --- .../app/_components/harness-selector.tsx | 27 +- evalboard/app/_components/window-selector.tsx | 33 --- evalboard/app/_overview/window-summary.tsx | 15 +- evalboard/app/page.tsx | 88 +++---- evalboard/app/path-to-ga/page.tsx | 72 +++--- evalboard/app/runs/[id]/run-view.tsx | 11 +- evalboard/app/trends/trends-view.tsx | 9 +- evalboard/app/watchlist/watchlist-view.tsx | 7 +- evalboard/lib/__tests__/overview.test.ts | 115 +++++++++ evalboard/lib/__tests__/pass-rate.test.ts | 76 ++++++ evalboard/lib/format.ts | 11 - evalboard/lib/overview.ts | 232 +++++++++++------- evalboard/lib/pass-rate.ts | 53 ++++ 13 files changed, 523 insertions(+), 226 deletions(-) delete mode 100644 evalboard/app/_components/window-selector.tsx create mode 100644 evalboard/lib/__tests__/pass-rate.test.ts create mode 100644 evalboard/lib/pass-rate.ts diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index d82a9070..adfc28b1 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -48,37 +48,52 @@ export function HarnessSelector({ current == null || harnesses.includes(current) ? harnesses : [current, ...harnesses]; + // shrink-0 + whitespace-nowrap: a segment must keep its label on one line + // and never compress, so on a narrow screen the control scrolls (see the + // wrapper) instead of wrapping "Claude Code" onto two lines and stretching + // the whole page past the viewport. const segment = (active: boolean) => - `flex items-center gap-1.5 px-3 py-1 ${ + `flex shrink-0 items-center gap-1.5 whitespace-nowrap px-2.5 py-1 sm:px-3 ${ active ? "bg-studio-blue text-white" : "bg-white text-gray-700 hover:bg-gray-50" }`; + // Below `sm` the vendor logo carries the identity on its own (each button + // keeps the full name as its accessible label and tooltip) — five spelled-out + // segments don't fit on a phone, and the chart legend below names every line. + const label = (text: string) => ( + {text} + ); return ( -
+
{includeAll && ( )} - {opts.map((h) => { + {opts.map((h, i) => { const active = h === current; + const name = harnessShortLabel(h); return ( ); })} diff --git a/evalboard/app/_components/window-selector.tsx b/evalboard/app/_components/window-selector.tsx deleted file mode 100644 index 3394eeef..00000000 --- a/evalboard/app/_components/window-selector.tsx +++ /dev/null @@ -1,33 +0,0 @@ -"use client"; - -import { usePathname, useRouter, useSearchParams } from "next/navigation"; -import { WINDOWS, type Window } from "@/lib/reviews-types"; - -export function WindowSelector({ current }: { current: Window }) { - const router = useRouter(); - const pathname = usePathname(); - const searchParams = useSearchParams(); - const set = (w: Window) => { - const p = new URLSearchParams(searchParams.toString()); - p.set("window", w); - router.replace(`${pathname}?${p.toString()}`, { scroll: false }); - }; - return ( -
- {WINDOWS.map((w) => { - const active = w === current; - return ( - - ); - })} -
- ); -} diff --git a/evalboard/app/_overview/window-summary.tsx b/evalboard/app/_overview/window-summary.tsx index cd3f5629..c8413067 100644 --- a/evalboard/app/_overview/window-summary.tsx +++ b/evalboard/app/_overview/window-summary.tsx @@ -1,4 +1,5 @@ -import { fmtDuration, fmtUsd, passClass } from "@/lib/format"; +import { fmtDuration, fmtUsd } from "@/lib/format"; +import { passClass } from "@/lib/pass-rate"; import type { RunListingTotals } from "@/lib/overview"; import type { Window } from "@/lib/reviews-types"; @@ -33,10 +34,14 @@ function Tile({ } // Front-page window rollup: total spend + shape of the runs in scope. Every -// tile is summed over the same set — `runCount` is the matched-run count -// (`matchedCount`), which is exactly what `totals` folds in, so the Runs tile -// can never disagree with the Cost/Tasks/Pass/Compute tiles. The totals are -// scoped to matching tasks whenever a filter is active. +// tile is summed over the same set — `runCount` and `totals` both come from +// getWindowRollup, so the Runs tile can never disagree with the +// Cost/Tasks/Pass/Compute tiles. The totals are scoped to matching tasks +// whenever a filter is active. +// +// This describes the window the charts plot, NOT however far the run table below +// is paged out; the tiles carry the window in their sub-labels so the difference +// is legible. export function WindowSummary({ totals, window, diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index bf410cbd..2bffcb2e 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -3,13 +3,14 @@ import { getAdhocRunListing, getOverview, getRunListing, + getWindowRollup, listRecentHarnesses, type TagCount, } from "@/lib/overview"; import { parseHarnessScope } from "@/lib/harness"; -import { fmtDuration, fmtRunTime, fmtTimestamp, passClass } from "@/lib/format"; -import { WindowSelector } from "./_components/window-selector"; -import { WINDOWS, type Window } from "@/lib/reviews-types"; +import { fmtDuration, fmtRunTime, fmtTimestamp } from "@/lib/format"; +import { passClass } from "@/lib/pass-rate"; +import { type Window } from "@/lib/reviews-types"; import { DailySuccessChart } from "./_overview/daily-chart"; import { TurnBudgetChart } from "./_overview/turn-budget-chart"; import { WindowSummary } from "./_overview/window-summary"; @@ -25,10 +26,11 @@ export const dynamic = "force-dynamic"; const DEFAULT_LIMIT = 20; const ADHOC_LIMIT = 10; -function parseWindow(raw: string | string[] | undefined): Window { - const v = Array.isArray(raw) ? raw[0] : raw; - return WINDOWS.includes(v as Window) ? (v as Window) : "30d"; -} +// The charts and the summary tiles cover a fixed 30 days. There is no window +// control: the run table pages back through all of history on its own (see +// getRunListing), which is what a shorter window was really being used for, and +// a 30-day chart is the one that shows a trend rather than a few points. +const WINDOW: Window = "30d"; function parseTag(raw: string | string[] | undefined): string | null { const v = Array.isArray(raw) ? raw[0] : raw; @@ -50,9 +52,9 @@ function parseQ(raw: string | string[] | undefined): string | null { // Hard ceiling on how far the tables can be paged out. Both sections grow a // page at a time and never expose a "show all" jump: every extra row is another -// run.json read behind the window load, and an unbounded expansion on a busy -// window is the one interaction that can stall the page. A hand-typed -// `?limit=` above this clamps here rather than being honored. +// multi-MB run.json read, so an unbounded jump is the one interaction that can +// stall the page. Well above the store's run count, so in practice you can keep +// expanding to the oldest run; a hand-typed `?limit=` above this clamps here. const MAX_LIMIT = 500; function parsePagedLimit( @@ -106,7 +108,6 @@ export default async function Page({ searchParams, }: { searchParams: Promise<{ - window?: string; tag?: string; q?: string; h?: string; @@ -115,7 +116,6 @@ export default async function Page({ }>; }) { const params = await searchParams; - const window = parseWindow(params.window); const activeTag = parseTag(params.tag); const q = parseQ(params.q); // null = every harness, and that is the default: the page opens on the @@ -130,9 +130,10 @@ export default async function Page({ // filtered to the same set. Picking a harness therefore recomputes the // tiles and the charts and narrows the table, instead of re-scoping the // analytics block while the table silently kept showing every harness. - const [overview, listing, adhoc, harnesses] = await Promise.all([ - getOverview(window, activeTag, q, harness), - getRunListing(window, activeTag, q, limit, harness), + const [overview, rollup, listing, adhoc, harnesses] = await Promise.all([ + getOverview(WINDOW, activeTag, q, harness), + getWindowRollup(WINDOW, activeTag, q, harness), + getRunListing(activeTag, q, limit, harness), getAdhocRunListing(adhocLimit), listRecentHarnesses(), ]); @@ -142,11 +143,14 @@ export default async function Page({ const reviewTags = filterTagsByQuery(overview.reviewTags, q); const shownCount = listing.rows.length; - const matchedCount = listing.matchedCount; - const totalInWindow = listing.totalInWindow; - const tableTotalLabel = isFiltered ? matchedCount : totalInWindow; - // More rows exist AND the page cap still has room for them. - const hasMore = shownCount < Math.min(tableTotalLabel, MAX_LIMIT); + // Another page exists AND the page cap still has room for it. Under a + // filter there is no total to count against: proving how many older runs + // match would mean reading all of them, so the table reports what it has + // and offers another page while one is there. + const hasMore = listing.hasMore && shownCount < MAX_LIMIT; + const tableCountLabel = isFiltered + ? `${shownCount} matching` + : `${shownCount} of ${listing.totalCandidates}`; // Current URL params, normalized to scalars. Spread as the base of every // href so toggling one section (main `limit` or ad-hoc `alimit`) carries @@ -159,7 +163,6 @@ export default async function Page({ // scope through every self-link so it isn't reset by pagination/clear. const hParam = harness ?? undefined; const base = { - window, tag: activeTag, q, h: hParam, @@ -169,16 +172,13 @@ export default async function Page({ // Both tables grow one page at a time. There is deliberately no "show all" // link: each row is backed by a run.json read, so an unbounded jump is the - // one click that can hang the page on a busy window. - const nextPageSize = Math.min( - DEFAULT_LIMIT, - Math.min(tableTotalLabel, MAX_LIMIT) - shownCount, - ); + // one click that can hang the page. + const nextPageSize = Math.min(DEFAULT_LIMIT, MAX_LIMIT - shownCount); const showMoreHref = buildHref({ ...base, limit: shownCount + nextPageSize, }); - const clearAllHref = buildHref({ window, h: hParam }); + const clearAllHref = buildHref({ h: hParam }); // Ad-hoc section disclosure: rows are filtered (by `q`) then capped to // adhocLimit; page in another ADHOC_LIMIT while more match than are shown, @@ -206,14 +206,14 @@ export default async function Page({
{/* The analytics block — daily success / turn-budget charts, the - window selector, and the colored skill/review/tag rail — is an + harness selector, and the colored skill/review/tag rail — is an internal-only surface (see lib/edition.ts). The public OSS edition drops it so the front page is just the run list. */} {isInternal && ( @@ -244,12 +244,12 @@ export default async function Page({ )}{" "} - over the last {window} ·{" "} + over the last {WINDOW} ·{" "} {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} {" · "} @@ -262,20 +262,22 @@ export default async function Page({ {harness ? `${harnessShortLabel(harness)} run` : "run, one line per harness,"}{" "} - across the last {window} ·{" "} + across the last {WINDOW} ·{" "} {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} )}

-
+ {/* min-w-0 + flex-wrap: the two segmented controls are wider + than a phone together, so they stack rather than pushing + the whole page past the viewport. */} +
-
- {isFiltered - ? `${shownCount} of ${matchedCount} matching · ${totalInWindow} in ${window}` - : `${shownCount} of ${totalInWindow} in ${window}`} + {tableCountLabel} {isFiltered && ( - {shownCount} of {tableTotalLabel} + {tableCountLabel}
) : undefined @@ -402,7 +402,11 @@ export default async function Page({ key={r.id} className="border-b border-gray-100 last:border-b-0 hover:bg-gray-50 transition-colors" > - + {/* nowrap: the table already scrolls inside + its own container, so a narrow screen + should scroll it rather than break the + timestamp across three lines. */} + = 80) return "text-green-700"; - if (pct >= 50) return "text-gray-700"; - return "text-red-700"; -} +// Fixed 30-day window, matching the front page: no window control, since the +// readiness question is "where is this task now, and how did it get here" rather +// than "how did it look in a 1-day slice". +const WINDOW: Window = "30d"; export default async function PathToGaPage({ searchParams, }: { - searchParams: Promise<{ window?: string; h?: string }>; + searchParams: Promise<{ h?: string }>; }) { const params = await searchParams; - const window = parseWindow(params.window); - const harness = parseHarnessParam(params.h); + // null = every harness, and that is the default. The chart draws a line per + // harness, so the readiness of a task on each is comparable side by side + // rather than requiring four page loads. Picking a harness narrows the + // chart, the headline numbers, and the task table together. + const harness = parseHarnessScope(params.h); - // Scope to one harness — readiness of a task is per-harness, and a blended - // chart/pass-rate mixes incomparable regimes. const [overview, taskRows, harnesses] = await Promise.all([ - getOverview(window, TAG, null, harness), - getTagTaskBreakdown(window, TAG, harness), + getOverview(WINDOW, TAG, null, harness), + getTagTaskBreakdown(WINDOW, TAG, harness), listRecentHarnesses(), ]); @@ -63,13 +57,18 @@ export default async function PathToGaPage({

Score for every task tagged{" "} - {TAG} on{" "} - {harnessShortLabel(harness)}. + {TAG} + {harness + ? ` on ${harnessShortLabel(harness)}.` + : ", across every harness."}

-
- - +
+
@@ -82,7 +81,7 @@ export default async function PathToGaPage({ : "—"}
- avg pass rate over the last {window} + avg pass rate over the last {WINDOW}
@@ -112,13 +111,26 @@ export default async function PathToGaPage({ /> ) : (

- No {TAG} runs in the last {window}. + No {TAG} runs in the last {WINDOW}.

)}
-

Tasks

+
+

+ Tasks +

+ {/* Unscoped, a task's appearances span harnesses, so its rate + pools regimes that aren't strictly comparable. Say so + rather than let the number read as one harness's. */} + {!harness && ( + + pooled across harnesses · pick one above to separate + them + + )} +
@@ -193,7 +205,7 @@ export default async function PathToGaPage({ className="py-6 px-4 text-center text-sm text-gray-500" > No tasks tagged {TAG} in the last{" "} - {window}. + {WINDOW}. )} diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index 160e1303..0b213d89 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -5,6 +5,7 @@ import { useCallback, useMemo, useState } from "react"; import type { ActivationScore, TaskResultSummary } from "@/lib/runs"; import type { ReviewIndexEntry } from "@/lib/reviews-types"; import { fmtDuration, humanizeTaskId } from "@/lib/format"; +import { passBarClass, passClass } from "@/lib/pass-rate"; import { perTaskPassCounts, statusCategory } from "@/lib/status"; import { ChipLegend } from "@/app/_overview/tag-rail"; import { CollapsibleRail } from "@/app/_components/collapsible-rail"; @@ -407,7 +408,9 @@ export function RunView({ return ( <>
- + 0)}`} + > {pct.toFixed(0)}% @@ -424,9 +427,13 @@ export function RunView({ replicate runs
)} + {/* The meter was unconditionally green, so a 77% + run still read as healthy at a glance. It now + carries the same traffic-light cutoffs as the + number beside it. */}
0)}`} style={{ width: `${pct}%` }} />
diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 67654303..e7a2f103 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -8,6 +8,7 @@ import { } from "@/lib/trends"; import type { TagCount } from "@/lib/overview"; import { fmtRunTime, fmtDuration, humanizeTaskId } from "@/lib/format"; +import { passClassRatio } from "@/lib/pass-rate"; import { displayedTurns, fmtTurnsCount, @@ -45,10 +46,12 @@ function fmtCount(n: number | null): string { return n.toFixed(0); } +// Per-task rate across the recent runs, on the same traffic-light cutoffs as +// every other pass rate on the site (lib/pass-rate.ts). This used to be green +// only at a perfect 100%, which painted a task that passed 9 of its last 10 runs +// the same red as one that never passed. function passRateClass(rate: number, hasRuns: boolean): string { - if (!hasRuns) return "text-gray-500"; - if (rate >= 1.0) return "text-green-700"; - return "text-red-700"; + return passClassRatio(rate, hasRuns); } type SortKey = diff --git a/evalboard/app/watchlist/watchlist-view.tsx b/evalboard/app/watchlist/watchlist-view.tsx index 17b0c971..973d23c8 100644 --- a/evalboard/app/watchlist/watchlist-view.tsx +++ b/evalboard/app/watchlist/watchlist-view.tsx @@ -6,6 +6,7 @@ import type { ReactNode } from "react"; import Link from "next/link"; import { humanizeTaskId } from "@/lib/format"; +import { passBarClass, passClassRatio } from "@/lib/pass-rate"; import { HarnessSelector } from "@/app/_components/harness-selector"; import { harnessShortLabel } from "@/app/_components/harness-badge"; import { @@ -425,14 +426,14 @@ export function WatchlistView({ > {r.skill} - + {pct(r.passRate)} diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index eaa935d1..3fc6da01 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test, vi } from "vitest"; import { buildAdhocRows, collectPipelineRuns, + projectRunRow, summarizeListing, turnBudgetRateForTasks, type PerRun, @@ -474,3 +475,117 @@ describe("buildAdhocRows", () => { expect(row.tasksRun).toBe(2); }); }); + +// projectRunRow is the single definition of "does this run count, and with which +// tasks" — the summary tiles (getWindowRollup) and the paged run table +// (getRunListing) both go through it. They used to be one loop; if they ever +// disagreed, the tiles would describe a different set of runs than the table +// below them with nothing failing. +describe("projectRunRow", () => { + function run(id: string, tasks: RunOverviewTask[], extra?: Partial): PerRun { + return { + id, + overview: { + id, + tasks, + totalCostUsd: 10, + taskDurationSeconds: 100, + componentShas: [], + }, + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + title: null, + ...extra, + }; + } + + test("unfiltered, reports whole-run totals", () => { + const row = projectRunRow( + run("r", [ + task({ taskId: "a", status: "SUCCESS" }), + task({ taskId: "b", status: "FAILURE" }), + ]), + null, + null, + ); + expect(row).toMatchObject({ + id: "r", + tasksSucceeded: 1, + tasksRun: 2, + totalCostUsd: 10, + taskDurationSeconds: 100, + }); + }); + + test("drops a run with no task matching the tag", () => { + const row = projectRunRow( + run("r", [task({ taskId: "a", tags: ["smoke"] })]), + "path-to-ga", + null, + ); + expect(row).toBeNull(); + }); + + test("scopes cost and duration to the matching tasks", () => { + // The row's numbers have to describe the filtered slice, not the run — + // otherwise filtering to one tag still shows the whole run's spend. + const row = projectRunRow( + run("r", [ + task({ + taskId: "a", + tags: ["keep"], + totalCostUsd: 3, + durationSeconds: 30, + }), + task({ taskId: "b", totalCostUsd: 99, durationSeconds: 999 }), + ]), + "keep", + null, + ); + expect(row).toMatchObject({ + tasksRun: 1, + totalCostUsd: 3, + taskDurationSeconds: 30, + }); + }); + + test("nulls a scoped duration when any matching task lacks one", () => { + // A partial sum understates the slice, which reads as a fast run rather + // than an unmeasured one. + const row = projectRunRow( + run("r", [ + task({ taskId: "a", tags: ["keep"], durationSeconds: 30 }), + task({ taskId: "b", tags: ["keep"], durationSeconds: null }), + ]), + "keep", + null, + ); + expect(row?.taskDurationSeconds).toBeNull(); + }); + + test("a query matching only the run id keeps the whole-run slice", () => { + // Pinning a run by pasting a date fragment must not show 0 tasks just + // because no task name contains the date. + const row = projectRunRow( + run("2026-07-30_04-38-11", [task({ taskId: "a" })]), + null, + "07-30", + ); + expect(row).toMatchObject({ tasksRun: 1 }); + }); + + test("returns null for a run whose run.json could not be read", () => { + const r = run("r", []); + expect(projectRunRow({ ...r, overview: null }, null, null)).toBeNull(); + }); + + test("carries the harness through for the table's badge column", () => { + const r = run("r", [task({ taskId: "a" })]); + const withHarness = { + ...r, + overview: { ...r.overview!, harness: "codex" }, + }; + expect(projectRunRow(withHarness, null, null)?.harness).toBe("codex"); + }); +}); diff --git a/evalboard/lib/__tests__/pass-rate.test.ts b/evalboard/lib/__tests__/pass-rate.test.ts new file mode 100644 index 00000000..c8c0daab --- /dev/null +++ b/evalboard/lib/__tests__/pass-rate.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from "vitest"; +import { + PASS_GREEN_PCT, + PASS_RED_PCT, + passBarClass, + passClass, + passClassRatio, + passTone, +} from "../pass-rate"; + +// These cutoffs are shared with the nightly Slack rollup's traffic-light dots. +// A run that pings the channel red and then reads green on the page it links to +// is the failure this module exists to prevent, so the boundaries are pinned +// here rather than left to whatever each call site felt like. +describe("passTone", () => { + test("mirrors the Slack rollup's cutoffs", () => { + expect(PASS_GREEN_PCT).toBe(95); + expect(PASS_RED_PCT).toBe(85); + }); + + test("is inclusive at green and exclusive at red", () => { + expect(passTone(95)).toBe("good"); + expect(passTone(94.9)).toBe("warn"); + expect(passTone(85)).toBe("warn"); + expect(passTone(84.9)).toBe("bad"); + }); + + test("treats a measured 0% as bad, not as missing data", () => { + // Every task failed. A falsy check here would paint the worst possible + // run the same neutral gray as a run that hasn't started. + expect(passTone(0)).toBe("bad"); + expect(passClass(0)).toBe("text-red-700"); + }); + + test("distinguishes no data from a low rate", () => { + expect(passTone(null)).toBe("none"); + expect(passTone(undefined)).toBe("none"); + expect(passTone(50, false)).toBe("none"); + expect(passClass(null)).toBe("text-gray-500"); + }); + + test("does not paint a tone from a non-finite rate", () => { + // 0/0 in a caller's own division arrives here as NaN. + expect(passTone(Number.NaN)).toBe("none"); + }); + + test("clears 100% and anything above it", () => { + expect(passTone(100)).toBe("good"); + }); +}); + +describe("class helpers", () => { + test("bars are more saturated than text at the same tone", () => { + // Text at the 500 step is too light to read; a 2px-tall bar at the 700 + // step is too dark to distinguish from its track. + expect(passClass(99)).toBe("text-green-700"); + expect(passBarClass(99)).toBe("bg-green-500"); + expect(passBarClass(90)).toBe("bg-amber-500"); + expect(passBarClass(10)).toBe("bg-red-500"); + }); + + test("every tone maps to a distinct class", () => { + const classes = [passClass(99), passClass(90), passClass(10), passClass(null)]; + expect(new Set(classes).size).toBe(4); + }); + + test("the ratio variant scales 0-1 onto the same cutoffs", () => { + // trends/watchlist carry rates as fractions; feeding one in unscaled + // would put every task in the red band. + expect(passClassRatio(0.96)).toBe(passClass(96)); + expect(passClassRatio(0.9)).toBe(passClass(90)); + expect(passClassRatio(0.5)).toBe(passClass(50)); + expect(passClassRatio(1)).toBe("text-green-700"); + expect(passClassRatio(null)).toBe("text-gray-500"); + }); +}); diff --git a/evalboard/lib/format.ts b/evalboard/lib/format.ts index 833c5a96..d801c526 100644 --- a/evalboard/lib/format.ts +++ b/evalboard/lib/format.ts @@ -49,17 +49,6 @@ export function fmtUsd(n: number | null | undefined): string { return `$${n.toFixed(digits)}`; } -// Pass-rate → Tailwind text color. Shared by the front-page run table -// (page.tsx) and the window summary tile so the >=80 green / >=50 gray / else -// red thresholds live in one place. `hasTasks` distinguishes "0%" (red) from -// "no tasks yet" (neutral gray); pct is null exactly when hasTasks is false. -export function passClass(pct: number | null, hasTasks: boolean): string { - if (!hasTasks || pct == null) return "text-gray-500"; - if (pct >= 80) return "text-green-700"; - if (pct >= 50) return "text-gray-700"; - return "text-red-700"; -} - export function fmtDuration(s: number | null): string { if (s == null) return "—"; // Round once on the total to avoid `1m 60s` from rounding the remainder. diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 08d748bf..8ec074af 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -163,11 +163,21 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals { export interface RunListing { rows: RunListingRow[]; // after filter + limit, newest first - totalInWindow: number; - matchedCount: number; // post-filter, pre-limit - // Rollup across all matched runs (pre-limit) — powers the front-page - // window summary; independent of the `rows` limit. + // Every pipeline run in the store, not just the ones loaded for this page. + // Free to compute (it's the id count), so the table can say "20 of 94" + // without reading 94 run.json files. + totalCandidates: number; + // Another page exists behind `rows`. Derived from over-fetching by one + // rather than from a total match count: counting matches would mean loading + // every run in history on every render. + hasMore: boolean; +} + +// Rollup over a time window, independent of the table's pagination. The tiles +// read this so they describe the same set of runs the charts plot. +export interface WindowRollup { totals: RunListingTotals; + runCount: number; // matched pipeline runs in the window } const FETCH_CONCURRENCY = 16; @@ -698,102 +708,142 @@ export async function getTagTaskBreakdown( return rows.sort((a, b) => a.taskId.localeCompare(b.taskId)); } -export async function getRunListing( +// Project one loaded run into a table row under the active tag/q filter, or +// null when the run has nothing matching in it. Pure over an already-loaded +// PerRun, so both the lazy row scan (getRunListing) and the window rollup +// (getWindowRollup) decide "does this run count?" the same way — the two used to +// be one loop, and the tiles disagreeing with the table would be invisible. +// Exported for unit testing. +export function projectRunRow( + { id, overview, reviewTagsByTask }: PerRun, + tag: string | null, + needle: string | null, +): RunListingRow | null { + if (!overview) return null; + + // Default to the whole-run slice; narrow to matching tasks if a filter is + // active AND any task matches. When `q` matches only the run ID + // (date-fragment "pin a run" use case), we keep the whole-run slice so the + // row shows real totals rather than 0/—/—. + let scopedTasks = overview.tasks; + let scopedCost = overview.totalCostUsd; + let scopedDur = overview.taskDurationSeconds; + + if (tag != null || needle != null) { + const matching = overview.tasks.filter((t) => { + const passesTag = + tag == null || taskMatchesTag(t, reviewTagsByTask, tag); + const passesQ = + needle == null || taskMatchesQuery(t, reviewTagsByTask, needle); + return passesTag && passesQ; + }); + const idMatchesQ = needle != null && id.toLowerCase().includes(needle); + if (matching.length === 0 && !idMatchesQ) return null; + + if (matching.length > 0) { + // Scope cost/duration to matching tasks. Cost sums any task with a + // recorded value; duration is only meaningful when every matching + // task has a duration recorded (otherwise the partial sum would + // understate the slice — mirrors readRunOverview's whole-run rule). + let costSum = 0; + let costHasAny = false; + let durSum = 0; + let durAllPresent = true; + for (const t of matching) { + if (t.totalCostUsd != null) { + costSum += t.totalCostUsd; + costHasAny = true; + } + if (t.durationSeconds != null) { + durSum += t.durationSeconds; + } else { + durAllPresent = false; + } + } + scopedTasks = matching; + scopedCost = costHasAny ? costSum : null; + scopedDur = durAllPresent ? durSum : null; + } + } + + return { + id, + tasksSucceeded: scopedTasks.filter((t) => t.status === "SUCCESS").length, + tasksRun: scopedTasks.length, + totalCostUsd: scopedCost, + taskDurationSeconds: scopedDur, + harness: overview.harness ?? null, + }; +} + +// The tiles' rollup: every matched pipeline run in the window, whatever the +// table is currently paged to. Reads the same memoized window load the charts +// do, so the two always describe one set of runs. +export async function getWindowRollup( window: Window, tag: string | null, q: string | null, - limit: number | null, // null = unlimited - // When set, scope the listing (and therefore the window rollup that feeds - // the summary tiles) to one harness. null = all harnesses. Applied at the - // same seam as the chart's scope in getOverview, so the tiles, the charts, - // and the table always describe the same set of runs. harness: string | null = null, -): Promise { - // Exclude ad-hoc runs from the main listing — they appear in their own - // section (getAdhocRunListing). totalInWindow therefore counts only - // pipeline runs in scope, matching the chart above it. - const perRun = (await loadWindowData(window)).filter( - (r) => - !r.adhoc && - (harness == null || - normalizeHarness(r.overview?.harness) === harness), - ); - // Run IDs are timestamped — newest first by lexical compare. - const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); - const totalInWindow = sorted.length; - +): Promise { const needle = q?.trim().toLowerCase() || null; - const needsFilter = tag != null || needle != null; - - const matched: RunListingRow[] = []; - for (const { id, overview, reviewTagsByTask } of sorted) { - if (!overview) continue; + const rows: RunListingRow[] = []; + for (const r of await loadWindowData(window)) { + // Ad-hoc uploads aren't pipeline cadence — they have their own section + // and are excluded here exactly as they are from the charts. + if (r.adhoc) continue; + if (harness != null && normalizeHarness(r.overview?.harness) !== harness) + continue; + const row = projectRunRow(r, tag, needle); + if (row) rows.push(row); + } + return { totals: summarizeListing(rows), runCount: rows.length }; +} - // Default to the whole-run slice; narrow to matching tasks if a - // filter is active AND any task matches. When `q` matches only the - // run ID (date-fragment "pin a run" use case), we keep the whole-run - // slice so the row shows real totals rather than 0/—/—. - let scopedTasks = overview.tasks; - let scopedCost = overview.totalCostUsd; - let scopedDur = overview.taskDurationSeconds; - - if (needsFilter) { - const matching = overview.tasks.filter((t) => { - const passesTag = - tag == null || taskMatchesTag(t, reviewTagsByTask, tag); - const passesQ = - needle == null || - taskMatchesQuery(t, reviewTagsByTask, needle); - return passesTag && passesQ; - }); - const idMatchesQ = - needle != null && id.toLowerCase().includes(needle); - if (matching.length === 0 && !idMatchesQ) continue; - - if (matching.length > 0) { - // Scope cost/duration to matching tasks. Cost sums any task - // with a recorded value; duration is only meaningful when - // every matching task has a duration recorded (otherwise the - // partial sum would understate the slice — mirrors - // readRunOverview's whole-run rule). - let costSum = 0; - let costHasAny = false; - let durSum = 0; - let durAllPresent = true; - for (const t of matching) { - if (t.totalCostUsd != null) { - costSum += t.totalCostUsd; - costHasAny = true; - } - if (t.durationSeconds != null) { - durSum += t.durationSeconds; - } else { - durAllPresent = false; - } - } - scopedTasks = matching; - scopedCost = costHasAny ? costSum : null; - scopedDur = durAllPresent ? durSum : null; - } - } +// The run table. Unlike the rollup above, this is NOT date-bounded: it pages +// back through the whole store, a screenful at a time, so history older than the +// charts' window is still reachable without a time-window control. +// +// Rows are loaded lazily because a run.json is multi-MB — reading all of history +// to count matches would cost hundreds of MB of parsing per render. So the depth +// of the read is the depth of the page: `hasMore` comes from over-fetching a +// single row rather than from a total, and the only free count (every pipeline +// candidate in the store) is reported separately as `totalCandidates`. +export async function getRunListing( + tag: string | null, + q: string | null, + limit: number, + // When set, scope the listing to one harness — applied at the same seam as + // the chart's scope in getOverview, so the tiles, the charts, and the table + // always describe the same set of runs. null = all harnesses. + harness: string | null = null, +): Promise { + // Ad-hoc ids aren't date-shaped, so this drops them (they have their own + // section) without loading anything. Newest-first: ids are timestamps. + const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) != null); + const needle = q?.trim().toLowerCase() || null; + const hasFilter = tag != null || needle != null || harness != null; + const isMatch = hasFilter + ? (r: PerRun) => + (harness == null || + normalizeHarness(r.overview?.harness) === harness) && + projectRunRow(r, tag, needle) != null + : undefined; - matched.push({ - id, - tasksSucceeded: scopedTasks.filter((t) => t.status === "SUCCESS") - .length, - tasksRun: scopedTasks.length, - totalCostUsd: scopedCost, - taskDurationSeconds: scopedDur, - harness: overview.harness ?? null, - }); - } + // Over-fetch by one: if the extra row materializes, another page exists. + const loaded = await collectPipelineRuns( + ids, + limit + 1, + cachedLoadPerRun, + isMatch, + ); + const rows = loaded + .map((r) => projectRunRow(r, tag, needle)) + .filter((row): row is RunListingRow => row != null); - const rows = limit == null ? matched : matched.slice(0, limit); return { - rows, - totalInWindow, - matchedCount: matched.length, - totals: summarizeListing(matched), + rows: rows.slice(0, limit), + totalCandidates: ids.length, + hasMore: rows.length > limit, }; } diff --git a/evalboard/lib/pass-rate.ts b/evalboard/lib/pass-rate.ts new file mode 100644 index 00000000..719d99cd --- /dev/null +++ b/evalboard/lib/pass-rate.ts @@ -0,0 +1,53 @@ +// Traffic-light thresholds for a pass rate. +// +// These mirror the nightly Slack rollup's dots +// (coder_eval_uipath/eval_runner/scripts/ci/slack_summary.py, GREEN_PCT / +// RED_PCT) so a number that reads green in the channel reads green here. The +// channel is where a regression is first noticed and the evalboard is where it +// gets opened, so the two disagreeing about which numbers are healthy is worse +// than either bar being slightly off. Change these only alongside that script. +export const PASS_GREEN_PCT = 95; +export const PASS_RED_PCT = 85; + +// "none" is the absence of a measurement (no tasks ran), which is not the same +// as a measured 0% — that one is genuinely bad and must read red. +export type PassTone = "none" | "good" | "warn" | "bad"; + +export function passTone(pct: number | null | undefined, hasData = true): PassTone { + if (!hasData || pct == null || !Number.isFinite(pct)) return "none"; + if (pct >= PASS_GREEN_PCT) return "good"; + if (pct < PASS_RED_PCT) return "bad"; + return "warn"; +} + +const TEXT_CLASS: Record = { + none: "text-gray-500", + good: "text-green-700", + warn: "text-amber-700", + bad: "text-red-700", +}; + +// Fills (progress bars, meters) need more saturation than text to read at all, +// so they take the 500 step rather than 700. +const BAR_CLASS: Record = { + none: "bg-gray-300", + good: "bg-green-500", + warn: "bg-amber-500", + bad: "bg-red-500", +}; + +// Pass rate as a percent (0-100) -> Tailwind text color. `hasData` distinguishes +// "no tasks yet" (neutral) from a real 0%. +export function passClass(pct: number | null | undefined, hasData = true): string { + return TEXT_CLASS[passTone(pct, hasData)]; +} + +export function passBarClass(pct: number | null | undefined, hasData = true): string { + return BAR_CLASS[passTone(pct, hasData)]; +} + +// Same, for the tables that carry a rate as a 0-1 fraction (trends, watchlist) +// rather than a percent. +export function passClassRatio(rate: number | null | undefined, hasData = true): string { + return passClass(rate == null ? null : rate * 100, hasData); +} From 404fccdfd759c0df041f17e3d2b33669905fd77e Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 30 Jul 2026 15:43:55 -0700 Subject: [PATCH 5/5] fix(evalboard): honest scoped counts, and one definition of a run's scope Self-review turned up three places where the page could state something untrue, plus dead code with comments justifying it. Scoped counts were wrong. `isFiltered` only looked at tag/q, so selecting a harness left the run table reading "20 of 122" where 122 counted every harness, and left the tiles labelled as if nothing were narrowed. Split into `isFiltered` (tasks within a run) and `isNarrowed` (the run set, harness included), and every sub-label now names the window even when scoped so a narrowed view can't read as all-time. The charts and the tiles could describe different runs. getOverview and getWindowRollup each had their own copy of "does this run count, over which tasks", and they disagreed on a `q` that matched only a run id. Extracted `scopeRunTasks` as the single seam and folded the rollup into getOverview's existing pass, so the two are now the same loop rather than two loops that agree by convention. Drops getWindowRollup, WindowRollup, and a Promise.all slot. Pagination could strand history. `hasMore` came from over-fetching one row, but collectPipelineRuns gives up at a multiple of the limit, so once the store passes ~170 runs a filtered page one would report "no more" and hide every older match behind a link that vanished. The listing now probes every candidate. Also: the coder_eval chip no longer links pre-0.8.2 runs at a release tag that was never cut; chart dataKeys are index-prefixed so two harness ids that sanitize alike can't collide onto one line; tag chips stop emitting a `?window=` nobody reads; and the turn-budget caption says that runs with no budgeted task are omitted rather than plotted at 0. Removed as filler: a dead ALL_HARNESSES re-export, the redundant `hasData` parameter on all four pass-rate helpers (null already says it), a one-line forwarder in trends, two one-line limit-parser wrappers, and a second copy of the harness ordering logic. Co-Authored-By: Claude Opus 5 (1M context) --- .../app/_components/harness-selector.tsx | 5 - .../__tests__/harness-legend.test.tsx | 4 +- .../__tests__/harness-series.test.ts | 34 +-- evalboard/app/_overview/harness-series.ts | 24 +- evalboard/app/_overview/tag-rail.tsx | 13 +- evalboard/app/_overview/window-summary.tsx | 20 +- evalboard/app/page.tsx | 60 ++--- evalboard/app/runs/[id]/run-view.tsx | 7 +- evalboard/app/trends/trends-view.tsx | 14 +- evalboard/lib/__tests__/overview.test.ts | 51 ++++ evalboard/lib/__tests__/pass-rate.test.ts | 1 - evalboard/lib/overview.ts | 238 +++++++++--------- evalboard/lib/pass-rate.ts | 25 +- evalboard/lib/runs.ts | 9 +- 14 files changed, 266 insertions(+), 239 deletions(-) diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index adfc28b1..9e7acc0a 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -1,7 +1,6 @@ "use client"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; -import { ALL_HARNESSES } from "@/lib/harness"; import { HarnessBadge, harnessShortLabel } from "./harness-badge"; // Segmented control for a page's harness scope. Sets `?h=` while @@ -100,7 +99,3 @@ export function HarnessSelector({
); } - -// Re-exported so a caller can build an `?h=all` link without importing the leaf -// constants module directly. -export { ALL_HARNESSES }; diff --git a/evalboard/app/_overview/__tests__/harness-legend.test.tsx b/evalboard/app/_overview/__tests__/harness-legend.test.tsx index 361d1291..f55698f4 100644 --- a/evalboard/app/_overview/__tests__/harness-legend.test.tsx +++ b/evalboard/app/_overview/__tests__/harness-legend.test.tsx @@ -5,9 +5,9 @@ import { HarnessLegend, HarnessTooltip } from "../harness-legend"; import { seriesKey } from "../harness-series"; function series(...harnesses: string[]) { - return harnesses.map((harness) => ({ + return harnesses.map((harness, i) => ({ harness, - dataKey: seriesKey(harness), + dataKey: seriesKey(harness, i), color: harnessColor(harness), })); } diff --git a/evalboard/app/_overview/__tests__/harness-series.test.ts b/evalboard/app/_overview/__tests__/harness-series.test.ts index 2930d6ac..399bbd02 100644 --- a/evalboard/app/_overview/__tests__/harness-series.test.ts +++ b/evalboard/app/_overview/__tests__/harness-series.test.ts @@ -27,13 +27,16 @@ describe("seriesKey", () => { test("prefixes and sanitizes so recharts can't read it as a path", () => { // A dataKey containing "." is a nested-object path in recharts, which // would resolve to undefined and silently draw no line. - expect(seriesKey("gpt-5.5")).not.toContain("."); - expect(seriesKey("claude-code")).toBe("h_claude_code"); + expect(seriesKey("gpt-5.5", 0)).not.toContain("."); + expect(seriesKey("claude-code", 0)).toBe("h0_claude_code"); }); - test("is injective across the known harness ids", () => { - const ids = ["claude-code", "codex", "antigravity", "delegate-sdk"]; - expect(new Set(ids.map(seriesKey)).size).toBe(ids.length); + test("is injective even when two ids sanitize alike", () => { + // Without the index prefix "a.b" and "a-b" both become "h_a_b", and one + // harness's line would silently take over the other's column. + const ids = ["claude-code", "codex", "gpt-5.5", "gpt-5-5"]; + const keys = ids.map((id, i) => seriesKey(id, i)); + expect(new Set(keys).size).toBe(ids.length); }); }); @@ -45,13 +48,14 @@ describe("pivotByHarness", () => { "successRate", ); expect(series.map((s) => s.harness)).toEqual(["claude-code", "codex"]); + const [cc, cx] = series.map((s) => s.dataKey); expect(rows).toEqual([ - { timestamp: 100, h_claude_code: 90 }, - { timestamp: 200, h_codex: 70 }, + { timestamp: 100, [cc]: 90 }, + { timestamp: 200, [cx]: 70 }, ]); // The absent key on each row is what connectNulls bridges — that is how // interleaved runs become one continuous line per harness. - expect(rows[0]).not.toHaveProperty("h_codex"); + expect(rows[0]).not.toHaveProperty(cx); }); test("colors come from the harness, not the series index", () => { @@ -67,14 +71,13 @@ describe("pivotByHarness", () => { }); test("merges two harnesses that share a timestamp into one row", () => { - const { rows } = pivotByHarness( + const { rows, series } = pivotByHarness( [point("claude-code", 100, 90), point("codex", 100, 70)], ["claude-code", "codex"], "successRate", ); - expect(rows).toEqual([ - { timestamp: 100, h_claude_code: 90, h_codex: 70 }, - ]); + const [cc, cx] = series.map((s) => s.dataKey); + expect(rows).toEqual([{ timestamp: 100, [cc]: 90, [cx]: 70 }]); }); test("sorts rows by timestamp regardless of input order", () => { @@ -91,12 +94,13 @@ describe("pivotByHarness", () => { // by the metric key — reading the wrong one would put the success rate // on the turn-budget chart with no visible error. const pts = [point("codex", 100, 90, 40)]; + const key = seriesKey("codex", 0); expect(pivotByHarness(pts, ["codex"], "successRate").rows[0]).toEqual({ timestamp: 100, - h_codex: 90, + [key]: 90, }); expect(pivotByHarness(pts, ["codex"], "turnBudgetRate").rows[0]).toEqual( - { timestamp: 100, h_codex: 40 }, + { timestamp: 100, [key]: 40 }, ); }); @@ -132,7 +136,7 @@ describe("pivotByHarness", () => { "successRate", ); expect(series).toHaveLength(1); - expect(rows[0].h_codex).toBe(0); + expect(rows[0][series[0].dataKey]).toBe(0); }); test("handles an empty window", () => { diff --git a/evalboard/app/_overview/harness-series.ts b/evalboard/app/_overview/harness-series.ts index 369cbcd7..1b50f111 100644 --- a/evalboard/app/_overview/harness-series.ts +++ b/evalboard/app/_overview/harness-series.ts @@ -16,9 +16,7 @@ export type HarnessMetric = "successRate" | "turnBudgetRate"; export interface HarnessSeries { harness: string; - // Column name in the pivoted rows. Prefixed and sanitized because recharts - // reads a dataKey containing "." as a nested object path, which would - // silently resolve to undefined for a harness id with a dot in it. + // Column name in the pivoted rows. See seriesKey. dataKey: string; color: string; } @@ -28,8 +26,13 @@ export interface HarnessChartData { series: HarnessSeries[]; } -export function seriesKey(harness: string): string { - return `h_${harness.replace(/[^\w]/g, "_")}`; +// Sanitized because recharts reads a dataKey containing "." as a nested object +// path, which would resolve to undefined and silently draw no line. Prefixed +// with the series index because sanitizing alone is not injective — `a.b` and +// `a-b` would collide onto one column and one harness's data would vanish +// under the other's. +export function seriesKey(harness: string, index: number): string { + return `h${index}_${harness.replace(/[^\w]/g, "_")}`; } export function pivotByHarness( @@ -37,11 +40,12 @@ export function pivotByHarness( harnesses: string[], metric: HarnessMetric, ): HarnessChartData { - const series: HarnessSeries[] = harnesses.map((harness) => ({ + const series: HarnessSeries[] = harnesses.map((harness, i) => ({ harness, - dataKey: seriesKey(harness), + dataKey: seriesKey(harness, i), color: harnessColor(harness), })); + const keyByHarness = new Map(series.map((s) => [s.harness, s.dataKey])); // Merge on timestamp so two harnesses that happen to start a run in the same // second share one row instead of producing two points recharts would render @@ -49,12 +53,16 @@ export function pivotByHarness( const byTimestamp = new Map>(); for (const p of points) { if (p[metric] == null) continue; + // A point whose harness isn't in the requested series list has no column + // to land in; dropping it keeps a stale id out of the chart. + const key = keyByHarness.get(p.harness); + if (!key) continue; let row = byTimestamp.get(p.timestamp); if (!row) { row = { timestamp: p.timestamp }; byTimestamp.set(p.timestamp, row); } - row[seriesKey(p.harness)] = p[metric] as number; + row[key] = p[metric] as number; } const rows = [...byTimestamp.values()].sort( diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx index b3674418..97b0ef09 100644 --- a/evalboard/app/_overview/tag-rail.tsx +++ b/evalboard/app/_overview/tag-rail.tsx @@ -1,6 +1,5 @@ import Link from "next/link"; import type { TagCount } from "@/lib/overview"; -import type { Window } from "@/lib/reviews-types"; type Variant = "neutral" | "rose" | "indigo"; @@ -33,12 +32,10 @@ const STYLES: Record< function hrefForTag( basePath: string, tag: string | null, - window: Window | null, q: string | null, harness: string | null, ): string { const params = new URLSearchParams(); - if (window) params.set("window", window); if (tag) params.set("tag", tag); if (q) params.set("q", q); // Preserve the active harness scope across tag clicks. null is the @@ -56,7 +53,6 @@ function TagChip({ variant, active, basePath, - window, q, harness, }: { @@ -65,14 +61,13 @@ function TagChip({ variant: Variant; active: boolean; basePath: string; - window: Window | null; q: string | null; harness: string | null; }) { const s = STYLES[variant]; return ( @@ -117,7 +112,6 @@ export function MergedTagRail({ reviewTags, activeTag, basePath = "/", - window = null, q = null, harness = null, limit = 24, @@ -129,8 +123,6 @@ export function MergedTagRail({ // Path the chip links point at — "/" for the overview, "/trends" for the // trends page. Query string is preserved. basePath?: string; - // Null on pages that don't expose a window selector (e.g. /trends). - window?: Window | null; q?: string | null; // Active harness scope to preserve in chip links (null = not harness-scoped). harness?: string | null; @@ -155,7 +147,6 @@ export function MergedTagRail({ variant="indigo" active={tc.tag === activeTag} basePath={basePath} - window={window} q={q} harness={harness} /> @@ -168,7 +159,6 @@ export function MergedTagRail({ variant="rose" active={tc.tag === activeTag} basePath={basePath} - window={window} q={q} harness={harness} /> @@ -181,7 +171,6 @@ export function MergedTagRail({ variant="neutral" active={tc.tag === activeTag} basePath={basePath} - window={window} q={q} harness={harness} /> diff --git a/evalboard/app/_overview/window-summary.tsx b/evalboard/app/_overview/window-summary.tsx index c8413067..29c06245 100644 --- a/evalboard/app/_overview/window-summary.tsx +++ b/evalboard/app/_overview/window-summary.tsx @@ -34,14 +34,14 @@ function Tile({ } // Front-page window rollup: total spend + shape of the runs in scope. Every -// tile is summed over the same set — `runCount` and `totals` both come from -// getWindowRollup, so the Runs tile can never disagree with the -// Cost/Tasks/Pass/Compute tiles. The totals are scoped to matching tasks -// whenever a filter is active. +// tile is summed over the same set — `runCount` and `totals` both come out of +// getOverview's single pass, so the Runs tile can never disagree with the +// Cost/Tasks/Pass/Compute tiles or with the charts. The totals are scoped to +// matching tasks whenever a filter is active. // // This describes the window the charts plot, NOT however far the run table below -// is paged out; the tiles carry the window in their sub-labels so the difference -// is legible. +// is paged out (the table reads back past this window), so every sub-label names +// the window even when scoped — otherwise a narrowed view reads as all-time. export function WindowSummary({ totals, window, @@ -57,7 +57,9 @@ export function WindowSummary({ totals.tasksRun > 0 ? (totals.tasksSucceeded / totals.tasksRun) * 100 : null; - const scope = isFiltered ? "matching runs" : `runs · last ${window}`; + const scope = isFiltered + ? `matching · last ${window}` + : `runs · last ${window}`; return (
@@ -69,7 +71,7 @@ export function WindowSummary({ totals.costPartial ? "some runs missing cost" : isFiltered - ? "matching tasks" + ? `matching tasks · ${window}` : `across ${window}` } /> @@ -86,7 +88,7 @@ export function WindowSummary({ 0)} + valueClass={passClass(pct)} /> {/* The analytics block — daily success / turn-budget charts, the @@ -293,7 +285,8 @@ export default async function Page({

% of budgeted tasks that stayed within 1.5× their expected turns (a budgeted task that failed counts as - over budget) + over budget) · runs with no budgeted task are omitted + rather than plotted at 0 {activeTag || q ? " · scoped to the active filter" : ""}

0, - )}`} + className={`font-medium ${passClass(pct)}`} > {pct != null ? `${pct.toFixed(0)}%` @@ -569,10 +558,7 @@ export default async function Page({
{ warn.mockRestore(); }); + test("probeAll reaches a match sitting past the scan cap", async () => { + // The paged run table derives "Show more" from finding one extra row, so + // a scan that gave up at the cap would report "no more" and strand every + // older matching run. Only r1 matches, 30 ids deep — well past + // limit x HARNESS_SCAN_FACTOR. + const ids = Array.from({ length: 30 }, (_, i) => `r${30 - i}`); + const load = vi.fn(async (id: string) => run(id)); + const isMatch = (r: PerRun) => r.id === "r1"; + + expect(await collectPipelineRuns(ids, 1, load, isMatch)).toEqual([]); + const out = await collectPipelineRuns(ids, 1, load, isMatch, true); + expect(out.map((r) => r.id)).toEqual(["r1"]); + }); + test("limit=0 and empty ids short-circuit without loading", async () => { const load = vi.fn(async (id: string) => run(id)); expect(await collectPipelineRuns(["r1"], 0, load)).toEqual([]); @@ -588,4 +603,40 @@ describe("projectRunRow", () => { }; expect(projectRunRow(withHarness, null, null)?.harness).toBe("codex"); }); + + // getOverview reads the task LIST out of the same call (for the turn-budget + // rate), while the table only ever sees the counts. If the two disagreed + // about which tasks are in scope, the charts and the tiles would describe + // different runs with nothing on the page to show it. + describe("scopeRunTasks, the seam both share", () => { + test("hands back the matching tasks themselves, not just a count", () => { + const scoped = scopeRunTasks( + run("r", [ + task({ taskId: "keep-me", tags: ["keep"] }), + task({ taskId: "drop-me" }), + ]), + "keep", + null, + ); + expect(scoped?.tasks.map((t) => t.taskId)).toEqual(["keep-me"]); + }); + + test("agrees with projectRunRow on which runs are in scope", () => { + const r = run("r", [task({ taskId: "a", tags: ["other"] })]); + expect(scopeRunTasks(r, "keep", null)).toBeNull(); + expect(projectRunRow(r, "keep", null)).toBeNull(); + }); + + test("the run-id fallback widens back to every task", () => { + const scoped = scopeRunTasks( + run("2026-07-30_04-38-11", [ + task({ taskId: "a" }), + task({ taskId: "b" }), + ]), + null, + "07-30", + ); + expect(scoped?.tasks).toHaveLength(2); + }); + }); }); diff --git a/evalboard/lib/__tests__/pass-rate.test.ts b/evalboard/lib/__tests__/pass-rate.test.ts index c8c0daab..5d50a696 100644 --- a/evalboard/lib/__tests__/pass-rate.test.ts +++ b/evalboard/lib/__tests__/pass-rate.test.ts @@ -35,7 +35,6 @@ describe("passTone", () => { test("distinguishes no data from a low rate", () => { expect(passTone(null)).toBe("none"); expect(passTone(undefined)).toBe("none"); - expect(passTone(50, false)).toBe("none"); expect(passClass(null)).toBe("text-gray-500"); }); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 8ec074af..e02215d6 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,12 +15,7 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; -import { - DEFAULT_HARNESS, - KNOWN_HARNESSES, - normalizeHarness, - orderHarnesses, -} from "./harness"; +import { DEFAULT_HARNESS, normalizeHarness, orderHarnesses } from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { @@ -95,6 +90,11 @@ export interface OverviewData { harnesses: string[]; windowStart: number; // ms — chart x-domain start windowEnd: number; // ms — chart x-domain end + // The summary tiles' rollup, folded over the same runs `runs` plots so the + // two can't disagree. Independent of the run table's pagination, which reads + // back past this window (see getRunListing). + totals: RunListingTotals; + runCount: number; // matched pipeline runs in the window skills: TagCount[]; taskTags: TagCount[]; reviewTags: TagCount[]; @@ -173,13 +173,6 @@ export interface RunListing { hasMore: boolean; } -// Rollup over a time window, independent of the table's pagination. The tiles -// read this so they describe the same set of runs the charts plot. -export interface WindowRollup { - totals: RunListingTotals; - runCount: number; // matched pipeline runs in the window -} - const FETCH_CONCURRENCY = 16; const WINDOW_DAYS: Record = { @@ -317,13 +310,7 @@ async function listRecentHarnessesInner(): Promise { for (const r of perRun) { if (r.overview) seen.add(normalizeHarness(r.overview.harness)); } - // Known harnesses first (stable display order), then any newcomers - // (alphabetical) so the list is deterministic but self-extending. - const known = KNOWN_HARNESSES.filter((h) => seen.has(h)); - const extras = [...seen] - .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h)) - .sort(); - const ordered = [...known, ...extras]; + const ordered = orderHarnesses(seen); // Never hand back an empty list — the default must always be selectable. return ordered.length > 0 ? ordered : [DEFAULT_HARNESS]; } @@ -380,9 +367,16 @@ export async function collectPipelineRuns( // Optional extra predicate (AND-ed with usability) — e.g. a harness filter. // When set, the scan reaches further back since matches are sparser. isMatch?: (r: PerRun) => boolean, + // Probe every candidate instead of stopping at a multiple of `limit`. A + // capped scan can't tell "no older match exists" from "stopped looking", so + // the paged run table — whose "Show more" link comes from finding one extra + // row — has to probe fully or it silently hides older matching runs. + probeAll = false, ): Promise { const scanFactor = isMatch ? HARNESS_SCAN_FACTOR : RECENT_SCAN_FACTOR; - const maxScan = Math.min(ids.length, limit * scanFactor); + const maxScan = probeAll + ? ids.length + : Math.min(ids.length, limit * scanFactor); const usable = isMatch ? (r: PerRun) => isUsablePipelineRun(r) && isMatch(r) : isUsablePipelineRun; @@ -572,44 +566,34 @@ export async function getOverview( ); const needle = q?.trim().toLowerCase() || null; - // ---- Per-run chart points ---- - // One point per run plotted at its own timestamp, no daily averaging. - // When tag or q is active, scope each run's rate to only matching tasks. + // ---- Per-run chart points + the window rollup ---- + // One chart point per run plotted at its own timestamp, no daily averaging, + // and one rollup row per run for the summary tiles. Both come out of the + // same scopeRunTasks call in the same loop, so the tiles can never describe + // a different set of runs than the charts plot. const runPoints: RunPoint[] = []; - + const rows: RunListingRow[] = []; const seenHarnesses = new Set(); - for (const { id, overview, reviewTagsByTask } of perRun) { + for (const r of perRun) { + const { id, overview } = r; if (!overview || overview.tasks.length === 0) continue; const date = parseRunIdDate(id); if (!date) continue; + const scoped = scopeRunTasks(r, tag, needle); + if (!scoped || scoped.tasks.length === 0) continue; - let matching = overview.tasks; - if (tag) { - matching = matching.filter((t) => - taskMatchesTag(t, reviewTagsByTask, tag), - ); - } - if (needle) { - matching = matching.filter((t) => - taskMatchesQuery(t, reviewTagsByTask, needle), - ); - } - if (matching.length === 0) continue; + const row = rowFromScoped(id, scoped, overview.harness); + rows.push(row); - const succeeded = matching.filter( - (t) => t.status === "SUCCESS", - ).length; - const rate = (succeeded / matching.length) * 100; const runHarness = normalizeHarness(overview.harness); seenHarnesses.add(runHarness); - runPoints.push({ runId: id, timestamp: date.getTime(), harness: runHarness, - successRate: rate, - turnBudgetRate: turnBudgetRateForTasks(matching), + successRate: (row.tasksSucceeded / row.tasksRun) * 100, + turnBudgetRate: turnBudgetRateForTasks(scoped.tasks), }); } runPoints.sort((a, b) => a.timestamp - b.timestamp); @@ -628,6 +612,8 @@ export async function getOverview( harnesses: orderHarnesses(seenHarnesses), windowStart, windowEnd, + totals: summarizeListing(rows), + runCount: rows.length, skills, taskTags, reviewTags, @@ -708,100 +694,103 @@ export async function getTagTaskBreakdown( return rows.sort((a, b) => a.taskId.localeCompare(b.taskId)); } -// Project one loaded run into a table row under the active tag/q filter, or -// null when the run has nothing matching in it. Pure over an already-loaded -// PerRun, so both the lazy row scan (getRunListing) and the window rollup -// (getWindowRollup) decide "does this run count?" the same way — the two used to -// be one loop, and the tiles disagreeing with the table would be invisible. -// Exported for unit testing. -export function projectRunRow( +// The slice of a run that the active tag/q filter selects: which tasks count, +// and the cost/duration summed over exactly those. null means the run has +// nothing matching and drops out entirely. +// +// This is the ONE definition of "does this run count, and over which tasks", +// shared by the chart points, the summary tiles, and the run table. Each used to +// carry its own copy, and a disagreement between them would have been invisible +// on the page. +export interface ScopedRun { + tasks: RunOverviewTask[]; + totalCostUsd: number | null; + taskDurationSeconds: number | null; +} + +export function scopeRunTasks( { id, overview, reviewTagsByTask }: PerRun, tag: string | null, needle: string | null, -): RunListingRow | null { +): ScopedRun | null { if (!overview) return null; - - // Default to the whole-run slice; narrow to matching tasks if a filter is - // active AND any task matches. When `q` matches only the run ID - // (date-fragment "pin a run" use case), we keep the whole-run slice so the - // row shows real totals rather than 0/—/—. - let scopedTasks = overview.tasks; - let scopedCost = overview.totalCostUsd; - let scopedDur = overview.taskDurationSeconds; - - if (tag != null || needle != null) { - const matching = overview.tasks.filter((t) => { - const passesTag = - tag == null || taskMatchesTag(t, reviewTagsByTask, tag); - const passesQ = - needle == null || taskMatchesQuery(t, reviewTagsByTask, needle); - return passesTag && passesQ; - }); + const wholeRun: ScopedRun = { + tasks: overview.tasks, + totalCostUsd: overview.totalCostUsd, + taskDurationSeconds: overview.taskDurationSeconds, + }; + if (tag == null && needle == null) return wholeRun; + + const matching = overview.tasks.filter((t) => { + const passesTag = + tag == null || taskMatchesTag(t, reviewTagsByTask, tag); + const passesQ = + needle == null || taskMatchesQuery(t, reviewTagsByTask, needle); + return passesTag && passesQ; + }); + if (matching.length === 0) { + // A `q` that matches only the run ID is the date-fragment "pin a run" + // case: keep the whole-run slice so the row shows real totals rather + // than 0/—/—. Anything else has nothing to show. const idMatchesQ = needle != null && id.toLowerCase().includes(needle); - if (matching.length === 0 && !idMatchesQ) return null; - - if (matching.length > 0) { - // Scope cost/duration to matching tasks. Cost sums any task with a - // recorded value; duration is only meaningful when every matching - // task has a duration recorded (otherwise the partial sum would - // understate the slice — mirrors readRunOverview's whole-run rule). - let costSum = 0; - let costHasAny = false; - let durSum = 0; - let durAllPresent = true; - for (const t of matching) { - if (t.totalCostUsd != null) { - costSum += t.totalCostUsd; - costHasAny = true; - } - if (t.durationSeconds != null) { - durSum += t.durationSeconds; - } else { - durAllPresent = false; - } - } - scopedTasks = matching; - scopedCost = costHasAny ? costSum : null; - scopedDur = durAllPresent ? durSum : null; + return idMatchesQ ? wholeRun : null; + } + + // Cost sums any matching task that recorded one; duration is only meaningful + // when every matching task has one (otherwise the partial sum would + // understate the slice — mirrors readRunOverview's whole-run rule). + let costSum = 0; + let costHasAny = false; + let durSum = 0; + let durAllPresent = true; + for (const t of matching) { + if (t.totalCostUsd != null) { + costSum += t.totalCostUsd; + costHasAny = true; + } + if (t.durationSeconds != null) { + durSum += t.durationSeconds; + } else { + durAllPresent = false; } } + return { + tasks: matching, + totalCostUsd: costHasAny ? costSum : null, + taskDurationSeconds: durAllPresent ? durSum : null, + }; +} +function rowFromScoped( + id: string, + scoped: ScopedRun, + harness: string | null | undefined, +): RunListingRow { return { id, - tasksSucceeded: scopedTasks.filter((t) => t.status === "SUCCESS").length, - tasksRun: scopedTasks.length, - totalCostUsd: scopedCost, - taskDurationSeconds: scopedDur, - harness: overview.harness ?? null, + tasksSucceeded: scoped.tasks.filter((t) => t.status === "SUCCESS") + .length, + tasksRun: scoped.tasks.length, + totalCostUsd: scoped.totalCostUsd, + taskDurationSeconds: scoped.taskDurationSeconds, + harness: harness ?? null, }; } -// The tiles' rollup: every matched pipeline run in the window, whatever the -// table is currently paged to. Reads the same memoized window load the charts -// do, so the two always describe one set of runs. -export async function getWindowRollup( - window: Window, +// One loaded run as a table row, or null when the filter excludes it. Exported +// for unit testing. +export function projectRunRow( + run: PerRun, tag: string | null, - q: string | null, - harness: string | null = null, -): Promise { - const needle = q?.trim().toLowerCase() || null; - const rows: RunListingRow[] = []; - for (const r of await loadWindowData(window)) { - // Ad-hoc uploads aren't pipeline cadence — they have their own section - // and are excluded here exactly as they are from the charts. - if (r.adhoc) continue; - if (harness != null && normalizeHarness(r.overview?.harness) !== harness) - continue; - const row = projectRunRow(r, tag, needle); - if (row) rows.push(row); - } - return { totals: summarizeListing(rows), runCount: rows.length }; + needle: string | null, +): RunListingRow | null { + const scoped = scopeRunTasks(run, tag, needle); + return scoped ? rowFromScoped(run.id, scoped, run.overview?.harness) : null; } -// The run table. Unlike the rollup above, this is NOT date-bounded: it pages -// back through the whole store, a screenful at a time, so history older than the -// charts' window is still reachable without a time-window control. +// The run table. Unlike getOverview's window rollup, this is NOT date-bounded: +// it pages back through the whole store, a screenful at a time, so history older +// than the charts' window is still reachable without a time-window control. // // Rows are loaded lazily because a run.json is multi-MB — reading all of history // to count matches would cost hundreds of MB of parsing per render. So the depth @@ -830,11 +819,14 @@ export async function getRunListing( : undefined; // Over-fetch by one: if the extra row materializes, another page exists. + // probeAll, because a scan that gave up at a cap would report `hasMore: + // false` and strand every older matching run behind a link that vanished. const loaded = await collectPipelineRuns( ids, limit + 1, cachedLoadPerRun, isMatch, + true, ); const rows = loaded .map((r) => projectRunRow(r, tag, needle)) diff --git a/evalboard/lib/pass-rate.ts b/evalboard/lib/pass-rate.ts index 719d99cd..d1ff242b 100644 --- a/evalboard/lib/pass-rate.ts +++ b/evalboard/lib/pass-rate.ts @@ -9,12 +9,13 @@ export const PASS_GREEN_PCT = 95; export const PASS_RED_PCT = 85; -// "none" is the absence of a measurement (no tasks ran), which is not the same -// as a measured 0% — that one is genuinely bad and must read red. +// "none" is the absence of a measurement, which callers signal by passing null. +// That is not the same as a measured 0% — that one is genuinely bad and must +// read red. export type PassTone = "none" | "good" | "warn" | "bad"; -export function passTone(pct: number | null | undefined, hasData = true): PassTone { - if (!hasData || pct == null || !Number.isFinite(pct)) return "none"; +export function passTone(pct: number | null | undefined): PassTone { + if (pct == null || !Number.isFinite(pct)) return "none"; if (pct >= PASS_GREEN_PCT) return "good"; if (pct < PASS_RED_PCT) return "bad"; return "warn"; @@ -36,18 +37,18 @@ const BAR_CLASS: Record = { bad: "bg-red-500", }; -// Pass rate as a percent (0-100) -> Tailwind text color. `hasData` distinguishes -// "no tasks yet" (neutral) from a real 0%. -export function passClass(pct: number | null | undefined, hasData = true): string { - return TEXT_CLASS[passTone(pct, hasData)]; +// Pass rate as a percent (0-100) -> Tailwind text color. Pass null when there is +// nothing to measure so it reads neutral rather than red. +export function passClass(pct: number | null | undefined): string { + return TEXT_CLASS[passTone(pct)]; } -export function passBarClass(pct: number | null | undefined, hasData = true): string { - return BAR_CLASS[passTone(pct, hasData)]; +export function passBarClass(pct: number | null | undefined): string { + return BAR_CLASS[passTone(pct)]; } // Same, for the tables that carry a rate as a 0-1 fraction (trends, watchlist) // rather than a percent. -export function passClassRatio(rate: number | null | undefined, hasData = true): string { - return passClass(rate == null ? null : rate * 100, hasData); +export function passClassRatio(rate: number | null | undefined): string { + return passClass(rate == null ? null : rate * 100); } diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 0c09d9ed..e4f98842 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -493,11 +493,12 @@ const COMPONENTS: { nonShaUrl: null, // Releases are tagged `v` by the Release workflow, so the // version resolves to an exact release page. Runs from before tagging - // began (v0.8.2) recorded a placeholder `0.1.0` and will 404 — the honest - // answer, since no such release exists, and not worth a version floor - // that would rot. + // began (v0.8.2) recorded a hardcoded `0.1.0` that was never released; + // render those as plain text rather than a link that 404s. versionUrl: (v) => - `https://github.com/UiPath/coder_eval/releases/tag/v${v}`, + v === "0.1.0" + ? null + : `https://github.com/UiPath/coder_eval/releases/tag/v${v}`, // The framework identifies itself by the released package version // (env_info `coder_eval`, e.g. "0.9.1"), which is what the suite // actually pins — so that is the chip, and it links to the release.
0, - )}`} + className={`font-medium ${passClass(pct)}`} > {pct != null ? `${pct.toFixed(0)}%` diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index 0b213d89..1aac1316 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -405,11 +405,14 @@ export function RunView({ const totalN = hasRepeats ? metrics.taskTotal : metrics.total; + // null when nothing ran, so an empty run reads neutral + // rather than as a measured 0%. + const tone = totalN > 0 ? pct : null; return ( <>
0)}`} + className={`text-2xl font-semibold tabular-nums ${passClass(tone)}`} > {pct.toFixed(0)}% @@ -433,7 +436,7 @@ export function RunView({ number beside it. */}
0)}`} + className={`h-full ${passBarClass(tone)}`} style={{ width: `${pct}%` }} />
diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index e7a2f103..0eacba8f 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -46,14 +46,6 @@ function fmtCount(n: number | null): string { return n.toFixed(0); } -// Per-task rate across the recent runs, on the same traffic-light cutoffs as -// every other pass rate on the site (lib/pass-rate.ts). This used to be green -// only at a perfect 100%, which painted a task that passed 9 of its last 10 runs -// the same red as one that never passed. -function passRateClass(rate: number, hasRuns: boolean): string { - return passClassRatio(rate, hasRuns); -} - type SortKey = | "task" | "runs" @@ -382,7 +374,11 @@ function TaskRow({ pending: boolean; onToggle: () => void; }) { - const rateClass = passRateClass(t.passRate, t.totalRuns > 0); + // Same traffic-light cutoffs as every other pass rate on the site + // (lib/pass-rate.ts). This used to be green only at a perfect 100%, which + // painted a task that passed 9 of its last 10 runs the same red as one that + // never passed. + const rateClass = passClassRatio(t.totalRuns > 0 ? t.passRate : null); return ( <>