diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index dd3912db..c5f1df85 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -107,3 +107,45 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule caught them. The cleanup plan explicitly deferred this as YAGNI for the one-time purge, but any future doc rename/deletion re-opens the same blind spot — caught in the 2026-07-03 open-source-docs-cleanup implementation run. + +## From 2026-07-30 evalboard multi-harness charts + pricing SSOT + +- [ ] **CI never runs the `evalboard/` test suite — the highest-value gap here.** + `grep -rn evalboard .github/workflows/ .pre-commit-config.yaml Makefile` yields + exactly one hit: a comment in `pr-checks.yml` saying `evalboard/pnpm-lock.yaml` + is *out of scope*. `pnpm verify` (`tsc --noEmit && vitest run && next build`) + exists in `evalboard/package.json` but nothing invokes it. Consequence: the + pricing drift guard that the whole generated-SSOT design rests on is never + executed. A Python dev edits a rate in `src/coder_eval/pricing.py`, never opens + `evalboard/`, and the PR goes green while every rendered USD figure keeps the + stale rate — precisely how the 7 wrong rates (Opus 4.6/4.7/4.8 at 3× actual) + accumulated in the first place. Flagged independently by two reviewers. + *Not done here because adding a required check to `pr-checks.yml` is a repo-wide + CI-policy change affecting every contributor's PR, outside a plan scoped to + `evalboard/`.* Ready to apply: a job mirroring the existing `setup-node` steps + (`pnpm install --frozen-lockfile` + `pnpm verify` with `working-directory: + evalboard`), ideally non-blocking first. Alternatively a Python-side CExxx rule + asserting `lib/pricing.generated.ts` is current, which would put the guard on + the side that actually edits rates. +- [ ] **One shared normalizer fixture for `normalizeModel` (TS) and + `_normalize_model` (Python).** They have drifted in *both* directions: TS strips + a trailing `-YYYYMMDD`, Python does not — so `claude-opus-4-6-20250514` prices + in the evalboard and is unpriced in the backend, and the frontend "estimated" + cost can disagree with the backend's authoritative Cost for dated ids. Neither + side's tests can see the other. Needs a decision on which behaviour is the + reference before a fixture can be written; the plan lists it under Open + Questions. Note also that plugin rates registered via `register_pricing` (e.g. + `coder_eval_uipath`) can never reach the generated table at all. +- [ ] **No test can see `app/page.tsx`'s self-link scope threading.** `hParam`, + `base` and `buildHref` are module-private inside an async server component and + there is no test for that file. The plan calls a dropped `?h=` the phase's main + risk — a future seventh self-link spelled `buildHref({ window, tag })` silently + resets the user's harness scope with the whole suite green. Guarding it means + extracting `buildHref` + a `selfLinkBase` helper into a testable module. +- [ ] **"Any chart with ≥2 series must have a hover-attribution test."** Nothing + in the suite mounted a chart tooltip before this run, which is how two real + misattribution bugs survived a spec review: recharts silently ignores + `shared={false}` on `LineChart`, and with the default `allowDuplicatedCategory` + each `` indexes its own points with an index into the concatenation of + all series. Closed for the two overview charts; nothing enforces it for the next + multi-series chart someone adds. diff --git a/evalboard/app/_components/__tests__/harness-badge.test.tsx b/evalboard/app/_components/__tests__/harness-badge.test.tsx new file mode 100644 index 00000000..cdd297e5 --- /dev/null +++ b/evalboard/app/_components/__tests__/harness-badge.test.tsx @@ -0,0 +1,51 @@ +import { render } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +// next/image needs no runtime here, but stub it to a plain so the rendered +// attributes are asserted directly rather than through the loader's transform. +vi.mock("next/image", () => ({ + default: (props: Record) => ( + + ), +})); + +const { HarnessBadge } = await import("../harness-badge"); + +describe("HarnessBadge", () => { + test("renders the vendor logo for a harness that has one", () => { + const { container } = render(); + const img = container.querySelector("img"); + expect(img).not.toBeNull(); + expect(img!.getAttribute("src")).toBe("/harness/codex.png"); + expect(img!.getAttribute("title")).toBe("Codex · OpenAI"); + }); + + test("the logo is decorative — empty alt, so the name isn't announced twice", () => { + // Every call site renders harnessShortLabel beside the badge; alt text + // here would make a screen reader say "Codex · OpenAI Codex". + const { container } = render(); + expect(container.querySelector("img")!.getAttribute("alt")).toBe(""); + }); + + test("renders NOTHING for an unknown harness — the B2 fix", () => { + // It used to print the raw id, which callers then printed again beside it. + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); + + test("renders nothing for a known harness with no logo yet", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + test("a missing harness folds to claude-code (legacy unstamped runs)", () => { + for (const harness of [null, undefined]) { + const { container } = render(); + expect(container.querySelector("img")!.getAttribute("src")).toBe( + "/harness/claude-code.png", + ); + } + }); +}); diff --git a/evalboard/app/_components/__tests__/harness-selector.test.tsx b/evalboard/app/_components/__tests__/harness-selector.test.tsx new file mode 100644 index 00000000..f24222d5 --- /dev/null +++ b/evalboard/app/_components/__tests__/harness-selector.test.tsx @@ -0,0 +1,207 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { harnessColor } from "@/lib/harness"; + +// The selector reads router/params hooks; stub them so it renders in jsdom +// without a provider, and capture `replace` to assert the resulting URL. +const replace = vi.fn(); +let searchParams = new URLSearchParams(); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace }), + usePathname: () => "/", + useSearchParams: () => searchParams, +})); + +const { HarnessSelector } = await import("../harness-selector"); + +const HARNESSES = ["claude-code", "codex", "antigravity"]; + +beforeEach(() => { + replace.mockClear(); + searchParams = new URLSearchParams(); +}); + +// The vendor logo is decorative (alt=""), so a chip's accessible name is exactly +// its label — no duplication to work around. +function chip(label: string) { + return screen.getByRole("button", { name: label }); +} +function lastUrl(): string { + expect(replace).toHaveBeenCalledTimes(1); + return replace.mock.calls[0][0] as string; +} + +describe("the All chip (allowAll)", () => { + test("renders and is the only pressed chip when current is null", () => { + render( + , + ); + expect(chip("All")).toBeInTheDocument(); + const pressed = screen + .getAllByRole("button") + .filter((b) => b.getAttribute("aria-pressed") === "true"); + expect(pressed).toHaveLength(1); + expect(pressed[0]).toHaveTextContent("All"); + }); + + test("clicking All removes h and preserves every other param", () => { + searchParams = new URLSearchParams("window=7d&tag=foo&h=codex"); + render( + , + ); + fireEvent.click(chip("All")); + const url = lastUrl(); + expect(url).not.toContain("h="); + expect(url).toContain("window=7d"); + expect(url).toContain("tag=foo"); + }); + + test("clicking All when h is the only param leaves no trailing ?", () => { + searchParams = new URLSearchParams("h=codex"); + render( + , + ); + fireEvent.click(chip("All")); + expect(lastUrl()).toBe("/"); + }); + + test("a harness chip is pressed instead of All once one is selected", () => { + render( + , + ); + expect(chip("All")).toHaveAttribute("aria-pressed", "false"); + expect(chip("Codex")).toHaveAttribute("aria-pressed", "true"); + }); + + test("the All chip carries no colour swatch — it is not a series", () => { + render( + , + ); + expect(chip("All").querySelectorAll("span[style]")).toHaveLength(0); + }); +}); + +describe("selecting a harness", () => { + test("clicking a chip sets h to that harness", () => { + render( + , + ); + fireEvent.click(chip("Codex")); + expect(lastUrl()).toContain("h=codex"); + }); + + test("selecting preserves the other params", () => { + searchParams = new URLSearchParams("window=14d&q=needle"); + render( + , + ); + fireEvent.click(chip("Antigravity")); + const url = lastUrl(); + expect(url).toContain("h=antigravity"); + expect(url).toContain("window=14d"); + expect(url).toContain("q=needle"); + }); + + test("each harness chip carries a swatch matching its series colour", () => { + render( + , + ); + // Assert against harnessColor rather than hardcoded hexes, so the test + // stays honest if the palette is re-validated — and so a chip and its + // chart line are provably the same colour. + for (const [harness, label] of [ + ["claude-code", "Claude Code"], + ["codex", "Codex"], + ["antigravity", "Antigravity"], + ] as const) { + const swatch = chip(label).querySelector( + "span[style]", + ) as HTMLElement | null; + expect(swatch, `${label} chip has no swatch`).not.toBeNull(); + expect(swatch!.style.backgroundColor).toBe( + hexToRgb(harnessColor(harness)), + ); + } + }); +}); + +describe("without allowAll (the /trends, /watchlist, /path-to-ga shape)", () => { + test("no All chip is rendered", () => { + render(); + expect(screen.queryByRole("button", { name: "All" })).toBeNull(); + }); + + test("the current harness is still marked active", () => { + render(); + expect(chip("Claude Code")).toHaveAttribute("aria-pressed", "true"); + }); +}); + +describe("an active harness absent from the recent window", () => { + test("is still rendered and active (a deep-linked ?h= reads as selected)", () => { + render( + , + ); + expect(chip("Delegate SDK")).toHaveAttribute("aria-pressed", "true"); + }); + + test("the null (All) case does not prepend a phantom chip", () => { + render( + , + ); + // All + exactly the two supplied harnesses. + expect(screen.getAllByRole("button")).toHaveLength(3); + }); +}); + +function hexToRgb(hex: string): string { + const n = parseInt(hex.slice(1), 16); + return `rgb(${(n >> 16) & 255}, ${(n >> 8) & 255}, ${n & 255})`; +} + +// B2: a harness with no vendor mark used to have its id rendered twice — once by +// HarnessBadge's text fallback and once by the chip's own harnessShortLabel. +describe("a harness with no vendor logo (B2)", () => { + test("its label appears exactly once in the chip", () => { + render( + , + ); + // textContent equality, not containment: "delegate-sdkDelegate SDK" + // would satisfy a contains() assertion. + expect(chip("Delegate SDK").textContent).toBe("Delegate SDK"); + }); + + test("an unknown harness likewise shows its id exactly once", () => { + render( + , + ); + expect(chip("some-future-harness").textContent).toBe( + "some-future-harness", + ); + }); + + test("a known harness keeps its logo AND exactly one label", () => { + render(); + const c = chip("Codex"); + expect(c.querySelector("img")).not.toBeNull(); + expect(c.textContent).toBe("Codex"); + }); +}); diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index da16b704..8e34f17d 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -1,47 +1,40 @@ import Image from "next/image"; +import { harnessRow, normalizeHarness } from "@/lib/harness"; // Canonical harness constants live in the leaf lib/harness module (shared by // the server data layer); re-exported here so existing badge importers are // unaffected. -export { KNOWN_HARNESSES, type KnownHarness } from "@/lib/harness"; - -// Vendor logo + labels for a run's harness (RunConfig). Renders the recognizable -// vendor mark instead of raw "claude-code"/"codex"/"antigravity" text, -// mirroring the Slack rollup's vendor emoji. A missing harness defaults to -// claude-code (the nightly). This is an internal-only column — the caller -// gates it behind isInternal (see lib/edition.ts). -const HARNESS_LOGO: Record = { - "claude-code": { - src: "/harness/claude-code.png", - label: "Claude Code · Anthropic", - short: "Claude Code", - }, - codex: { src: "/harness/codex.png", label: "Codex · OpenAI", short: "Codex" }, - antigravity: { - src: "/harness/antigravity.png", - label: "Antigravity · Google Gemini", - short: "Antigravity", - }, -}; - -// Short human label for a harness id ("Claude Code"), for selectors and prose. -// Unknown ids fall through to the raw id. -export function harnessShortLabel(harness: string): string { - return HARNESS_LOGO[harness]?.short ?? harness; -} +export { + KNOWN_HARNESSES, + harnessShortLabel, + type KnownHarness, +} from "@/lib/harness"; +// Vendor logo for a run's harness (RunConfig). Renders the recognizable vendor +// mark instead of raw "claude-code"/"codex"/"antigravity" text, mirroring the +// Slack rollup's vendor emoji. Labels, logo paths and the claude-code default +// all come from the HARNESSES table — this component is only the . A +// missing harness folds to claude-code (the nightly). This is an internal-only +// column — the caller gates it behind isInternal (see lib/edition.ts). export function HarnessBadge({ harness }: { harness?: string | null }) { - const key = harness ?? "claude-code"; - const logo = HARNESS_LOGO[key]; - // Unknown harness: show the raw id rather than a misleading logo. - if (!logo) { - return {key}; - } + const row = harnessRow(normalizeHarness(harness)); + // No vendor mark for this harness — an unknown one, or a known one with no + // logo yet: render NOTHING. "No logo → no image" lives here and only here. + // + // This deliberately does not fall back to printing the raw id: every caller + // already renders harnessShortLabel beside the badge, so a text fallback + // showed the id twice ("delegate-sdk delegate-sdk"). Pair the badge with + // harnessShortLabel rather than expecting it to name the harness itself. + if (!row?.logo) return null; return ( {logo.label}` -// while preserving the active q/tag params, mirroring WindowSelector. Each -// segment shows the vendor logo + short label so it reads like the per-run -// harness badge on the runs tables. +const ACTIVE_CLS = "bg-studio-blue text-white"; +const INACTIVE_CLS = "bg-white text-gray-700 hover:bg-gray-50"; + +// Segmented control for the harness scope. Sets `?h=` while preserving +// the active q/tag/window params, mirroring WindowSelector. Each segment shows +// the vendor logo + a colour swatch + the short label, so a chip reads like both +// the per-run harness badge on the runs tables and its line in the charts. +// +// `current == null` means "all harnesses". Only pages that support that scope +// pass `allowAll`; /trends, /watchlist and /path-to-ga keep exactly-one-harness +// semantics and are unaffected. export function HarnessSelector({ current, harnesses, + allowAll = false, }: { - current: string; + current: string | null; harnesses: readonly string[]; + allowAll?: boolean; }) { const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - const set = (h: string) => { + // One navigation helper for both setting and clearing the scope, so the + // trailing-`?` guard exists once and no URL is built twice. + const nav = (mutate: (p: URLSearchParams) => void) => { const p = new URLSearchParams(searchParams.toString()); - p.set("h", h); - router.replace(`${pathname}?${p.toString()}`, { scroll: false }); + mutate(p); + // Length-check the serialized string, NOT `p.size`: that property is + // Safari 17+ only, and where it's undefined the ternary would take the + // falsy branch and navigate to a bare pathname — silently discarding + // window/tag/q along with the scope. Matches buildHref and SearchBox. + const qs = p.toString(); + router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); }; // Always show the active harness, even if it has aged out of the recent // window (so a deep-linked `?h=` still reads as selected rather than absent). - const opts = harnesses.includes(current) - ? harnesses - : [current, ...harnesses]; + // Guarded for null: in the All view there is no active harness to prepend. + const opts = + current == null || harnesses.includes(current) + ? harnesses + : [current, ...harnesses]; return (
+ {allowAll && ( + + )} {opts.map((h) => { const active = h === current; return ( ); diff --git a/evalboard/app/_components/legend.tsx b/evalboard/app/_components/legend.tsx new file mode 100644 index 00000000..20329a5e --- /dev/null +++ b/evalboard/app/_components/legend.tsx @@ -0,0 +1,44 @@ +// Shared legend/swatch primitives, deliberately recharts-free and in their own +// module. LegendStrip is used by the overview charts' harness legend AND by the +// tag rail's ChipLegend — and the tag rail renders on pages with no chart at all +// (/runs/[id] via run-view, /trends via trends-view). Importing it from +// chart-common.tsx instead drags recharts into those bundles: measured at +// +105 kB First Load JS on /runs/[id] (163 -> 268 kB) and on /trends +// (119 -> 225 kB). Keep this file free of chart imports. + +import { harnessColor } from "@/lib/harness"; + +// The colour dot that ties a harness to its series line and its selector chip. +// Inline style, not a Tailwind class: the colour is a runtime value, and +// Tailwind's JIT only emits classes it can see as literal strings at build time, +// so a dynamically-built `bg-[${hex}]` would be purged and render colourless. +export function HarnessSwatch({ harness }: { harness: string }) { + return ( + + ); +} + +// The generic "dot + label" row. Colours arrive as values so the same primitive +// serves both the fixed tag-rail legend and the runtime-coloured harness legend. +export function LegendStrip({ + entries, +}: { + entries: { key: string; color: string; label: string }[]; +}) { + return ( +
+ {entries.map((e) => ( + + + {e.label} + + ))} +
+ ); +} diff --git a/evalboard/app/_overview/__tests__/chart-common.test.tsx b/evalboard/app/_overview/__tests__/chart-common.test.tsx new file mode 100644 index 00000000..97c1c948 --- /dev/null +++ b/evalboard/app/_overview/__tests__/chart-common.test.tsx @@ -0,0 +1,347 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { cloneElement, type ReactElement } from "react"; +import { describe, expect, test, vi } from "vitest"; +import { harnessColor } from "@/lib/harness"; +import { harnessesInPoints, type RunPoint } from "@/lib/overview"; + +vi.mock("recharts", async () => { + const actual = await vi.importActual("recharts"); + return { + ...actual, + // jsdom has no layout, so the real ResponsiveContainer measures 0x0 and + // renders nothing (and needs ResizeObserver). Hand the chart a fixed size. + // The cast is needed because ReactElement's default prop type is unknown. + ResponsiveContainer: ({ children }: { children: ReactElement }) => + cloneElement( + children as ReactElement<{ width: number; height: number }>, + { width: 600, height: 300 }, + ), + }; +}); + +// Import the components AFTER the mock is registered. +const { DailySuccessChart } = await import("../daily-chart"); +const { TurnBudgetChart } = await import("../turn-budget-chart"); +const { ChipLegend } = await import("../tag-rail"); + +function point(overrides: Partial = {}): RunPoint { + return { + runId: "2026-07-30_00-00-00", + timestamp: Date.UTC(2026, 6, 30), + harness: "claude-code", + successRate: 90, + turnBudgetRate: 80, + ...overrides, + }; +} + +const WINDOW = { windowStart: Date.UTC(2026, 6, 1), windowEnd: Date.UTC(2026, 6, 31) }; + +function curves(container: HTMLElement): Element[] { + return [...container.querySelectorAll(".recharts-line-curve")]; +} +function dots(container: HTMLElement): Element[] { + return [...container.querySelectorAll(".recharts-dot")]; +} + +// recharts only emits a .recharts-line-curve path when a series has at least two +// points — a single point has no segment to draw (it still renders its dot). So +// curve-counting tests give every harness two points; dot-counting ones don't need to. +function twoPoints(harness: string, day: number): RunPoint[] { + return [ + point({ harness, timestamp: Date.UTC(2026, 6, day) }), + point({ harness, timestamp: Date.UTC(2026, 6, day + 1) }), + ]; +} + +describe("RunPointLineChart — one line per harness", () => { + test("three harnesses render three curves, each in its own brand colour", () => { + const { container } = render( + , + ); + const paths = curves(container); + expect(paths).toHaveLength(3); + // Assert against harnessColor, never hardcoded hexes, so this stays + // honest if the palette is re-validated. + expect(new Set(paths.map((p) => p.getAttribute("stroke")))).toEqual( + new Set([ + harnessColor("claude-code"), + harnessColor("codex"), + harnessColor("antigravity"), + ]), + ); + }); + + test("dot count equals the total point count across series", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(3); + }); + + test("a single harness renders one curve and NO legend", () => { + // One series is already named by the section subtitle; a one-entry + // legend box would be noise. + const { container } = render( + , + ); + expect(curves(container)).toHaveLength(1); + expect(screen.queryByText("Claude Code")).toBeNull(); + }); + + test("two or more harnesses render one legend entry each, by short label", () => { + render( + , + ); + expect(screen.getByText("Claude Code")).toBeInTheDocument(); + expect(screen.getByText("Codex")).toBeInTheDocument(); + }); + + test("an unknown harness still plots, with the neutral colour and its raw id", () => { + const { container } = render( + , + ); + expect(curves(container)).toHaveLength(2); + expect( + curves(container).map((p) => p.getAttribute("stroke")), + ).toContain(harnessColor("some-future-harness")); + expect(screen.getByText("some-future-harness")).toBeInTheDocument(); + }); + + test("empty data renders without throwing and draws no curves", () => { + // A 30d window with no runs, or an over-narrow tag filter. + const { container } = render(); + expect(curves(container)).toHaveLength(0); + }); + + test("a harness with a single point draws a lone dot", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(3); + }); +}); + +describe("the two wrappers plot different metrics", () => { + // The one assertion that would have caught a wrong-dataKey copy-paste back + // when this logic was duplicated across two files. Kept even though the + // shared implementation now makes that mistake structurally unlikely. + const nullBudget = [ + point({ harness: "claude-code", successRate: 100, turnBudgetRate: null }), + ]; + + test("TurnBudgetChart plots turnBudgetRate — a null budget yields no dot", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(0); + }); + + test("DailySuccessChart plots successRate — the same point DOES yield a dot", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(1); + }); +}); + +// The most dangerous failure mode this chart has: reporting one harness's rate +// under another harness's date. Nothing else in the suite mounts a tooltip, and +// that gap is exactly how two real attribution bugs survived a first review — +// recharts ignores `shared={false}` on LineChart, and with the default +// allowDuplicatedCategory each indexed its OWN points with an index into +// the CONCATENATION of all series. Hover every dot and assert the triple. +describe("tooltip attributes the hovered point to the right harness", () => { + // claude-code and codex interleave in time, so an index- or nearest-x-based + // lookup would visibly cross the wires. + const data = [ + point({ harness: "claude-code", timestamp: Date.UTC(2026, 6, 5), successRate: 20 }), + point({ harness: "claude-code", timestamp: Date.UTC(2026, 6, 25), successRate: 21 }), + point({ harness: "codex", timestamp: Date.UTC(2026, 6, 10), successRate: 95 }), + point({ harness: "codex", timestamp: Date.UTC(2026, 6, 14), successRate: 96 }), + ]; + const expected = [ + ["Claude Code", "2026-07-05 00:00 UTC", "20.0% success"], + ["Claude Code", "2026-07-25 00:00 UTC", "21.0% success"], + ["Codex", "2026-07-10 00:00 UTC", "95.0% success"], + ["Codex", "2026-07-14 00:00 UTC", "96.0% success"], + ]; + + test.each(expected.map((e, i) => [i, ...e] as const))( + "dot %i reports %s / %s / %s", + (index, harness, date, rate) => { + const { container } = render( + , + ); + const dot = [...container.querySelectorAll(".recharts-dot")][index]; + const cx = Number(dot.getAttribute("cx")); + const cy = Number(dot.getAttribute("cy")); + // recharts reads pageX/pageY; send both pairs so this works + // regardless of which the version consults. + fireEvent.mouseMove(container.querySelector(".recharts-wrapper")!, { + pageX: cx, + pageY: cy, + clientX: cx, + clientY: cy, + }); + const text = + container.querySelector(".recharts-tooltip-wrapper") + ?.textContent ?? ""; + expect(text).toContain(harness); + expect(text).toContain(date); + expect(text).toContain(rate); + // Exactly one highlighted mark: with the combined-index bug this was + // 2 (and would be 4 with four harnesses), lighting up an unrelated + // point on every other series. + expect( + container.querySelectorAll(".recharts-active-dot"), + ).toHaveLength(1); + }, + ); +}); + +describe("connectNulls is scoped to one harness", () => { + test("a gap in one harness is not bridged across another harness's run", () => { + // The headline correctness gain of per-series data: with a single + // over the flat array, claude-code's null at Jul 10 would have been + // bridged straight through codex's Jul 12 run. + const { container } = render( + , + ); + // One curve per harness, and each curve's geometry is built only from its + // own points — so no claude-code segment can pass through a codex dot. + const paths = curves(container); + expect(paths).toHaveLength(2); + const codexDotCount = 2; + // claude-code contributes 2 non-null points, codex 2 → 4 dots total. + expect(dots(container)).toHaveLength(2 + codexDotCount); + }); +}); + +describe("ChipLegend after the LegendStrip re-base", () => { + test("renders its three labels with the Tailwind-400 hexes it used to carry", () => { + // Regression on the class -> hex swap: rendered output must be unchanged. + const { container } = render(); + for (const label of ["skill", "review", "tag"]) { + expect(screen.getByText(label)).toBeInTheDocument(); + } + const colors = [...container.querySelectorAll("span[style]")].map((s) => + (s as HTMLElement).style.backgroundColor, + ); + expect(colors).toEqual([ + "rgb(129, 140, 248)", // #818cf8 indigo-400 + "rgb(251, 113, 133)", // #fb7185 rose-400 + "rgb(156, 163, 175)", // #9ca3af gray-400 + ]); + }); +}); + +// The page's subtitle says "all N harnesses" using OverviewData.harnesses, while +// the chart draws groupByHarness(overview.runs). Those are two derivations of +// what should be one set — this pins them together over the same input, so a +// future edit that points the label at a different list (the recent-runs +// discovery list is one token away) cannot claim a count the chart contradicts. +describe("the drawn series set equals harnessesInPoints", () => { + test.each([ + ["one harness", ["claude-code"]], + ["two harnesses", ["claude-code", "codex"]], + ["all four, out of table order", ["delegate-sdk", "codex", "antigravity", "claude-code"]], + ["including an unknown harness", ["some-future-harness", "claude-code"]], + ])("%s", (_label, harnesses) => { + const data = harnesses.flatMap((h, i) => twoPoints(h, 2 + i * 3)); + const { container } = render(); + expect(harnessesInPoints(data)).toHaveLength(harnesses.length); + expect(curves(container)).toHaveLength(harnessesInPoints(data).length); + }); +}); + +// groupByHarness deliberately keeps points whose metric is null (grouping must +// not filter), so the legend has to do the filtering — otherwise it names a +// harness that draws no line, which reads as a missing line or as 0%. +describe("the legend names only harnesses that actually plot", () => { + const mixed = [ + // claude-code has a turn budget; codex's runs carry none. + ...[10, 11].map((d) => + point({ + harness: "claude-code", + timestamp: Date.UTC(2026, 6, d), + successRate: 90, + turnBudgetRate: 80, + }), + ), + ...[13, 14].map((d) => + point({ + harness: "codex", + timestamp: Date.UTC(2026, 6, d), + successRate: 70, + turnBudgetRate: null, + }), + ), + ]; + + test("the success chart names both — both plot successRate", () => { + render(); + expect(screen.getByText("Claude Code")).toBeInTheDocument(); + expect(screen.getByText("Codex")).toBeInTheDocument(); + }); + + test("the turn-budget chart draws and names only the harness with data", () => { + const { container } = render( + , + ); + // recharts still emits a element for the all-null series, but with + // no `d` — nothing is drawn. Count geometry, not elements. + const drawn = curves(container).filter((c) => c.getAttribute("d")); + expect(drawn).toHaveLength(1); + expect(drawn[0].getAttribute("stroke")).toBe(harnessColor("claude-code")); + // Only the plotting harness contributes dots. + expect(dots(container)).toHaveLength(2); + // And no legend at all: with one series actually plotting, HarnessLegend + // suppresses itself, so the invisible Codex line is never named. Before + // the plotted-filter it appeared here with no line to point at. + expect(screen.queryByText("Codex")).toBeNull(); + expect(screen.queryByText("Claude Code")).toBeNull(); + }); +}); diff --git a/evalboard/app/_overview/__tests__/tag-rail.test.tsx b/evalboard/app/_overview/__tests__/tag-rail.test.tsx new file mode 100644 index 00000000..bd192d3b --- /dev/null +++ b/evalboard/app/_overview/__tests__/tag-rail.test.tsx @@ -0,0 +1,84 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, test } from "vitest"; +import { MergedTagRail } from "../tag-rail"; + +// MergedTagRail renders next/link, which needs no router for an href assertion. +const TAGS = { + skills: [{ tag: "activation", count: 3 }], + reviewTags: [{ tag: "flaky", count: 2 }], + taskTags: [{ tag: "python", count: 1 }], +}; + +function hrefs(): string[] { + return screen + .getAllByRole("link") + .map((a) => a.getAttribute("href") ?? ""); +} + +// The rule: an explicit scope is ALWAYS emitted; an absent `h` means all +// harnesses. Omitting `h=claude-code` "to keep URLs clean" is a scope reset now +// that absent no longer resolves to claude-code on the home page. +describe("tag chips preserve the harness scope", () => { + test("claude-code is emitted explicitly — the B1 regression", () => { + // This is the case the old `harness !== DEFAULT_HARNESS` guard dropped: + // an isolated Claude Code view silently widened to All on any tag click. + render( + , + ); + const links = hrefs(); + expect(links.length).toBeGreaterThan(0); + for (const href of links) { + expect(href).toContain("h=claude-code"); + } + }); + + test("a non-default harness is preserved (unchanged behaviour)", () => { + render(); + for (const href of hrefs()) { + expect(href).toContain("h=codex"); + } + }); + + test("the All scope (null) emits no h at all, keeping URLs clean", () => { + render(); + for (const href of hrefs()) { + expect(href).not.toContain("h="); + } + }); + + test("an active chip links to the CLEARED tag and still carries the scope", () => { + // Clicking an active chip must remove the tag filter without also + // widening the harness scope. + render( + , + ); + const active = screen.getByRole("link", { name: /activation/ }); + const href = active.getAttribute("href") ?? ""; + expect(href).not.toContain("tag=activation"); + expect(href).toContain("h=codex"); + expect(href).toContain("window=7d"); + }); + + test("on /trends the URL simply becomes explicit — same behaviour", () => { + // parseHarnessParam maps absent -> claude-code there, so emitting it + // changes nothing except the URL's shape. Asserted so a future reader + // knows the explicitness is intended, not an accident. + render( + , + ); + for (const href of hrefs()) { + expect(href).toMatch(/^\/trends\?/); + expect(href).toContain("h=claude-code"); + } + }); +}); diff --git a/evalboard/app/_overview/chart-common.tsx b/evalboard/app/_overview/chart-common.tsx new file mode 100644 index 00000000..960fecad --- /dev/null +++ b/evalboard/app/_overview/chart-common.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { + CartesianGrid, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; +import { groupByHarness, harnessColor, harnessShortLabel } from "@/lib/harness"; +import type { RunPoint } from "@/lib/overview"; +import { HarnessSwatch, LegendStrip } from "../_components/legend"; + +// MM-DD tick label on the axis; full date+time appears in the tooltip. +function shortLabel(ms: number): string { + const d = new Date(ms); + const m = String(d.getUTCMonth() + 1).padStart(2, "0"); + const day = String(d.getUTCDate()).padStart(2, "0"); + return `${m}-${day}`; +} + +function fullLabel(ms: number): string { + const d = new Date(ms); + const y = d.getUTCFullYear(); + const m = String(d.getUTCMonth() + 1).padStart(2, "0"); + const day = String(d.getUTCDate()).padStart(2, "0"); + const h = String(d.getUTCHours()).padStart(2, "0"); + const min = String(d.getUTCMinutes()).padStart(2, "0"); + return `${y}-${m}-${day} ${h}:${min} UTC`; +} + +// Names each plotted harness. Returns null below two series: a single line is +// already named by the section subtitle, so a one-entry legend box is just noise. +// With two or more, this is what carries identity — the dataviz rule that >=2 +// series always get a legend is unconditional. +function HarnessLegend({ harnesses }: { harnesses: string[] }) { + if (harnesses.length < 2) return null; + return ( + ({ + key: h, + color: harnessColor(h), + label: harnessShortLabel(h), + }))} + /> + ); +} + +// Which RunPoint field a chart plots. Derived from RunPoint, and narrowed to the +// rate metrics, so a typo can't plot a timestamp and a field rename breaks here. +type MetricKey = Extract; + +function ChartTooltip({ + active, + payload, + dataKey, + describe, +}: { + active?: boolean; + payload?: Array<{ payload: RunPoint }>; + dataKey: MetricKey; + describe: (value: number | null) => string; +}) { + if (!active || !payload?.length) return null; + const point = payload[0].payload; + return ( +
+
+ + {harnessShortLabel(point.harness)} +
+
+ {fullLabel(point.timestamp)} +
+ {/* Read through dataKey, so the narrated value IS the plotted one — + a wrapper cannot describe one metric while charting another. */} +
+ {describe(point[dataKey])} +
+
+ ); +} + +// The one chart implementation behind both overview charts. They differ only in +// which metric they plot, whether nulls are bridged, and one tooltip sentence — +// so those are the only parameters, and the multi-series conversion, legend, +// tooltip, axes and height exist exactly once. +// +// One per harness, each carrying its own `data`, sharing a numeric time +// x-axis. Colour comes from the harness id, never from series order, so a chart +// line and a selector chip can never disagree. +export function RunPointLineChart({ + data, + windowStart, + windowEnd, + dataKey, + connectNulls, + describe, +}: { + data: RunPoint[]; + windowStart: number; + windowEnd: number; + dataKey: MetricKey; + connectNulls: boolean; + // Narrates the plotted value. Takes the value, not the point, so it is + // structurally impossible to describe a different metric than the one drawn. + describe: (value: number | null) => string; +}) { + const series = groupByHarness(data); + // Name only the harnesses that actually draw something on THIS chart. + // groupByHarness deliberately keeps points whose metric is null (grouping + // must not filter), but turnBudgetRate is legitimately null for a whole + // harness when no task in its runs carries an expected_turns budget — and a + // legend entry for an invisible line reads as a missing line, or worse as 0%. + const plotted = series + .filter((s) => s.points.some((p) => p[dataKey] != null)) + .map((s) => s.harness); + return ( +
+ + UTC + + {/* Left-aligned in its own row, so it cannot collide with the + absolutely-positioned UTC badge in the top-right. */} + +
+ + + + then indexes its OWN points with that + // combined index: hovering one harness's dot lights + // up an unrelated point on every other series, and + // the tooltip resolves by nearest-x across all + // harnesses. Setting this false switches lookup to + // findEntryInArray(series, "timestamp", activeLabel), + // so both the tooltip and the active dot resolve to + // the hovered series' own point at that timestamp. + allowDuplicatedCategory={false} + /> + 70 dip + // look catastrophic and shifts as the window's min + // changes, breaking cross-window comparison. + domain={[0, 100]} + ticks={[0, 25, 50, 75, 100]} + tick={{ fontSize: 11, fill: "#6b7280" }} + tickLine={false} + axisLine={false} + width={36} + /> + + } + cursor={{ + stroke: "#e5e7eb", + strokeDasharray: "3 3", + }} + /> + {series.map((s) => ( + + ))} + + +
+
+ ); +} diff --git a/evalboard/app/_overview/daily-chart.tsx b/evalboard/app/_overview/daily-chart.tsx index 721dd655..1ba3c3ee 100644 --- a/evalboard/app/_overview/daily-chart.tsx +++ b/evalboard/app/_overview/daily-chart.tsx @@ -1,57 +1,10 @@ "use client"; -import { - CartesianGrid, - Line, - LineChart, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; import type { RunPoint } from "@/lib/overview"; +import { RunPointLineChart } from "./chart-common"; -// MM-DD tick label on the axis; full date+time appears in the tooltip. -function shortLabel(ms: number): string { - const d = new Date(ms); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - return `${m}-${day}`; -} - -function fullLabel(ms: number): string { - const d = new Date(ms); - const y = d.getUTCFullYear(); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - const h = String(d.getUTCHours()).padStart(2, "0"); - const min = String(d.getUTCMinutes()).padStart(2, "0"); - return `${y}-${m}-${day} ${h}:${min} UTC`; -} - -function CustomTooltip({ - active, - payload, -}: { - active?: boolean; - payload?: Array<{ payload: RunPoint }>; -}) { - if (!active || !payload?.length) return null; - const point = payload[0].payload; - return ( -
-
- {fullLabel(point.timestamp)} -
-
- {point.successRate != null - ? `${point.successRate.toFixed(1)}% success` - : "no tasks"} -
-
- ); -} - +// Per-run success rate, one coloured line per harness. Everything but the metric, +// the null-bridging rule and the tooltip sentence lives in RunPointLineChart. export function DailySuccessChart({ data, windowStart, @@ -62,51 +15,19 @@ export function DailySuccessChart({ windowEnd: number; }) { return ( -
- - UTC - - - - - - - } - cursor={{ stroke: "#e5e7eb", strokeDasharray: "3 3" }} - /> - - - -
+ + rate != null ? `${rate.toFixed(1)}% success` : "no tasks" + } + /> ); } diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx index 2bf1f9be..505026b8 100644 --- a/evalboard/app/_overview/tag-rail.tsx +++ b/evalboard/app/_overview/tag-rail.tsx @@ -1,7 +1,7 @@ import Link from "next/link"; import type { TagCount } from "@/lib/overview"; -import { DEFAULT_HARNESS } from "@/lib/harness"; import type { Window } from "@/lib/reviews-types"; +import { LegendStrip } from "../_components/legend"; type Variant = "neutral" | "rose" | "indigo"; @@ -42,10 +42,17 @@ function hrefForTag( if (window) params.set("window", window); if (tag) params.set("tag", tag); if (q) params.set("q", q); - // Preserve the active harness scope across tag clicks (omit the default to - // keep URLs clean). Without this, filtering by a tag would silently reset a - // codex/antigravity view back to claude-code. - if (harness && harness !== DEFAULT_HARNESS) params.set("h", harness); + // Preserve the active harness scope across tag clicks. The rule, stated once + // because the next page to build a self-link will copy it: an explicit scope + // is ALWAYS emitted; an absent `h` means all harnesses. + // + // Omitting `h=claude-code` to shorten the URL is what this used to do, and it + // is now a scope reset: on `/`, absent `h` no longer resolves to claude-code, + // so dropping it silently widens an isolated Claude Code view back to All on + // every tag click. On `/trends`, where parseHarnessParam still maps absent to + // claude-code, the URL merely becomes explicit — same behaviour, every + // consumer, no per-page exception. + if (harness) params.set("h", harness); const qs = params.toString(); return qs ? `${basePath}?${qs}` : basePath; } @@ -197,22 +204,19 @@ export function MergedTagRail({ // Tiny legend strip explaining what the chip colors mean. Placed above // the chip rail so users don't have to hover-discover the convention. +// +// Shares LegendStrip with the charts' harness legend rather than mirroring its +// markup. LegendStrip takes colours as values, so these are the literal hexes +// of the Tailwind classes this used to carry (indigo-400, rose-400, gray-400) — +// rendered output is unchanged. Keeps its zero-arg API for all three consumers. export function ChipLegend() { - const entries: Array<[string, string, string]> = [ - ["skill", "bg-indigo-400", "skill"], - ["review", "bg-rose-400", "review"], - ["tag", "bg-gray-400", "tag"], - ]; return ( -
- {entries.map(([key, dotCls, label]) => ( - - - {label} - - ))} -
+ ); } diff --git a/evalboard/app/_overview/turn-budget-chart.tsx b/evalboard/app/_overview/turn-budget-chart.tsx index f274f32e..d7e9a168 100644 --- a/evalboard/app/_overview/turn-budget-chart.tsx +++ b/evalboard/app/_overview/turn-budget-chart.tsx @@ -1,60 +1,11 @@ "use client"; -import { - CartesianGrid, - Line, - LineChart, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; import type { RunPoint } from "@/lib/overview"; +import { RunPointLineChart } from "./chart-common"; -// MM-DD tick label on the axis; full date+time appears in the tooltip. -function shortLabel(ms: number): string { - const d = new Date(ms); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - return `${m}-${day}`; -} - -function fullLabel(ms: number): string { - const d = new Date(ms); - const y = d.getUTCFullYear(); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - const h = String(d.getUTCHours()).padStart(2, "0"); - const min = String(d.getUTCMinutes()).padStart(2, "0"); - return `${y}-${m}-${day} ${h}:${min} UTC`; -} - -function CustomTooltip({ - active, - payload, -}: { - active?: boolean; - payload?: Array<{ payload: RunPoint }>; -}) { - if (!active || !payload?.length) return null; - const point = payload[0].payload; - return ( -
-
- {fullLabel(point.timestamp)} -
-
- {point.turnBudgetRate != null - ? `${point.turnBudgetRate.toFixed(1)}% within turn budget (budgeted failures count as over)` - : "no tasks with a turn budget"} -
-
- ); -} - -// Sibling of DailySuccessChart: same axes/styling, plotting the per-run -// "within expected turns" rate instead of the success rate. Driven by the -// same windowed RunPoint[] so the shared window selector controls both. +// Sibling of DailySuccessChart: same axes and styling, plotting the per-run +// "within expected turns" rate instead of the success rate. Driven by the same +// windowed RunPoint[] so the shared window selector controls both. export function TurnBudgetChart({ data, windowStart, @@ -65,53 +16,20 @@ export function TurnBudgetChart({ windowEnd: number; }) { return ( -
- - UTC - - - - - - - } - cursor={{ stroke: "#e5e7eb", strokeDasharray: "3 3" }} - /> - - - -
+ + rate != null + ? `${rate.toFixed(1)}% within turn budget (budgeted failures count as over)` + : "no tasks with a turn budget" + } + /> ); } diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index d8326a0a..cb26aa66 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -6,7 +6,11 @@ import { listRecentHarnesses, type TagCount, } from "@/lib/overview"; -import { parseHarnessParam, DEFAULT_HARNESS } from "@/lib/harness"; +import { + normalizeHarness, + orderHarnesses, + parseHarnessScope, +} from "@/lib/harness"; import { fmtDuration, fmtRunTime, fmtTimestamp, passClass } from "@/lib/format"; import { WindowSelector } from "./_components/window-selector"; import { WINDOWS, type Window } from "@/lib/reviews-types"; @@ -110,19 +114,22 @@ export default async function Page({ const window = parseWindow(params.window); const activeTag = parseTag(params.tag); const q = parseQ(params.q); - const harness = parseHarnessParam(params.h); + // null = all harnesses. An absent `?h=` shows every harness at once. + const harness = parseHarnessScope(params.h); const limit = parseLimit(params.limit); const adhocLimit = parseAdhocLimit(params.alimit); const isFiltered = activeTag != null || q != null; - // The analytics block (chart + rails) is scoped to one harness so the - // success line stops zigzagging across incomparable harnesses. The run - // LIST stays all-harness — seeing every recent run is the page's job, and - // the Harness column already disambiguates each row. + // `?h=` scopes the WHOLE page — charts, rails, the run table, its totals and + // the ad-hoc section — so every number on screen describes the same set of + // runs. With it absent the page shows all harnesses, drawing one + // colour-coded line per harness; the old single-harness default existed to + // stop the success line zigzagging between incomparable harnesses, which is + // solved by separating the series rather than by narrowing the view. const [overview, listing, adhoc, harnesses] = await Promise.all([ getOverview(window, activeTag, q, harness), - getRunListing(window, activeTag, q, limit), - getAdhocRunListing(adhocLimit), + getRunListing(window, activeTag, q, limit, harness), + getAdhocRunListing(adhocLimit, harness), listRecentHarnesses(), ]); @@ -143,9 +150,11 @@ export default async function Page({ const rawAlimit = Array.isArray(params.alimit) ? params.alimit[0] : params.alimit; - // Omit the default harness from URLs to keep them clean; carry a non-default - // scope through every self-link so it isn't reset by pagination/clear. - const hParam = harness === DEFAULT_HARNESS ? undefined : harness; + // Carry an explicit scope through every self-link so it isn't reset by + // pagination / clear / tag clicks. Emit it whenever there IS one — omitting + // `h=claude-code` because it happens to be the default would silently widen + // an isolated view back to All, since absent now means "all harnesses". + const hParam = harness ?? undefined; const base = { window, tag: activeTag, @@ -170,16 +179,56 @@ export default async function Page({ const adhocShowAllHref = buildHref({ ...base, alimit: "all" }); const adhocShowLessHref = buildHref({ ...base, alimit: undefined }); + // Computed once and used in BOTH subtitle branches, so "the chart says what + // it is scoped to" is enforced by construction rather than by two prose + // edits. The count comes from the server's windowed answer + // (OverviewData.harnesses is derived from the same points the chart groups), + // never a local recomputation — so "all N" always equals the number of lines + // drawn. Counts of 0 and 1 are special-cased: "all 0 harnesses" reads as a + // bug, and "all 1 harnesses" is both ungrammatical and less informative than + // naming the one harness — especially since the legend hides itself below two + // series, so nothing else on screen would name it. + const scopeLabel = harness + ? harnessShortLabel(harness) + : overview.harnesses.length === 1 + ? harnessShortLabel(overview.harnesses[0]) + : overview.harnesses.length > 1 + ? `all ${overview.harnesses.length} harnesses` + : "all harnesses"; + return (
-
-

- Recent runs -

-

- Click a run to drill into tasks, criteria, artifacts, - and logs. -

+ {/* The harness scope sits in the PAGE header, not the analytics + header, because `?h=` scopes the whole page — charts, rails, run + table, totals and ad-hoc section. A control that narrows + everything belongs beside the page title, not inside one of the + sections it filters. */} +
+
+

+ Recent runs +

+

+ Click a run to drill into tasks, criteria, artifacts, + and logs. +

+
+
{/* The analytics block — daily success / turn-budget charts, the - window selector, and the colored skill/review/tag rail — is an - internal-only surface (see lib/edition.ts). The public OSS - edition drops it so the front page is just the run list. */} - {isInternal && ( + window selector, and the colored skill/review/tag rail. Shown in + EVERY edition: it renders whatever runs the instance is pointed + at, so an OSS clone charts its own results and nothing about it + is UiPath-specific. (It was internal-only until 2026-07-30.) */}
@@ -221,7 +270,7 @@ export default async function Page({ )}{" "} - over the last {window} ·{" "} + for {scopeLabel} over the last {window} ·{" "} {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} {" · "} @@ -235,21 +284,14 @@ export default async function Page({ ) : ( <> - Success rate per{" "} - {harnessShortLabel(harness)} run across the - last {window} · {overview.runs.length} run + Success rate per run for {scopeLabel} over + the last {window} · {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} )}

-
- - -
+
- )}
@@ -393,9 +434,21 @@ export default async function Page({ {isInternal && ( - + {/* Badge + label: the badge renders + nothing for a harness with no + vendor mark, so the label is what + keeps this cell from looking + broken — and it names the harness + for screen readers, since the + logo is decorative. */} + + + {harnessShortLabel( + normalizeHarness(r.harness), + )} + )} diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts index 8dcb064c..e34665e1 100644 --- a/evalboard/lib/__tests__/harness.test.ts +++ b/evalboard/lib/__tests__/harness.test.ts @@ -1,5 +1,17 @@ import { describe, expect, test } from "vitest"; -import { DEFAULT_HARNESS, parseHarnessParam } from "../harness"; +import { + DEFAULT_HARNESS, + HARNESSES, + KNOWN_HARNESSES, + UNKNOWN_HARNESS_COLOR, + groupByHarness, + harnessColor, + harnessShortLabel, + knownIndex, + orderHarnesses, + parseHarnessParam, + parseHarnessScope, +} from "../harness"; // parseHarnessParam gates the untrusted `?h=` query param at every // harness-scoped route, and its result decides which runs are counted toward @@ -47,3 +59,296 @@ describe("parseHarnessParam", () => { expect(parseHarnessParam("a".repeat(65))).toBe(DEFAULT_HARNESS); }); }); + +// parseHarnessScope is the underlying `?h=` gate; parseHarnessParam (above) is +// derived from it. `null` means "all harnesses" — the home page's default. +describe("parseHarnessScope", () => { + test("returns null for absent / empty input (no implicit default)", () => { + expect(parseHarnessScope(undefined)).toBeNull(); + expect(parseHarnessScope([])).toBeNull(); + expect(parseHarnessScope("")).toBeNull(); + expect(parseHarnessScope(" ")).toBeNull(); + }); + + test("picks the first element of an array param", () => { + expect(parseHarnessScope(["codex", "antigravity"])).toBe("codex"); + }); + + test("trims surrounding whitespace before validating", () => { + expect(parseHarnessScope(" codex ")).toBe("codex"); + expect(parseHarnessScope([" antigravity "])).toBe("antigravity"); + }); + + test("passes through any syntactically-valid id, including unknown ones", () => { + for (const id of [ + "claude-code", + "codex", + "antigravity", + "delegate-sdk", + "some-future-harness", + "gpt-5.5", + "some_harness", + ]) { + expect(parseHarnessScope(id)).toBe(id); + } + }); + + test("returns null for a malformed id — never a partial match", () => { + for (const bad of ["has space", "a/b", "bad;rm", "a b c", "x!"]) { + expect(parseHarnessScope(bad)).toBeNull(); + } + }); + + test("enforces the 1-64 char length bound", () => { + expect(parseHarnessScope("a".repeat(64))).toBe("a".repeat(64)); + expect(parseHarnessScope("a".repeat(65))).toBeNull(); + }); +}); + +// The derivation itself: one charset, two surfaces. If these ever disagree, the +// `?h=` contract has drifted between the home page and /trends. +describe("parseHarnessParam is parseHarnessScope ?? DEFAULT_HARNESS", () => { + test("holds across valid, absent, and malformed inputs", () => { + const inputs: Array = [ + undefined, + [], + "", + " ", + "codex", + " codex ", + ["codex", "antigravity"], + "delegate-sdk", + "unknown-harness", + "has space", + "a/b", + "bad;rm", + "a".repeat(64), + "a".repeat(65), + ]; + for (const raw of inputs) { + expect(parseHarnessParam(raw)).toBe( + parseHarnessScope(raw) ?? DEFAULT_HARNESS, + ); + } + }); +}); + +// Each colour was chosen AGAINST the others by validator sweep on the all-pairs +// gate, so pin all four explicitly: a silent edit or reorder of HARNESSES must +// fail a test, not just a review. +describe("harnessColor", () => { + test("pins each harness to its vendor brand colour", () => { + expect(harnessColor("claude-code")).toBe("#D97757"); + expect(harnessColor("codex")).toBe("#000000"); + expect(harnessColor("antigravity")).toBe("#4285F4"); + expect(harnessColor("delegate-sdk")).toBe("#991B1B"); + }); + + test("all four brand colours are distinct", () => { + const colors = KNOWN_HARNESSES.map(harnessColor); + expect(new Set(colors).size).toBe(colors.length); + }); + + test("an unknown harness gets the neutral grey, never undefined", () => { + expect(harnessColor("some-future-harness")).toBe(UNKNOWN_HARNESS_COLOR); + expect(harnessColor("")).toBe(UNKNOWN_HARNESS_COLOR); + }); + + test("a degenerate id cannot reach a prototype member", () => { + // findIndex over the table, not a Record lookup: harnessColor("toString") + // must be a real hex, not Function.prototype.toString. + for (const id of ["toString", "constructor", "__proto__", "valueOf"]) { + expect(harnessColor(id)).toBe(UNKNOWN_HARNESS_COLOR); + } + }); + + test("the unknown grey is not equal to any brand colour", () => { + // Guards the grey-vs-black interaction: both are achromatic, so a future + // palette edit must not collapse them into the same value. + for (const h of KNOWN_HARNESSES) { + expect(harnessColor(h)).not.toBe(UNKNOWN_HARNESS_COLOR); + } + }); +}); + +describe("HARNESSES table integrity", () => { + test("ids are unique", () => { + const ids = HARNESSES.map((h) => h.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + test("colors are unique, case-insensitively", () => { + // Compare lowercased: the table mixes cases (#D97757 vs #991B1B), so a + // raw Set would count two case-variants of one colour as distinct. + const colors = HARNESSES.map((h) => h.color.toLowerCase()); + expect(new Set(colors).size).toBe(colors.length); + }); + + test("every row carries a short label, a full label and a colour", () => { + for (const h of HARNESSES) { + expect(h.short).toBeTruthy(); + expect(h.label).toBeTruthy(); + expect(h.color).toMatch(/^#[0-9a-fA-F]{6}$/); + } + }); + + test("every id survives the ?h= charset, so every chip is clickable", () => { + // The one cross-invariant in this module: a selector chip writes its id + // into `?h=`, and parseHarnessScope rejects anything outside this charset. + // An id with a space or a slash would render a chip that silently + // reset the scope on click, with nothing else failing. + for (const h of HARNESSES) { + expect(parseHarnessScope(h.id)).toBe(h.id); + } + }); +}); + +describe("harnessShortLabel", () => { + test("returns the table's short label for a known harness", () => { + expect(harnessShortLabel("claude-code")).toBe("Claude Code"); + expect(harnessShortLabel("codex")).toBe("Codex"); + expect(harnessShortLabel("antigravity")).toBe("Antigravity"); + expect(harnessShortLabel("delegate-sdk")).toBe("Delegate SDK"); + }); + + test("falls through to the raw id for an unknown harness", () => { + expect(harnessShortLabel("some-future-harness")).toBe( + "some-future-harness", + ); + }); +}); + +describe("knownIndex", () => { + test("gives the row position for each known harness", () => { + expect(knownIndex("claude-code")).toBe(0); + expect(knownIndex("codex")).toBe(1); + expect(knownIndex("antigravity")).toBe(2); + expect(knownIndex("delegate-sdk")).toBe(3); + }); + + test("gives -1 for anything else", () => { + expect(knownIndex("some-future-harness")).toBe(-1); + expect(knownIndex("")).toBe(-1); + expect(knownIndex("toString")).toBe(-1); + }); +}); + +describe("orderHarnesses", () => { + test("puts known harnesses in table order regardless of input order", () => { + expect( + orderHarnesses(["antigravity", "delegate-sdk", "claude-code", "codex"]), + ).toEqual(["claude-code", "codex", "antigravity", "delegate-sdk"]); + }); + + test("sorts unknown newcomers alphabetically after the known ones", () => { + expect( + orderHarnesses(["zebra-harness", "codex", "apex-harness", "claude-code"]), + ).toEqual(["claude-code", "codex", "apex-harness", "zebra-harness"]); + }); + + test("dedupes", () => { + expect(orderHarnesses(["codex", "codex", "claude-code", "codex"])).toEqual([ + "claude-code", + "codex", + ]); + }); + + test("an empty input yields an empty list, NOT the default harness", () => { + // An empty window must draw zero series rather than one phantom + // claude-code line; the non-empty fallback belongs to the selector. + expect(orderHarnesses([])).toEqual([]); + expect(orderHarnesses(new Set())).toEqual([]); + }); + + test("accepts any iterable, not just an array", () => { + expect(orderHarnesses(new Set(["codex", "claude-code"]))).toEqual([ + "claude-code", + "codex", + ]); + }); +}); + +describe("groupByHarness", () => { + const pt = (harness: string, timestamp: number) => ({ harness, timestamp }); + + test("an empty input yields no groups", () => { + expect(groupByHarness([])).toEqual([]); + }); + + test("a single harness yields one group in input order", () => { + const points = [pt("codex", 1), pt("codex", 2), pt("codex", 3)]; + expect(groupByHarness(points)).toEqual([ + { harness: "codex", points }, + ]); + }); + + test("interleaved input splits into correct groups, in table order", () => { + const groups = groupByHarness([ + pt("codex", 1), + pt("claude-code", 2), + pt("antigravity", 3), + pt("codex", 4), + pt("claude-code", 5), + ]); + expect(groups.map((g) => g.harness)).toEqual([ + "claude-code", + "codex", + "antigravity", + ]); + expect(groups.map((g) => g.points.length)).toEqual([2, 2, 1]); + }); + + test("an unknown harness sorts last", () => { + const groups = groupByHarness([ + pt("some-future-harness", 1), + pt("claude-code", 2), + ]); + expect(groups.map((g) => g.harness)).toEqual([ + "claude-code", + "some-future-harness", + ]); + }); + + test("preserves within-group order — monotonic x is what stops a zigzag", () => { + const groups = groupByHarness([ + pt("codex", 10), + pt("claude-code", 20), + pt("codex", 30), + pt("claude-code", 40), + pt("codex", 50), + ]); + expect( + groups.find((g) => g.harness === "codex")!.points.map((p) => p.timestamp), + ).toEqual([10, 30, 50]); + expect( + groups + .find((g) => g.harness === "claude-code")! + .points.map((p) => p.timestamp), + ).toEqual([20, 40]); + }); + + test("retains points with null values — grouping must not filter", () => { + const points = [ + { harness: "codex", successRate: null }, + { harness: "codex", successRate: 90 }, + ]; + expect(groupByHarness(points)[0].points).toHaveLength(2); + }); + + test("hands back fresh arrays, never an alias of the caller's", () => { + // A consumer that sorts a series in place (recharts wants monotonic x) + // must not reorder the caller's array as a side effect. The + // single-harness case is where an aliasing shortcut would be tempting. + const points = [pt("codex", 1), pt("codex", 2)]; + const groups = groupByHarness(points); + expect(groups[0].points).not.toBe(points); + expect(groups[0].points).toEqual(points); + }); + + test("does not mutate the caller's array", () => { + const points = [pt("codex", 1), pt("claude-code", 2)]; + const snapshot = [...points]; + groupByHarness(points); + expect(points).toEqual(snapshot); + }); +}); diff --git a/evalboard/lib/__tests__/module-boundaries.test.ts b/evalboard/lib/__tests__/module-boundaries.test.ts new file mode 100644 index 00000000..f941ba21 --- /dev/null +++ b/evalboard/lib/__tests__/module-boundaries.test.ts @@ -0,0 +1,78 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; + +// Two bundle-correctness invariants that were previously enforced only by +// comments, each worth a measured ~105 kB of First Load JS. Both are violated by +// adding one import, and neither shows up in tsc, the test suite, or a passing +// `next build` — only in the bundle size, which nobody reads on a green PR. + +const here = dirname(fileURLToPath(import.meta.url)); +const root = resolve(here, "../.."); + +function readSource(rel: string): string { + return readFileSync(resolve(root, rel), "utf8"); +} + +// Matches `import … from "x"`, `import "x"`, and `export … from "x"`. +function moduleSpecifiers(src: string): string[] { + return [...src.matchAll(/(?:^|\n)\s*(?:import|export)[\s\S]*?from\s+"([^"]+)"/g)] + .map((m) => m[1]) + .concat( + [...src.matchAll(/(?:^|\n)\s*import\s+"([^"]+)"/g)].map((m) => m[1]), + ); +} + +describe("lib/harness.ts is a true leaf", () => { + // Client components ("use client" charts, the selector, the badge) all import + // this module for values, not just types. lib/overview.ts transitively reaches + // node:fs, node:path and @azure/storage-blob, so a single VALUE import from + // there would drag that whole graph into the browser bundle — which is why + // groupByHarness lives in this module rather than beside its only consumer. + // Type-only imports erase at compile time; values do not. + test("has no imports at all", () => { + const specs = moduleSpecifiers(readSource("lib/harness.ts")); + expect( + specs, + "lib/harness.ts must stay dependency-free — see the KEEP IT A LEAF note at its top", + ).toEqual([]); + }); +}); + +describe("app/_components/legend.tsx stays recharts-free", () => { + // tag-rail.tsx's ChipLegend consumes LegendStrip, and the tag rail renders on + // pages with no chart at all (/runs/[id] via run-view, /trends via + // trends-view). Importing these primitives from chart-common.tsx instead + // measured +105 kB First Load JS on /runs/[id] (163 -> 268 kB) and on /trends + // (119 -> 225 kB). + const FORBIDDEN = ["recharts", "chart-common"]; + + test("imports neither recharts nor the chart module", () => { + const specs = moduleSpecifiers(readSource("app/_components/legend.tsx")); + for (const spec of specs) { + for (const bad of FORBIDDEN) { + expect( + spec.includes(bad), + `legend.tsx must not import ${spec} — it renders on chart-less pages`, + ).toBe(false); + } + } + }); + + test("its only dependency is the leaf harness module", () => { + // Keeps the guard above honest: if legend.tsx grows a dependency, that + // dependency's own graph has to be re-checked, so force the decision here. + expect(moduleSpecifiers(readSource("app/_components/legend.tsx"))).toEqual( + ["@/lib/harness"], + ); + }); + + test("the tag rail takes LegendStrip from the recharts-free module", () => { + // The regression path: re-pointing this import at chart-common is what + // pulls recharts onto /runs/[id] and /trends. + const specs = moduleSpecifiers(readSource("app/_overview/tag-rail.tsx")); + expect(specs).toContain("../_components/legend"); + expect(specs.some((s) => s.includes("chart-common"))).toBe(false); + }); +}); diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index eaa935d1..837e9d1b 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -2,12 +2,14 @@ import { describe, expect, test, vi } from "vitest"; import { buildAdhocRows, collectPipelineRuns, + harnessesInPoints, summarizeListing, turnBudgetRateForTasks, type PerRun, type RunListingRow, + type RunPoint, } from "../overview"; -import { normalizeHarness } from "../harness"; +import { normalizeHarness, orderHarnesses } from "../harness"; import type { RunOverviewTask } from "../runs"; function task(overrides: Partial): RunOverviewTask { @@ -474,3 +476,190 @@ describe("buildAdhocRows", () => { expect(row.tasksRun).toBe(2); }); }); + +// OverviewData.harnesses is the server-side answer to "which harnesses am I +// looking at", read by the page for its scope label and by the charts for their +// series. Derived from the WINDOWED, FILTERED points — not from the discovery +// list — so it can never name a harness this window contains no runs for. +describe("harnessesInPoints", () => { + function point(overrides: Partial = {}): RunPoint { + return { + runId: "2026-07-30_00-00-00", + timestamp: 0, + harness: "claude-code", + successRate: null, + turnBudgetRate: null, + ...overrides, + }; + } + + test("an empty window yields [] — not [DEFAULT_HARNESS]", () => { + // A phantom claude-code entry here would make the page claim "all 1 + // harnesses" and the chart draw one empty series for a window with no runs. + expect(harnessesInPoints([])).toEqual([]); + }); + + test("returns the distinct harnesses present, deduped", () => { + expect( + harnessesInPoints([ + point({ harness: "codex" }), + point({ harness: "codex" }), + point({ harness: "claude-code" }), + ]), + ).toEqual(["claude-code", "codex"]); + }); + + test("orders known harnesses by the shared display order, not by first appearance", () => { + expect( + harnessesInPoints([ + point({ harness: "antigravity" }), + point({ harness: "claude-code" }), + point({ harness: "codex" }), + ]), + ).toEqual(["claude-code", "codex", "antigravity"]); + }); + + test("sorts an unknown harness after the known ones", () => { + expect( + harnessesInPoints([ + point({ harness: "some-future-harness" }), + point({ harness: "codex" }), + ]), + ).toEqual(["codex", "some-future-harness"]); + }); + + test("a single-harness window yields exactly that harness", () => { + expect(harnessesInPoints([point({ harness: "codex" })])).toEqual(["codex"]); + }); +}); + +// The switcher's chip set must be STABLE under selection. It is fed +// windowHarnesses, not harnesses: the latter is derived from the filtered +// points, so with `?h=codex` active it collapses to ["codex"] and every other +// chip would vanish the moment one was selected — stranding the user on a scope +// they cannot leave without hand-editing the URL. These pin the distinction. +describe("harnesses vs windowHarnesses", () => { + function run(harness: string | null, id = "2026-07-30_00-00-00"): PerRun { + return { + id, + overview: { + harness, + tasks: [task({ status: "SUCCESS" })], + totalCostUsd: null, + taskDurationSeconds: null, + startedAt: null, + } as unknown as PerRun["overview"], + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + }; + } + + // The derivation getOverview applies for windowHarnesses: every non-adhoc + // run in the window with a readable overview, BEFORE the harness filter. + function windowHarnessesOf(runs: PerRun[]): string[] { + return orderHarnesses( + runs + .filter((r) => !r.adhoc && r.overview) + .map((r) => normalizeHarness(r.overview!.harness)), + ); + } + + const window = [ + run("claude-code", "2026-07-28_00-00-00"), + run("codex", "2026-07-29_00-00-00"), + run("delegate-sdk", "2026-07-30_00-00-00"), + ]; + + test("windowHarnesses lists every harness in the window", () => { + expect(windowHarnessesOf(window)).toEqual([ + "claude-code", + "codex", + "delegate-sdk", + ]); + }); + + test("it is IDENTICAL whichever harness is selected — the anti-shift rule", () => { + // Contrast the two derivations over the SAME scoped input. The old one + // (post-filter) collapses to the selected harness and deletes the other + // chips; the new one (pre-filter) is invariant. Comparing the pre-filter + // result to itself would be vacuous, so apply the real harness filter + // and show only one of the two survives it. + const unscoped = windowHarnessesOf(window); + expect(unscoped).toHaveLength(3); + + for (const scope of ["claude-code", "codex", "delegate-sdk"]) { + const afterHarnessFilter = window.filter( + (r) => normalizeHarness(r.overview!.harness) === scope, + ); + + // What the chips used to be built from — collapses to one. + const oldWay = windowHarnessesOf(afterHarnessFilter); + expect(oldWay).toEqual([scope]); + + // What they are built from now: computed before that filter, so the + // full set survives and every chip stays clickable. + expect( + windowHarnessesOf(window), + `chip set shifted when scoped to ${scope}`, + ).toEqual(unscoped); + } + }); + + test("by contrast, harnesses (the plotted set) DOES narrow with the scope", () => { + // This is correct and is what "all N harnesses" counts — it must equal + // the number of lines drawn, which is 1 when scoped. + const scopedPoints = [ + { harness: "codex" } as RunPoint, + ]; + expect(harnessesInPoints(scopedPoints)).toEqual(["codex"]); + }); + + test("a legacy unstamped run folds into claude-code, not a phantom chip", () => { + expect(windowHarnessesOf([run(null)])).toEqual(["claude-code"]); + }); + + test("ad-hoc runs contribute no chip", () => { + const adhoc = { ...run("delegate-sdk"), adhoc: true }; + expect(windowHarnessesOf([run("codex"), adhoc])).toEqual(["codex"]); + }); +}); + +// `?h=` scopes the WHOLE page, so the run table, its totals and the ad-hoc +// section all narrow with it — not just the charts. These pin the two filters +// getRunListing and getAdhocRunListing apply, over the same shape of input. +describe("harness scoping of the run listings", () => { + function scoped(runs: PerRun[], harness: string | null): PerRun[] { + return runs.filter( + (r) => + harness == null || + normalizeHarness(r.overview?.harness) === harness, + ); + } + + const runs: PerRun[] = [ + { id: "a", overview: { harness: "claude-code" } as PerRun["overview"], reviewTagCounts: {}, reviewTagsByTask: {} }, + { id: "b", overview: { harness: "codex" } as PerRun["overview"], reviewTagCounts: {}, reviewTagsByTask: {} }, + { id: "c", overview: { harness: null } as PerRun["overview"], reviewTagCounts: {}, reviewTagsByTask: {} }, + ]; + + test("null lists every run — the All scope", () => { + expect(scoped(runs, null).map((r) => r.id)).toEqual(["a", "b", "c"]); + }); + + test("a harness lists only its own runs", () => { + expect(scoped(runs, "codex").map((r) => r.id)).toEqual(["b"]); + }); + + test("an unstamped legacy run counts as claude-code, not as its own scope", () => { + // normalizeHarness folds null -> claude-code, so run "c" must appear + // under the Claude Code scope rather than becoming unreachable. + expect(scoped(runs, "claude-code").map((r) => r.id)).toEqual(["a", "c"]); + }); + + test("a harness with no runs yields an empty table, not everything", () => { + // The failure mode worth guarding: a falsy-check bug here would make an + // unmatched scope fall through and list every run under the wrong label. + expect(scoped(runs, "delegate-sdk")).toEqual([]); + }); +}); diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 648f49f6..75d8a353 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -1,97 +1,110 @@ -import { readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; import { describe, expect, test } from "vitest"; -import { PRICING } from "../pricing"; +import { readTable } from "../../scripts/gen-pricing.mjs"; +import { PRICING, resolvePricing } from "../pricing"; -// Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative -// Python table in src/coder_eval/pricing.py. It exists so the frontend's -// "estimated" USD figures agree with the backend's authoritative Cost on the -// same tokens. +// Drift guard: lib/pricing.generated.ts is generated from the authoritative +// Python table (src/coder_eval/pricing.py) by scripts/gen-pricing.mjs. Nobody +// types a rate on this side, so the only failure mode left is a STALE artifact — +// pricing.py changed and `pnpm gen:pricing` wasn't run. // -// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts -// must exist in pricing.py with identical rates (a frontend rate that disagrees -// with the backend, or prices a model the backend doesn't, fails the build). -// The frontend is NOT required to mirror every backend model — it only needs to -// price the ones it displays, and the backend legitimately prices models the -// evalboard never renders. (Exact-match was too strict: it forced unrelated -// backend model additions into this file to keep the build green.) - -const here = dirname(fileURLToPath(import.meta.url)); -const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py"); - -// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), -const ROW_RE = - /"([^"]+)":\s*ModelPricing\(\s*([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\s*\)/g; - -function parsePythonTable(): Record< - string, - [number, number, number, number] -> { - const src = readFileSync(PY_PATH, "utf8"); - const out: Record = {}; - for (const m of src.matchAll(ROW_RE)) { - out[m[1]] = [ - Number(m[2]), - Number(m[3]), - Number(m[4]), - Number(m[5]), - ]; - } - return out; -} +// Semantics are EXACT-MATCH, which the old hand-copied mirror couldn't sustain +// (it needed an explicit not-mirrored allowlist that 17 models drifted past, two +// of whose own entries were models real runs use — silently rendering "—" for +// their cost). Generation makes exact-match free: one assertion subsumes +// orphans, missing entries, and rate drift, with no allowlist to maintain. +// +// This imports readTable, NOT parsePythonTable, and the distinction is the whole +// guard: parsePythonTable on both sides would compare two products of the same +// regex, so a row that regex cannot match is missing from the artifact AND from +// the expectation — deep-equal passes while the board renders "—" for that model +// forever. readTable cross-checks the matched-row count against the number of +// `ModelPricing(` constructions in the file and throws if they disagree. -describe("pricing.ts ↔ pricing.py parity", () => { - const py = parsePythonTable(); +describe("pricing.generated.ts ↔ pricing.py parity", () => { + // Throws if the parser skipped a row, so this doubles as the coverage check. + const py = readTable(); test("parses a non-trivial Python table", () => { // Guard against a regex/path regression silently passing the test. expect(Object.keys(py).length).toBeGreaterThan(10); }); - test("every model in lib/pricing.ts exists in pricing.py", () => { - const orphans = Object.keys(PRICING).filter((m) => !(m in py)); + test("the generated table is current with pricing.py", () => { + const generated = Object.fromEntries( + Object.entries(PRICING).map(([model, p]) => [ + model, + [ + p.inputPerMTok, + p.outputPerMTok, + p.cacheWritePerMTok, + p.cacheReadPerMTok, + ], + ]), + ); expect( - orphans, - `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`, - ).toEqual([]); + generated, + "lib/pricing.generated.ts is stale — run `pnpm gen:pricing`", + ).toEqual(py); }); +}); - test("shared models have identical input/output/cacheWrite/cacheRead rates", () => { - for (const [model, ts] of Object.entries(PRICING)) { - const rates = py[model]; - expect(rates, `not priced in pricing.py: ${model}`).toBeDefined(); - expect([ - ts.inputPerMTok, - ts.outputPerMTok, - ts.cacheWritePerMTok, - ts.cacheReadPerMTok, - ]).toEqual(rates); +// Behaviour the generated table has to preserve. Deliberately NOT a second copy +// of the rate card: every expectation below is derived from pricing.py via +// `py[...]`, never a literal. The test above already proves every rate matches, +// so hardcoding one here would mean a legitimate vendor repricing breaks the +// suite even after a correct regeneration — exactly the two-places problem +// generation exists to remove. What these pin is lookup *logic* the deep-equal +// cannot see: the undated fallback, a genuine zero surviving, float precision. +describe("resolvePricing over the generated table", () => { + const py = readTable(); + + test("a deleted dated key still prices via the undated fallback", () => { + // "claude-opus-4-6-20250514" was a dead entry (absent from pricing.py) + // and redundant: resolvePricing strips a trailing -YYYYMMDD. Deleting it + // must change no rendered figure — the dated id still resolves, and to + // exactly the undated key's rate. + for (const undated of ["claude-opus-4-6", "claude-sonnet-4-6"]) { + const dated = resolvePricing(`${undated}-20250514`); + expect(dated, `${undated}-20250514 no longer resolves`).not.toBeNull(); + expect(dated).toEqual(resolvePricing(undated)); } }); - // Python-priced models we deliberately do NOT mirror to the frontend: heavy - // frontier Claude/GPT variants the evalboard never runs, so pricing them here - // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model - // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build — - // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids - // that previously rendered "—" for cost). - const DELIBERATELY_UNMIRRORED = new Set([ - "claude-sonnet-5", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.4-pro", - "gpt-5.5-pro", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - ]); + test("a model the old allowlist suppressed is now priced", () => { + // By far the most-run model in the recorded run data, yet it sat in the + // old not-mirrored allowlist — so the evalboard rendered "—" for cost on + // runs it actually executed. This is the regression that must not return. + expect(resolvePricing("gpt-5.6-terra")).not.toBeNull(); + }); - test("every pricing.py model is mirrored in pricing.ts or explicitly unmirrored", () => { - const missing = Object.keys(py).filter((m) => !(m in PRICING) && !DELIBERATELY_UNMIRRORED.has(m)); - expect( - missing, - `priced in pricing.py but missing from pricing.ts — mirror it or add to DELIBERATELY_UNMIRRORED: ${missing.join(", ")}`, - ).toEqual([]); + test("a legitimately-zero rate survives as 0, not undefined", () => { + // Bedrock publishes no prompt-cache rate for the open-weight models, so + // cache-read is a real 0 — generation must not drop it as falsy. Asserted + // against pricing.py's own value, and separately that it IS zero there, + // so the test still means something if that rate ever changes. + for (const model of ["deepseek.v3.2", "zai.glm-5"]) { + expect(py[model][3]).toBe(0); + expect(resolvePricing(model)?.cacheReadPerMTok).toBe(py[model][3]); + } + }); + + test("fractional rates round-trip exactly through generation", () => { + // The generator emits numbers via string interpolation, so a precision + // loss would show up as generated !== parsed for these long decimals. + for (const model of ["z-ai/glm-5.2", "deepseek/deepseek-v4-pro"]) { + const p = resolvePricing(model); + expect(p).not.toBeNull(); + expect([ + p!.inputPerMTok, + p!.outputPerMTok, + p!.cacheWritePerMTok, + p!.cacheReadPerMTok, + ]).toEqual(py[model]); + // And that at least one of them really is a long decimal, so this + // test cannot pass vacuously if pricing.py switches to round numbers. + expect( + py[model].some((r) => String(r).replace("0.", "").length >= 4), + ).toBe(true); + } }); }); diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66302cf1..4125e6c7 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -36,7 +36,11 @@ describe("resolvePricing", () => { }); test("knows the current default opus id", () => { - expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75); + // Rate-agnostic on purpose: pricing-parity.test.ts already proves every + // rate equals pricing.py, so pinning a number here would break the suite + // on a legitimate vendor repricing. What matters at this seam is that the + // current default Opus id resolves at all. + expect(resolvePricing("claude-opus-4-8")).not.toBeNull(); }); test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => { diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts index bf36cb28..047687c7 100644 --- a/evalboard/lib/harness.ts +++ b/evalboard/lib/harness.ts @@ -1,31 +1,213 @@ -// Canonical harness (coder-eval AgentKind) constants. Leaf module with no -// server- or client-only dependencies, so both the client badge/selector and -// the server data layer can import it without pulling node-only code into the -// client bundle or creating an import cycle. - -// The harnesses the nightly rotates through, in preferred display order. This -// is only a display-order hint / default source — the switcher is data-driven -// (see listRecentHarnesses), so a new harness like "delegate" surfaces -// automatically without editing this list. -export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; -export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; +// Canonical harness (coder-eval AgentKind) knowledge. Leaf module with no +// server- or client-only dependencies, so both the client badge/selector/charts +// and the server data layer can import it without pulling node-only code into +// the client bundle or creating an import cycle. +// +// KEEP IT A LEAF: no `next/*`, no `node:*`, and nothing from ./runs, ./blob, or +// ./overview. `groupByHarness` lives here rather than in lib/overview.ts for +// exactly this reason — overview.ts reaches node:fs and @azure/storage-blob +// transitively, and a *value* import from a "use client" chart would drag that +// whole graph into the browser bundle (type-only imports erase; values do not). + +// One ordered table is the single source for a harness's id, display order, +// labels, logo, and series colour. These used to be two parallel lists in two +// files — display order here, labels + logo paths in harness-badge.tsx — and +// adding colour plus a 4th harness would have made it four lists whose indices +// had to be hand-aligned. Row order IS the display order, so there is no slot +// map to keep in sync. +// +// COLOURS — each harness wears its vendor's brand colour, so a line reads as its +// harness without a legend lookup. The set was chosen by validator sweep on the +// strict all-pairs gate, not by eye, which changed two of the four: +// +// node /scripts/validate_palette.js \ +// "#D97757,#000000,#4285F4,#991B1B" --mode light --surface "#ffffff" --pairs all +// [FAIL] Lightness band outside band: [["#000000",0]] +// [FAIL] Chroma floor below floor (reads gray): [["#000000",0]] +// [PASS] CVD separation worst all-pairs #991B1B<->#D97757 dE 22.6 (deutan) +// [PASS] Normal-vision floor worst all-pairs #991B1B<->#D97757 dE 23.3 +// [PASS] Contrast vs surface all 4 >= 3:1 +// +// The two FAILs on pure black are WAIVED DELIBERATELY — do not "fix" them by +// substituting a dark green (#047857 was evaluated: ALL PASS, and rejected +// because vendor recognition outweighed a proxy gate). Both failing checks are +// *proxies* for "will these hues be distinguishable" and assume a chromatic +// categorical set. Every check that measures distinguishability directly — CVD +// separation, normal-vision floor, contrast — passes, and black is not even the +// worst pair (violet<->blue is, comfortably above both floors). Black separates +// on *lightness*, the one channel colour-vision deficiency does not touch, and +// sits at 21:1 on white. The residual risk is semantic, not perceptual: black +// reads as ink if the plot area ever gains a black element, so revisit then. +// Delegate's slot must be a DARK red. Everything in the red/orange family is +// competing with Anthropic coral, so brightness is what decides it — swept +// against the fixed three (higher dE is better; the floors are 6-8 for CVD and +// 15 for normal vision): +// +// #EF4444 red-500 CVD 3.3 / NV 9.3 unusable +// #FA4616 UiPath brand CVD 5.1 / NV 9.5 unusable +// #FF0000 pure red CVD 5.5 / NV 13.8 fails +// #E11D48 rose-600 CVD 8.0 / NV 14.0 fails the NV floor +// #DC2626 red-600 CVD 9.1 / NV 13.2 fails the NV floor +// #B91C1C red-700 CVD 16.2 / NV 18.0 passes, tighter margin +// #991B1B red-800 CVD 22.6 / NV 23.3 SHIPPED — best margin +// +// So the naive reds all collide with coral, several of them worse than the +// UiPath brand orange that was rejected for exactly this reason. red-800 is far +// enough down the lightness axis to separate cleanly, and it beats the violet +// (#4a3aa7) this replaced, whose worst pair was dE 18.5 against Google blue. +export const HARNESSES = [ + { + id: "claude-code", + short: "Claude Code", + label: "Claude Code · Anthropic", + logo: "/harness/claude-code.png", + color: "#D97757", // Anthropic coral + }, + { + id: "codex", + short: "Codex", + label: "Codex · OpenAI", + logo: "/harness/codex.png", + color: "#000000", // OpenAI black — proxy gates waived, see above + }, + { + id: "antigravity", + short: "Antigravity", + label: "Antigravity · Google Gemini", + logo: "/harness/antigravity.png", + color: "#4285F4", // Google blue + }, + { + id: "delegate-sdk", + short: "Delegate SDK", + label: "Delegate SDK · UiPath", + logo: null, // no vendor mark shipped yet + color: "#991B1B", // deep red — see the red sweep above + }, +] as const satisfies readonly { + id: string; + short: string; + label: string; + logo: string | null; + color: string; +}[]; + +export type KnownHarness = (typeof HARNESSES)[number]["id"]; + +// Display-order hint / default source, derived from the table. Deliberately NOT +// a whitelist — the switcher is data-driven (see listRecentHarnesses), so a new +// harness surfaces automatically without editing anything here. +export const KNOWN_HARNESSES: readonly KnownHarness[] = HARNESSES.map( + (h) => h.id, +); // A run with no RunConfig harness predates the stamp; every such run was // claude-code (the only nightly harness before codex/antigravity), so -// null/undefined folds to the default. Mirrors HarnessBadge's default. +// null/undefined folds to the default. HarnessBadge and the run table both +// route through this, so it is the one definition of that fold. export const DEFAULT_HARNESS = "claude-code"; export function normalizeHarness(harness: string | null | undefined): string { return harness ?? DEFAULT_HARNESS; } -// Accept any syntactically-plausible harness id, defaulting to claude-code when -// absent or malformed. Deliberately NOT whitelisted against KNOWN_HARNESSES so -// a newly-added harness (e.g. "delegate") is selectable the moment its runs -// exist — the value is only ever compared for equality against a run's stamped -// harness, never used in a path or query, so a bounded charset is enough. -export function parseHarnessParam(raw: string | string[] | undefined): string { +// The one expression of "is this harness known", and its position in display +// order. Nothing else re-derives it. -1 when unknown. +export function knownIndex(h: string): number { + return HARNESSES.findIndex((x) => x.id === h); +} + +// The table row for a harness, or undefined when unknown. A linear find rather +// than a Record lookup so a degenerate id ("toString", "constructor") cannot +// resolve to an inherited prototype member. +export function harnessRow(h: string) { + return HARNESSES.find((x) => x.id === h); +} + +// Short human label ("Claude Code") for selectors and prose; unknown ids fall +// through to the raw id. +export function harnessShortLabel(h: string): string { + return harnessRow(h)?.short ?? h; +} + +// Neutral grey for any harness not in the table — reads as "untracked" rather +// than claiming a brand. 4.8:1 on white, and it separates from the black series +// on lightness (L 0.55 vs 0). A lighter grey is not an option: #9ca3af is 2.1:1, +// too faint for a 2px line. +// +// CAVEAT: this is also the axis-tick fill, so a 5th harness's line would render +// in exactly the tick-label grey and could read as chart chrome rather than data. +// Unreachable while the table covers every harness that runs; if a 5th appears, +// give it a table row with its own colour rather than letting it fall through. +export const UNKNOWN_HARNESS_COLOR = "#6b7280"; + +// Pure and total: colour follows the ENTITY, never its rank in the visible set, +// so a selector chip and a chart line can never disagree, and the four known +// harnesses never repaint when the visible set changes. +export function harnessColor(h: string): string { + return harnessRow(h)?.color ?? UNKNOWN_HARNESS_COLOR; +} + +// Known harnesses first in table order, then any newcomers alphabetically, so +// the result is deterministic but self-extending. Dedupes, so a raw array of +// per-point harnesses works. +// +// Deliberately does NOT apply the "never empty -> [DEFAULT_HARNESS]" rule: that +// belongs to the selector-feeding call site, not to grouping chart points, where +// an empty window must yield zero series rather than one phantom Claude Code +// series. +export function orderHarnesses(seen: Iterable): string[] { + return [...new Set(seen)].sort((a, b) => { + const ia = knownIndex(a); + const ib = knownIndex(b); + if (ia >= 0 && ib >= 0) return ia - ib; + if (ia >= 0) return -1; + if (ib >= 0) return 1; + return a.localeCompare(b); + }); +} + +// Split points into one group per distinct harness, groups ordered by +// orderHarnesses. Generic over the minimal structural constraint so this module +// needs no import from the data layer. +// +// Each group preserves incoming point order, which is load-bearing: recharts +// needs monotonic x within a series or it draws a zigzag. getOverview already +// sorts ascending by timestamp before handing points over. +export function groupByHarness( + points: readonly T[], +): Array<{ harness: string; points: T[] }> { + const byHarness = new Map(); + for (const p of points) { + const group = byHarness.get(p.harness); + if (group) group.push(p); + else byHarness.set(p.harness, [p]); + } + return orderHarnesses(byHarness.keys()).map((harness) => ({ + harness, + // Non-null: every key came from the map itself. + points: byHarness.get(harness)!, + })); +} + +// The `?h=` gate. Accept any syntactically-plausible harness id; `null` means +// "all harnesses", which is what the home page renders when `?h=` is absent. +// Deliberately NOT whitelisted against KNOWN_HARNESSES so a newly-added harness +// is selectable the moment its runs exist — the value is only ever compared for +// equality against a run's stamped harness, never used in a path or query, so a +// bounded charset is enough. +export function parseHarnessScope( + raw: string | string[] | undefined, +): string | null { const v = Array.isArray(raw) ? raw[0] : raw; - if (typeof v !== "string") return DEFAULT_HARNESS; + if (typeof v !== "string") return null; const trimmed = v.trim(); - return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS; + return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : null; +} + +// The single-harness parser for /trends, /watchlist and /path-to-ga, which keep +// "exactly one harness" semantics. Derived from parseHarnessScope so the charset +// and length bound exist once and cannot drift apart; observably identical to +// the previous standalone implementation for every input. +export function parseHarnessParam(raw: string | string[] | undefined): string { + return parseHarnessScope(raw) ?? DEFAULT_HARNESS; } diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 4e6cfa6b..f2bf039a 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,12 +15,17 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; -import { DEFAULT_HARNESS, KNOWN_HARNESSES, normalizeHarness } from "./harness"; +import { DEFAULT_HARNESS, normalizeHarness, orderHarnesses } from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { runId: string; timestamp: number; // ms since epoch (UTC); used as the chart x-coordinate + // Which harness produced this run, already normalized — legacy runs that + // predate the RunConfig stamp fold to claude-code via normalizeHarness. + // Required, not optional: getOverview always has a value, so optionality + // would only invite `?? DEFAULT_HARNESS` fallbacks at every consumer. + harness: string; successRate: number | null; // % of budgeted tasks whose visible turns stayed within 1.5× their // expected_turns budget. Only tasks carrying a positive expected_turns @@ -82,12 +87,28 @@ export interface OverviewData { runs: RunPoint[]; // one point per run, no daily aggregation windowStart: number; // ms — chart x-domain start windowEnd: number; // ms — chart x-domain end + // The harnesses actually PLOTTED — derived from the windowed, filtered + // points, so it always equals the number of lines the chart draws. That is + // what the page's "all N harnesses" subtitle counts. Empty window → []. + harnesses: string[]; + // Every harness present in the window, ignoring the harness/tag/q filters. + // Feeds the switcher, which must offer a stable set: `harnesses` above + // collapses to just the active one when `?h=` is set, so using it for the + // chips would delete every other chip the moment one is selected. + windowHarnesses: string[]; skills: TagCount[]; taskTags: TagCount[]; reviewTags: TagCount[]; activeTag: string | null; } +// The harnesses present in a set of chart points, in display order. Extracted so +// the derivation is unit-testable without touching the blob layer (getOverview +// itself is not unit-tested here, matching this module's convention). +export function harnessesInPoints(points: readonly RunPoint[]): string[] { + return orderHarnesses(points.map((p) => p.harness)); +} + export interface RunListingRow { id: string; // When a tag/q filter is active, every metric is scoped to matching tasks. @@ -295,13 +316,13 @@ async function listRecentHarnessesInner(): Promise { if (r.overview) seen.add(normalizeHarness(r.overview.harness)); } // Known harnesses first (stable display order), then any newcomers - // (alphabetical) so the list is deterministic but self-extending. - const known = KNOWN_HARNESSES.filter((h) => seen.has(h)); - const extras = [...seen] - .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h)) - .sort(); - const ordered = [...known, ...extras]; + // (alphabetical) so the list is deterministic but self-extending. One shared + // comparator with the chart's series ordering, so the switcher and the + // legend can never disagree about where a newcomer sorts. + const ordered = orderHarnesses(seen); // Never hand back an empty list — the default must always be selectable. + // This rule belongs to the switcher, not to orderHarnesses: an empty window + // must yield zero chart series rather than one phantom claude-code line. return ordered.length > 0 ? ordered : [DEFAULT_HARNESS]; } @@ -541,11 +562,21 @@ export async function getOverview( // Ad-hoc runs never feed the daily chart or the tag rails — they're not // pipeline cadence. (Non-date-named ones are already pruned upstream by // listRunIdsInWindow; this also drops date-named runs flagged adhoc.) - const perRun = (await loadWindowData(window)).filter( + const windowRuns = (await loadWindowData(window)).filter((r) => !r.adhoc); + // Captured BEFORE the harness filter, and deliberately ignoring tag/q: this + // is what the switcher offers, so it must not depend on what is currently + // selected. Deriving it after the filter would collapse it to the active + // harness and make every other chip vanish on selection — stranding the user + // on a scope they cannot leave without editing the URL. + const windowHarnesses = orderHarnesses( + windowRuns + .filter((r) => r.overview) + .map((r) => normalizeHarness(r.overview!.harness)), + ); + const perRun = windowRuns.filter( (r) => - !r.adhoc && - (harness == null || - normalizeHarness(r.overview?.harness) === harness), + harness == null || + normalizeHarness(r.overview?.harness) === harness, ); const needle = q?.trim().toLowerCase() || null; @@ -580,6 +611,7 @@ export async function getOverview( runPoints.push({ runId: id, timestamp: date.getTime(), + harness: normalizeHarness(overview.harness), successRate: rate, turnBudgetRate: turnBudgetRateForTasks(matching), }); @@ -599,6 +631,8 @@ export async function getOverview( runs: runPoints, windowStart, windowEnd, + harnesses: harnessesInPoints(runPoints), + windowHarnesses, skills, taskTags, reviewTags, @@ -684,11 +718,20 @@ export async function getRunListing( tag: string | null, q: string | null, limit: number | null, // null = unlimited + // When set, list only that harness's runs. null = all harnesses. Applied + // alongside the ad-hoc exclusion so totalInWindow and the totals rollup are + // both scoped — "N in 30d" must count what the table can actually show. + harness: string | null = null, ): Promise { // Exclude ad-hoc runs from the main listing — they appear in their own // section (getAdhocRunListing). totalInWindow therefore counts only // pipeline runs, matching the chart above it. - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); // Run IDs are timestamped — newest first by lexical compare. const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); const totalInWindow = sorted.length; @@ -832,8 +875,20 @@ export function buildAdhocRows( // not the id, so we can't window by id and still show the most recent): the // ad-hoc set is small by construction (manual uploads only) and per-run reads // are memoized for 5 min, so a warm front page pays no extra IO. -export async function getAdhocRunListing(limit: number | null): Promise { +export async function getAdhocRunListing( + limit: number | null, + // Scoped by the same `?h=` as the rest of the page: an ad-hoc run carries a + // harness stamp like any other, and leaving this section unfiltered would + // show codex uploads under a Claude Code scope with no column to explain why. + harness: string | null = null, +): Promise { const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) == null); const perRun = await mapWithConcurrency(ids, FETCH_CONCURRENCY, cachedLoadPerRun); - return buildAdhocRows(perRun, limit); + const scoped = + harness == null + ? perRun + : perRun.filter( + (r) => normalizeHarness(r.overview?.harness) === harness, + ); + return buildAdhocRows(scoped, limit); } diff --git a/evalboard/lib/pricing-types.ts b/evalboard/lib/pricing-types.ts new file mode 100644 index 00000000..120fba36 --- /dev/null +++ b/evalboard/lib/pricing-types.ts @@ -0,0 +1,12 @@ +// The per-MTok rate shape, shared by the generated table +// (lib/pricing.generated.ts) and the hand-written lookup/cost helpers in +// lib/pricing.ts. It lives in its own leaf module so the generated file can +// import it without a cycle (pricing.ts → generated → pricing.ts). +// Consumers should keep importing `Pricing` from lib/pricing.ts. + +export interface Pricing { + inputPerMTok: number; + outputPerMTok: number; + cacheWritePerMTok: number; + cacheReadPerMTok: number; +} diff --git a/evalboard/lib/pricing.generated.ts b/evalboard/lib/pricing.generated.ts new file mode 100644 index 00000000..fbbefc29 --- /dev/null +++ b/evalboard/lib/pricing.generated.ts @@ -0,0 +1,345 @@ +// GENERATED BY scripts/gen-pricing.mjs — DO NOT EDIT. +// Source of truth: src/coder_eval/pricing.py. +// To change a rate: edit that file, then run `pnpm gen:pricing`. +// Row order mirrors pricing.py so regeneration diffs stay minimal. + +import type { Pricing } from "./pricing-types"; + +export const GENERATED_PRICING: Record = { + "claude-fable-5": { + inputPerMTok: 10, + outputPerMTok: 50, + cacheWritePerMTok: 12.5, + cacheReadPerMTok: 1, + }, + "claude-opus-5": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-8": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-7": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-6": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-5": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-5-20251101": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-1": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-opus-4": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-opus-4-20250514": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-sonnet-5": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-6": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-5": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-5-20250929": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-20250514": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-haiku-4-5": { + inputPerMTok: 1, + outputPerMTok: 5, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.1, + }, + "claude-haiku-4-5-20251001": { + inputPerMTok: 1, + outputPerMTok: 5, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.1, + }, + "claude-haiku-3-5": { + inputPerMTok: 0.8, + outputPerMTok: 4, + cacheWritePerMTok: 1, + cacheReadPerMTok: 0.08, + }, + "claude-3-7-sonnet-20250219": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-5-sonnet-20241022": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-5-sonnet-20240620": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-opus-20240229": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-3-sonnet-20240229": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-haiku-20240307": { + inputPerMTok: 0.25, + outputPerMTok: 1.25, + cacheWritePerMTok: 0.3, + cacheReadPerMTok: 0.03, + }, + "gpt-5-codex": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex-max": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex-mini": { + inputPerMTok: 0.25, + outputPerMTok: 2, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "codex-mini-latest": { + inputPerMTok: 1.5, + outputPerMTok: 6, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.375, + }, + "gpt-5.3-codex": { + inputPerMTok: 1.75, + outputPerMTok: 14, + cacheWritePerMTok: 1.75, + cacheReadPerMTok: 0.175, + }, + "gpt-5.2-codex": { + inputPerMTok: 1.75, + outputPerMTok: 14, + cacheWritePerMTok: 1.75, + cacheReadPerMTok: 0.175, + }, + "gpt-5.4": { + inputPerMTok: 2.5, + outputPerMTok: 15, + cacheWritePerMTok: 2.5, + cacheReadPerMTok: 0.25, + }, + "gpt-5.5": { + inputPerMTok: 5, + outputPerMTok: 30, + cacheWritePerMTok: 5, + cacheReadPerMTok: 0.5, + }, + "gpt-5.5-pro": { + inputPerMTok: 30, + outputPerMTok: 180, + cacheWritePerMTok: 30, + cacheReadPerMTok: 3, + }, + "gpt-5.4-pro": { + inputPerMTok: 30, + outputPerMTok: 180, + cacheWritePerMTok: 30, + cacheReadPerMTok: 3, + }, + "gpt-5.4-mini": { + inputPerMTok: 0.75, + outputPerMTok: 4.5, + cacheWritePerMTok: 0.75, + cacheReadPerMTok: 0.075, + }, + "gpt-5.4-nano": { + inputPerMTok: 0.2, + outputPerMTok: 1.25, + cacheWritePerMTok: 0.2, + cacheReadPerMTok: 0.02, + }, + "gpt-5.6-sol": { + inputPerMTok: 5, + outputPerMTok: 30, + cacheWritePerMTok: 5, + cacheReadPerMTok: 0.5, + }, + "gpt-5.6-terra": { + inputPerMTok: 2.5, + outputPerMTok: 15, + cacheWritePerMTok: 2.5, + cacheReadPerMTok: 0.25, + }, + "gpt-5.6-luna": { + inputPerMTok: 1, + outputPerMTok: 6, + cacheWritePerMTok: 1, + cacheReadPerMTok: 0.1, + }, + "gemini-3.6-flash": { + inputPerMTok: 1.5, + outputPerMTok: 7.5, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.15, + }, + "gemini-3.5-flash": { + inputPerMTok: 1.5, + outputPerMTok: 9, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.15, + }, + "gemini-3.5-flash-lite": { + inputPerMTok: 0.3, + outputPerMTok: 2.5, + cacheWritePerMTok: 0.3, + cacheReadPerMTok: 0.03, + }, + "gemini-3.1-pro-preview": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "gemini-3.1-pro-preview-customtools": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "gemini-3.1-flash-lite": { + inputPerMTok: 0.25, + outputPerMTok: 1.5, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "gemini-3.1-flash-lite-preview": { + inputPerMTok: 0.25, + outputPerMTok: 1.5, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "gemini-3-flash-preview": { + inputPerMTok: 0.5, + outputPerMTok: 3, + cacheWritePerMTok: 0.5, + cacheReadPerMTok: 0.05, + }, + "gemini-3-pro-preview": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "deepseek.v3.2": { + inputPerMTok: 0.74, + outputPerMTok: 2.22, + cacheWritePerMTok: 0.74, + cacheReadPerMTok: 0, + }, + "zai.glm-5": { + inputPerMTok: 1.2, + outputPerMTok: 3.84, + cacheWritePerMTok: 1.2, + cacheReadPerMTok: 0, + }, + "moonshotai.kimi-k2.5": { + inputPerMTok: 0.72, + outputPerMTok: 3.6, + cacheWritePerMTok: 0.72, + cacheReadPerMTok: 0, + }, + "moonshotai/kimi-k3": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3, + cacheReadPerMTok: 0.3, + }, + "z-ai/glm-5.2": { + inputPerMTok: 0.7168, + outputPerMTok: 2.2528, + cacheWritePerMTok: 0.7168, + cacheReadPerMTok: 0.13312, + }, + "deepseek/deepseek-v4-pro": { + inputPerMTok: 0.435, + outputPerMTok: 0.87, + cacheWritePerMTok: 0.435, + cacheReadPerMTok: 0.003625, + }, +}; diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 93ac23c8..431e2fd5 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -1,84 +1,24 @@ -// Per-million-token prices and cost math. Ported from -// src/coder_eval/pricing.py — keep in sync when that table changes. -// Source: Anthropic / OpenAI / Google public pricing. +// Per-million-token prices and cost math. // -// This is the single source of truth for rates on the frontend: the +// Rates are NOT written here: they are generated from the authoritative Python +// table (src/coder_eval/pricing.py) into lib/pricing.generated.ts. To change a +// rate, edit that file and run `pnpm gen:pricing` — never hand-edit a rate on +// this side. lib/__tests__/pricing-parity.test.ts fails the build if the +// generated artifact drifts from pricing.py. +// +// This module is the single source of truth for rates *on the frontend*: the // cascade-aware thinking-cost simulator (lib/thinkingSim.ts) and the // per-message cost column (lib/runs.ts) both price against this table, so a -// model added or repriced here updates both at once. +// model added or repriced in pricing.py updates both at once. -export interface Pricing { - inputPerMTok: number; - outputPerMTok: number; - cacheWritePerMTok: number; - cacheReadPerMTok: number; -} +import { GENERATED_PRICING } from "./pricing.generated"; +import type { Pricing } from "./pricing-types"; -// Exported so a unit test can assert key-and-rate parity against the -// authoritative Python table (src/coder_eval/pricing.py) and fail the -// build on drift — this hand-copied mirror is otherwise guarded only by a -// comment. Not part of the consumer API; use resolvePricing() instead. -export const PRICING: Record = { - "claude-opus-4-8": p(15, 75, 18.75, 1.5), - "claude-opus-4-7": p(15, 75, 18.75, 1.5), - "claude-opus-4-6": p(15, 75, 18.75, 1.5), - "claude-opus-4-6-20250514": p(15, 75, 18.75, 1.5), - "claude-opus-4-5-20251101": p(15, 75, 18.75, 1.5), - "claude-opus-4-20250514": p(15, 75, 18.75, 1.5), - "claude-sonnet-4-6": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-6-20250514": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-5-20250929": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-20250514": p(3, 15, 3.75, 0.3), - "claude-haiku-4-5-20251001": p(0.8, 4, 1, 0.08), - "claude-3-7-sonnet-20250219": p(3, 15, 3.75, 0.3), - "claude-3-5-sonnet-20241022": p(3, 15, 3.75, 0.3), - "claude-3-5-sonnet-20240620": p(3, 15, 3.75, 0.3), - "claude-3-opus-20240229": p(15, 75, 18.75, 1.5), - "claude-3-sonnet-20240229": p(3, 15, 3.75, 0.3), - "claude-3-haiku-20240307": p(0.25, 1.25, 0.3, 0.03), - "gpt-5-codex": p(1.25, 10, 1.25, 0.125), - "gpt-5": p(1.25, 10, 1.25, 0.125), - "gpt-5.3-codex": p(1.75, 14, 1.75, 0.175), - "gpt-5.4": p(2.5, 15, 2.5, 0.25), - "gpt-5.5": p(5, 30, 5, 0.5), - // Google Gemini (AntigravityAgent). Gemini bills no separate cache-write - // fee (cache_write == input, effectively unused); cache_read is the cached- - // input rate. Pro's >200K-token tier is higher — this flat rate reads low - // for very-large-context runs, fine for typical eval tasks. - "gemini-3-pro-preview": p(2, 12, 2, 0.2), - "gemini-3.1-pro-preview": p(2, 12, 2, 0.2), - "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2), - "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15), - "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15), - // OpenRouter open-weight models (litellm backend). Mirror of pricing.py; - // these providers cache implicitly (cache_write == input, unused) so only - // cache_read carries a discounted rate. NOTE: OpenRouter routes per-request, - // so these rates are only accurate when the litellm config pins the provider - // (sort: price) — otherwise the billed rate can differ from the headline. - "moonshotai/kimi-k3": p(3, 15, 3, 0.3), - "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534), - "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625), - // Bedrock open-weight models (litellm backend, eu-north-1). Mirror of pricing.py. - // The recorded model_used arrives prefixed (e.g. "converse/zai.glm-5"), so - // resolvePricing strips the routing/region prefixes before lookup. - "deepseek.v3.2": p(0.74, 2.22, 0.74, 0), - "zai.glm-5": p(1.2, 3.84, 1.2, 0), - "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0), -}; +export type { Pricing } from "./pricing-types"; -function p( - input: number, - output: number, - cacheWrite: number, - cacheRead: number, -): Pricing { - return { - inputPerMTok: input, - outputPerMTok: output, - cacheWritePerMTok: cacheWrite, - cacheReadPerMTok: cacheRead, - }; -} +// Exported so the parity test can assert the generated artifact is current +// against pricing.py. Not part of the consumer API; use resolvePricing() instead. +export const PRICING: Record = GENERATED_PRICING; // Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare // pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the diff --git a/evalboard/package.json b/evalboard/package.json index 63e3d802..0b24eb65 100644 --- a/evalboard/package.json +++ b/evalboard/package.json @@ -5,6 +5,7 @@ "scripts": { "dev": "next dev -p 3030", "dev:local": "EVALBOARD_LOCAL_RUNS_DIR=../runs next dev -p 3030", + "gen:pricing": "node scripts/gen-pricing.mjs", "build": "next build", "start": "next start -p 3030", "typecheck": "tsc --noEmit", diff --git a/evalboard/scripts/gen-pricing.d.mts b/evalboard/scripts/gen-pricing.d.mts new file mode 100644 index 00000000..b6fb56d6 --- /dev/null +++ b/evalboard/scripts/gen-pricing.d.mts @@ -0,0 +1,15 @@ +// Types for the plain-JS generator, so lib/__tests__/pricing-parity.test.ts can +// import its parser under `allowJs: false`. Declares only what that test +// consumes — everything else in gen-pricing.mjs is module-private, so this file +// stays a one-function surface rather than a second mirror to keep in sync. + +export declare function parsePythonTable(): Record< + string, + [number, number, number, number] +>; + +/** parsePythonTable + the coverage cross-check. Throws if a row was skipped. */ +export declare function readTable(): Record< + string, + [number, number, number, number] +>; diff --git a/evalboard/scripts/gen-pricing.mjs b/evalboard/scripts/gen-pricing.mjs new file mode 100644 index 00000000..da8b12ad --- /dev/null +++ b/evalboard/scripts/gen-pricing.mjs @@ -0,0 +1,166 @@ +// Generates lib/pricing.generated.ts from the authoritative Python rate table +// (src/coder_eval/pricing.py), so per-MTok rates are written in exactly one +// file in one language. Run `pnpm gen:pricing` after editing pricing.py. +// +// Also the single home of the Python-table parser: lib/__tests__/pricing-parity.test.ts +// imports parsePythonTable from here rather than re-declaring the regex, so the +// generator and its drift guard can never disagree about what pricing.py says. +// +// That shared parser is why ROW_RE's coverage has to be checked structurally +// rather than trusted: it only matches rows written as a literal +// `ModelPricing(a, b, c, d)`, and a row it fails to match is missing from BOTH +// the generated table and the parity test's expectation — so deep-equal passes +// while the board silently renders "—" for that model's cost. readTable() +// therefore cross-checks the matched-row count against the number of +// `ModelPricing(` constructions in the file and refuses to emit on a mismatch. + +import { readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Resolve both paths from the script's own location, never cwd, so the +// generator behaves identically from `evalboard/` or the repo root. +const here = dirname(fileURLToPath(import.meta.url)); +const PY_PATH = resolve(here, "../../src/coder_eval/pricing.py"); +const OUT_PATH = resolve(here, "../lib/pricing.generated.ts"); + +// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), +const ROW_RE = + /"([^"]+)":\s*ModelPricing\(\s*([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\s*\)/g; + +// Every `ModelPricing(` in the file should be one matched table row. +const CONSTRUCTOR_RE = /ModelPricing\(/g; + +// Below this, assume a regex or path regression rather than a genuinely tiny +// table — emitting an empty/near-empty table would silently unprice the board. +const MIN_ROWS = 10; + +function readSource() { + try { + return readFileSync(PY_PATH, "utf8"); + } catch (err) { + throw new Error( + `cannot read the Python pricing table at ${PY_PATH}: ${err instanceof Error ? err.message : err}`, + ); + } +} + +/** + * Parse the Python rate table into `{ model: [input, output, cacheWrite, cacheRead] }`, + * preserving pricing.py's insertion order. + * + * Uses a null-prototype object so a degenerate key (`__proto__`) becomes an own + * property instead of silently setting the prototype and dropping the row. + * @returns {Record} + */ +export function parsePythonTable() { + const src = readSource(); + /** @type {Record} */ + const out = Object.create(null); + for (const m of src.matchAll(ROW_RE)) { + const rates = [Number(m[2]), Number(m[3]), Number(m[4]), Number(m[5])]; + if (!rates.every(Number.isFinite)) { + throw new Error( + `non-finite rate for "${m[1]}" in ${PY_PATH}: [${rates.join(", ")}]. ` + + `Emitting NaN would typecheck and render "$NaN" on the board.`, + ); + } + out[m[1]] = /** @type {[number, number, number, number]} */ (rates); + } + return out; +} + +/** + * Parse, then verify the parse actually covered the file. + * + * The drift guard MUST use this rather than parsePythonTable: comparing two + * products of the same parser cannot detect a row the parser skipped, because + * the row is missing from the artifact AND from the expectation. This is the + * only function that can tell "the mirror is faithful" apart from "the mirror + * and the expectation are equally blind". + * @returns {Record} + */ +export function readTable() { + const src = readSource(); + const table = parsePythonTable(); + const rows = Object.keys(table).length; + if (rows < MIN_ROWS) { + throw new Error( + `parsed only ${rows} rows from ${PY_PATH} (expected at least ${MIN_ROWS}). ` + + `The regex or the path has regressed — refusing to emit a partial table.`, + ); + } + const constructions = (src.match(CONSTRUCTOR_RE) ?? []).length; + if (constructions !== rows) { + throw new Error( + `${PY_PATH} has ${constructions} ModelPricing(...) constructions but ROW_RE matched ${rows} rows.\n` + + `A rate row ROW_RE cannot parse (keyword arguments, a variable, a computed expression, ` + + `scientific notation) would be dropped from the generated table AND invisible to the parity ` + + `test, silently unpricing that model. Either rewrite the row as a literal ` + + `\`"model": ModelPricing(a, b, c, d)\`, or widen ROW_RE. ` + + `(If a ModelPricing(...) legitimately lives outside the table, teach this check to skip it.)`, + ); + } + return table; +} + +/** + * Render the generated TypeScript module. Every key is quoted unconditionally + * via JSON.stringify — `z-ai/glm-5.2`, `deepseek.v3.2`, and `gpt-5.1-codex-max` + * all need it, and deciding per key costs more than it saves. Rates are + * interpolated as-is, so a legitimate `0` (Bedrock open-weight cache-read) is + * emitted as `0`; parsePythonTable has already guaranteed each is finite. + * @param {Record} table + * @returns {string} + */ +function renderModule(table) { + const rows = Object.entries(table) + .map( + ([model, [input, output, cacheWrite, cacheRead]]) => + ` ${JSON.stringify(model)}: {\n` + + ` inputPerMTok: ${input},\n` + + ` outputPerMTok: ${output},\n` + + ` cacheWritePerMTok: ${cacheWrite},\n` + + ` cacheReadPerMTok: ${cacheRead},\n` + + ` },`, + ) + .join("\n"); + return ( + `// GENERATED BY scripts/gen-pricing.mjs — DO NOT EDIT.\n` + + `// Source of truth: src/coder_eval/pricing.py.\n` + + `// To change a rate: edit that file, then run \`pnpm gen:pricing\`.\n` + + `// Row order mirrors pricing.py so regeneration diffs stay minimal.\n` + + `\n` + + `import type { Pricing } from "./pricing-types";\n` + + `\n` + + `export const GENERATED_PRICING: Record = {\n` + + `${rows}\n` + + `};\n` + ); +} + +function main() { + let table; + try { + table = readTable(); + } catch (err) { + // Every guard above throws rather than emitting, so a failed run leaves + // the previous artifact intact instead of writing a partial table. + console.error( + `gen-pricing: ${err instanceof Error ? err.message : err}`, + ); + process.exit(1); + } + writeFileSync(OUT_PATH, renderModule(table), "utf8"); + console.log( + `gen-pricing: wrote ${Object.keys(table).length} models to ${OUT_PATH}`, + ); +} + +// Only emit when run as a script; importers (the parity test) get the parser +// alone. realpathSync, not resolve: Node realpaths the main module for +// import.meta.url, so a lexical compare silently no-ops (exit 0, nothing +// written) whenever a symlink is anywhere in the invocation path. +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +}