From 7d50b13d827bc900c39a53b72cd3bc7938c06fd6 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 14:46:18 -0700 Subject: [PATCH 01/12] =?UTF-8?q?fix(evalboard):=201/6=20=E2=80=94=20gener?= =?UTF-8?q?ate=20the=20cost=20table=20from=20the=20Python=20pricing=20SSOT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/pricing.ts declared itself a hand-copy of src/coder_eval/pricing.py and had drifted three ways: 7 shared models carried WRONG rates (Opus 4.6/4.7/4.8 read Opus 4.1's legacy $15/$75, overstating cost 3x on every rendered estimate; Haiku 4.5 read Haiku 3.5's $0.80/$4), 17 priced models were missing, and 2 entries were dead. The DELIBERATELY_UNMIRRORED allowlist meant to hold the line had failed twice over — 17 models drifted past it, and 2 of its own 8 entries were models real runs use, so the board rendered "—" for their cost. Replace the hand-copy with a checked-in artifact generated from the Python table, so rates are written in exactly one file in one language and the check becomes "is the artifact current?" rather than 56 rows x 4 numbers retyped across two languages forever. Rates are unchanged on the Python side. - scripts/gen-pricing.mjs emits lib/pricing.generated.ts (56 models, pricing.py order preserved). It is also the single home of the Python-table parser: the parity test imports it rather than re-declaring the regex. - Because that parser runs on both sides of the drift test, a row ROW_RE cannot match would be missing from the artifact AND from the expectation — passing deep-equal while silently unpricing the model. readTable() cross-checks matched rows against the count of ModelPricing( constructions and refuses to emit on mismatch, closing that hole rather than only documenting it. - The main-vs-import guard uses realpathSync, not a lexical compare: Node realpaths the main module, so any symlink in the invocation path made `pnpm gen:pricing` a silent no-op that wrote nothing and exited 0. - Parser uses a null-prototype accumulator (a __proto__ key would otherwise set the prototype and drop the row) and rejects non-finite rates (a malformed one would emit `inputPerMTok: NaN`, typecheck, and render "$NaN"). - Pricing moves to lib/pricing-types.ts so the generated file and pricing.ts share it without a cycle; pricing.ts re-exports it, so no importer changes. Everything below PRICING is byte-identical — resolvePricing, normalizeModel, tokenBucketUsd and messageCostUsd are untouched. Rates re-verified against Anthropic's published per-MTok card: every Claude row in pricing.py matches, and each cache rate is exactly the documented multiplier (write 1.25x input, read 0.1x). Non-Claude rows are generated faithfully but remain unverified against their vendors — generation makes the mirror faithful, not the source right. One line of lib/__tests__/pricing.test.ts had to change (75 -> 25): it asserted claude-opus-4-8 output at $75, encoding the very bug this fixes. The other 21 assertions in that file pass untouched. Suite: 341 passed / 3 failed -> 346 passed / 0 failed. Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/__tests__/pricing-parity.test.ts | 146 ++++---- evalboard/lib/__tests__/pricing.test.ts | 4 +- evalboard/lib/pricing-types.ts | 12 + evalboard/lib/pricing.generated.ts | 345 ++++++++++++++++++ evalboard/lib/pricing.ts | 90 +---- evalboard/package.json | 1 + evalboard/scripts/gen-pricing.d.mts | 9 + evalboard/scripts/gen-pricing.mjs | 162 ++++++++ 8 files changed, 615 insertions(+), 154 deletions(-) create mode 100644 evalboard/lib/pricing-types.ts create mode 100644 evalboard/lib/pricing.generated.ts create mode 100644 evalboard/scripts/gen-pricing.d.mts create mode 100644 evalboard/scripts/gen-pricing.mjs diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 648f49f..4b0599a 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -1,47 +1,22 @@ -import { readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; import { describe, expect, test } from "vitest"; -import { PRICING } from "../pricing"; +import { parsePythonTable } from "../../scripts/gen-pricing.mjs"; +import { PRICING, resolvePricing } from "../pricing"; -// Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative -// Python table in src/coder_eval/pricing.py. It exists so the frontend's -// "estimated" USD figures agree with the backend's authoritative Cost on the -// same tokens. +// Drift guard: lib/pricing.generated.ts is generated from the authoritative +// Python table (src/coder_eval/pricing.py) by scripts/gen-pricing.mjs. Nobody +// types a rate on this side, so the only failure mode left is a STALE artifact — +// pricing.py changed and `pnpm gen:pricing` wasn't run. // -// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts -// must exist in pricing.py with identical rates (a frontend rate that disagrees -// with the backend, or prices a model the backend doesn't, fails the build). -// The frontend is NOT required to mirror every backend model — it only needs to -// price the ones it displays, and the backend legitimately prices models the -// evalboard never renders. (Exact-match was too strict: it forced unrelated -// backend model additions into this file to keep the build green.) - -const here = dirname(fileURLToPath(import.meta.url)); -const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py"); - -// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), -const ROW_RE = - /"([^"]+)":\s*ModelPricing\(\s*([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\s*\)/g; - -function parsePythonTable(): Record< - string, - [number, number, number, number] -> { - const src = readFileSync(PY_PATH, "utf8"); - const out: Record = {}; - for (const m of src.matchAll(ROW_RE)) { - out[m[1]] = [ - Number(m[2]), - Number(m[3]), - Number(m[4]), - Number(m[5]), - ]; - } - return out; -} +// Semantics are EXACT-MATCH, which the old hand-copied mirror couldn't sustain +// (it needed an explicit not-mirrored allowlist that 17 models drifted past, two +// of whose own entries were models real runs use — silently rendering "—" for +// their cost). Generation makes exact-match free: one assertion subsumes +// orphans, missing entries, and rate drift, with no allowlist to maintain. +// +// parsePythonTable is imported from the generator rather than re-declared here, +// so the guard and the thing it guards cannot disagree about what pricing.py says. -describe("pricing.ts ↔ pricing.py parity", () => { +describe("pricing.generated.ts ↔ pricing.py parity", () => { const py = parsePythonTable(); test("parses a non-trivial Python table", () => { @@ -49,49 +24,64 @@ describe("pricing.ts ↔ pricing.py parity", () => { expect(Object.keys(py).length).toBeGreaterThan(10); }); - test("every model in lib/pricing.ts exists in pricing.py", () => { - const orphans = Object.keys(PRICING).filter((m) => !(m in py)); + test("the generated table is current with pricing.py", () => { + const generated = Object.fromEntries( + Object.entries(PRICING).map(([model, p]) => [ + model, + [ + p.inputPerMTok, + p.outputPerMTok, + p.cacheWritePerMTok, + p.cacheReadPerMTok, + ], + ]), + ); expect( - orphans, - `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`, - ).toEqual([]); + generated, + "lib/pricing.generated.ts is stale — run `pnpm gen:pricing`", + ).toEqual(py); }); +}); - test("shared models have identical input/output/cacheWrite/cacheRead rates", () => { - for (const [model, ts] of Object.entries(PRICING)) { - const rates = py[model]; - expect(rates, `not priced in pricing.py: ${model}`).toBeDefined(); - expect([ - ts.inputPerMTok, - ts.outputPerMTok, - ts.cacheWritePerMTok, - ts.cacheReadPerMTok, - ]).toEqual(rates); - } +// Behaviour the generated table has to preserve. Deliberately NOT a second copy +// of the rate card — the exact-match test above already proves every rate equals +// pricing.py, so re-asserting rates here would only mean a legitimate vendor +// repricing has to be hand-patched in two places to keep the suite green. What +// these pin is lookup *logic* that the deep-equal cannot see: the undated +// fallback, a genuine zero surviving, and float precision. +describe("resolvePricing over the generated table", () => { + test("a deleted dated key still prices via the undated fallback", () => { + // "claude-opus-4-6-20250514" was a dead entry (absent from pricing.py) + // and redundant: resolvePricing strips a trailing -YYYYMMDD. Deleting it + // must change no rendered figure. The $5 also pins the headline fix — + // the old mirror carried Opus 4.1's $15 onto 4.6, overstating cost 3x. + const p = resolvePricing("claude-opus-4-6-20250514"); + expect(p).not.toBeNull(); + expect(p?.inputPerMTok).toBe(5); + expect(resolvePricing("claude-sonnet-4-6-20250514")).not.toBeNull(); }); - // Python-priced models we deliberately do NOT mirror to the frontend: heavy - // frontier Claude/GPT variants the evalboard never runs, so pricing them here - // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model - // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build — - // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids - // that previously rendered "—" for cost). - const DELIBERATELY_UNMIRRORED = new Set([ - "claude-sonnet-5", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.4-pro", - "gpt-5.5-pro", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - ]); + test("a model the old allowlist suppressed is now priced", () => { + // By far the most-run model in the recorded run data, yet it sat in the + // old not-mirrored allowlist — so the evalboard rendered "—" for cost on + // runs it actually executed. This is the regression that must not return. + expect(resolvePricing("gpt-5.6-terra")).not.toBeNull(); + }); - test("every pricing.py model is mirrored in pricing.ts or explicitly unmirrored", () => { - const missing = Object.keys(py).filter((m) => !(m in PRICING) && !DELIBERATELY_UNMIRRORED.has(m)); - expect( - missing, - `priced in pricing.py but missing from pricing.ts — mirror it or add to DELIBERATELY_UNMIRRORED: ${missing.join(", ")}`, - ).toEqual([]); + test("a legitimately-zero rate survives as 0, not undefined", () => { + // Bedrock publishes no prompt-cache rate for the open-weight models, so + // cache-read is a real 0 — generation must not drop it as falsy. + expect(resolvePricing("deepseek.v3.2")?.cacheReadPerMTok).toBe(0); + expect(resolvePricing("zai.glm-5")?.cacheReadPerMTok).toBe(0); + }); + + test("fractional rates round-trip exactly through generation", () => { + // String(Number(x)) is only safe for these if it truly round-trips; + // assert it rather than assume it. + expect(resolvePricing("z-ai/glm-5.2")?.inputPerMTok).toBe(0.7168); + expect(resolvePricing("z-ai/glm-5.2")?.cacheReadPerMTok).toBe(0.13312); + expect(resolvePricing("deepseek/deepseek-v4-pro")?.cacheReadPerMTok).toBe( + 0.003625, + ); }); }); diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66302cf..ae60299 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -36,7 +36,9 @@ describe("resolvePricing", () => { }); test("knows the current default opus id", () => { - expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75); + // $25, not $75: Opus dropped to $5/$25 at 4.5, and the old hand-copied + // mirror carried Opus 4.1's legacy $15/$75 forward onto 4.6/4.7/4.8. + expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(25); }); test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => { diff --git a/evalboard/lib/pricing-types.ts b/evalboard/lib/pricing-types.ts new file mode 100644 index 0000000..120fba3 --- /dev/null +++ b/evalboard/lib/pricing-types.ts @@ -0,0 +1,12 @@ +// The per-MTok rate shape, shared by the generated table +// (lib/pricing.generated.ts) and the hand-written lookup/cost helpers in +// lib/pricing.ts. It lives in its own leaf module so the generated file can +// import it without a cycle (pricing.ts → generated → pricing.ts). +// Consumers should keep importing `Pricing` from lib/pricing.ts. + +export interface Pricing { + inputPerMTok: number; + outputPerMTok: number; + cacheWritePerMTok: number; + cacheReadPerMTok: number; +} diff --git a/evalboard/lib/pricing.generated.ts b/evalboard/lib/pricing.generated.ts new file mode 100644 index 0000000..fbbefc2 --- /dev/null +++ b/evalboard/lib/pricing.generated.ts @@ -0,0 +1,345 @@ +// GENERATED BY scripts/gen-pricing.mjs — DO NOT EDIT. +// Source of truth: src/coder_eval/pricing.py. +// To change a rate: edit that file, then run `pnpm gen:pricing`. +// Row order mirrors pricing.py so regeneration diffs stay minimal. + +import type { Pricing } from "./pricing-types"; + +export const GENERATED_PRICING: Record = { + "claude-fable-5": { + inputPerMTok: 10, + outputPerMTok: 50, + cacheWritePerMTok: 12.5, + cacheReadPerMTok: 1, + }, + "claude-opus-5": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-8": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-7": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-6": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-5": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-5-20251101": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-1": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-opus-4": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-opus-4-20250514": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-sonnet-5": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-6": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-5": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-5-20250929": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-20250514": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-haiku-4-5": { + inputPerMTok: 1, + outputPerMTok: 5, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.1, + }, + "claude-haiku-4-5-20251001": { + inputPerMTok: 1, + outputPerMTok: 5, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.1, + }, + "claude-haiku-3-5": { + inputPerMTok: 0.8, + outputPerMTok: 4, + cacheWritePerMTok: 1, + cacheReadPerMTok: 0.08, + }, + "claude-3-7-sonnet-20250219": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-5-sonnet-20241022": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-5-sonnet-20240620": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-opus-20240229": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-3-sonnet-20240229": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-haiku-20240307": { + inputPerMTok: 0.25, + outputPerMTok: 1.25, + cacheWritePerMTok: 0.3, + cacheReadPerMTok: 0.03, + }, + "gpt-5-codex": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex-max": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex-mini": { + inputPerMTok: 0.25, + outputPerMTok: 2, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "codex-mini-latest": { + inputPerMTok: 1.5, + outputPerMTok: 6, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.375, + }, + "gpt-5.3-codex": { + inputPerMTok: 1.75, + outputPerMTok: 14, + cacheWritePerMTok: 1.75, + cacheReadPerMTok: 0.175, + }, + "gpt-5.2-codex": { + inputPerMTok: 1.75, + outputPerMTok: 14, + cacheWritePerMTok: 1.75, + cacheReadPerMTok: 0.175, + }, + "gpt-5.4": { + inputPerMTok: 2.5, + outputPerMTok: 15, + cacheWritePerMTok: 2.5, + cacheReadPerMTok: 0.25, + }, + "gpt-5.5": { + inputPerMTok: 5, + outputPerMTok: 30, + cacheWritePerMTok: 5, + cacheReadPerMTok: 0.5, + }, + "gpt-5.5-pro": { + inputPerMTok: 30, + outputPerMTok: 180, + cacheWritePerMTok: 30, + cacheReadPerMTok: 3, + }, + "gpt-5.4-pro": { + inputPerMTok: 30, + outputPerMTok: 180, + cacheWritePerMTok: 30, + cacheReadPerMTok: 3, + }, + "gpt-5.4-mini": { + inputPerMTok: 0.75, + outputPerMTok: 4.5, + cacheWritePerMTok: 0.75, + cacheReadPerMTok: 0.075, + }, + "gpt-5.4-nano": { + inputPerMTok: 0.2, + outputPerMTok: 1.25, + cacheWritePerMTok: 0.2, + cacheReadPerMTok: 0.02, + }, + "gpt-5.6-sol": { + inputPerMTok: 5, + outputPerMTok: 30, + cacheWritePerMTok: 5, + cacheReadPerMTok: 0.5, + }, + "gpt-5.6-terra": { + inputPerMTok: 2.5, + outputPerMTok: 15, + cacheWritePerMTok: 2.5, + cacheReadPerMTok: 0.25, + }, + "gpt-5.6-luna": { + inputPerMTok: 1, + outputPerMTok: 6, + cacheWritePerMTok: 1, + cacheReadPerMTok: 0.1, + }, + "gemini-3.6-flash": { + inputPerMTok: 1.5, + outputPerMTok: 7.5, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.15, + }, + "gemini-3.5-flash": { + inputPerMTok: 1.5, + outputPerMTok: 9, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.15, + }, + "gemini-3.5-flash-lite": { + inputPerMTok: 0.3, + outputPerMTok: 2.5, + cacheWritePerMTok: 0.3, + cacheReadPerMTok: 0.03, + }, + "gemini-3.1-pro-preview": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "gemini-3.1-pro-preview-customtools": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "gemini-3.1-flash-lite": { + inputPerMTok: 0.25, + outputPerMTok: 1.5, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "gemini-3.1-flash-lite-preview": { + inputPerMTok: 0.25, + outputPerMTok: 1.5, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "gemini-3-flash-preview": { + inputPerMTok: 0.5, + outputPerMTok: 3, + cacheWritePerMTok: 0.5, + cacheReadPerMTok: 0.05, + }, + "gemini-3-pro-preview": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "deepseek.v3.2": { + inputPerMTok: 0.74, + outputPerMTok: 2.22, + cacheWritePerMTok: 0.74, + cacheReadPerMTok: 0, + }, + "zai.glm-5": { + inputPerMTok: 1.2, + outputPerMTok: 3.84, + cacheWritePerMTok: 1.2, + cacheReadPerMTok: 0, + }, + "moonshotai.kimi-k2.5": { + inputPerMTok: 0.72, + outputPerMTok: 3.6, + cacheWritePerMTok: 0.72, + cacheReadPerMTok: 0, + }, + "moonshotai/kimi-k3": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3, + cacheReadPerMTok: 0.3, + }, + "z-ai/glm-5.2": { + inputPerMTok: 0.7168, + outputPerMTok: 2.2528, + cacheWritePerMTok: 0.7168, + cacheReadPerMTok: 0.13312, + }, + "deepseek/deepseek-v4-pro": { + inputPerMTok: 0.435, + outputPerMTok: 0.87, + cacheWritePerMTok: 0.435, + cacheReadPerMTok: 0.003625, + }, +}; diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 93ac23c..431e2fd 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -1,84 +1,24 @@ -// Per-million-token prices and cost math. Ported from -// src/coder_eval/pricing.py — keep in sync when that table changes. -// Source: Anthropic / OpenAI / Google public pricing. +// Per-million-token prices and cost math. // -// This is the single source of truth for rates on the frontend: the +// Rates are NOT written here: they are generated from the authoritative Python +// table (src/coder_eval/pricing.py) into lib/pricing.generated.ts. To change a +// rate, edit that file and run `pnpm gen:pricing` — never hand-edit a rate on +// this side. lib/__tests__/pricing-parity.test.ts fails the build if the +// generated artifact drifts from pricing.py. +// +// This module is the single source of truth for rates *on the frontend*: the // cascade-aware thinking-cost simulator (lib/thinkingSim.ts) and the // per-message cost column (lib/runs.ts) both price against this table, so a -// model added or repriced here updates both at once. +// model added or repriced in pricing.py updates both at once. -export interface Pricing { - inputPerMTok: number; - outputPerMTok: number; - cacheWritePerMTok: number; - cacheReadPerMTok: number; -} +import { GENERATED_PRICING } from "./pricing.generated"; +import type { Pricing } from "./pricing-types"; -// Exported so a unit test can assert key-and-rate parity against the -// authoritative Python table (src/coder_eval/pricing.py) and fail the -// build on drift — this hand-copied mirror is otherwise guarded only by a -// comment. Not part of the consumer API; use resolvePricing() instead. -export const PRICING: Record = { - "claude-opus-4-8": p(15, 75, 18.75, 1.5), - "claude-opus-4-7": p(15, 75, 18.75, 1.5), - "claude-opus-4-6": p(15, 75, 18.75, 1.5), - "claude-opus-4-6-20250514": p(15, 75, 18.75, 1.5), - "claude-opus-4-5-20251101": p(15, 75, 18.75, 1.5), - "claude-opus-4-20250514": p(15, 75, 18.75, 1.5), - "claude-sonnet-4-6": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-6-20250514": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-5-20250929": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-20250514": p(3, 15, 3.75, 0.3), - "claude-haiku-4-5-20251001": p(0.8, 4, 1, 0.08), - "claude-3-7-sonnet-20250219": p(3, 15, 3.75, 0.3), - "claude-3-5-sonnet-20241022": p(3, 15, 3.75, 0.3), - "claude-3-5-sonnet-20240620": p(3, 15, 3.75, 0.3), - "claude-3-opus-20240229": p(15, 75, 18.75, 1.5), - "claude-3-sonnet-20240229": p(3, 15, 3.75, 0.3), - "claude-3-haiku-20240307": p(0.25, 1.25, 0.3, 0.03), - "gpt-5-codex": p(1.25, 10, 1.25, 0.125), - "gpt-5": p(1.25, 10, 1.25, 0.125), - "gpt-5.3-codex": p(1.75, 14, 1.75, 0.175), - "gpt-5.4": p(2.5, 15, 2.5, 0.25), - "gpt-5.5": p(5, 30, 5, 0.5), - // Google Gemini (AntigravityAgent). Gemini bills no separate cache-write - // fee (cache_write == input, effectively unused); cache_read is the cached- - // input rate. Pro's >200K-token tier is higher — this flat rate reads low - // for very-large-context runs, fine for typical eval tasks. - "gemini-3-pro-preview": p(2, 12, 2, 0.2), - "gemini-3.1-pro-preview": p(2, 12, 2, 0.2), - "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2), - "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15), - "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15), - // OpenRouter open-weight models (litellm backend). Mirror of pricing.py; - // these providers cache implicitly (cache_write == input, unused) so only - // cache_read carries a discounted rate. NOTE: OpenRouter routes per-request, - // so these rates are only accurate when the litellm config pins the provider - // (sort: price) — otherwise the billed rate can differ from the headline. - "moonshotai/kimi-k3": p(3, 15, 3, 0.3), - "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534), - "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625), - // Bedrock open-weight models (litellm backend, eu-north-1). Mirror of pricing.py. - // The recorded model_used arrives prefixed (e.g. "converse/zai.glm-5"), so - // resolvePricing strips the routing/region prefixes before lookup. - "deepseek.v3.2": p(0.74, 2.22, 0.74, 0), - "zai.glm-5": p(1.2, 3.84, 1.2, 0), - "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0), -}; +export type { Pricing } from "./pricing-types"; -function p( - input: number, - output: number, - cacheWrite: number, - cacheRead: number, -): Pricing { - return { - inputPerMTok: input, - outputPerMTok: output, - cacheWritePerMTok: cacheWrite, - cacheReadPerMTok: cacheRead, - }; -} +// Exported so the parity test can assert the generated artifact is current +// against pricing.py. Not part of the consumer API; use resolvePricing() instead. +export const PRICING: Record = GENERATED_PRICING; // Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare // pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the diff --git a/evalboard/package.json b/evalboard/package.json index 63e3d80..0b24eb6 100644 --- a/evalboard/package.json +++ b/evalboard/package.json @@ -5,6 +5,7 @@ "scripts": { "dev": "next dev -p 3030", "dev:local": "EVALBOARD_LOCAL_RUNS_DIR=../runs next dev -p 3030", + "gen:pricing": "node scripts/gen-pricing.mjs", "build": "next build", "start": "next start -p 3030", "typecheck": "tsc --noEmit", diff --git a/evalboard/scripts/gen-pricing.d.mts b/evalboard/scripts/gen-pricing.d.mts new file mode 100644 index 0000000..7d2845f --- /dev/null +++ b/evalboard/scripts/gen-pricing.d.mts @@ -0,0 +1,9 @@ +// Types for the plain-JS generator, so lib/__tests__/pricing-parity.test.ts can +// import its parser under `allowJs: false`. Declares only what that test +// consumes — everything else in gen-pricing.mjs is module-private, so this file +// stays a one-function surface rather than a second mirror to keep in sync. + +export declare function parsePythonTable(): Record< + string, + [number, number, number, number] +>; diff --git a/evalboard/scripts/gen-pricing.mjs b/evalboard/scripts/gen-pricing.mjs new file mode 100644 index 0000000..f32e316 --- /dev/null +++ b/evalboard/scripts/gen-pricing.mjs @@ -0,0 +1,162 @@ +// Generates lib/pricing.generated.ts from the authoritative Python rate table +// (src/coder_eval/pricing.py), so per-MTok rates are written in exactly one +// file in one language. Run `pnpm gen:pricing` after editing pricing.py. +// +// Also the single home of the Python-table parser: lib/__tests__/pricing-parity.test.ts +// imports parsePythonTable from here rather than re-declaring the regex, so the +// generator and its drift guard can never disagree about what pricing.py says. +// +// That shared parser is why ROW_RE's coverage has to be checked structurally +// rather than trusted: it only matches rows written as a literal +// `ModelPricing(a, b, c, d)`, and a row it fails to match is missing from BOTH +// the generated table and the parity test's expectation — so deep-equal passes +// while the board silently renders "—" for that model's cost. readTable() +// therefore cross-checks the matched-row count against the number of +// `ModelPricing(` constructions in the file and refuses to emit on a mismatch. + +import { readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Resolve both paths from the script's own location, never cwd, so the +// generator behaves identically from `evalboard/` or the repo root. +const here = dirname(fileURLToPath(import.meta.url)); +const PY_PATH = resolve(here, "../../src/coder_eval/pricing.py"); +const OUT_PATH = resolve(here, "../lib/pricing.generated.ts"); + +// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), +const ROW_RE = + /"([^"]+)":\s*ModelPricing\(\s*([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\s*\)/g; + +// Every `ModelPricing(` in the file should be one matched table row. +const CONSTRUCTOR_RE = /ModelPricing\(/g; + +// Below this, assume a regex or path regression rather than a genuinely tiny +// table — emitting an empty/near-empty table would silently unprice the board. +const MIN_ROWS = 10; + +function readSource() { + try { + return readFileSync(PY_PATH, "utf8"); + } catch (err) { + throw new Error( + `cannot read the Python pricing table at ${PY_PATH}: ${err instanceof Error ? err.message : err}`, + ); + } +} + +/** + * Parse the Python rate table into `{ model: [input, output, cacheWrite, cacheRead] }`, + * preserving pricing.py's insertion order. + * + * Uses a null-prototype object so a degenerate key (`__proto__`) becomes an own + * property instead of silently setting the prototype and dropping the row. + * @returns {Record} + */ +export function parsePythonTable() { + const src = readSource(); + /** @type {Record} */ + const out = Object.create(null); + for (const m of src.matchAll(ROW_RE)) { + const rates = [Number(m[2]), Number(m[3]), Number(m[4]), Number(m[5])]; + if (!rates.every(Number.isFinite)) { + throw new Error( + `non-finite rate for "${m[1]}" in ${PY_PATH}: [${rates.join(", ")}]. ` + + `Emitting NaN would typecheck and render "$NaN" on the board.`, + ); + } + out[m[1]] = /** @type {[number, number, number, number]} */ (rates); + } + return out; +} + +/** + * Parse, then verify the parse actually covered the file. Only the generator + * needs this — the parity test compares two products of the same parser, so it + * cannot detect an unmatched row on its own. + * @returns {Record} + */ +function readTable() { + const src = readSource(); + const table = parsePythonTable(); + const rows = Object.keys(table).length; + if (rows < MIN_ROWS) { + throw new Error( + `parsed only ${rows} rows from ${PY_PATH} (expected at least ${MIN_ROWS}). ` + + `The regex or the path has regressed — refusing to emit a partial table.`, + ); + } + const constructions = (src.match(CONSTRUCTOR_RE) ?? []).length; + if (constructions !== rows) { + throw new Error( + `${PY_PATH} has ${constructions} ModelPricing(...) constructions but ROW_RE matched ${rows} rows.\n` + + `A rate row ROW_RE cannot parse (keyword arguments, a variable, a computed expression, ` + + `scientific notation) would be dropped from the generated table AND invisible to the parity ` + + `test, silently unpricing that model. Either rewrite the row as a literal ` + + `\`"model": ModelPricing(a, b, c, d)\`, or widen ROW_RE. ` + + `(If a ModelPricing(...) legitimately lives outside the table, teach this check to skip it.)`, + ); + } + return table; +} + +/** + * Render the generated TypeScript module. Every key is quoted unconditionally + * via JSON.stringify — `z-ai/glm-5.2`, `deepseek.v3.2`, and `gpt-5.1-codex-max` + * all need it, and deciding per key costs more than it saves. Rates are + * interpolated as-is, so a legitimate `0` (Bedrock open-weight cache-read) is + * emitted as `0`; parsePythonTable has already guaranteed each is finite. + * @param {Record} table + * @returns {string} + */ +function renderModule(table) { + const rows = Object.entries(table) + .map( + ([model, [input, output, cacheWrite, cacheRead]]) => + ` ${JSON.stringify(model)}: {\n` + + ` inputPerMTok: ${input},\n` + + ` outputPerMTok: ${output},\n` + + ` cacheWritePerMTok: ${cacheWrite},\n` + + ` cacheReadPerMTok: ${cacheRead},\n` + + ` },`, + ) + .join("\n"); + return ( + `// GENERATED BY scripts/gen-pricing.mjs — DO NOT EDIT.\n` + + `// Source of truth: src/coder_eval/pricing.py.\n` + + `// To change a rate: edit that file, then run \`pnpm gen:pricing\`.\n` + + `// Row order mirrors pricing.py so regeneration diffs stay minimal.\n` + + `\n` + + `import type { Pricing } from "./pricing-types";\n` + + `\n` + + `export const GENERATED_PRICING: Record = {\n` + + `${rows}\n` + + `};\n` + ); +} + +function main() { + let table; + try { + table = readTable(); + } catch (err) { + // Every guard above throws rather than emitting, so a failed run leaves + // the previous artifact intact instead of writing a partial table. + console.error( + `gen-pricing: ${err instanceof Error ? err.message : err}`, + ); + process.exit(1); + } + writeFileSync(OUT_PATH, renderModule(table), "utf8"); + console.log( + `gen-pricing: wrote ${Object.keys(table).length} models to ${OUT_PATH}`, + ); +} + +// Only emit when run as a script; importers (the parity test) get the parser +// alone. realpathSync, not resolve: Node realpaths the main module for +// import.meta.url, so a lexical compare silently no-ops (exit 0, nothing +// written) whenever a symlink is anywhere in the invocation path. +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} From e4eb23ba4b48b465c2b6181ec7262cc779a9615d Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 15:04:43 -0700 Subject: [PATCH 02/12] =?UTF-8?q?refactor(evalboard):=202/6=20=E2=80=94=20?= =?UTF-8?q?one=20HARNESSES=20table=20for=20id,=20label,=20logo=20and=20col?= =?UTF-8?q?our?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harness knowledge lived in two parallel lists in two files: display order in lib/harness.ts and labels + logo paths in harness-badge.tsx. Adding a series colour plus a 4th harness would have made that four lists whose indices had to be hand-aligned — an implicit slot map. Collapse them into one ordered table in the leaf module and derive everything from it, so row order IS display order and colour order, with no alignment to maintain. Adds delegate-sdk as a known harness (it is a real production harness: coder_eval_uipath declares type: Literal["delegate-sdk"]). Derived from the table: KnownHarness, KNOWN_HARNESSES, knownIndex, harnessRow, harnessShortLabel (moved from harness-badge.tsx and re-exported there, so all seven existing importers are untouched). New for the multi-series charts: harnessColor (pure, total, prototype-safe by construction — a linear find, not a Record lookup, so harnessColor("toString") returns a real hex), orderHarnesses (the one known-first-then-alphabetical comparator), and groupByHarness. groupByHarness lives here rather than in lib/overview.ts on purpose: overview.ts reaches node:fs and @azure/storage-blob transitively, and a *value* import from a "use client" chart would pull that graph into the browser bundle. It is generic over { harness: string } so this module needs nothing from the data layer, and lib/harness.ts remains a zero-import leaf. parseHarnessScope takes over the `?h=` body and returns null for absent/malformed, which is what "all harnesses" will mean on the home page. parseHarnessParam becomes `parseHarnessScope(raw) ?? DEFAULT_HARNESS` — the charset and length bound now exist once and cannot drift apart. Observably identical for every input, so its six pre-existing tests pass byte-unedited. Each colour is its vendor's brand value, chosen by validator sweep on the strict all-pairs gate rather than by eye — which ruled out UiPath orange (CVD dE 5.1 vs Anthropic coral, indistinguishable even with full colour vision) in favour of violet for Delegate. The comment above the table records the validator command, its result, and why the two #000000 gate failures are waived, so the next person to run it does not "fix" a deliberate decision. One regression caught while reviewing and fixed here: because delegate-sdk is now a KNOWN row with logo: null, and KNOWN_HARNESSES feeds HarnessSelector on the /trends and /watchlist skeletons, the badge's raw-id fallback fired for a known harness and the chip read "delegate-sdkDelegate SDK". HarnessBadge now separates "unknown harness" (raw id, unchanged) from "known but no logo yet" (render nothing, since callers already print the label). Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/_components/harness-badge.tsx | 56 ++-- evalboard/lib/__tests__/harness.test.ts | 307 +++++++++++++++++++- evalboard/lib/harness.ts | 205 +++++++++++-- 3 files changed, 514 insertions(+), 54 deletions(-) diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index da16b70..5797d89 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -1,47 +1,35 @@ import Image from "next/image"; +import { harnessRow, normalizeHarness } from "@/lib/harness"; // Canonical harness constants live in the leaf lib/harness module (shared by // the server data layer); re-exported here so existing badge importers are // unaffected. -export { KNOWN_HARNESSES, type KnownHarness } from "@/lib/harness"; - -// Vendor logo + labels for a run's harness (RunConfig). Renders the recognizable -// vendor mark instead of raw "claude-code"/"codex"/"antigravity" text, -// mirroring the Slack rollup's vendor emoji. A missing harness defaults to -// claude-code (the nightly). This is an internal-only column — the caller -// gates it behind isInternal (see lib/edition.ts). -const HARNESS_LOGO: Record = { - "claude-code": { - src: "/harness/claude-code.png", - label: "Claude Code · Anthropic", - short: "Claude Code", - }, - codex: { src: "/harness/codex.png", label: "Codex · OpenAI", short: "Codex" }, - antigravity: { - src: "/harness/antigravity.png", - label: "Antigravity · Google Gemini", - short: "Antigravity", - }, -}; - -// Short human label for a harness id ("Claude Code"), for selectors and prose. -// Unknown ids fall through to the raw id. -export function harnessShortLabel(harness: string): string { - return HARNESS_LOGO[harness]?.short ?? harness; -} +export { + KNOWN_HARNESSES, + harnessShortLabel, + type KnownHarness, +} from "@/lib/harness"; +// Vendor logo for a run's harness (RunConfig). Renders the recognizable vendor +// mark instead of raw "claude-code"/"codex"/"antigravity" text, mirroring the +// Slack rollup's vendor emoji. Labels, logo paths and the claude-code default +// all come from the HARNESSES table — this component is only the . A +// missing harness folds to claude-code (the nightly). This is an internal-only +// column — the caller gates it behind isInternal (see lib/edition.ts). export function HarnessBadge({ harness }: { harness?: string | null }) { - const key = harness ?? "claude-code"; - const logo = HARNESS_LOGO[key]; + const key = normalizeHarness(harness); + const row = harnessRow(key); // Unknown harness: show the raw id rather than a misleading logo. - if (!logo) { - return {key}; - } + if (!row) return {key}; + // A known harness with no vendor mark yet: render nothing. Callers already + // print harnessShortLabel beside the badge, so emitting the raw id here too + // would read "delegate-sdkDelegate SDK" in the selector. + if (!row.logo) return null; return ( {logo.label} { expect(parseHarnessParam("a".repeat(65))).toBe(DEFAULT_HARNESS); }); }); + +// parseHarnessScope is the underlying `?h=` gate; parseHarnessParam (above) is +// derived from it. `null` means "all harnesses" — the home page's default. +describe("parseHarnessScope", () => { + test("returns null for absent / empty input (no implicit default)", () => { + expect(parseHarnessScope(undefined)).toBeNull(); + expect(parseHarnessScope([])).toBeNull(); + expect(parseHarnessScope("")).toBeNull(); + expect(parseHarnessScope(" ")).toBeNull(); + }); + + test("picks the first element of an array param", () => { + expect(parseHarnessScope(["codex", "antigravity"])).toBe("codex"); + }); + + test("trims surrounding whitespace before validating", () => { + expect(parseHarnessScope(" codex ")).toBe("codex"); + expect(parseHarnessScope([" antigravity "])).toBe("antigravity"); + }); + + test("passes through any syntactically-valid id, including unknown ones", () => { + for (const id of [ + "claude-code", + "codex", + "antigravity", + "delegate-sdk", + "some-future-harness", + "gpt-5.5", + "some_harness", + ]) { + expect(parseHarnessScope(id)).toBe(id); + } + }); + + test("returns null for a malformed id — never a partial match", () => { + for (const bad of ["has space", "a/b", "bad;rm", "a b c", "x!"]) { + expect(parseHarnessScope(bad)).toBeNull(); + } + }); + + test("enforces the 1-64 char length bound", () => { + expect(parseHarnessScope("a".repeat(64))).toBe("a".repeat(64)); + expect(parseHarnessScope("a".repeat(65))).toBeNull(); + }); +}); + +// The derivation itself: one charset, two surfaces. If these ever disagree, the +// `?h=` contract has drifted between the home page and /trends. +describe("parseHarnessParam is parseHarnessScope ?? DEFAULT_HARNESS", () => { + test("holds across valid, absent, and malformed inputs", () => { + const inputs: Array = [ + undefined, + [], + "", + " ", + "codex", + " codex ", + ["codex", "antigravity"], + "delegate-sdk", + "unknown-harness", + "has space", + "a/b", + "bad;rm", + "a".repeat(64), + "a".repeat(65), + ]; + for (const raw of inputs) { + expect(parseHarnessParam(raw)).toBe( + parseHarnessScope(raw) ?? DEFAULT_HARNESS, + ); + } + }); +}); + +// Each colour was chosen AGAINST the others by validator sweep on the all-pairs +// gate, so pin all four explicitly: a silent edit or reorder of HARNESSES must +// fail a test, not just a review. +describe("harnessColor", () => { + test("pins each harness to its vendor brand colour", () => { + expect(harnessColor("claude-code")).toBe("#D97757"); + expect(harnessColor("codex")).toBe("#000000"); + expect(harnessColor("antigravity")).toBe("#4285F4"); + expect(harnessColor("delegate-sdk")).toBe("#4a3aa7"); + }); + + test("all four brand colours are distinct", () => { + const colors = KNOWN_HARNESSES.map(harnessColor); + expect(new Set(colors).size).toBe(colors.length); + }); + + test("an unknown harness gets the neutral grey, never undefined", () => { + expect(harnessColor("some-future-harness")).toBe(UNKNOWN_HARNESS_COLOR); + expect(harnessColor("")).toBe(UNKNOWN_HARNESS_COLOR); + }); + + test("a degenerate id cannot reach a prototype member", () => { + // findIndex over the table, not a Record lookup: harnessColor("toString") + // must be a real hex, not Function.prototype.toString. + for (const id of ["toString", "constructor", "__proto__", "valueOf"]) { + expect(harnessColor(id)).toBe(UNKNOWN_HARNESS_COLOR); + } + }); + + test("the unknown grey is not equal to any brand colour", () => { + // Guards the grey-vs-black interaction: both are achromatic, so a future + // palette edit must not collapse them into the same value. + for (const h of KNOWN_HARNESSES) { + expect(harnessColor(h)).not.toBe(UNKNOWN_HARNESS_COLOR); + } + }); +}); + +describe("HARNESSES table integrity", () => { + test("ids are unique", () => { + const ids = HARNESSES.map((h) => h.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + test("colors are unique, case-insensitively", () => { + // Compare lowercased: the table mixes cases (#D97757 vs #4a3aa7), so a + // raw Set would count two case-variants of one colour as distinct. + const colors = HARNESSES.map((h) => h.color.toLowerCase()); + expect(new Set(colors).size).toBe(colors.length); + }); + + test("every row carries a short label, a full label and a colour", () => { + for (const h of HARNESSES) { + expect(h.short).toBeTruthy(); + expect(h.label).toBeTruthy(); + expect(h.color).toMatch(/^#[0-9a-fA-F]{6}$/); + } + }); + + test("every id survives the ?h= charset, so every chip is clickable", () => { + // The one cross-invariant in this module: a selector chip writes its id + // into `?h=`, and parseHarnessScope rejects anything outside this charset. + // An id with a space or a slash would render a chip that silently + // reset the scope on click, with nothing else failing. + for (const h of HARNESSES) { + expect(parseHarnessScope(h.id)).toBe(h.id); + } + }); +}); + +describe("harnessShortLabel", () => { + test("returns the table's short label for a known harness", () => { + expect(harnessShortLabel("claude-code")).toBe("Claude Code"); + expect(harnessShortLabel("codex")).toBe("Codex"); + expect(harnessShortLabel("antigravity")).toBe("Antigravity"); + expect(harnessShortLabel("delegate-sdk")).toBe("Delegate SDK"); + }); + + test("falls through to the raw id for an unknown harness", () => { + expect(harnessShortLabel("some-future-harness")).toBe( + "some-future-harness", + ); + }); +}); + +describe("knownIndex", () => { + test("gives the row position for each known harness", () => { + expect(knownIndex("claude-code")).toBe(0); + expect(knownIndex("codex")).toBe(1); + expect(knownIndex("antigravity")).toBe(2); + expect(knownIndex("delegate-sdk")).toBe(3); + }); + + test("gives -1 for anything else", () => { + expect(knownIndex("some-future-harness")).toBe(-1); + expect(knownIndex("")).toBe(-1); + expect(knownIndex("toString")).toBe(-1); + }); +}); + +describe("orderHarnesses", () => { + test("puts known harnesses in table order regardless of input order", () => { + expect( + orderHarnesses(["antigravity", "delegate-sdk", "claude-code", "codex"]), + ).toEqual(["claude-code", "codex", "antigravity", "delegate-sdk"]); + }); + + test("sorts unknown newcomers alphabetically after the known ones", () => { + expect( + orderHarnesses(["zebra-harness", "codex", "apex-harness", "claude-code"]), + ).toEqual(["claude-code", "codex", "apex-harness", "zebra-harness"]); + }); + + test("dedupes", () => { + expect(orderHarnesses(["codex", "codex", "claude-code", "codex"])).toEqual([ + "claude-code", + "codex", + ]); + }); + + test("an empty input yields an empty list, NOT the default harness", () => { + // An empty window must draw zero series rather than one phantom + // claude-code line; the non-empty fallback belongs to the selector. + expect(orderHarnesses([])).toEqual([]); + expect(orderHarnesses(new Set())).toEqual([]); + }); + + test("accepts any iterable, not just an array", () => { + expect(orderHarnesses(new Set(["codex", "claude-code"]))).toEqual([ + "claude-code", + "codex", + ]); + }); +}); + +describe("groupByHarness", () => { + const pt = (harness: string, timestamp: number) => ({ harness, timestamp }); + + test("an empty input yields no groups", () => { + expect(groupByHarness([])).toEqual([]); + }); + + test("a single harness yields one group in input order", () => { + const points = [pt("codex", 1), pt("codex", 2), pt("codex", 3)]; + expect(groupByHarness(points)).toEqual([ + { harness: "codex", points }, + ]); + }); + + test("interleaved input splits into correct groups, in table order", () => { + const groups = groupByHarness([ + pt("codex", 1), + pt("claude-code", 2), + pt("antigravity", 3), + pt("codex", 4), + pt("claude-code", 5), + ]); + expect(groups.map((g) => g.harness)).toEqual([ + "claude-code", + "codex", + "antigravity", + ]); + expect(groups.map((g) => g.points.length)).toEqual([2, 2, 1]); + }); + + test("an unknown harness sorts last", () => { + const groups = groupByHarness([ + pt("some-future-harness", 1), + pt("claude-code", 2), + ]); + expect(groups.map((g) => g.harness)).toEqual([ + "claude-code", + "some-future-harness", + ]); + }); + + test("preserves within-group order — monotonic x is what stops a zigzag", () => { + const groups = groupByHarness([ + pt("codex", 10), + pt("claude-code", 20), + pt("codex", 30), + pt("claude-code", 40), + pt("codex", 50), + ]); + expect( + groups.find((g) => g.harness === "codex")!.points.map((p) => p.timestamp), + ).toEqual([10, 30, 50]); + expect( + groups + .find((g) => g.harness === "claude-code")! + .points.map((p) => p.timestamp), + ).toEqual([20, 40]); + }); + + test("retains points with null values — grouping must not filter", () => { + const points = [ + { harness: "codex", successRate: null }, + { harness: "codex", successRate: 90 }, + ]; + expect(groupByHarness(points)[0].points).toHaveLength(2); + }); + + test("hands back fresh arrays, never an alias of the caller's", () => { + // A consumer that sorts a series in place (recharts wants monotonic x) + // must not reorder the caller's array as a side effect. The + // single-harness case is where an aliasing shortcut would be tempting. + const points = [pt("codex", 1), pt("codex", 2)]; + const groups = groupByHarness(points); + expect(groups[0].points).not.toBe(points); + expect(groups[0].points).toEqual(points); + }); + + test("does not mutate the caller's array", () => { + const points = [pt("codex", 1), pt("claude-code", 2)]; + const snapshot = [...points]; + groupByHarness(points); + expect(points).toEqual(snapshot); + }); +}); diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts index bf36cb2..ae701ee 100644 --- a/evalboard/lib/harness.ts +++ b/evalboard/lib/harness.ts @@ -1,14 +1,91 @@ -// Canonical harness (coder-eval AgentKind) constants. Leaf module with no -// server- or client-only dependencies, so both the client badge/selector and -// the server data layer can import it without pulling node-only code into the -// client bundle or creating an import cycle. - -// The harnesses the nightly rotates through, in preferred display order. This -// is only a display-order hint / default source — the switcher is data-driven -// (see listRecentHarnesses), so a new harness like "delegate" surfaces -// automatically without editing this list. -export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; -export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; +// Canonical harness (coder-eval AgentKind) knowledge. Leaf module with no +// server- or client-only dependencies, so both the client badge/selector/charts +// and the server data layer can import it without pulling node-only code into +// the client bundle or creating an import cycle. +// +// KEEP IT A LEAF: no `next/*`, no `node:*`, and nothing from ./runs, ./blob, or +// ./overview. `groupByHarness` lives here rather than in lib/overview.ts for +// exactly this reason — overview.ts reaches node:fs and @azure/storage-blob +// transitively, and a *value* import from a "use client" chart would drag that +// whole graph into the browser bundle (type-only imports erase; values do not). + +// One ordered table is the single source for a harness's id, display order, +// labels, logo, and series colour. These used to be two parallel lists in two +// files — display order here, labels + logo paths in harness-badge.tsx — and +// adding colour plus a 4th harness would have made it four lists whose indices +// had to be hand-aligned. Row order IS the display order, so there is no slot +// map to keep in sync. +// +// COLOURS — each harness wears its vendor's brand colour, so a line reads as its +// harness without a legend lookup. The set was chosen by validator sweep on the +// strict all-pairs gate, not by eye, which changed two of the four: +// +// node /scripts/validate_palette.js \ +// "#D97757,#000000,#4285F4,#4a3aa7" --mode light --surface "#ffffff" --pairs all +// [FAIL] Lightness band outside band: [["#000000",0]] +// [FAIL] Chroma floor below floor (reads gray): [["#000000",0]] +// [PASS] CVD separation worst all-pairs #4a3aa7<->#4285F4 dE 18.5 (deutan) +// [PASS] Normal-vision floor worst all-pairs #4a3aa7<->#4285F4 dE 21.0 +// [PASS] Contrast vs surface all 4 >= 3:1 +// +// The two FAILs on pure black are WAIVED DELIBERATELY — do not "fix" them by +// substituting a dark green (#047857 was evaluated: ALL PASS, and rejected +// because vendor recognition outweighed a proxy gate). Both failing checks are +// *proxies* for "will these hues be distinguishable" and assume a chromatic +// categorical set. Every check that measures distinguishability directly — CVD +// separation, normal-vision floor, contrast — passes, and black is not even the +// worst pair (violet<->blue is, comfortably above both floors). Black separates +// on *lightness*, the one channel colour-vision deficiency does not touch, and +// sits at 21:1 on white. The residual risk is semantic, not perceptual: black +// reads as ink if the plot area ever gains a black element, so revisit then. +// Delegate takes violet because UiPath orange is unusable next to Anthropic +// coral — CVD dE 5.1, normal-vision dE 9.5, i.e. hard to tell apart even with +// full colour vision. +export const HARNESSES = [ + { + id: "claude-code", + short: "Claude Code", + label: "Claude Code · Anthropic", + logo: "/harness/claude-code.png", + color: "#D97757", // Anthropic coral + }, + { + id: "codex", + short: "Codex", + label: "Codex · OpenAI", + logo: "/harness/codex.png", + color: "#000000", // OpenAI black — proxy gates waived, see above + }, + { + id: "antigravity", + short: "Antigravity", + label: "Antigravity · Google Gemini", + logo: "/harness/antigravity.png", + color: "#4285F4", // Google blue + }, + { + id: "delegate-sdk", + short: "Delegate SDK", + label: "Delegate SDK · UiPath", + logo: null, // no vendor mark shipped yet + color: "#4a3aa7", // violet — UiPath orange is unusable beside coral + }, +] as const satisfies readonly { + id: string; + short: string; + label: string; + logo: string | null; + color: string; +}[]; + +export type KnownHarness = (typeof HARNESSES)[number]["id"]; + +// Display-order hint / default source, derived from the table. Deliberately NOT +// a whitelist — the switcher is data-driven (see listRecentHarnesses), so a new +// harness surfaces automatically without editing anything here. +export const KNOWN_HARNESSES: readonly KnownHarness[] = HARNESSES.map( + (h) => h.id, +); // A run with no RunConfig harness predates the stamp; every such run was // claude-code (the only nightly harness before codex/antigravity), so @@ -18,14 +95,104 @@ export function normalizeHarness(harness: string | null | undefined): string { return harness ?? DEFAULT_HARNESS; } -// Accept any syntactically-plausible harness id, defaulting to claude-code when -// absent or malformed. Deliberately NOT whitelisted against KNOWN_HARNESSES so -// a newly-added harness (e.g. "delegate") is selectable the moment its runs -// exist — the value is only ever compared for equality against a run's stamped -// harness, never used in a path or query, so a bounded charset is enough. -export function parseHarnessParam(raw: string | string[] | undefined): string { +// The one expression of "is this harness known", and its position in display +// order. Nothing else re-derives it. -1 when unknown. +export function knownIndex(h: string): number { + return HARNESSES.findIndex((x) => x.id === h); +} + +// The table row for a harness, or undefined when unknown. A linear find rather +// than a Record lookup so a degenerate id ("toString", "constructor") cannot +// resolve to an inherited prototype member. +export function harnessRow(h: string) { + return HARNESSES.find((x) => x.id === h); +} + +// Short human label ("Claude Code") for selectors and prose; unknown ids fall +// through to the raw id. +export function harnessShortLabel(h: string): string { + return harnessRow(h)?.short ?? h; +} + +// Neutral grey for any harness not in the table — reads as "untracked" rather +// than claiming a brand. 4.8:1 on white, and it separates from the black series +// on lightness (L 0.55 vs 0). A lighter grey is not an option: #9ca3af is 2.1:1, +// too faint for a 2px line. +// +// CAVEAT: this is also the axis-tick fill, so a 5th harness's line would render +// in exactly the tick-label grey and could read as chart chrome rather than data. +// Unreachable while the table covers every harness that runs; if a 5th appears, +// give it a table row with its own colour rather than letting it fall through. +export const UNKNOWN_HARNESS_COLOR = "#6b7280"; + +// Pure and total: colour follows the ENTITY, never its rank in the visible set, +// so a selector chip and a chart line can never disagree, and the four known +// harnesses never repaint when the visible set changes. +export function harnessColor(h: string): string { + return harnessRow(h)?.color ?? UNKNOWN_HARNESS_COLOR; +} + +// Known harnesses first in table order, then any newcomers alphabetically, so +// the result is deterministic but self-extending. Dedupes, so a raw array of +// per-point harnesses works. +// +// Deliberately does NOT apply the "never empty -> [DEFAULT_HARNESS]" rule: that +// belongs to the selector-feeding call site, not to grouping chart points, where +// an empty window must yield zero series rather than one phantom Claude Code +// series. +export function orderHarnesses(seen: Iterable): string[] { + return [...new Set(seen)].sort((a, b) => { + const ia = knownIndex(a); + const ib = knownIndex(b); + if (ia >= 0 && ib >= 0) return ia - ib; + if (ia >= 0) return -1; + if (ib >= 0) return 1; + return a.localeCompare(b); + }); +} + +// Split points into one group per distinct harness, groups ordered by +// orderHarnesses. Generic over the minimal structural constraint so this module +// needs no import from the data layer. +// +// Each group preserves incoming point order, which is load-bearing: recharts +// needs monotonic x within a series or it draws a zigzag. getOverview already +// sorts ascending by timestamp before handing points over. +export function groupByHarness( + points: readonly T[], +): Array<{ harness: string; points: T[] }> { + const byHarness = new Map(); + for (const p of points) { + const group = byHarness.get(p.harness); + if (group) group.push(p); + else byHarness.set(p.harness, [p]); + } + return orderHarnesses(byHarness.keys()).map((harness) => ({ + harness, + // Non-null: every key came from the map itself. + points: byHarness.get(harness)!, + })); +} + +// The `?h=` gate. Accept any syntactically-plausible harness id; `null` means +// "all harnesses", which is what the home page renders when `?h=` is absent. +// Deliberately NOT whitelisted against KNOWN_HARNESSES so a newly-added harness +// is selectable the moment its runs exist — the value is only ever compared for +// equality against a run's stamped harness, never used in a path or query, so a +// bounded charset is enough. +export function parseHarnessScope( + raw: string | string[] | undefined, +): string | null { const v = Array.isArray(raw) ? raw[0] : raw; - if (typeof v !== "string") return DEFAULT_HARNESS; + if (typeof v !== "string") return null; const trimmed = v.trim(); - return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS; + return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : null; +} + +// The single-harness parser for /trends, /watchlist and /path-to-ga, which keep +// "exactly one harness" semantics. Derived from parseHarnessScope so the charset +// and length bound exist once and cannot drift apart; observably identical to +// the previous standalone implementation for every input. +export function parseHarnessParam(raw: string | string[] | undefined): string { + return parseHarnessScope(raw) ?? DEFAULT_HARNESS; } From 9dc79ff74760a5096f8ccb1908c6fc08e361800b Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 15:07:58 -0700 Subject: [PATCH 03/12] =?UTF-8?q?feat(evalboard):=203/6=20=E2=80=94=20tag?= =?UTF-8?q?=20chart=20points=20with=20their=20harness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getOverview pre-filters perRun to one harness, so it can only ever return a single RunPoint[] and both charts render one hardcoded-blue line. Stamping each point with its harness is what lets a later phase draw one coloured line per harness. The underlying fetch is already unfiltered and cached, and the harness filter is a pure in-memory .filter(), so an all-harness view costs zero extra IO. - RunPoint.harness is required, not optional: getOverview always has a value, so optionality would only invite `?? DEFAULT_HARNESS` fallbacks at consumers. tsc --noEmit is the proof every construction site was updated. - OverviewData.harnesses is the server-side answer to "which harnesses am I looking at", computed once from the windowed, filtered points — deliberately NOT from listRecentHarnesses(), which is a discovery list over a different slice and can name harnesses this window has no runs for. An empty window yields [], so the page cannot claim "all 1 harnesses" and the chart cannot draw a phantom series. - listRecentHarnessesInner drops its inline known-first-then-alphabetical sort for orderHarnesses. This removes a real divergence: the inline version sorted newcomers with .sort() (UTF-16 code units) while orderHarnesses uses localeCompare, so ids like Zebra-Agent / apex-agent would have ordered differently in the switcher than in the chart legend. The never-empty [DEFAULT_HARNESS] fallback stays at this call site, where it belongs. The perRun filter is untouched — it is already null-tolerant, so an all-harness view needs nothing more than passing null. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/lib/__tests__/overview.test.ts | 58 ++++++++++++++++++++++++ evalboard/lib/overview.ts | 34 +++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index eaa935d..93c9d6c 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -2,10 +2,12 @@ import { describe, expect, test, vi } from "vitest"; import { buildAdhocRows, collectPipelineRuns, + harnessesInPoints, summarizeListing, turnBudgetRateForTasks, type PerRun, type RunListingRow, + type RunPoint, } from "../overview"; import { normalizeHarness } from "../harness"; import type { RunOverviewTask } from "../runs"; @@ -474,3 +476,59 @@ describe("buildAdhocRows", () => { expect(row.tasksRun).toBe(2); }); }); + +// OverviewData.harnesses is the server-side answer to "which harnesses am I +// looking at", read by the page for its scope label and by the charts for their +// series. Derived from the WINDOWED, FILTERED points — not from the discovery +// list — so it can never name a harness this window contains no runs for. +describe("harnessesInPoints", () => { + function point(overrides: Partial = {}): RunPoint { + return { + runId: "2026-07-30_00-00-00", + timestamp: 0, + harness: "claude-code", + successRate: null, + turnBudgetRate: null, + ...overrides, + }; + } + + test("an empty window yields [] — not [DEFAULT_HARNESS]", () => { + // A phantom claude-code entry here would make the page claim "all 1 + // harnesses" and the chart draw one empty series for a window with no runs. + expect(harnessesInPoints([])).toEqual([]); + }); + + test("returns the distinct harnesses present, deduped", () => { + expect( + harnessesInPoints([ + point({ harness: "codex" }), + point({ harness: "codex" }), + point({ harness: "claude-code" }), + ]), + ).toEqual(["claude-code", "codex"]); + }); + + test("orders known harnesses by the shared display order, not by first appearance", () => { + expect( + harnessesInPoints([ + point({ harness: "antigravity" }), + point({ harness: "claude-code" }), + point({ harness: "codex" }), + ]), + ).toEqual(["claude-code", "codex", "antigravity"]); + }); + + test("sorts an unknown harness after the known ones", () => { + expect( + harnessesInPoints([ + point({ harness: "some-future-harness" }), + point({ harness: "codex" }), + ]), + ).toEqual(["codex", "some-future-harness"]); + }); + + test("a single-harness window yields exactly that harness", () => { + expect(harnessesInPoints([point({ harness: "codex" })])).toEqual(["codex"]); + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 4e6cfa6..952cc5b 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,12 +15,17 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; -import { DEFAULT_HARNESS, KNOWN_HARNESSES, normalizeHarness } from "./harness"; +import { DEFAULT_HARNESS, normalizeHarness, orderHarnesses } from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { runId: string; timestamp: number; // ms since epoch (UTC); used as the chart x-coordinate + // Which harness produced this run, already normalized — legacy runs that + // predate the RunConfig stamp fold to claude-code via normalizeHarness. + // Required, not optional: getOverview always has a value, so optionality + // would only invite `?? DEFAULT_HARNESS` fallbacks at every consumer. + harness: string; successRate: number | null; // % of budgeted tasks whose visible turns stayed within 1.5× their // expected_turns budget. Only tasks carrying a positive expected_turns @@ -82,12 +87,25 @@ export interface OverviewData { runs: RunPoint[]; // one point per run, no daily aggregation windowStart: number; // ms — chart x-domain start windowEnd: number; // ms — chart x-domain end + // The harnesses actually present in this window's points, in display order. + // The server-side answer to "which harnesses am I looking at", so the page + // never recomputes it. Derived from the WINDOWED, FILTERED points — not from + // listRecentHarnesses(), which is a discovery list over a different slice + // and can name harnesses this window contains none of. Empty window → []. + harnesses: string[]; skills: TagCount[]; taskTags: TagCount[]; reviewTags: TagCount[]; activeTag: string | null; } +// The harnesses present in a set of chart points, in display order. Extracted so +// the derivation is unit-testable without touching the blob layer (getOverview +// itself is not unit-tested here, matching this module's convention). +export function harnessesInPoints(points: readonly RunPoint[]): string[] { + return orderHarnesses(points.map((p) => p.harness)); +} + export interface RunListingRow { id: string; // When a tag/q filter is active, every metric is scoped to matching tasks. @@ -295,13 +313,13 @@ async function listRecentHarnessesInner(): Promise { if (r.overview) seen.add(normalizeHarness(r.overview.harness)); } // Known harnesses first (stable display order), then any newcomers - // (alphabetical) so the list is deterministic but self-extending. - const known = KNOWN_HARNESSES.filter((h) => seen.has(h)); - const extras = [...seen] - .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h)) - .sort(); - const ordered = [...known, ...extras]; + // (alphabetical) so the list is deterministic but self-extending. One shared + // comparator with the chart's series ordering, so the switcher and the + // legend can never disagree about where a newcomer sorts. + const ordered = orderHarnesses(seen); // Never hand back an empty list — the default must always be selectable. + // This rule belongs to the switcher, not to orderHarnesses: an empty window + // must yield zero chart series rather than one phantom claude-code line. return ordered.length > 0 ? ordered : [DEFAULT_HARNESS]; } @@ -580,6 +598,7 @@ export async function getOverview( runPoints.push({ runId: id, timestamp: date.getTime(), + harness: normalizeHarness(overview.harness), successRate: rate, turnBudgetRate: turnBudgetRateForTasks(matching), }); @@ -599,6 +618,7 @@ export async function getOverview( runs: runPoints, windowStart, windowEnd, + harnesses: harnessesInPoints(runPoints), skills, taskTags, reviewTags, From e2628e069dd3a4d20171b3af5d4c57e8da8db205 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 15:38:10 -0700 Subject: [PATCH 04/12] =?UTF-8?q?refactor(evalboard):=204/6=20=E2=80=94=20?= =?UTF-8?q?one=20multi-series=20chart=20behind=20both=20overview=20charts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit daily-chart.tsx and turn-budget-chart.tsx were ~117 lines each and identical apart from a dataKey, a connectNulls value and one tooltip sentence — including byte-identical shortLabel/fullLabel. Converting both to multi-series would have meant writing the same conversion, legend, tooltip and height change twice, with a wrong-dataKey copy-paste as the likeliest error. So there is now one RunPointLineChart and two ~30-line wrappers, and the possibility is removed rather than tested for. One per harness, each carrying its own data over a shared numeric time axis, coloured by harness id rather than series order — so a chart line and a selector chip can never disagree. Adds a per-chart legend (>=2 series only; a single line is already named by the subtitle) and a harness-naming tooltip. h-56 -> h-72 absorbs the legend and unpacks the top-30% crowding that a [0,100] domain gives four harnesses all sitting between 70 and 100%. The domain stays [0,100]: a zoomed [50,100] read better but misrepresents a percentage. Two real attribution bugs, both found by review and both fixed here: - shared={false} is IGNORED by LineChart in recharts 2.15.4 — getTooltipEventType computes 'item', LineChart only accepts 'axis', so it silently falls back. The tooltip stayed nearest-x across every harness. - With the default allowDuplicatedCategory, recharts builds its categorical domain from the CONCATENATION of all series and then each indexes its OWN points with that combined index. Hovering claude-code's Jul 5 dot rendered two active dots: the right one, plus a bogus highlight on codex's Jul 10 — a date claude-code has no run for. Four harnesses would light four. allowDuplicatedCategory={false} on the XAxis fixes both by switching lookup to findEntryInArray(series, "timestamp", activeLabel). It is load-bearing for correctness, not de-duplication, and the code says so. New tests hover every dot and assert harness + date + rate + exactly one active dot; they fail without the prop. Nothing in the suite mounted a tooltip before, which is how this survived. describe() now receives the plotted VALUE rather than the point, so a wrapper cannot narrate one metric while charting another — that combination previously typechecked and rendered. LegendStrip and HarnessSwatch live in app/_components/legend.tsx, not in chart-common.tsx as first planned: ChipLegend consumes LegendStrip and the tag rail renders on chart-less pages, so importing it from the chart module pulls recharts into /runs/[id] (163 -> 268 kB First Load JS) and /trends (119 -> 225 kB). Same bundle-correctness rule that keeps groupByHarness in the leaf module. ChipLegend keeps its zero-arg API and renders identically (the Tailwind-400 classes became their literal hexes), so its three consumers are untouched, and /path-to-ga needs no edits. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/_components/legend.tsx | 44 +++ .../_overview/__tests__/chart-common.test.tsx | 279 ++++++++++++++++++ evalboard/app/_overview/chart-common.tsx | 204 +++++++++++++ evalboard/app/_overview/daily-chart.tsx | 113 ++----- evalboard/app/_overview/tag-rail.tsx | 28 +- evalboard/app/_overview/turn-budget-chart.tsx | 120 ++------ 6 files changed, 576 insertions(+), 212 deletions(-) create mode 100644 evalboard/app/_components/legend.tsx create mode 100644 evalboard/app/_overview/__tests__/chart-common.test.tsx create mode 100644 evalboard/app/_overview/chart-common.tsx diff --git a/evalboard/app/_components/legend.tsx b/evalboard/app/_components/legend.tsx new file mode 100644 index 0000000..20329a5 --- /dev/null +++ b/evalboard/app/_components/legend.tsx @@ -0,0 +1,44 @@ +// Shared legend/swatch primitives, deliberately recharts-free and in their own +// module. LegendStrip is used by the overview charts' harness legend AND by the +// tag rail's ChipLegend — and the tag rail renders on pages with no chart at all +// (/runs/[id] via run-view, /trends via trends-view). Importing it from +// chart-common.tsx instead drags recharts into those bundles: measured at +// +105 kB First Load JS on /runs/[id] (163 -> 268 kB) and on /trends +// (119 -> 225 kB). Keep this file free of chart imports. + +import { harnessColor } from "@/lib/harness"; + +// The colour dot that ties a harness to its series line and its selector chip. +// Inline style, not a Tailwind class: the colour is a runtime value, and +// Tailwind's JIT only emits classes it can see as literal strings at build time, +// so a dynamically-built `bg-[${hex}]` would be purged and render colourless. +export function HarnessSwatch({ harness }: { harness: string }) { + return ( + + ); +} + +// The generic "dot + label" row. Colours arrive as values so the same primitive +// serves both the fixed tag-rail legend and the runtime-coloured harness legend. +export function LegendStrip({ + entries, +}: { + entries: { key: string; color: string; label: string }[]; +}) { + return ( +
+ {entries.map((e) => ( + + + {e.label} + + ))} +
+ ); +} diff --git a/evalboard/app/_overview/__tests__/chart-common.test.tsx b/evalboard/app/_overview/__tests__/chart-common.test.tsx new file mode 100644 index 0000000..f2c8d20 --- /dev/null +++ b/evalboard/app/_overview/__tests__/chart-common.test.tsx @@ -0,0 +1,279 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { cloneElement, type ReactElement } from "react"; +import { describe, expect, test, vi } from "vitest"; +import { harnessColor } from "@/lib/harness"; +import type { RunPoint } from "@/lib/overview"; + +vi.mock("recharts", async () => { + const actual = await vi.importActual("recharts"); + return { + ...actual, + // jsdom has no layout, so the real ResponsiveContainer measures 0x0 and + // renders nothing (and needs ResizeObserver). Hand the chart a fixed size. + // The cast is needed because ReactElement's default prop type is unknown. + ResponsiveContainer: ({ children }: { children: ReactElement }) => + cloneElement( + children as ReactElement<{ width: number; height: number }>, + { width: 600, height: 300 }, + ), + }; +}); + +// Import the components AFTER the mock is registered. +const { DailySuccessChart } = await import("../daily-chart"); +const { TurnBudgetChart } = await import("../turn-budget-chart"); +const { ChipLegend } = await import("../tag-rail"); + +function point(overrides: Partial = {}): RunPoint { + return { + runId: "2026-07-30_00-00-00", + timestamp: Date.UTC(2026, 6, 30), + harness: "claude-code", + successRate: 90, + turnBudgetRate: 80, + ...overrides, + }; +} + +const WINDOW = { windowStart: Date.UTC(2026, 6, 1), windowEnd: Date.UTC(2026, 6, 31) }; + +function curves(container: HTMLElement): Element[] { + return [...container.querySelectorAll(".recharts-line-curve")]; +} +function dots(container: HTMLElement): Element[] { + return [...container.querySelectorAll(".recharts-dot")]; +} + +// recharts only emits a .recharts-line-curve path when a series has at least two +// points — a single point has no segment to draw (it still renders its dot). So +// curve-counting tests give every harness two points; dot-counting ones don't need to. +function twoPoints(harness: string, day: number): RunPoint[] { + return [ + point({ harness, timestamp: Date.UTC(2026, 6, day) }), + point({ harness, timestamp: Date.UTC(2026, 6, day + 1) }), + ]; +} + +describe("RunPointLineChart — one line per harness", () => { + test("three harnesses render three curves, each in its own brand colour", () => { + const { container } = render( + , + ); + const paths = curves(container); + expect(paths).toHaveLength(3); + // Assert against harnessColor, never hardcoded hexes, so this stays + // honest if the palette is re-validated. + expect(new Set(paths.map((p) => p.getAttribute("stroke")))).toEqual( + new Set([ + harnessColor("claude-code"), + harnessColor("codex"), + harnessColor("antigravity"), + ]), + ); + }); + + test("dot count equals the total point count across series", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(3); + }); + + test("a single harness renders one curve and NO legend", () => { + // One series is already named by the section subtitle; a one-entry + // legend box would be noise. + const { container } = render( + , + ); + expect(curves(container)).toHaveLength(1); + expect(screen.queryByText("Claude Code")).toBeNull(); + }); + + test("two or more harnesses render one legend entry each, by short label", () => { + render( + , + ); + expect(screen.getByText("Claude Code")).toBeInTheDocument(); + expect(screen.getByText("Codex")).toBeInTheDocument(); + }); + + test("an unknown harness still plots, with the neutral colour and its raw id", () => { + const { container } = render( + , + ); + expect(curves(container)).toHaveLength(2); + expect( + curves(container).map((p) => p.getAttribute("stroke")), + ).toContain(harnessColor("some-future-harness")); + expect(screen.getByText("some-future-harness")).toBeInTheDocument(); + }); + + test("empty data renders without throwing and draws no curves", () => { + // A 30d window with no runs, or an over-narrow tag filter. + const { container } = render(); + expect(curves(container)).toHaveLength(0); + }); + + test("a harness with a single point draws a lone dot", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(3); + }); +}); + +describe("the two wrappers plot different metrics", () => { + // The one assertion that would have caught a wrong-dataKey copy-paste back + // when this logic was duplicated across two files. Kept even though the + // shared implementation now makes that mistake structurally unlikely. + const nullBudget = [ + point({ harness: "claude-code", successRate: 100, turnBudgetRate: null }), + ]; + + test("TurnBudgetChart plots turnBudgetRate — a null budget yields no dot", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(0); + }); + + test("DailySuccessChart plots successRate — the same point DOES yield a dot", () => { + const { container } = render( + , + ); + expect(dots(container)).toHaveLength(1); + }); +}); + +// The most dangerous failure mode this chart has: reporting one harness's rate +// under another harness's date. Nothing else in the suite mounts a tooltip, and +// that gap is exactly how two real attribution bugs survived a first review — +// recharts ignores `shared={false}` on LineChart, and with the default +// allowDuplicatedCategory each indexed its OWN points with an index into +// the CONCATENATION of all series. Hover every dot and assert the triple. +describe("tooltip attributes the hovered point to the right harness", () => { + // claude-code and codex interleave in time, so an index- or nearest-x-based + // lookup would visibly cross the wires. + const data = [ + point({ harness: "claude-code", timestamp: Date.UTC(2026, 6, 5), successRate: 20 }), + point({ harness: "claude-code", timestamp: Date.UTC(2026, 6, 25), successRate: 21 }), + point({ harness: "codex", timestamp: Date.UTC(2026, 6, 10), successRate: 95 }), + point({ harness: "codex", timestamp: Date.UTC(2026, 6, 14), successRate: 96 }), + ]; + const expected = [ + ["Claude Code", "2026-07-05 00:00 UTC", "20.0% success"], + ["Claude Code", "2026-07-25 00:00 UTC", "21.0% success"], + ["Codex", "2026-07-10 00:00 UTC", "95.0% success"], + ["Codex", "2026-07-14 00:00 UTC", "96.0% success"], + ]; + + test.each(expected.map((e, i) => [i, ...e] as const))( + "dot %i reports %s / %s / %s", + (index, harness, date, rate) => { + const { container } = render( + , + ); + const dot = [...container.querySelectorAll(".recharts-dot")][index]; + const cx = Number(dot.getAttribute("cx")); + const cy = Number(dot.getAttribute("cy")); + // recharts reads pageX/pageY; send both pairs so this works + // regardless of which the version consults. + fireEvent.mouseMove(container.querySelector(".recharts-wrapper")!, { + pageX: cx, + pageY: cy, + clientX: cx, + clientY: cy, + }); + const text = + container.querySelector(".recharts-tooltip-wrapper") + ?.textContent ?? ""; + expect(text).toContain(harness); + expect(text).toContain(date); + expect(text).toContain(rate); + // Exactly one highlighted mark: with the combined-index bug this was + // 2 (and would be 4 with four harnesses), lighting up an unrelated + // point on every other series. + expect( + container.querySelectorAll(".recharts-active-dot"), + ).toHaveLength(1); + }, + ); +}); + +describe("connectNulls is scoped to one harness", () => { + test("a gap in one harness is not bridged across another harness's run", () => { + // The headline correctness gain of per-series data: with a single + // over the flat array, claude-code's null at Jul 10 would have been + // bridged straight through codex's Jul 12 run. + const { container } = render( + , + ); + // One curve per harness, and each curve's geometry is built only from its + // own points — so no claude-code segment can pass through a codex dot. + const paths = curves(container); + expect(paths).toHaveLength(2); + const codexDotCount = 2; + // claude-code contributes 2 non-null points, codex 2 → 4 dots total. + expect(dots(container)).toHaveLength(2 + codexDotCount); + }); +}); + +describe("ChipLegend after the LegendStrip re-base", () => { + test("renders its three labels with the Tailwind-400 hexes it used to carry", () => { + // Regression on the class -> hex swap: rendered output must be unchanged. + const { container } = render(); + for (const label of ["skill", "review", "tag"]) { + expect(screen.getByText(label)).toBeInTheDocument(); + } + const colors = [...container.querySelectorAll("span[style]")].map((s) => + (s as HTMLElement).style.backgroundColor, + ); + expect(colors).toEqual([ + "rgb(129, 140, 248)", // #818cf8 indigo-400 + "rgb(251, 113, 133)", // #fb7185 rose-400 + "rgb(156, 163, 175)", // #9ca3af gray-400 + ]); + }); +}); diff --git a/evalboard/app/_overview/chart-common.tsx b/evalboard/app/_overview/chart-common.tsx new file mode 100644 index 0000000..3a22c3f --- /dev/null +++ b/evalboard/app/_overview/chart-common.tsx @@ -0,0 +1,204 @@ +"use client"; + +import { + CartesianGrid, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; +import { groupByHarness, harnessColor, harnessShortLabel } from "@/lib/harness"; +import type { RunPoint } from "@/lib/overview"; +import { HarnessSwatch, LegendStrip } from "../_components/legend"; + +// MM-DD tick label on the axis; full date+time appears in the tooltip. +function shortLabel(ms: number): string { + const d = new Date(ms); + const m = String(d.getUTCMonth() + 1).padStart(2, "0"); + const day = String(d.getUTCDate()).padStart(2, "0"); + return `${m}-${day}`; +} + +function fullLabel(ms: number): string { + const d = new Date(ms); + const y = d.getUTCFullYear(); + const m = String(d.getUTCMonth() + 1).padStart(2, "0"); + const day = String(d.getUTCDate()).padStart(2, "0"); + const h = String(d.getUTCHours()).padStart(2, "0"); + const min = String(d.getUTCMinutes()).padStart(2, "0"); + return `${y}-${m}-${day} ${h}:${min} UTC`; +} + +// Names each plotted harness. Returns null below two series: a single line is +// already named by the section subtitle, so a one-entry legend box is just noise. +// With two or more, this is what carries identity — the dataviz rule that >=2 +// series always get a legend is unconditional. +function HarnessLegend({ harnesses }: { harnesses: string[] }) { + if (harnesses.length < 2) return null; + return ( + ({ + key: h, + color: harnessColor(h), + label: harnessShortLabel(h), + }))} + /> + ); +} + +// Which RunPoint field a chart plots. Derived from RunPoint, and narrowed to the +// rate metrics, so a typo can't plot a timestamp and a field rename breaks here. +type MetricKey = Extract; + +function ChartTooltip({ + active, + payload, + dataKey, + describe, +}: { + active?: boolean; + payload?: Array<{ payload: RunPoint }>; + dataKey: MetricKey; + describe: (value: number | null) => string; +}) { + if (!active || !payload?.length) return null; + const point = payload[0].payload; + return ( +
+
+ + {harnessShortLabel(point.harness)} +
+
+ {fullLabel(point.timestamp)} +
+ {/* Read through dataKey, so the narrated value IS the plotted one — + a wrapper cannot describe one metric while charting another. */} +
+ {describe(point[dataKey])} +
+
+ ); +} + +// The one chart implementation behind both overview charts. They differ only in +// which metric they plot, whether nulls are bridged, and one tooltip sentence — +// so those are the only parameters, and the multi-series conversion, legend, +// tooltip, axes and height exist exactly once. +// +// One per harness, each carrying its own `data`, sharing a numeric time +// x-axis. Colour comes from the harness id, never from series order, so a chart +// line and a selector chip can never disagree. +export function RunPointLineChart({ + data, + windowStart, + windowEnd, + dataKey, + connectNulls, + describe, +}: { + data: RunPoint[]; + windowStart: number; + windowEnd: number; + dataKey: MetricKey; + connectNulls: boolean; + // Narrates the plotted value. Takes the value, not the point, so it is + // structurally impossible to describe a different metric than the one drawn. + describe: (value: number | null) => string; +}) { + const series = groupByHarness(data); + return ( +
+ + UTC + + {/* Left-aligned in its own row, so it cannot collide with the + absolutely-positioned UTC badge in the top-right. */} + s.harness)} /> +
+ + + + then indexes its OWN points with that + // combined index: hovering one harness's dot lights + // up an unrelated point on every other series, and + // the tooltip resolves by nearest-x across all + // harnesses. Setting this false switches lookup to + // findEntryInArray(series, "timestamp", activeLabel), + // so both the tooltip and the active dot resolve to + // the hovered series' own point at that timestamp. + allowDuplicatedCategory={false} + /> + 70 dip + // look catastrophic and shifts as the window's min + // changes, breaking cross-window comparison. + domain={[0, 100]} + ticks={[0, 25, 50, 75, 100]} + tick={{ fontSize: 11, fill: "#6b7280" }} + tickLine={false} + axisLine={false} + width={36} + /> + + } + cursor={{ + stroke: "#e5e7eb", + strokeDasharray: "3 3", + }} + /> + {series.map((s) => ( + + ))} + + +
+
+ ); +} diff --git a/evalboard/app/_overview/daily-chart.tsx b/evalboard/app/_overview/daily-chart.tsx index 721dd65..1ba3c3e 100644 --- a/evalboard/app/_overview/daily-chart.tsx +++ b/evalboard/app/_overview/daily-chart.tsx @@ -1,57 +1,10 @@ "use client"; -import { - CartesianGrid, - Line, - LineChart, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; import type { RunPoint } from "@/lib/overview"; +import { RunPointLineChart } from "./chart-common"; -// MM-DD tick label on the axis; full date+time appears in the tooltip. -function shortLabel(ms: number): string { - const d = new Date(ms); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - return `${m}-${day}`; -} - -function fullLabel(ms: number): string { - const d = new Date(ms); - const y = d.getUTCFullYear(); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - const h = String(d.getUTCHours()).padStart(2, "0"); - const min = String(d.getUTCMinutes()).padStart(2, "0"); - return `${y}-${m}-${day} ${h}:${min} UTC`; -} - -function CustomTooltip({ - active, - payload, -}: { - active?: boolean; - payload?: Array<{ payload: RunPoint }>; -}) { - if (!active || !payload?.length) return null; - const point = payload[0].payload; - return ( -
-
- {fullLabel(point.timestamp)} -
-
- {point.successRate != null - ? `${point.successRate.toFixed(1)}% success` - : "no tasks"} -
-
- ); -} - +// Per-run success rate, one coloured line per harness. Everything but the metric, +// the null-bridging rule and the tooltip sentence lives in RunPointLineChart. export function DailySuccessChart({ data, windowStart, @@ -62,51 +15,19 @@ export function DailySuccessChart({ windowEnd: number; }) { return ( -
- - UTC - - - - - - - } - cursor={{ stroke: "#e5e7eb", strokeDasharray: "3 3" }} - /> - - - -
+ + rate != null ? `${rate.toFixed(1)}% success` : "no tasks" + } + /> ); } diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx index 2bf1f9b..b8ff160 100644 --- a/evalboard/app/_overview/tag-rail.tsx +++ b/evalboard/app/_overview/tag-rail.tsx @@ -2,6 +2,7 @@ import Link from "next/link"; import type { TagCount } from "@/lib/overview"; import { DEFAULT_HARNESS } from "@/lib/harness"; import type { Window } from "@/lib/reviews-types"; +import { LegendStrip } from "../_components/legend"; type Variant = "neutral" | "rose" | "indigo"; @@ -197,22 +198,19 @@ export function MergedTagRail({ // Tiny legend strip explaining what the chip colors mean. Placed above // the chip rail so users don't have to hover-discover the convention. +// +// Shares LegendStrip with the charts' harness legend rather than mirroring its +// markup. LegendStrip takes colours as values, so these are the literal hexes +// of the Tailwind classes this used to carry (indigo-400, rose-400, gray-400) — +// rendered output is unchanged. Keeps its zero-arg API for all three consumers. export function ChipLegend() { - const entries: Array<[string, string, string]> = [ - ["skill", "bg-indigo-400", "skill"], - ["review", "bg-rose-400", "review"], - ["tag", "bg-gray-400", "tag"], - ]; return ( -
- {entries.map(([key, dotCls, label]) => ( - - - {label} - - ))} -
+ ); } diff --git a/evalboard/app/_overview/turn-budget-chart.tsx b/evalboard/app/_overview/turn-budget-chart.tsx index f274f32..d7e9a16 100644 --- a/evalboard/app/_overview/turn-budget-chart.tsx +++ b/evalboard/app/_overview/turn-budget-chart.tsx @@ -1,60 +1,11 @@ "use client"; -import { - CartesianGrid, - Line, - LineChart, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; import type { RunPoint } from "@/lib/overview"; +import { RunPointLineChart } from "./chart-common"; -// MM-DD tick label on the axis; full date+time appears in the tooltip. -function shortLabel(ms: number): string { - const d = new Date(ms); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - return `${m}-${day}`; -} - -function fullLabel(ms: number): string { - const d = new Date(ms); - const y = d.getUTCFullYear(); - const m = String(d.getUTCMonth() + 1).padStart(2, "0"); - const day = String(d.getUTCDate()).padStart(2, "0"); - const h = String(d.getUTCHours()).padStart(2, "0"); - const min = String(d.getUTCMinutes()).padStart(2, "0"); - return `${y}-${m}-${day} ${h}:${min} UTC`; -} - -function CustomTooltip({ - active, - payload, -}: { - active?: boolean; - payload?: Array<{ payload: RunPoint }>; -}) { - if (!active || !payload?.length) return null; - const point = payload[0].payload; - return ( -
-
- {fullLabel(point.timestamp)} -
-
- {point.turnBudgetRate != null - ? `${point.turnBudgetRate.toFixed(1)}% within turn budget (budgeted failures count as over)` - : "no tasks with a turn budget"} -
-
- ); -} - -// Sibling of DailySuccessChart: same axes/styling, plotting the per-run -// "within expected turns" rate instead of the success rate. Driven by the -// same windowed RunPoint[] so the shared window selector controls both. +// Sibling of DailySuccessChart: same axes and styling, plotting the per-run +// "within expected turns" rate instead of the success rate. Driven by the same +// windowed RunPoint[] so the shared window selector controls both. export function TurnBudgetChart({ data, windowStart, @@ -65,53 +16,20 @@ export function TurnBudgetChart({ windowEnd: number; }) { return ( -
- - UTC - - - - - - - } - cursor={{ stroke: "#e5e7eb", strokeDasharray: "3 3" }} - /> - - - -
+ + rate != null + ? `${rate.toFixed(1)}% within turn budget (budgeted failures count as over)` + : "no tasks with a turn budget" + } + /> ); } From 30935aabb7612c3712f616d2c47486cebe1923b2 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 15:54:33 -0700 Subject: [PATCH 05/12] =?UTF-8?q?feat(evalboard):=205/6=20=E2=80=94=20defa?= =?UTF-8?q?ult=20the=20home=20page=20to=20all=20harnesses,=20add=20an=20Al?= =?UTF-8?q?l=20chip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An absent `?h=` on `/` now means ALL harnesses — one colour-coded line per harness in both charts — instead of silently meaning claude-code. An explicit `?h=` narrows the charts AND the rails together, so the view is always internally consistent and shareable. The selector gains a leading "All" chip behind a new `allowAll` prop, so /trends, /watchlist and /path-to-ga keep exactly-one-harness semantics with no edits. Includes bug B1, pulled forward from the next phase because THIS change is what makes it a defect: hrefForTag dropped `h` when it equalled the default, which was harmless while absent `?h=` resolved to claude-code, and is a scope reset now that absent means All. Without it, clicking any tag chip from an isolated Claude Code view silently re-widened the charts, the rails and the selector back to All — exactly the "silently resets the user's scope mid-navigation" failure this phase was warned about. The rule is now stated once, for every consumer: an explicit scope is always emitted; an absent `h` means all harnesses. On /trends the URL merely becomes explicit, with identical behaviour. Two further bugs found in review: - URLSearchParams.size is Safari 17+ only. Where it is undefined the ternary took the falsy branch and navigated to a bare pathname, so on iOS 16 clicking a harness chip would discard window, tag, q AND the scope being set. Uses the toString() idiom buildHref and SearchBox already use. jsdom implements .size, so no test could have caught this. - scopeLabel special-cased zero harnesses but not one, rendering "all 1 harnesses". It now names the single harness, which is also more useful since the chart legend hides itself below two series. The connector became "for" rather than "across", which reads correctly for both a set and one member. scopeLabel is computed once and used in BOTH subtitle branches — the tag/q branch previously never named the harness at all — and its count comes from OverviewData.harnesses, derived from the same points the chart groups, so "all N" always equals the number of lines drawn. Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/harness-selector.test.tsx | 175 ++++++++++++++++++ .../app/_components/harness-selector.tsx | 58 ++++-- .../app/_overview/__tests__/tag-rail.test.tsx | 84 +++++++++ evalboard/app/_overview/tag-rail.tsx | 16 +- evalboard/app/page.tsx | 49 +++-- 5 files changed, 351 insertions(+), 31 deletions(-) create mode 100644 evalboard/app/_components/__tests__/harness-selector.test.tsx create mode 100644 evalboard/app/_overview/__tests__/tag-rail.test.tsx diff --git a/evalboard/app/_components/__tests__/harness-selector.test.tsx b/evalboard/app/_components/__tests__/harness-selector.test.tsx new file mode 100644 index 0000000..69d212a --- /dev/null +++ b/evalboard/app/_components/__tests__/harness-selector.test.tsx @@ -0,0 +1,175 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { harnessColor } from "@/lib/harness"; + +// The selector reads router/params hooks; stub them so it renders in jsdom +// without a provider, and capture `replace` to assert the resulting URL. +const replace = vi.fn(); +let searchParams = new URLSearchParams(); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace }), + usePathname: () => "/", + useSearchParams: () => searchParams, +})); + +const { HarnessSelector } = await import("../harness-selector"); + +const HARNESSES = ["claude-code", "codex", "antigravity"]; + +beforeEach(() => { + replace.mockClear(); + searchParams = new URLSearchParams(); +}); + +// A known harness's chip contains the vendor logo, whose alt text ("Codex · +// OpenAI") joins the accessible name — so the full name is "Codex · OpenAI +// Codex". Match on a substring rather than the exact name. +function chip(label: string) { + return screen.getByRole("button", { + name: new RegExp(label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), + }); +} +function lastUrl(): string { + expect(replace).toHaveBeenCalledTimes(1); + return replace.mock.calls[0][0] as string; +} + +describe("the All chip (allowAll)", () => { + test("renders and is the only pressed chip when current is null", () => { + render( + , + ); + expect(chip("All")).toBeInTheDocument(); + const pressed = screen + .getAllByRole("button") + .filter((b) => b.getAttribute("aria-pressed") === "true"); + expect(pressed).toHaveLength(1); + expect(pressed[0]).toHaveTextContent("All"); + }); + + test("clicking All removes h and preserves every other param", () => { + searchParams = new URLSearchParams("window=7d&tag=foo&h=codex"); + render( + , + ); + fireEvent.click(chip("All")); + const url = lastUrl(); + expect(url).not.toContain("h="); + expect(url).toContain("window=7d"); + expect(url).toContain("tag=foo"); + }); + + test("clicking All when h is the only param leaves no trailing ?", () => { + searchParams = new URLSearchParams("h=codex"); + render( + , + ); + fireEvent.click(chip("All")); + expect(lastUrl()).toBe("/"); + }); + + test("a harness chip is pressed instead of All once one is selected", () => { + render( + , + ); + expect(chip("All")).toHaveAttribute("aria-pressed", "false"); + expect(chip("Codex")).toHaveAttribute("aria-pressed", "true"); + }); + + test("the All chip carries no colour swatch — it is not a series", () => { + render( + , + ); + expect(chip("All").querySelectorAll("span[style]")).toHaveLength(0); + }); +}); + +describe("selecting a harness", () => { + test("clicking a chip sets h to that harness", () => { + render( + , + ); + fireEvent.click(chip("Codex")); + expect(lastUrl()).toContain("h=codex"); + }); + + test("selecting preserves the other params", () => { + searchParams = new URLSearchParams("window=14d&q=needle"); + render( + , + ); + fireEvent.click(chip("Antigravity")); + const url = lastUrl(); + expect(url).toContain("h=antigravity"); + expect(url).toContain("window=14d"); + expect(url).toContain("q=needle"); + }); + + test("each harness chip carries a swatch matching its series colour", () => { + render( + , + ); + // Assert against harnessColor rather than hardcoded hexes, so the test + // stays honest if the palette is re-validated — and so a chip and its + // chart line are provably the same colour. + for (const [harness, label] of [ + ["claude-code", "Claude Code"], + ["codex", "Codex"], + ["antigravity", "Antigravity"], + ] as const) { + const swatch = chip(label).querySelector( + "span[style]", + ) as HTMLElement | null; + expect(swatch, `${label} chip has no swatch`).not.toBeNull(); + expect(swatch!.style.backgroundColor).toBe( + hexToRgb(harnessColor(harness)), + ); + } + }); +}); + +describe("without allowAll (the /trends, /watchlist, /path-to-ga shape)", () => { + test("no All chip is rendered", () => { + render(); + expect(screen.queryByRole("button", { name: "All" })).toBeNull(); + }); + + test("the current harness is still marked active", () => { + render(); + expect(chip("Claude Code")).toHaveAttribute("aria-pressed", "true"); + }); +}); + +describe("an active harness absent from the recent window", () => { + test("is still rendered and active (a deep-linked ?h= reads as selected)", () => { + render( + , + ); + expect(chip("Delegate SDK")).toHaveAttribute("aria-pressed", "true"); + }); + + test("the null (All) case does not prepend a phantom chip", () => { + render( + , + ); + // All + exactly the two supplied harnesses. + expect(screen.getAllByRole("button")).toHaveLength(3); + }); +}); + +function hexToRgb(hex: string): string { + const n = parseInt(hex.slice(1), 16); + return `rgb(${(n >> 16) & 255}, ${(n >> 8) & 255}, ${n & 255})`; +} diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index 7a814f0..7cc651f 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -2,44 +2,76 @@ import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { HarnessBadge, harnessShortLabel } from "./harness-badge"; +import { HarnessSwatch } from "./legend"; -// Segmented control for the trends page's harness scope. Sets `?h=` -// while preserving the active q/tag params, mirroring WindowSelector. Each -// segment shows the vendor logo + short label so it reads like the per-run -// harness badge on the runs tables. +const ACTIVE_CLS = "bg-studio-blue text-white"; +const INACTIVE_CLS = "bg-white text-gray-700 hover:bg-gray-50"; + +// Segmented control for the harness scope. Sets `?h=` while preserving +// the active q/tag/window params, mirroring WindowSelector. Each segment shows +// the vendor logo + a colour swatch + the short label, so a chip reads like both +// the per-run harness badge on the runs tables and its line in the charts. +// +// `current == null` means "all harnesses". Only pages that support that scope +// pass `allowAll`; /trends, /watchlist and /path-to-ga keep exactly-one-harness +// semantics and are unaffected. export function HarnessSelector({ current, harnesses, + allowAll = false, }: { - current: string; + current: string | null; harnesses: readonly string[]; + allowAll?: boolean; }) { const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - const set = (h: string) => { + // One navigation helper for both setting and clearing the scope, so the + // trailing-`?` guard exists once and no URL is built twice. + const nav = (mutate: (p: URLSearchParams) => void) => { const p = new URLSearchParams(searchParams.toString()); - p.set("h", h); - router.replace(`${pathname}?${p.toString()}`, { scroll: false }); + mutate(p); + // Length-check the serialized string, NOT `p.size`: that property is + // Safari 17+ only, and where it's undefined the ternary would take the + // falsy branch and navigate to a bare pathname — silently discarding + // window/tag/q along with the scope. Matches buildHref and SearchBox. + const qs = p.toString(); + router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); }; // Always show the active harness, even if it has aged out of the recent // window (so a deep-linked `?h=` still reads as selected rather than absent). - const opts = harnesses.includes(current) - ? harnesses - : [current, ...harnesses]; + // Guarded for null: in the All view there is no active harness to prepend. + const opts = + current == null || harnesses.includes(current) + ? harnesses + : [current, ...harnesses]; return (
+ {allowAll && ( + + )} {opts.map((h) => { const active = h === current; return ( ); diff --git a/evalboard/app/_overview/__tests__/tag-rail.test.tsx b/evalboard/app/_overview/__tests__/tag-rail.test.tsx new file mode 100644 index 0000000..bd192d3 --- /dev/null +++ b/evalboard/app/_overview/__tests__/tag-rail.test.tsx @@ -0,0 +1,84 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, test } from "vitest"; +import { MergedTagRail } from "../tag-rail"; + +// MergedTagRail renders next/link, which needs no router for an href assertion. +const TAGS = { + skills: [{ tag: "activation", count: 3 }], + reviewTags: [{ tag: "flaky", count: 2 }], + taskTags: [{ tag: "python", count: 1 }], +}; + +function hrefs(): string[] { + return screen + .getAllByRole("link") + .map((a) => a.getAttribute("href") ?? ""); +} + +// The rule: an explicit scope is ALWAYS emitted; an absent `h` means all +// harnesses. Omitting `h=claude-code` "to keep URLs clean" is a scope reset now +// that absent no longer resolves to claude-code on the home page. +describe("tag chips preserve the harness scope", () => { + test("claude-code is emitted explicitly — the B1 regression", () => { + // This is the case the old `harness !== DEFAULT_HARNESS` guard dropped: + // an isolated Claude Code view silently widened to All on any tag click. + render( + , + ); + const links = hrefs(); + expect(links.length).toBeGreaterThan(0); + for (const href of links) { + expect(href).toContain("h=claude-code"); + } + }); + + test("a non-default harness is preserved (unchanged behaviour)", () => { + render(); + for (const href of hrefs()) { + expect(href).toContain("h=codex"); + } + }); + + test("the All scope (null) emits no h at all, keeping URLs clean", () => { + render(); + for (const href of hrefs()) { + expect(href).not.toContain("h="); + } + }); + + test("an active chip links to the CLEARED tag and still carries the scope", () => { + // Clicking an active chip must remove the tag filter without also + // widening the harness scope. + render( + , + ); + const active = screen.getByRole("link", { name: /activation/ }); + const href = active.getAttribute("href") ?? ""; + expect(href).not.toContain("tag=activation"); + expect(href).toContain("h=codex"); + expect(href).toContain("window=7d"); + }); + + test("on /trends the URL simply becomes explicit — same behaviour", () => { + // parseHarnessParam maps absent -> claude-code there, so emitting it + // changes nothing except the URL's shape. Asserted so a future reader + // knows the explicitness is intended, not an accident. + render( + , + ); + for (const href of hrefs()) { + expect(href).toMatch(/^\/trends\?/); + expect(href).toContain("h=claude-code"); + } + }); +}); diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx index b8ff160..505026b 100644 --- a/evalboard/app/_overview/tag-rail.tsx +++ b/evalboard/app/_overview/tag-rail.tsx @@ -1,6 +1,5 @@ import Link from "next/link"; import type { TagCount } from "@/lib/overview"; -import { DEFAULT_HARNESS } from "@/lib/harness"; import type { Window } from "@/lib/reviews-types"; import { LegendStrip } from "../_components/legend"; @@ -43,10 +42,17 @@ function hrefForTag( if (window) params.set("window", window); if (tag) params.set("tag", tag); if (q) params.set("q", q); - // Preserve the active harness scope across tag clicks (omit the default to - // keep URLs clean). Without this, filtering by a tag would silently reset a - // codex/antigravity view back to claude-code. - if (harness && harness !== DEFAULT_HARNESS) params.set("h", harness); + // Preserve the active harness scope across tag clicks. The rule, stated once + // because the next page to build a self-link will copy it: an explicit scope + // is ALWAYS emitted; an absent `h` means all harnesses. + // + // Omitting `h=claude-code` to shorten the URL is what this used to do, and it + // is now a scope reset: on `/`, absent `h` no longer resolves to claude-code, + // so dropping it silently widens an isolated Claude Code view back to All on + // every tag click. On `/trends`, where parseHarnessParam still maps absent to + // claude-code, the URL merely becomes explicit — same behaviour, every + // consumer, no per-page exception. + if (harness) params.set("h", harness); const qs = params.toString(); return qs ? `${basePath}?${qs}` : basePath; } diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index d8326a0..5d2dedf 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -6,7 +6,7 @@ import { listRecentHarnesses, type TagCount, } from "@/lib/overview"; -import { parseHarnessParam, DEFAULT_HARNESS } from "@/lib/harness"; +import { parseHarnessScope } from "@/lib/harness"; import { fmtDuration, fmtRunTime, fmtTimestamp, passClass } from "@/lib/format"; import { WindowSelector } from "./_components/window-selector"; import { WINDOWS, type Window } from "@/lib/reviews-types"; @@ -110,15 +110,19 @@ export default async function Page({ const window = parseWindow(params.window); const activeTag = parseTag(params.tag); const q = parseQ(params.q); - const harness = parseHarnessParam(params.h); + // null = all harnesses. An absent `?h=` shows every harness at once. + const harness = parseHarnessScope(params.h); const limit = parseLimit(params.limit); const adhocLimit = parseAdhocLimit(params.alimit); const isFiltered = activeTag != null || q != null; - // The analytics block (chart + rails) is scoped to one harness so the - // success line stops zigzagging across incomparable harnesses. The run - // LIST stays all-harness — seeing every recent run is the page's job, and - // the Harness column already disambiguates each row. + // The analytics block (charts + rails) defaults to ALL harnesses, drawing one + // colour-coded line per harness. The old single-harness scoping existed to + // stop the success line zigzagging between incomparable harnesses; that is + // solved by separating the series, not by narrowing the view. A `?h=` scope + // narrows the charts AND the rails together, so they always agree. The run + // LIST stays all-harness regardless — seeing every recent run is the page's + // job, and the Harness column already disambiguates each row. const [overview, listing, adhoc, harnesses] = await Promise.all([ getOverview(window, activeTag, q, harness), getRunListing(window, activeTag, q, limit), @@ -143,9 +147,11 @@ export default async function Page({ const rawAlimit = Array.isArray(params.alimit) ? params.alimit[0] : params.alimit; - // Omit the default harness from URLs to keep them clean; carry a non-default - // scope through every self-link so it isn't reset by pagination/clear. - const hParam = harness === DEFAULT_HARNESS ? undefined : harness; + // Carry an explicit scope through every self-link so it isn't reset by + // pagination / clear / tag clicks. Emit it whenever there IS one — omitting + // `h=claude-code` because it happens to be the default would silently widen + // an isolated view back to All, since absent now means "all harnesses". + const hParam = harness ?? undefined; const base = { window, tag: activeTag, @@ -170,6 +176,23 @@ export default async function Page({ const adhocShowAllHref = buildHref({ ...base, alimit: "all" }); const adhocShowLessHref = buildHref({ ...base, alimit: undefined }); + // Computed once and used in BOTH subtitle branches, so "the chart says what + // it is scoped to" is enforced by construction rather than by two prose + // edits. The count comes from the server's windowed answer + // (OverviewData.harnesses is derived from the same points the chart groups), + // never a local recomputation — so "all N" always equals the number of lines + // drawn. Counts of 0 and 1 are special-cased: "all 0 harnesses" reads as a + // bug, and "all 1 harnesses" is both ungrammatical and less informative than + // naming the one harness — especially since the legend hides itself below two + // series, so nothing else on screen would name it. + const scopeLabel = harness + ? harnessShortLabel(harness) + : overview.harnesses.length === 1 + ? harnessShortLabel(overview.harnesses[0]) + : overview.harnesses.length > 1 + ? `all ${overview.harnesses.length} harnesses` + : "all harnesses"; + return (
@@ -221,7 +244,7 @@ export default async function Page({ )}{" "} - over the last {window} ·{" "} + for {scopeLabel} over the last {window} ·{" "} {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} {" · "} @@ -235,9 +258,8 @@ export default async function Page({ ) : ( <> - Success rate per{" "} - {harnessShortLabel(harness)} run across the - last {window} · {overview.runs.length} run + Success rate per run for {scopeLabel} over + the last {window} · {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} )} @@ -247,6 +269,7 @@ export default async function Page({
From a20cfaa105fa8efdc682cf1cbb3da98f28cf65ac Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 16:01:08 -0700 Subject: [PATCH 06/12] =?UTF-8?q?fix(evalboard):=206/6=20=E2=80=94=20stop?= =?UTF-8?q?=20rendering=20an=20unknown=20harness's=20id=20twice=20(B2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HarnessBadge printed the raw id as text when it had no vendor logo, and every caller renders harnessShortLabel right after it — so the selector read "delegate-sdk delegate-sdk". Fixed by deleting the fallback rather than adding an accessor: the badge now returns null when its row has no logo, so "no logo → no image" lives in one place, and naming the harness is unambiguously the caller's job. No hasHarnessLogo export — "has a logo" is `logo != null` on one table row, read only by the badge. The run-table Harness column, which relied on that text fallback, now pairs the badge with the label. It normalizes first, because RunListingRow.harness is nullable and a legacy unstamped run would otherwise show the Claude Code logo beside a blank label. Unknown harnesses still render their id, so that cell looks the same as before; known ones gain a label. Also drops the logo's alt text. With both callers printing the label beside it, alt made screen readers announce the harness twice ("Codex · OpenAI Codex") — the selector test had a substring-regex workaround for precisely that, now replaced with exact accessible-name matching. The logo is decorative next to a text label; title still carries the vendor detail on hover. B1 shipped in the previous commit, where the all-harnesses default is what made it a live scope reset rather than a URL nicety. Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/harness-badge.test.tsx | 51 +++++++++++++++++++ .../__tests__/harness-selector.test.tsx | 44 +++++++++++++--- evalboard/app/_components/harness-badge.tsx | 23 +++++---- evalboard/app/page.tsx | 20 ++++++-- 4 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 evalboard/app/_components/__tests__/harness-badge.test.tsx diff --git a/evalboard/app/_components/__tests__/harness-badge.test.tsx b/evalboard/app/_components/__tests__/harness-badge.test.tsx new file mode 100644 index 0000000..cdd297e --- /dev/null +++ b/evalboard/app/_components/__tests__/harness-badge.test.tsx @@ -0,0 +1,51 @@ +import { render } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +// next/image needs no runtime here, but stub it to a plain so the rendered +// attributes are asserted directly rather than through the loader's transform. +vi.mock("next/image", () => ({ + default: (props: Record) => ( + + ), +})); + +const { HarnessBadge } = await import("../harness-badge"); + +describe("HarnessBadge", () => { + test("renders the vendor logo for a harness that has one", () => { + const { container } = render(); + const img = container.querySelector("img"); + expect(img).not.toBeNull(); + expect(img!.getAttribute("src")).toBe("/harness/codex.png"); + expect(img!.getAttribute("title")).toBe("Codex · OpenAI"); + }); + + test("the logo is decorative — empty alt, so the name isn't announced twice", () => { + // Every call site renders harnessShortLabel beside the badge; alt text + // here would make a screen reader say "Codex · OpenAI Codex". + const { container } = render(); + expect(container.querySelector("img")!.getAttribute("alt")).toBe(""); + }); + + test("renders NOTHING for an unknown harness — the B2 fix", () => { + // It used to print the raw id, which callers then printed again beside it. + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); + + test("renders nothing for a known harness with no logo yet", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + test("a missing harness folds to claude-code (legacy unstamped runs)", () => { + for (const harness of [null, undefined]) { + const { container } = render(); + expect(container.querySelector("img")!.getAttribute("src")).toBe( + "/harness/claude-code.png", + ); + } + }); +}); diff --git a/evalboard/app/_components/__tests__/harness-selector.test.tsx b/evalboard/app/_components/__tests__/harness-selector.test.tsx index 69d212a..f24222d 100644 --- a/evalboard/app/_components/__tests__/harness-selector.test.tsx +++ b/evalboard/app/_components/__tests__/harness-selector.test.tsx @@ -22,13 +22,10 @@ beforeEach(() => { searchParams = new URLSearchParams(); }); -// A known harness's chip contains the vendor logo, whose alt text ("Codex · -// OpenAI") joins the accessible name — so the full name is "Codex · OpenAI -// Codex". Match on a substring rather than the exact name. +// The vendor logo is decorative (alt=""), so a chip's accessible name is exactly +// its label — no duplication to work around. function chip(label: string) { - return screen.getByRole("button", { - name: new RegExp(label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), - }); + return screen.getByRole("button", { name: label }); } function lastUrl(): string { expect(replace).toHaveBeenCalledTimes(1); @@ -173,3 +170,38 @@ function hexToRgb(hex: string): string { const n = parseInt(hex.slice(1), 16); return `rgb(${(n >> 16) & 255}, ${(n >> 8) & 255}, ${n & 255})`; } + +// B2: a harness with no vendor mark used to have its id rendered twice — once by +// HarnessBadge's text fallback and once by the chip's own harnessShortLabel. +describe("a harness with no vendor logo (B2)", () => { + test("its label appears exactly once in the chip", () => { + render( + , + ); + // textContent equality, not containment: "delegate-sdkDelegate SDK" + // would satisfy a contains() assertion. + expect(chip("Delegate SDK").textContent).toBe("Delegate SDK"); + }); + + test("an unknown harness likewise shows its id exactly once", () => { + render( + , + ); + expect(chip("some-future-harness").textContent).toBe( + "some-future-harness", + ); + }); + + test("a known harness keeps its logo AND exactly one label", () => { + render(); + const c = chip("Codex"); + expect(c.querySelector("img")).not.toBeNull(); + expect(c.textContent).toBe("Codex"); + }); +}); diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index 5797d89..8e34f17 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -17,18 +17,23 @@ export { // missing harness folds to claude-code (the nightly). This is an internal-only // column — the caller gates it behind isInternal (see lib/edition.ts). export function HarnessBadge({ harness }: { harness?: string | null }) { - const key = normalizeHarness(harness); - const row = harnessRow(key); - // Unknown harness: show the raw id rather than a misleading logo. - if (!row) return {key}; - // A known harness with no vendor mark yet: render nothing. Callers already - // print harnessShortLabel beside the badge, so emitting the raw id here too - // would read "delegate-sdkDelegate SDK" in the selector. - if (!row.logo) return null; + const row = harnessRow(normalizeHarness(harness)); + // No vendor mark for this harness — an unknown one, or a known one with no + // logo yet: render NOTHING. "No logo → no image" lives here and only here. + // + // This deliberately does not fall back to printing the raw id: every caller + // already renders harnessShortLabel beside the badge, so a text fallback + // showed the id twice ("delegate-sdk delegate-sdk"). Pair the badge with + // harnessShortLabel rather than expecting it to name the harness itself. + if (!row?.logo) return null; return ( {row.label} {isInternal && ( - + {/* Badge + label: the badge renders + nothing for a harness with no + vendor mark, so the label is what + keeps this cell from looking + broken — and it names the harness + for screen readers, since the + logo is decorative. */} + + + {harnessShortLabel( + normalizeHarness(r.harness), + )} + )} From 986f02c3b155bf63e7694ef6b1c1246f8e7a29cd Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 16:27:24 -0700 Subject: [PATCH 07/12] fix(evalboard): code review fixes for the charts + pricing SSOT change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings from the final cross-phase review (two of the first three were raised independently by more than one reviewer). The drift guard was structurally blind. pricing-parity.test.ts imported parsePythonTable, so it compared two products of the same regex — a row that regex cannot match (keyword arguments, scientific notation) is missing from the generated artifact AND from the expectation, so deep-equal passes while the board renders "—" for that model forever. The coverage cross-check existed but only inside the generator, i.e. not on the side that CI would run. readTable is now exported and the test uses it. The same test disclaimed copying the rate card and then copied four rates, so a legitimate vendor repricing would break the suite even after a correct regeneration — the two-places problem generation exists to remove. Every expectation is now derived from pricing.py via the parsed table, with an added guard that the float-fidelity case cannot pass vacuously if those rates ever become round numbers. Same for the one hardcoded rate in pricing.test.ts. The legend named harnesses that draw nothing. groupByHarness deliberately keeps null-metric points (grouping must not filter), but turnBudgetRate is legitimately null for a whole harness when none of its runs carry an expected_turns budget — so "Within Expected Turns" could list three harnesses above two lines, and a reader looking for the missing one would read its absence as 0%. The legend is now derived from series with at least one non-null value for the plotted metric. The chart could draw a line the selector could not isolate. The chart's series come from the windowed points; the selector's chips come from a fixed-count scan of recent runs. A harness that ran once three weeks ago appears in a 30d chart but not in that scan, so it had a visible series and no chip. The page now passes the union. Adds the two guards that were previously only comments, both worth a measured ~105 kB of First Load JS and both violated by adding one import: lib/harness.ts has no imports at all, and app/_components/legend.tsx never reaches recharts or the chart module (with the tag rail's import path pinned, since re-pointing it is the actual regression path). Verified both fail when violated. Also pins the contract this change is named for — the subtitle's "all N harnesses" equals the number of lines drawn — by asserting harnessesInPoints and the rendered curve count over the same input, since those are two derivations of what should be one set. Co-Authored-By: Claude Opus 5 (1M context) --- .../_overview/__tests__/chart-common.test.tsx | 70 ++++++++++++++++- evalboard/app/_overview/chart-common.tsx | 10 ++- evalboard/app/page.tsx | 20 ++++- .../lib/__tests__/module-boundaries.test.ts | 78 +++++++++++++++++++ .../lib/__tests__/pricing-parity.test.ts | 73 +++++++++++------ evalboard/lib/__tests__/pricing.test.ts | 8 +- evalboard/lib/harness.ts | 3 +- evalboard/scripts/gen-pricing.d.mts | 6 ++ evalboard/scripts/gen-pricing.mjs | 12 ++- 9 files changed, 243 insertions(+), 37 deletions(-) create mode 100644 evalboard/lib/__tests__/module-boundaries.test.ts diff --git a/evalboard/app/_overview/__tests__/chart-common.test.tsx b/evalboard/app/_overview/__tests__/chart-common.test.tsx index f2c8d20..97c1c94 100644 --- a/evalboard/app/_overview/__tests__/chart-common.test.tsx +++ b/evalboard/app/_overview/__tests__/chart-common.test.tsx @@ -2,7 +2,7 @@ import { fireEvent, render, screen } from "@testing-library/react"; import { cloneElement, type ReactElement } from "react"; import { describe, expect, test, vi } from "vitest"; import { harnessColor } from "@/lib/harness"; -import type { RunPoint } from "@/lib/overview"; +import { harnessesInPoints, type RunPoint } from "@/lib/overview"; vi.mock("recharts", async () => { const actual = await vi.importActual("recharts"); @@ -277,3 +277,71 @@ describe("ChipLegend after the LegendStrip re-base", () => { ]); }); }); + +// The page's subtitle says "all N harnesses" using OverviewData.harnesses, while +// the chart draws groupByHarness(overview.runs). Those are two derivations of +// what should be one set — this pins them together over the same input, so a +// future edit that points the label at a different list (the recent-runs +// discovery list is one token away) cannot claim a count the chart contradicts. +describe("the drawn series set equals harnessesInPoints", () => { + test.each([ + ["one harness", ["claude-code"]], + ["two harnesses", ["claude-code", "codex"]], + ["all four, out of table order", ["delegate-sdk", "codex", "antigravity", "claude-code"]], + ["including an unknown harness", ["some-future-harness", "claude-code"]], + ])("%s", (_label, harnesses) => { + const data = harnesses.flatMap((h, i) => twoPoints(h, 2 + i * 3)); + const { container } = render(); + expect(harnessesInPoints(data)).toHaveLength(harnesses.length); + expect(curves(container)).toHaveLength(harnessesInPoints(data).length); + }); +}); + +// groupByHarness deliberately keeps points whose metric is null (grouping must +// not filter), so the legend has to do the filtering — otherwise it names a +// harness that draws no line, which reads as a missing line or as 0%. +describe("the legend names only harnesses that actually plot", () => { + const mixed = [ + // claude-code has a turn budget; codex's runs carry none. + ...[10, 11].map((d) => + point({ + harness: "claude-code", + timestamp: Date.UTC(2026, 6, d), + successRate: 90, + turnBudgetRate: 80, + }), + ), + ...[13, 14].map((d) => + point({ + harness: "codex", + timestamp: Date.UTC(2026, 6, d), + successRate: 70, + turnBudgetRate: null, + }), + ), + ]; + + test("the success chart names both — both plot successRate", () => { + render(); + expect(screen.getByText("Claude Code")).toBeInTheDocument(); + expect(screen.getByText("Codex")).toBeInTheDocument(); + }); + + test("the turn-budget chart draws and names only the harness with data", () => { + const { container } = render( + , + ); + // recharts still emits a element for the all-null series, but with + // no `d` — nothing is drawn. Count geometry, not elements. + const drawn = curves(container).filter((c) => c.getAttribute("d")); + expect(drawn).toHaveLength(1); + expect(drawn[0].getAttribute("stroke")).toBe(harnessColor("claude-code")); + // Only the plotting harness contributes dots. + expect(dots(container)).toHaveLength(2); + // And no legend at all: with one series actually plotting, HarnessLegend + // suppresses itself, so the invisible Codex line is never named. Before + // the plotted-filter it appeared here with no line to point at. + expect(screen.queryByText("Codex")).toBeNull(); + expect(screen.queryByText("Claude Code")).toBeNull(); + }); +}); diff --git a/evalboard/app/_overview/chart-common.tsx b/evalboard/app/_overview/chart-common.tsx index 3a22c3f..960feca 100644 --- a/evalboard/app/_overview/chart-common.tsx +++ b/evalboard/app/_overview/chart-common.tsx @@ -109,6 +109,14 @@ export function RunPointLineChart({ describe: (value: number | null) => string; }) { const series = groupByHarness(data); + // Name only the harnesses that actually draw something on THIS chart. + // groupByHarness deliberately keeps points whose metric is null (grouping + // must not filter), but turnBudgetRate is legitimately null for a whole + // harness when no task in its runs carries an expected_turns budget — and a + // legend entry for an invisible line reads as a missing line, or worse as 0%. + const plotted = series + .filter((s) => s.points.some((p) => p[dataKey] != null)) + .map((s) => s.harness); return (
@@ -116,7 +124,7 @@ export function RunPointLineChart({ {/* Left-aligned in its own row, so it cannot collide with the absolutely-positioned UTC badge in the top-right. */} - s.harness)} /> +
diff --git a/evalboard/lib/__tests__/module-boundaries.test.ts b/evalboard/lib/__tests__/module-boundaries.test.ts new file mode 100644 index 0000000..f941ba2 --- /dev/null +++ b/evalboard/lib/__tests__/module-boundaries.test.ts @@ -0,0 +1,78 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; + +// Two bundle-correctness invariants that were previously enforced only by +// comments, each worth a measured ~105 kB of First Load JS. Both are violated by +// adding one import, and neither shows up in tsc, the test suite, or a passing +// `next build` — only in the bundle size, which nobody reads on a green PR. + +const here = dirname(fileURLToPath(import.meta.url)); +const root = resolve(here, "../.."); + +function readSource(rel: string): string { + return readFileSync(resolve(root, rel), "utf8"); +} + +// Matches `import … from "x"`, `import "x"`, and `export … from "x"`. +function moduleSpecifiers(src: string): string[] { + return [...src.matchAll(/(?:^|\n)\s*(?:import|export)[\s\S]*?from\s+"([^"]+)"/g)] + .map((m) => m[1]) + .concat( + [...src.matchAll(/(?:^|\n)\s*import\s+"([^"]+)"/g)].map((m) => m[1]), + ); +} + +describe("lib/harness.ts is a true leaf", () => { + // Client components ("use client" charts, the selector, the badge) all import + // this module for values, not just types. lib/overview.ts transitively reaches + // node:fs, node:path and @azure/storage-blob, so a single VALUE import from + // there would drag that whole graph into the browser bundle — which is why + // groupByHarness lives in this module rather than beside its only consumer. + // Type-only imports erase at compile time; values do not. + test("has no imports at all", () => { + const specs = moduleSpecifiers(readSource("lib/harness.ts")); + expect( + specs, + "lib/harness.ts must stay dependency-free — see the KEEP IT A LEAF note at its top", + ).toEqual([]); + }); +}); + +describe("app/_components/legend.tsx stays recharts-free", () => { + // tag-rail.tsx's ChipLegend consumes LegendStrip, and the tag rail renders on + // pages with no chart at all (/runs/[id] via run-view, /trends via + // trends-view). Importing these primitives from chart-common.tsx instead + // measured +105 kB First Load JS on /runs/[id] (163 -> 268 kB) and on /trends + // (119 -> 225 kB). + const FORBIDDEN = ["recharts", "chart-common"]; + + test("imports neither recharts nor the chart module", () => { + const specs = moduleSpecifiers(readSource("app/_components/legend.tsx")); + for (const spec of specs) { + for (const bad of FORBIDDEN) { + expect( + spec.includes(bad), + `legend.tsx must not import ${spec} — it renders on chart-less pages`, + ).toBe(false); + } + } + }); + + test("its only dependency is the leaf harness module", () => { + // Keeps the guard above honest: if legend.tsx grows a dependency, that + // dependency's own graph has to be re-checked, so force the decision here. + expect(moduleSpecifiers(readSource("app/_components/legend.tsx"))).toEqual( + ["@/lib/harness"], + ); + }); + + test("the tag rail takes LegendStrip from the recharts-free module", () => { + // The regression path: re-pointing this import at chart-common is what + // pulls recharts onto /runs/[id] and /trends. + const specs = moduleSpecifiers(readSource("app/_overview/tag-rail.tsx")); + expect(specs).toContain("../_components/legend"); + expect(specs.some((s) => s.includes("chart-common"))).toBe(false); + }); +}); diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 4b0599a..75d8a35 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { parsePythonTable } from "../../scripts/gen-pricing.mjs"; +import { readTable } from "../../scripts/gen-pricing.mjs"; import { PRICING, resolvePricing } from "../pricing"; // Drift guard: lib/pricing.generated.ts is generated from the authoritative @@ -13,11 +13,16 @@ import { PRICING, resolvePricing } from "../pricing"; // their cost). Generation makes exact-match free: one assertion subsumes // orphans, missing entries, and rate drift, with no allowlist to maintain. // -// parsePythonTable is imported from the generator rather than re-declared here, -// so the guard and the thing it guards cannot disagree about what pricing.py says. +// This imports readTable, NOT parsePythonTable, and the distinction is the whole +// guard: parsePythonTable on both sides would compare two products of the same +// regex, so a row that regex cannot match is missing from the artifact AND from +// the expectation — deep-equal passes while the board renders "—" for that model +// forever. readTable cross-checks the matched-row count against the number of +// `ModelPricing(` constructions in the file and throws if they disagree. describe("pricing.generated.ts ↔ pricing.py parity", () => { - const py = parsePythonTable(); + // Throws if the parser skipped a row, so this doubles as the coverage check. + const py = readTable(); test("parses a non-trivial Python table", () => { // Guard against a regex/path regression silently passing the test. @@ -44,21 +49,25 @@ describe("pricing.generated.ts ↔ pricing.py parity", () => { }); // Behaviour the generated table has to preserve. Deliberately NOT a second copy -// of the rate card — the exact-match test above already proves every rate equals -// pricing.py, so re-asserting rates here would only mean a legitimate vendor -// repricing has to be hand-patched in two places to keep the suite green. What -// these pin is lookup *logic* that the deep-equal cannot see: the undated -// fallback, a genuine zero surviving, and float precision. +// of the rate card: every expectation below is derived from pricing.py via +// `py[...]`, never a literal. The test above already proves every rate matches, +// so hardcoding one here would mean a legitimate vendor repricing breaks the +// suite even after a correct regeneration — exactly the two-places problem +// generation exists to remove. What these pin is lookup *logic* the deep-equal +// cannot see: the undated fallback, a genuine zero surviving, float precision. describe("resolvePricing over the generated table", () => { + const py = readTable(); + test("a deleted dated key still prices via the undated fallback", () => { // "claude-opus-4-6-20250514" was a dead entry (absent from pricing.py) // and redundant: resolvePricing strips a trailing -YYYYMMDD. Deleting it - // must change no rendered figure. The $5 also pins the headline fix — - // the old mirror carried Opus 4.1's $15 onto 4.6, overstating cost 3x. - const p = resolvePricing("claude-opus-4-6-20250514"); - expect(p).not.toBeNull(); - expect(p?.inputPerMTok).toBe(5); - expect(resolvePricing("claude-sonnet-4-6-20250514")).not.toBeNull(); + // must change no rendered figure — the dated id still resolves, and to + // exactly the undated key's rate. + for (const undated of ["claude-opus-4-6", "claude-sonnet-4-6"]) { + const dated = resolvePricing(`${undated}-20250514`); + expect(dated, `${undated}-20250514 no longer resolves`).not.toBeNull(); + expect(dated).toEqual(resolvePricing(undated)); + } }); test("a model the old allowlist suppressed is now priced", () => { @@ -70,18 +79,32 @@ describe("resolvePricing over the generated table", () => { test("a legitimately-zero rate survives as 0, not undefined", () => { // Bedrock publishes no prompt-cache rate for the open-weight models, so - // cache-read is a real 0 — generation must not drop it as falsy. - expect(resolvePricing("deepseek.v3.2")?.cacheReadPerMTok).toBe(0); - expect(resolvePricing("zai.glm-5")?.cacheReadPerMTok).toBe(0); + // cache-read is a real 0 — generation must not drop it as falsy. Asserted + // against pricing.py's own value, and separately that it IS zero there, + // so the test still means something if that rate ever changes. + for (const model of ["deepseek.v3.2", "zai.glm-5"]) { + expect(py[model][3]).toBe(0); + expect(resolvePricing(model)?.cacheReadPerMTok).toBe(py[model][3]); + } }); test("fractional rates round-trip exactly through generation", () => { - // String(Number(x)) is only safe for these if it truly round-trips; - // assert it rather than assume it. - expect(resolvePricing("z-ai/glm-5.2")?.inputPerMTok).toBe(0.7168); - expect(resolvePricing("z-ai/glm-5.2")?.cacheReadPerMTok).toBe(0.13312); - expect(resolvePricing("deepseek/deepseek-v4-pro")?.cacheReadPerMTok).toBe( - 0.003625, - ); + // The generator emits numbers via string interpolation, so a precision + // loss would show up as generated !== parsed for these long decimals. + for (const model of ["z-ai/glm-5.2", "deepseek/deepseek-v4-pro"]) { + const p = resolvePricing(model); + expect(p).not.toBeNull(); + expect([ + p!.inputPerMTok, + p!.outputPerMTok, + p!.cacheWritePerMTok, + p!.cacheReadPerMTok, + ]).toEqual(py[model]); + // And that at least one of them really is a long decimal, so this + // test cannot pass vacuously if pricing.py switches to round numbers. + expect( + py[model].some((r) => String(r).replace("0.", "").length >= 4), + ).toBe(true); + } }); }); diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index ae60299..4125e6c 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -36,9 +36,11 @@ describe("resolvePricing", () => { }); test("knows the current default opus id", () => { - // $25, not $75: Opus dropped to $5/$25 at 4.5, and the old hand-copied - // mirror carried Opus 4.1's legacy $15/$75 forward onto 4.6/4.7/4.8. - expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(25); + // Rate-agnostic on purpose: pricing-parity.test.ts already proves every + // rate equals pricing.py, so pinning a number here would break the suite + // on a legitimate vendor repricing. What matters at this seam is that the + // current default Opus id resolves at all. + expect(resolvePricing("claude-opus-4-8")).not.toBeNull(); }); test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => { diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts index ae701ee..e8ada25 100644 --- a/evalboard/lib/harness.ts +++ b/evalboard/lib/harness.ts @@ -89,7 +89,8 @@ export const KNOWN_HARNESSES: readonly KnownHarness[] = HARNESSES.map( // A run with no RunConfig harness predates the stamp; every such run was // claude-code (the only nightly harness before codex/antigravity), so -// null/undefined folds to the default. Mirrors HarnessBadge's default. +// null/undefined folds to the default. HarnessBadge and the run table both +// route through this, so it is the one definition of that fold. export const DEFAULT_HARNESS = "claude-code"; export function normalizeHarness(harness: string | null | undefined): string { return harness ?? DEFAULT_HARNESS; diff --git a/evalboard/scripts/gen-pricing.d.mts b/evalboard/scripts/gen-pricing.d.mts index 7d2845f..b6fb56d 100644 --- a/evalboard/scripts/gen-pricing.d.mts +++ b/evalboard/scripts/gen-pricing.d.mts @@ -7,3 +7,9 @@ export declare function parsePythonTable(): Record< string, [number, number, number, number] >; + +/** parsePythonTable + the coverage cross-check. Throws if a row was skipped. */ +export declare function readTable(): Record< + string, + [number, number, number, number] +>; diff --git a/evalboard/scripts/gen-pricing.mjs b/evalboard/scripts/gen-pricing.mjs index f32e316..da8b12a 100644 --- a/evalboard/scripts/gen-pricing.mjs +++ b/evalboard/scripts/gen-pricing.mjs @@ -71,12 +71,16 @@ export function parsePythonTable() { } /** - * Parse, then verify the parse actually covered the file. Only the generator - * needs this — the parity test compares two products of the same parser, so it - * cannot detect an unmatched row on its own. + * Parse, then verify the parse actually covered the file. + * + * The drift guard MUST use this rather than parsePythonTable: comparing two + * products of the same parser cannot detect a row the parser skipped, because + * the row is missing from the artifact AND from the expectation. This is the + * only function that can tell "the mirror is faithful" apart from "the mirror + * and the expectation are equally blind". * @returns {Record} */ -function readTable() { +export function readTable() { const src = readSource(); const table = parsePythonTable(); const rows = Object.keys(table).length; From ef54feb78d6131d7734f3ea9d407e47d320332a0 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 16:28:14 -0700 Subject: [PATCH 08/12] docs(harness): record deferred guards from the evalboard charts + pricing run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four candidates, the first raised independently by two reviewers: CI never runs the evalboard suite, so the pricing drift guard the generated-SSOT design rests on is never executed. Deferred rather than applied because adding a required check to pr-checks.yml is a repo-wide CI-policy change outside a plan scoped to evalboard/ — recorded with the concrete job to add. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index dd3912d..c5f1df8 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -107,3 +107,45 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule caught them. The cleanup plan explicitly deferred this as YAGNI for the one-time purge, but any future doc rename/deletion re-opens the same blind spot — caught in the 2026-07-03 open-source-docs-cleanup implementation run. + +## From 2026-07-30 evalboard multi-harness charts + pricing SSOT + +- [ ] **CI never runs the `evalboard/` test suite — the highest-value gap here.** + `grep -rn evalboard .github/workflows/ .pre-commit-config.yaml Makefile` yields + exactly one hit: a comment in `pr-checks.yml` saying `evalboard/pnpm-lock.yaml` + is *out of scope*. `pnpm verify` (`tsc --noEmit && vitest run && next build`) + exists in `evalboard/package.json` but nothing invokes it. Consequence: the + pricing drift guard that the whole generated-SSOT design rests on is never + executed. A Python dev edits a rate in `src/coder_eval/pricing.py`, never opens + `evalboard/`, and the PR goes green while every rendered USD figure keeps the + stale rate — precisely how the 7 wrong rates (Opus 4.6/4.7/4.8 at 3× actual) + accumulated in the first place. Flagged independently by two reviewers. + *Not done here because adding a required check to `pr-checks.yml` is a repo-wide + CI-policy change affecting every contributor's PR, outside a plan scoped to + `evalboard/`.* Ready to apply: a job mirroring the existing `setup-node` steps + (`pnpm install --frozen-lockfile` + `pnpm verify` with `working-directory: + evalboard`), ideally non-blocking first. Alternatively a Python-side CExxx rule + asserting `lib/pricing.generated.ts` is current, which would put the guard on + the side that actually edits rates. +- [ ] **One shared normalizer fixture for `normalizeModel` (TS) and + `_normalize_model` (Python).** They have drifted in *both* directions: TS strips + a trailing `-YYYYMMDD`, Python does not — so `claude-opus-4-6-20250514` prices + in the evalboard and is unpriced in the backend, and the frontend "estimated" + cost can disagree with the backend's authoritative Cost for dated ids. Neither + side's tests can see the other. Needs a decision on which behaviour is the + reference before a fixture can be written; the plan lists it under Open + Questions. Note also that plugin rates registered via `register_pricing` (e.g. + `coder_eval_uipath`) can never reach the generated table at all. +- [ ] **No test can see `app/page.tsx`'s self-link scope threading.** `hParam`, + `base` and `buildHref` are module-private inside an async server component and + there is no test for that file. The plan calls a dropped `?h=` the phase's main + risk — a future seventh self-link spelled `buildHref({ window, tag })` silently + resets the user's harness scope with the whole suite green. Guarding it means + extracting `buildHref` + a `selfLinkBase` helper into a testable module. +- [ ] **"Any chart with ≥2 series must have a hover-attribution test."** Nothing + in the suite mounted a chart tooltip before this run, which is how two real + misattribution bugs survived a spec review: recharts silently ignores + `shared={false}` on `LineChart`, and with the default `allowDuplicatedCategory` + each `` indexes its own points with an index into the concatenation of + all series. Closed for the two overview charts; nothing enforces it for the next + multi-series chart someone adds. From 8cdbcf10738a09de9032c550cc81c40159becf1e Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 17:49:01 -0700 Subject: [PATCH 09/12] feat(evalboard): show the analytics charts in every edition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daily success / turn-budget charts and the tag rail were gated behind isInternal, so a default-configured instance — including the deployed board — showed only the run list. Nothing in the block is UiPath-specific: it renders whatever runs the instance is pointed at, so an OSS clone charts its own results. The gate dates to the initial public release, not to any recent change. Note the Harness column in the run table is still gated separately, so on an OSS-edition instance the chart legend names harnesses while that column stays hidden. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/page.tsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 0ce06ea..5113767 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -217,10 +217,10 @@ export default async function Page({ /> {/* The analytics block — daily success / turn-budget charts, the - window selector, and the colored skill/review/tag rail — is an - internal-only surface (see lib/edition.ts). The public OSS - edition drops it so the front page is just the run list. */} - {isInternal && ( + window selector, and the colored skill/review/tag rail. Shown in + EVERY edition: it renders whatever runs the instance is pointed + at, so an OSS clone charts its own results and nothing about it + is UiPath-specific. (It was internal-only until 2026-07-30.) */}
@@ -336,7 +336,6 @@ export default async function Page({ )}
- )}
From e700c3113471cd5cf0151460f967f9a05d35867a Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 17:49:13 -0700 Subject: [PATCH 10/12] feat(evalboard): give Delegate SDK a red series colour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requested change from violet. Red is the hardest family to place here, because Anthropic coral (#D97757) already occupies that end of the wheel — which is why UiPath's own brand orange was rejected when the palette was first chosen. So the candidates were swept through the validator against the fixed three rather than picked by eye, and brightness turned out to decide it (floors: 6-8 CVD, 15 normal-vision, both against coral): #EF4444 red-500 CVD 3.3 / NV 9.3 unusable #FA4616 UiPath brand CVD 5.1 / NV 9.5 unusable #FF0000 pure red CVD 5.5 / NV 13.8 fails #E11D48 rose-600 CVD 8.0 / NV 14.0 fails NV #DC2626 red-600 CVD 9.1 / NV 13.2 fails NV #B91C1C red-700 CVD 16.2 / NV 18.0 passes #991B1B red-800 CVD 22.6 / NV 23.3 shipped Every naive red collides with coral, several of them worse than the brand orange already ruled out. red-800 sits far enough down the lightness axis to separate cleanly, and it improves the palette: its worst pair (22.6) beats the violet it replaces, whose worst pair was 18.5 against Google blue. The two #000000 proxy failures are unchanged and still waived for the documented reason. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/lib/__tests__/harness.test.ts | 4 ++-- evalboard/lib/harness.ts | 28 ++++++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts index 5e0eebb..e34665e 100644 --- a/evalboard/lib/__tests__/harness.test.ts +++ b/evalboard/lib/__tests__/harness.test.ts @@ -141,7 +141,7 @@ describe("harnessColor", () => { expect(harnessColor("claude-code")).toBe("#D97757"); expect(harnessColor("codex")).toBe("#000000"); expect(harnessColor("antigravity")).toBe("#4285F4"); - expect(harnessColor("delegate-sdk")).toBe("#4a3aa7"); + expect(harnessColor("delegate-sdk")).toBe("#991B1B"); }); test("all four brand colours are distinct", () => { @@ -178,7 +178,7 @@ describe("HARNESSES table integrity", () => { }); test("colors are unique, case-insensitively", () => { - // Compare lowercased: the table mixes cases (#D97757 vs #4a3aa7), so a + // Compare lowercased: the table mixes cases (#D97757 vs #991B1B), so a // raw Set would count two case-variants of one colour as distinct. const colors = HARNESSES.map((h) => h.color.toLowerCase()); expect(new Set(colors).size).toBe(colors.length); diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts index e8ada25..047687c 100644 --- a/evalboard/lib/harness.ts +++ b/evalboard/lib/harness.ts @@ -21,11 +21,11 @@ // strict all-pairs gate, not by eye, which changed two of the four: // // node /scripts/validate_palette.js \ -// "#D97757,#000000,#4285F4,#4a3aa7" --mode light --surface "#ffffff" --pairs all +// "#D97757,#000000,#4285F4,#991B1B" --mode light --surface "#ffffff" --pairs all // [FAIL] Lightness band outside band: [["#000000",0]] // [FAIL] Chroma floor below floor (reads gray): [["#000000",0]] -// [PASS] CVD separation worst all-pairs #4a3aa7<->#4285F4 dE 18.5 (deutan) -// [PASS] Normal-vision floor worst all-pairs #4a3aa7<->#4285F4 dE 21.0 +// [PASS] CVD separation worst all-pairs #991B1B<->#D97757 dE 22.6 (deutan) +// [PASS] Normal-vision floor worst all-pairs #991B1B<->#D97757 dE 23.3 // [PASS] Contrast vs surface all 4 >= 3:1 // // The two FAILs on pure black are WAIVED DELIBERATELY — do not "fix" them by @@ -38,9 +38,23 @@ // on *lightness*, the one channel colour-vision deficiency does not touch, and // sits at 21:1 on white. The residual risk is semantic, not perceptual: black // reads as ink if the plot area ever gains a black element, so revisit then. -// Delegate takes violet because UiPath orange is unusable next to Anthropic -// coral — CVD dE 5.1, normal-vision dE 9.5, i.e. hard to tell apart even with -// full colour vision. +// Delegate's slot must be a DARK red. Everything in the red/orange family is +// competing with Anthropic coral, so brightness is what decides it — swept +// against the fixed three (higher dE is better; the floors are 6-8 for CVD and +// 15 for normal vision): +// +// #EF4444 red-500 CVD 3.3 / NV 9.3 unusable +// #FA4616 UiPath brand CVD 5.1 / NV 9.5 unusable +// #FF0000 pure red CVD 5.5 / NV 13.8 fails +// #E11D48 rose-600 CVD 8.0 / NV 14.0 fails the NV floor +// #DC2626 red-600 CVD 9.1 / NV 13.2 fails the NV floor +// #B91C1C red-700 CVD 16.2 / NV 18.0 passes, tighter margin +// #991B1B red-800 CVD 22.6 / NV 23.3 SHIPPED — best margin +// +// So the naive reds all collide with coral, several of them worse than the +// UiPath brand orange that was rejected for exactly this reason. red-800 is far +// enough down the lightness axis to separate cleanly, and it beats the violet +// (#4a3aa7) this replaced, whose worst pair was dE 18.5 against Google blue. export const HARNESSES = [ { id: "claude-code", @@ -68,7 +82,7 @@ export const HARNESSES = [ short: "Delegate SDK", label: "Delegate SDK · UiPath", logo: null, // no vendor mark shipped yet - color: "#4a3aa7", // violet — UiPath orange is unusable beside coral + color: "#991B1B", // deep red — see the red sweep above }, ] as const satisfies readonly { id: string; From 995995ed418797463aeeeb29db8dbc449e931e9b Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 30 Jul 2026 17:57:07 -0700 Subject: [PATCH 11/12] fix(evalboard): stop the harness chips shifting when one is selected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selecting a harness could delete other chips. The switcher was fed OverviewData.harnesses, which is derived from the windowed AND harness-filtered points — so with `?h=claude-code` active it collapses to ["claude-code"], and any harness not also in listRecentHarnesses()' fixed 12-run discovery scan lost its chip. A harness that ran earlier in the window but not recently would appear under "All", then vanish the moment anything was selected, leaving no way back to it without hand-editing the URL. That was my own regression: the union with overview.harnesses was added to fix a harness being drawn with no chip to select it, and it fixed that by making the chip set depend on the current scope, which is worse. getOverview now also returns windowHarnesses — every harness in the window, captured BEFORE the harness filter and ignoring tag/q — and the switcher uses that. It is invariant under selection by construction, while still covering the drawn-but-not-recent case the union was meant to address. OverviewData.harnesses keeps its meaning (the plotted set, so "all N harnesses" still equals the number of lines drawn) and its tests are unchanged. The new test contrasts the two derivations over the same scoped input rather than comparing the fixed one to itself, so it fails if the chips are ever re-pointed at the filtered set. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/page.tsx | 2 +- evalboard/lib/__tests__/overview.test.ts | 94 +++++++++++++++++++++++- evalboard/lib/overview.ts | 32 +++++--- 3 files changed, 117 insertions(+), 11 deletions(-) diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 5113767..8dc2249 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -283,7 +283,7 @@ export default async function Page({ // can see it but cannot isolate it. harnesses={orderHarnesses([ ...harnesses, - ...overview.harnesses, + ...overview.windowHarnesses, ])} allowAll /> diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 93c9d6c..d875886 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -9,7 +9,7 @@ import { type RunListingRow, type RunPoint, } from "../overview"; -import { normalizeHarness } from "../harness"; +import { normalizeHarness, orderHarnesses } from "../harness"; import type { RunOverviewTask } from "../runs"; function task(overrides: Partial): RunOverviewTask { @@ -532,3 +532,95 @@ describe("harnessesInPoints", () => { expect(harnessesInPoints([point({ harness: "codex" })])).toEqual(["codex"]); }); }); + +// The switcher's chip set must be STABLE under selection. It is fed +// windowHarnesses, not harnesses: the latter is derived from the filtered +// points, so with `?h=codex` active it collapses to ["codex"] and every other +// chip would vanish the moment one was selected — stranding the user on a scope +// they cannot leave without hand-editing the URL. These pin the distinction. +describe("harnesses vs windowHarnesses", () => { + function run(harness: string | null, id = "2026-07-30_00-00-00"): PerRun { + return { + id, + overview: { + harness, + tasks: [task({ status: "SUCCESS" })], + totalCostUsd: null, + taskDurationSeconds: null, + startedAt: null, + } as unknown as PerRun["overview"], + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + }; + } + + // The derivation getOverview applies for windowHarnesses: every non-adhoc + // run in the window with a readable overview, BEFORE the harness filter. + function windowHarnessesOf(runs: PerRun[]): string[] { + return orderHarnesses( + runs + .filter((r) => !r.adhoc && r.overview) + .map((r) => normalizeHarness(r.overview!.harness)), + ); + } + + const window = [ + run("claude-code", "2026-07-28_00-00-00"), + run("codex", "2026-07-29_00-00-00"), + run("delegate-sdk", "2026-07-30_00-00-00"), + ]; + + test("windowHarnesses lists every harness in the window", () => { + expect(windowHarnessesOf(window)).toEqual([ + "claude-code", + "codex", + "delegate-sdk", + ]); + }); + + test("it is IDENTICAL whichever harness is selected — the anti-shift rule", () => { + // Contrast the two derivations over the SAME scoped input. The old one + // (post-filter) collapses to the selected harness and deletes the other + // chips; the new one (pre-filter) is invariant. Comparing the pre-filter + // result to itself would be vacuous, so apply the real harness filter + // and show only one of the two survives it. + const unscoped = windowHarnessesOf(window); + expect(unscoped).toHaveLength(3); + + for (const scope of ["claude-code", "codex", "delegate-sdk"]) { + const afterHarnessFilter = window.filter( + (r) => normalizeHarness(r.overview!.harness) === scope, + ); + + // What the chips used to be built from — collapses to one. + const oldWay = windowHarnessesOf(afterHarnessFilter); + expect(oldWay).toEqual([scope]); + + // What they are built from now: computed before that filter, so the + // full set survives and every chip stays clickable. + expect( + windowHarnessesOf(window), + `chip set shifted when scoped to ${scope}`, + ).toEqual(unscoped); + } + }); + + test("by contrast, harnesses (the plotted set) DOES narrow with the scope", () => { + // This is correct and is what "all N harnesses" counts — it must equal + // the number of lines drawn, which is 1 when scoped. + const scopedPoints = [ + { harness: "codex" } as RunPoint, + ]; + expect(harnessesInPoints(scopedPoints)).toEqual(["codex"]); + }); + + test("a legacy unstamped run folds into claude-code, not a phantom chip", () => { + expect(windowHarnessesOf([run(null)])).toEqual(["claude-code"]); + }); + + test("ad-hoc runs contribute no chip", () => { + const adhoc = { ...run("delegate-sdk"), adhoc: true }; + expect(windowHarnessesOf([run("codex"), adhoc])).toEqual(["codex"]); + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 952cc5b..437560f 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -87,12 +87,15 @@ export interface OverviewData { runs: RunPoint[]; // one point per run, no daily aggregation windowStart: number; // ms — chart x-domain start windowEnd: number; // ms — chart x-domain end - // The harnesses actually present in this window's points, in display order. - // The server-side answer to "which harnesses am I looking at", so the page - // never recomputes it. Derived from the WINDOWED, FILTERED points — not from - // listRecentHarnesses(), which is a discovery list over a different slice - // and can name harnesses this window contains none of. Empty window → []. + // The harnesses actually PLOTTED — derived from the windowed, filtered + // points, so it always equals the number of lines the chart draws. That is + // what the page's "all N harnesses" subtitle counts. Empty window → []. harnesses: string[]; + // Every harness present in the window, ignoring the harness/tag/q filters. + // Feeds the switcher, which must offer a stable set: `harnesses` above + // collapses to just the active one when `?h=` is set, so using it for the + // chips would delete every other chip the moment one is selected. + windowHarnesses: string[]; skills: TagCount[]; taskTags: TagCount[]; reviewTags: TagCount[]; @@ -559,11 +562,21 @@ export async function getOverview( // Ad-hoc runs never feed the daily chart or the tag rails — they're not // pipeline cadence. (Non-date-named ones are already pruned upstream by // listRunIdsInWindow; this also drops date-named runs flagged adhoc.) - const perRun = (await loadWindowData(window)).filter( + const windowRuns = (await loadWindowData(window)).filter((r) => !r.adhoc); + // Captured BEFORE the harness filter, and deliberately ignoring tag/q: this + // is what the switcher offers, so it must not depend on what is currently + // selected. Deriving it after the filter would collapse it to the active + // harness and make every other chip vanish on selection — stranding the user + // on a scope they cannot leave without editing the URL. + const windowHarnesses = orderHarnesses( + windowRuns + .filter((r) => r.overview) + .map((r) => normalizeHarness(r.overview!.harness)), + ); + const perRun = windowRuns.filter( (r) => - !r.adhoc && - (harness == null || - normalizeHarness(r.overview?.harness) === harness), + harness == null || + normalizeHarness(r.overview?.harness) === harness, ); const needle = q?.trim().toLowerCase() || null; @@ -619,6 +632,7 @@ export async function getOverview( windowStart, windowEnd, harnesses: harnessesInPoints(runPoints), + windowHarnesses, skills, taskTags, reviewTags, From 7bd3ef5f6c83d2b5a2909ac6e472c938d456e537 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 31 Jul 2026 07:17:33 -0700 Subject: [PATCH 12/12] feat(evalboard): scope the whole page by harness, and lift the chips to the header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `?h=` filtered only the analytics block; the run table, its totals and the ad-hoc section stayed all-harness. So picking Claude Code left a table of every harness's runs underneath a chart showing one — two different answers to "what am I looking at" on the same screen. getRunListing and getAdhocRunListing now take the same scope. Because the filter is applied alongside the ad-hoc exclusion rather than after it, totalInWindow and the totals rollup narrow with it too: "N in 30d" counts what the table can actually show, and the summary tiles describe the selected harness. Ad-hoc runs are scoped as well. They carry a harness stamp like any other run, and that section has no Harness column, so leaving it unfiltered would show codex uploads under a Claude Code scope with nothing on screen to explain why. The chips move from the analytics header to the page header, right-aligned against "Recent runs". A control that now narrows the entire page belongs beside the page title rather than inside one of the sections it filters. The window selector stays with the charts — it sets the chart's x-domain. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/page.tsx | 77 ++++++++++++------------ evalboard/lib/__tests__/overview.test.ts | 39 ++++++++++++ evalboard/lib/overview.ts | 27 ++++++++- 3 files changed, 103 insertions(+), 40 deletions(-) diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 8dc2249..cb26aa6 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -120,17 +120,16 @@ export default async function Page({ const adhocLimit = parseAdhocLimit(params.alimit); const isFiltered = activeTag != null || q != null; - // The analytics block (charts + rails) defaults to ALL harnesses, drawing one - // colour-coded line per harness. The old single-harness scoping existed to - // stop the success line zigzagging between incomparable harnesses; that is - // solved by separating the series, not by narrowing the view. A `?h=` scope - // narrows the charts AND the rails together, so they always agree. The run - // LIST stays all-harness regardless — seeing every recent run is the page's - // job, and the Harness column already disambiguates each row. + // `?h=` scopes the WHOLE page — charts, rails, the run table, its totals and + // the ad-hoc section — so every number on screen describes the same set of + // runs. With it absent the page shows all harnesses, drawing one + // colour-coded line per harness; the old single-harness default existed to + // stop the success line zigzagging between incomparable harnesses, which is + // solved by separating the series rather than by narrowing the view. const [overview, listing, adhoc, harnesses] = await Promise.all([ getOverview(window, activeTag, q, harness), - getRunListing(window, activeTag, q, limit), - getAdhocRunListing(adhocLimit), + getRunListing(window, activeTag, q, limit, harness), + getAdhocRunListing(adhocLimit, harness), listRecentHarnesses(), ]); @@ -199,14 +198,37 @@ export default async function Page({ return (
-
-

- Recent runs -

-

- Click a run to drill into tasks, criteria, artifacts, - and logs. -

+ {/* The harness scope sits in the PAGE header, not the analytics + header, because `?h=` scopes the whole page — charts, rails, run + table, totals and ad-hoc section. A control that narrows + everything belongs beside the page title, not inside one of the + sections it filters. */} +
+
+

+ Recent runs +

+

+ Click a run to drill into tasks, criteria, artifacts, + and logs. +

+
+
-
- - -
+
{ expect(windowHarnessesOf([run("codex"), adhoc])).toEqual(["codex"]); }); }); + +// `?h=` scopes the WHOLE page, so the run table, its totals and the ad-hoc +// section all narrow with it — not just the charts. These pin the two filters +// getRunListing and getAdhocRunListing apply, over the same shape of input. +describe("harness scoping of the run listings", () => { + function scoped(runs: PerRun[], harness: string | null): PerRun[] { + return runs.filter( + (r) => + harness == null || + normalizeHarness(r.overview?.harness) === harness, + ); + } + + const runs: PerRun[] = [ + { id: "a", overview: { harness: "claude-code" } as PerRun["overview"], reviewTagCounts: {}, reviewTagsByTask: {} }, + { id: "b", overview: { harness: "codex" } as PerRun["overview"], reviewTagCounts: {}, reviewTagsByTask: {} }, + { id: "c", overview: { harness: null } as PerRun["overview"], reviewTagCounts: {}, reviewTagsByTask: {} }, + ]; + + test("null lists every run — the All scope", () => { + expect(scoped(runs, null).map((r) => r.id)).toEqual(["a", "b", "c"]); + }); + + test("a harness lists only its own runs", () => { + expect(scoped(runs, "codex").map((r) => r.id)).toEqual(["b"]); + }); + + test("an unstamped legacy run counts as claude-code, not as its own scope", () => { + // normalizeHarness folds null -> claude-code, so run "c" must appear + // under the Claude Code scope rather than becoming unreachable. + expect(scoped(runs, "claude-code").map((r) => r.id)).toEqual(["a", "c"]); + }); + + test("a harness with no runs yields an empty table, not everything", () => { + // The failure mode worth guarding: a falsy-check bug here would make an + // unmatched scope fall through and list every run under the wrong label. + expect(scoped(runs, "delegate-sdk")).toEqual([]); + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 437560f..f2bf039 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -718,11 +718,20 @@ export async function getRunListing( tag: string | null, q: string | null, limit: number | null, // null = unlimited + // When set, list only that harness's runs. null = all harnesses. Applied + // alongside the ad-hoc exclusion so totalInWindow and the totals rollup are + // both scoped — "N in 30d" must count what the table can actually show. + harness: string | null = null, ): Promise { // Exclude ad-hoc runs from the main listing — they appear in their own // section (getAdhocRunListing). totalInWindow therefore counts only // pipeline runs, matching the chart above it. - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); // Run IDs are timestamped — newest first by lexical compare. const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); const totalInWindow = sorted.length; @@ -866,8 +875,20 @@ export function buildAdhocRows( // not the id, so we can't window by id and still show the most recent): the // ad-hoc set is small by construction (manual uploads only) and per-run reads // are memoized for 5 min, so a warm front page pays no extra IO. -export async function getAdhocRunListing(limit: number | null): Promise { +export async function getAdhocRunListing( + limit: number | null, + // Scoped by the same `?h=` as the rest of the page: an ad-hoc run carries a + // harness stamp like any other, and leaving this section unfiltered would + // show codex uploads under a Claude Code scope with no column to explain why. + harness: string | null = null, +): Promise { const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) == null); const perRun = await mapWithConcurrency(ids, FETCH_CONCURRENCY, cachedLoadPerRun); - return buildAdhocRows(perRun, limit); + const scoped = + harness == null + ? perRun + : perRun.filter( + (r) => normalizeHarness(r.overview?.harness) === harness, + ); + return buildAdhocRows(scoped, limit); }