Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions packages/app/cypress/component/inference-chart-controls.cy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,36 @@ describe('Inference ChartControls', () => {
cy.get('@setSelectedYAxisMetric').should('have.been.calledOnce');
});

it('lists and selects the schema-v2 derived axes in the Measured Energy group', () => {
const options = [
{
key: 'y_measuredJPerSuccessfulQuery',
label: 'Measured Joules per Successful Query',
},
{
key: 'y_measuredWhPerSuccessfulQuery',
label: 'Measured Watt-hours per Successful Query',
},
{
key: 'y_measuredPowerPercentTdp',
label: 'Measured Average Power as Percent of TDP',
},
];

for (const option of options) {
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.contains('Measured Energy')
.parent()
.within(() => {
cy.contains('[role="option"]', option.label)
.scrollIntoView()
.should('be.visible')
.click();
});
cy.get('@setSelectedYAxisMetric').should('have.been.calledWith', option.key);
}
});

it('hides the GPU comparison section when no GPUs are selected', () => {
// Default mock: selectedGPUs = [] — GPU date range pickers should not render
cy.contains('Comparison Date Range').should('not.exist');
Expand Down
49 changes: 49 additions & 0 deletions packages/app/src/app/api/unofficial-run/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,55 @@ describe('normalizeArtifactRows', () => {
expect(m.mean_e2el).toBe(1.5);
});

it.each([
{
name: 'boolean verdict and junk-suffixed schema',
input: { power_valid: true, power_metric_schema_version: '2garbage' },
expectedVerdict: 0,
expectedSchema: undefined,
},
{
name: 'garbage verdict and valid numeric schema',
input: { power_valid: 'garbage', power_metric_schema_version: 2 },
expectedVerdict: 0,
expectedSchema: 2,
},
{
name: 'canonical numeric strings',
input: { power_valid: '1', power_metric_schema_version: '2' },
expectedVerdict: 1,
expectedSchema: 2,
},
{
name: 'explicit invalid verdict',
input: { power_valid: 0, power_metric_schema_version: 2 },
expectedVerdict: 0,
expectedSchema: 2,
},
{
name: 'legacy row without power contract fields',
input: {},
expectedVerdict: undefined,
expectedSchema: undefined,
},
])(
'normalizes overlay power contract discriminators: $name',
({ input, expectedVerdict, expectedSchema }) => {
const [row] = normalizeArtifactRows([rawRow(input)], '2026-03-01');

if (expectedVerdict === undefined) {
expect(row.metrics).not.toHaveProperty('power_valid');
} else {
expect(row.metrics.power_valid).toBe(expectedVerdict);
}
if (expectedSchema === undefined) {
expect(row.metrics).not.toHaveProperty('power_metric_schema_version');
} else {
expect(row.metrics.power_metric_schema_version).toBe(expectedSchema);
}
},
);

it('surfaces pipeline-parallelism fields in metrics (auto-capture)', () => {
// pp has no configs-table column: the frontend reads it from the metrics
// JSONB (rowToAggDataEntry), so the overlay route must keep passing it
Expand Down
15 changes: 15 additions & 0 deletions packages/app/src/components/ai-chart/types.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { describe, expect, it } from 'vitest';

import { validateSpec } from './types';

describe('validateSpec measured power axes', () => {
it.each([
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
])('preserves %s as a benchmark Y-axis metric', (yAxisMetric) => {
const spec = validateSpec({ dataSource: 'benchmarks', yAxisMetric });

expect(spec.yAxisMetric).toBe(yAxisMetric);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,30 @@ describe('interpolateMetricAtInteractivity', () => {
expect(result!).toBeLessThan(3.5);
});

it.each([
'measuredJPerSuccessfulQuery',
'measuredWhPerSuccessfulQuery',
'measuredPowerPercentTdp',
] as const)('interpolates the derived measured metric %s', (metricKey) => {
const points = [
makePoint({
x: 20,
tpPerGpu: { y: 800, roof: false },
[metricKey]: { y: 80, roof: false },
}),
makePoint({
x: 60,
tpPerGpu: { y: 400, roof: false },
[metricKey]: { y: 40, roof: false },
}),
];

const result = interpolateMetricAtInteractivity(points, 40, metricKey);
expect(result).not.toBeNull();
expect(result!).toBeGreaterThan(40);
expect(result!).toBeLessThan(80);
});

it('returns null when metric field is missing from data points', () => {
const points = [
makePoint({ x: 20, tpPerGpu: { y: 800, roof: false } }),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
reciprocalMetricAt,
} from '@/components/calculator/useThroughputData';
import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history';
import { getHardwareKey } from '@/lib/chart-utils';
import { buildMeasuredPowerChartFields, getHardwareKey } from '@/lib/chart-utils';
import { getGpuSpecs, isKnownGpu } from '@/lib/constants';
import { rowToAggDataEntry } from '@/lib/benchmark-transform';
import type { BenchmarkRow } from '@/lib/api';
Expand Down Expand Up @@ -74,24 +74,7 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null {
jTotal: wrapMetric(power > 0 && tput ? (power * 1000) / tput : 0),
...(outputTput ? { jOutput: wrapMetric(power > 0 ? (power * 1000) / outputTput : 0) } : {}),
...(inputTput ? { jInput: wrapMetric(power > 0 ? (power * 1000) / inputTput : 0) } : {}),
...(typeof entry.avg_power_w === 'number'
? { measuredAvgPower: { y: entry.avg_power_w, roof: false } }
: {}),
...(typeof entry.joules_per_output_token === 'number'
? { measuredJPerOutputToken: { y: entry.joules_per_output_token, roof: false } }
: {}),
...(typeof entry.joules_per_total_token === 'number'
? { measuredJPerTotalToken: { y: entry.joules_per_total_token, roof: false } }
: {}),
...(typeof entry.prefill_avg_power_w === 'number'
? { measuredPrefillAvgPower: { y: entry.prefill_avg_power_w, roof: false } }
: {}),
...(typeof entry.decode_avg_power_w === 'number'
? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } }
: {}),
...(typeof entry.joules_per_input_token === 'number'
? { measuredJPerInputToken: { y: entry.joules_per_input_token, roof: false } }
: {}),
...buildMeasuredPowerChartFields(entry, specs.tdp),
};
return point;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,24 @@
"y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)",
"y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗(含提示词)",
"y_measuredJPerTotalToken_roofline": "lower_right",
"y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y",
"y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)",
"y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(J/query)",
"y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query",
"y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗",
"y_measuredJPerSuccessfulQuery_roofline": "lower_right",
"y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y",
"y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)",
"y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(Wh/query)",
"y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query",
"y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗",
"y_measuredWhPerSuccessfulQuery_roofline": "lower_right",
"y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y",
"y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)",
"y_measuredPowerPercentTdp_labelZh": "实测平均功耗(TDP 占比)",
"y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP",
"y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比",
"y_measuredPowerPercentTdp_roofline": "lower_right",

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TDP axis has Pareto direction

Medium Severity

y_measuredPowerPercentTdp declares lower_right / lower_left rooflines even though this axis is a utilization gauge, not an efficiency frontier. That turns on Pareto / Optimal Only / best-per-SKU logic that treats lower % TDP as better, which misranks configs for this metric.

Additional Locations (2)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit bbd7f58. Configure here.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 WARNING (behavior vs. description mismatch, non-blocking): The PR description states "%TDP has no Pareto direction — it is a utilization gauge, not an efficiency frontier." But this config assigns y_measuredPowerPercentTdp_roofline (lower_right here, lower_left on the e2e block at line 369), and measured-power-direction.test.ts explicitly asserts it. So computeAllRooflines does build a Pareto frontier for %TDP, and "Optimal Only" filters to it.

I confirmed this at runtime: with the %TDP axis selected and Optimal Only on, the chart shows only the single frontier config (B200), not every deployment.

Why it matters: Reviewers trusting the PR body would expect all configs to remain visible under Optimal Only on this axis. The behavior is internally consistent (config + test + code agree) and defensible — %TDP is monotonic in avg_power_w per hardware, so it mirrors the existing measuredAvgPower frontier treatment. But the description is stale/contradictory.

Fix: Either drop the _roofline keys for y_measuredPowerPercentTdp (and relax the test) if a frontier is genuinely unwanted for a "gauge" axis, or update the PR description to match the shipped behavior. No code change needed if the frontier is intended.

"y_cost_limit": 5,
"y_latency_limit": 60
},
Expand Down Expand Up @@ -331,6 +349,24 @@
"y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)",
"y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗(含提示词)",
"y_measuredJPerTotalToken_roofline": "lower_left",
"y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y",
"y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)",
"y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(J/query)",
"y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query",
"y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗",
"y_measuredJPerSuccessfulQuery_roofline": "lower_left",
"y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y",
"y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)",
"y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(Wh/query)",
"y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query",
"y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗",
"y_measuredWhPerSuccessfulQuery_roofline": "lower_left",
"y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y",
"y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)",
"y_measuredPowerPercentTdp_labelZh": "实测平均功耗(TDP 占比)",
"y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP",
"y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比",
"y_measuredPowerPercentTdp_roofline": "lower_left",
"y_cost_limit": 5,
"y_latency_limit": 60
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ const MEASURED_POWER_METRICS = [
'y_measuredDecodeAvgPower',
] as const;

const DERIVED_POWER_METRICS = [
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
] as const;

const defs = chartDefinitions as unknown as ChartDefinition[];
const interactivityDef = defs.find((d) => d.chartType === 'interactivity')!;
const e2eDef = defs.find((d) => d.chartType === 'e2e')!;
Expand Down Expand Up @@ -133,4 +139,13 @@ describe('measured-power Pareto direction', () => {
}
}
});

it.each(DERIVED_POWER_METRICS)('%s is bilingual and lower-is-better', (metric) => {
for (const chartDef of [interactivityDef, e2eDef]) {
expect(chartDef[metric]).toMatch(/\.y$/u);
expect(chartDef[`${metric}_label`]).toBeTruthy();
expect(chartDef[`${metric}_labelZh`]).toBeTruthy();
expect(declaredDirection(chartDef, metric)).toMatch(/^lower_/u);
}
});
});
39 changes: 34 additions & 5 deletions packages/app/src/components/inference/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,21 +137,24 @@ export interface AggDataEntry {
'p99.9_e2el': number;
// Measured GPU telemetry (emitted by runner's aggregate_power.py).
// Optional because historical runs predate the fields.
power_valid?: number;
power_metric_schema_version?: number;
avg_power_w?: number;
joules_per_successful_query?: number;
joules_per_output_token?: number;
joules_per_total_token?: number;
// Multinode / disagg-only measured power. The aggregate_power.py runner
// emits per-role energy splits when the deployment has separate prefill
// and decode workers (single-node disagg or multinode disagg). Single-node
// aggregated configs leave these undefined.
// - prefill_avg_power_w / decode_avg_power_w: mean per-GPU draw (W) within each role
// - joules_per_input_token: prefill_energy / total_input_tokens (prefill GPUs only)
// The disagg decode-only J/output is carried by joules_per_output_token above
// (the runner overrides it to decode_energy / total_output_tokens on disagg) —
// there is no separate _decode field.
// Unprefixed joules fields are whole-deployment metrics in schema version 2.
// Explicit prefill/decode keys retain role-local energy breakdowns.
prefill_avg_power_w?: number;
decode_avg_power_w?: number;
joules_per_input_token?: number;
prefill_joules_per_input_token?: number;
decode_joules_per_output_token?: number;
// Cluster-wide GPU telemetry beyond power (temperature, utilization, memory).
// Emitted by aggregate_power.py when the perfmon CSVs include the matching
// sample columns. Optional because older runs (and runs without the relevant
Expand Down Expand Up @@ -315,6 +318,9 @@ export interface InferenceData extends Partial<Omit<AggDataEntry, AggDataConflic
measuredJPerOutputToken?: { y: number; roof: boolean };
measuredJPerTotalToken?: { y: number; roof: boolean };
measuredJPerInputToken?: { y: number; roof: boolean };
measuredJPerSuccessfulQuery?: { y: number; roof: boolean };
measuredWhPerSuccessfulQuery?: { y: number; roof: boolean };
measuredPowerPercentTdp?: { y: number; roof: boolean };
}

/** Why a chart-ready point was intentionally excluded from the visible plot. */
Expand Down Expand Up @@ -358,7 +364,10 @@ export type YAxisMetricKey =
| 'measuredDecodeAvgPower'
| 'measuredJPerOutputToken'
| 'measuredJPerTotalToken'
| 'measuredJPerInputToken';
| 'measuredJPerInputToken'
| 'measuredJPerSuccessfulQuery'
| 'measuredWhPerSuccessfulQuery'
| 'measuredPowerPercentTdp';

/**
* Defines the configuration and labels for a specific chart.
Expand Down Expand Up @@ -488,6 +497,26 @@ export interface ChartDefinition {
y_measuredJPerTotalToken_label?: string;
y_measuredJPerTotalToken_title?: string;
y_measuredJPerTotalToken_roofline?: 'upper_right' | 'upper_left' | 'lower_left' | 'lower_right';
y_measuredJPerSuccessfulQuery?: string;
y_measuredJPerSuccessfulQuery_label?: string;
y_measuredJPerSuccessfulQuery_title?: string;
y_measuredJPerSuccessfulQuery_roofline?:
| 'upper_right'
| 'upper_left'
| 'lower_left'
| 'lower_right';
y_measuredWhPerSuccessfulQuery?: string;
y_measuredWhPerSuccessfulQuery_label?: string;
y_measuredWhPerSuccessfulQuery_title?: string;
y_measuredWhPerSuccessfulQuery_roofline?:
| 'upper_right'
| 'upper_left'
| 'lower_left'
| 'lower_right';
y_measuredPowerPercentTdp?: string;
y_measuredPowerPercentTdp_label?: string;
y_measuredPowerPercentTdp_title?: string;
y_measuredPowerPercentTdp_roofline?: 'upper_right' | 'upper_left' | 'lower_left' | 'lower_right';
y_cost_limit?: number;
y_latency_limit?: number;
}
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/inference/ui/ChartControls.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ const METRIC_GROUPS: {
'y_measuredJPerInputToken',
'y_measuredJPerOutputToken',
'y_measuredJPerTotalToken',
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
Comment on lines +160 to +162

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Add an end-to-end dashboard test for the new axes

These lines expose three new dashboard selections, but the commit adds only a Cypress component test and unit tests; no cypress/e2e spec exercises selecting an axis against actual chart data. The repository's UI-testing requirement calls for an E2E test for new UI features, and without one the selector-to-chart flow—including official and unofficial data, rendering, and Optimal Only behavior—remains unverified.

AGENTS.md reference: AGENTS.md:L132-L139

Useful? React with 👍 / 👎.

],
},
{ label: 'Custom User Values', labelZh: '自定义值', metrics: ['y_costUser', 'y_powerUser'] },
Expand Down
Loading