diff --git a/packages/app/cypress/component/inference-chart-controls.cy.tsx b/packages/app/cypress/component/inference-chart-controls.cy.tsx index f6db52302..6932cbe54 100644 --- a/packages/app/cypress/component/inference-chart-controls.cy.tsx +++ b/packages/app/cypress/component/inference-chart-controls.cy.tsx @@ -41,6 +41,36 @@ describe('Inference ChartControls', () => { cy.get('@setSelectedYAxisMetric').should('have.been.calledOnce'); }); + it('lists and selects the schema-v2 derived axes in the Measured Energy group', () => { + const options = [ + { + key: 'y_measuredJPerSuccessfulQuery', + label: 'Measured Joules per Successful Query', + }, + { + key: 'y_measuredWhPerSuccessfulQuery', + label: 'Measured Watt-hours per Successful Query', + }, + { + key: 'y_measuredPowerPercentTdp', + label: 'Measured Average Power as Percent of TDP', + }, + ]; + + for (const option of options) { + cy.get('[data-testid="yaxis-metric-selector"]').click(); + cy.contains('Measured Energy') + .parent() + .within(() => { + cy.contains('[role="option"]', option.label) + .scrollIntoView() + .should('be.visible') + .click(); + }); + cy.get('@setSelectedYAxisMetric').should('have.been.calledWith', option.key); + } + }); + it('hides the GPU comparison section when no GPUs are selected', () => { // Default mock: selectedGPUs = [] — GPU date range pickers should not render cy.contains('Comparison Date Range').should('not.exist'); diff --git a/packages/app/src/app/api/unofficial-run/route.test.ts b/packages/app/src/app/api/unofficial-run/route.test.ts index b9c45b801..9dc323dcd 100644 --- a/packages/app/src/app/api/unofficial-run/route.test.ts +++ b/packages/app/src/app/api/unofficial-run/route.test.ts @@ -144,6 +144,55 @@ describe('normalizeArtifactRows', () => { expect(m.mean_e2el).toBe(1.5); }); + it.each([ + { + name: 'boolean verdict and junk-suffixed schema', + input: { power_valid: true, power_metric_schema_version: '2garbage' }, + expectedVerdict: 0, + expectedSchema: undefined, + }, + { + name: 'garbage verdict and valid numeric schema', + input: { power_valid: 'garbage', power_metric_schema_version: 2 }, + expectedVerdict: 0, + expectedSchema: 2, + }, + { + name: 'canonical numeric strings', + input: { power_valid: '1', power_metric_schema_version: '2' }, + expectedVerdict: 1, + expectedSchema: 2, + }, + { + name: 'explicit invalid verdict', + input: { power_valid: 0, power_metric_schema_version: 2 }, + expectedVerdict: 0, + expectedSchema: 2, + }, + { + name: 'legacy row without power contract fields', + input: {}, + expectedVerdict: undefined, + expectedSchema: undefined, + }, + ])( + 'normalizes overlay power contract discriminators: $name', + ({ input, expectedVerdict, expectedSchema }) => { + const [row] = normalizeArtifactRows([rawRow(input)], '2026-03-01'); + + if (expectedVerdict === undefined) { + expect(row.metrics).not.toHaveProperty('power_valid'); + } else { + expect(row.metrics.power_valid).toBe(expectedVerdict); + } + if (expectedSchema === undefined) { + expect(row.metrics).not.toHaveProperty('power_metric_schema_version'); + } else { + expect(row.metrics.power_metric_schema_version).toBe(expectedSchema); + } + }, + ); + it('surfaces pipeline-parallelism fields in metrics (auto-capture)', () => { // pp has no configs-table column: the frontend reads it from the metrics // JSONB (rowToAggDataEntry), so the overlay route must keep passing it diff --git a/packages/app/src/components/ai-chart/types.test.ts b/packages/app/src/components/ai-chart/types.test.ts new file mode 100644 index 000000000..1837e1c04 --- /dev/null +++ b/packages/app/src/components/ai-chart/types.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; + +import { validateSpec } from './types'; + +describe('validateSpec measured power axes', () => { + it.each([ + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', + 'y_measuredPowerPercentTdp', + ])('preserves %s as a benchmark Y-axis metric', (yAxisMetric) => { + const spec = validateSpec({ dataSource: 'benchmarks', yAxisMetric }); + + expect(spec.yAxisMetric).toBe(yAxisMetric); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts index a0e17a1dc..b718ca4e6 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts @@ -156,6 +156,30 @@ describe('interpolateMetricAtInteractivity', () => { expect(result!).toBeLessThan(3.5); }); + it.each([ + 'measuredJPerSuccessfulQuery', + 'measuredWhPerSuccessfulQuery', + 'measuredPowerPercentTdp', + ] as const)('interpolates the derived measured metric %s', (metricKey) => { + const points = [ + makePoint({ + x: 20, + tpPerGpu: { y: 800, roof: false }, + [metricKey]: { y: 80, roof: false }, + }), + makePoint({ + x: 60, + tpPerGpu: { y: 400, roof: false }, + [metricKey]: { y: 40, roof: false }, + }), + ]; + + const result = interpolateMetricAtInteractivity(points, 40, metricKey); + expect(result).not.toBeNull(); + expect(result!).toBeGreaterThan(40); + expect(result!).toBeLessThan(80); + }); + it('returns null when metric field is missing from data points', () => { const points = [ makePoint({ x: 20, tpPerGpu: { y: 800, roof: false } }), diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index bfef739d3..d733d71be 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -11,7 +11,7 @@ import { reciprocalMetricAt, } from '@/components/calculator/useThroughputData'; import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history'; -import { getHardwareKey } from '@/lib/chart-utils'; +import { buildMeasuredPowerChartFields, getHardwareKey } from '@/lib/chart-utils'; import { getGpuSpecs, isKnownGpu } from '@/lib/constants'; import { rowToAggDataEntry } from '@/lib/benchmark-transform'; import type { BenchmarkRow } from '@/lib/api'; @@ -74,24 +74,7 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { jTotal: wrapMetric(power > 0 && tput ? (power * 1000) / tput : 0), ...(outputTput ? { jOutput: wrapMetric(power > 0 ? (power * 1000) / outputTput : 0) } : {}), ...(inputTput ? { jInput: wrapMetric(power > 0 ? (power * 1000) / inputTput : 0) } : {}), - ...(typeof entry.avg_power_w === 'number' - ? { measuredAvgPower: { y: entry.avg_power_w, roof: false } } - : {}), - ...(typeof entry.joules_per_output_token === 'number' - ? { measuredJPerOutputToken: { y: entry.joules_per_output_token, roof: false } } - : {}), - ...(typeof entry.joules_per_total_token === 'number' - ? { measuredJPerTotalToken: { y: entry.joules_per_total_token, roof: false } } - : {}), - ...(typeof entry.prefill_avg_power_w === 'number' - ? { measuredPrefillAvgPower: { y: entry.prefill_avg_power_w, roof: false } } - : {}), - ...(typeof entry.decode_avg_power_w === 'number' - ? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } } - : {}), - ...(typeof entry.joules_per_input_token === 'number' - ? { measuredJPerInputToken: { y: entry.joules_per_input_token, roof: false } } - : {}), + ...buildMeasuredPowerChartFields(entry, specs.tdp), }; return point; } diff --git a/packages/app/src/components/inference/inference-chart-config.json b/packages/app/src/components/inference/inference-chart-config.json index 392909349..8c8b6018b 100644 --- a/packages/app/src/components/inference/inference-chart-config.json +++ b/packages/app/src/components/inference/inference-chart-config.json @@ -164,6 +164,24 @@ "y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)", "y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗（含提示词）", "y_measuredJPerTotalToken_roofline": "lower_right", + "y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y", + "y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)", + "y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（J/query）", + "y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query", + "y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗", + "y_measuredJPerSuccessfulQuery_roofline": "lower_right", + "y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y", + "y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)", + "y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（Wh/query）", + "y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query", + "y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗", + "y_measuredWhPerSuccessfulQuery_roofline": "lower_right", + "y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y", + "y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)", + "y_measuredPowerPercentTdp_labelZh": "实测平均功耗（TDP 占比）", + "y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP", + "y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比", + "y_measuredPowerPercentTdp_roofline": "lower_right", "y_cost_limit": 5, "y_latency_limit": 60 }, @@ -331,6 +349,24 @@ "y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)", "y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗（含提示词）", "y_measuredJPerTotalToken_roofline": "lower_left", + "y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y", + "y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)", + "y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（J/query）", + "y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query", + "y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗", + "y_measuredJPerSuccessfulQuery_roofline": "lower_left", + "y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y", + "y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)", + "y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（Wh/query）", + "y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query", + "y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗", + "y_measuredWhPerSuccessfulQuery_roofline": "lower_left", + "y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y", + "y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)", + "y_measuredPowerPercentTdp_labelZh": "实测平均功耗（TDP 占比）", + "y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP", + "y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比", + "y_measuredPowerPercentTdp_roofline": "lower_left", "y_cost_limit": 5, "y_latency_limit": 60 } diff --git a/packages/app/src/components/inference/measured-power-direction.test.ts b/packages/app/src/components/inference/measured-power-direction.test.ts index ae307e431..01b589a63 100644 --- a/packages/app/src/components/inference/measured-power-direction.test.ts +++ b/packages/app/src/components/inference/measured-power-direction.test.ts @@ -30,6 +30,12 @@ const MEASURED_POWER_METRICS = [ 'y_measuredDecodeAvgPower', ] as const; +const DERIVED_POWER_METRICS = [ + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', + 'y_measuredPowerPercentTdp', +] as const; + const defs = chartDefinitions as unknown as ChartDefinition[]; const interactivityDef = defs.find((d) => d.chartType === 'interactivity')!; const e2eDef = defs.find((d) => d.chartType === 'e2e')!; @@ -133,4 +139,13 @@ describe('measured-power Pareto direction', () => { } } }); + + it.each(DERIVED_POWER_METRICS)('%s is bilingual and lower-is-better', (metric) => { + for (const chartDef of [interactivityDef, e2eDef]) { + expect(chartDef[metric]).toMatch(/\.y$/u); + expect(chartDef[`${metric}_label`]).toBeTruthy(); + expect(chartDef[`${metric}_labelZh`]).toBeTruthy(); + expect(declaredDirection(chartDef, metric)).toMatch(/^lower_/u); + } + }); }); diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 71cb9d86f..231535865 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -137,7 +137,10 @@ export interface AggDataEntry { 'p99.9_e2el': number; // Measured GPU telemetry (emitted by runner's aggregate_power.py). // Optional because historical runs predate the fields. + power_valid?: number; + power_metric_schema_version?: number; avg_power_w?: number; + joules_per_successful_query?: number; joules_per_output_token?: number; joules_per_total_token?: number; // Multinode / disagg-only measured power. The aggregate_power.py runner @@ -145,13 +148,13 @@ export interface AggDataEntry { // and decode workers (single-node disagg or multinode disagg). Single-node // aggregated configs leave these undefined. // - prefill_avg_power_w / decode_avg_power_w: mean per-GPU draw (W) within each role - // - joules_per_input_token: prefill_energy / total_input_tokens (prefill GPUs only) - // The disagg decode-only J/output is carried by joules_per_output_token above - // (the runner overrides it to decode_energy / total_output_tokens on disagg) — - // there is no separate _decode field. + // Unprefixed joules fields are whole-deployment metrics in schema version 2. + // Explicit prefill/decode keys retain role-local energy breakdowns. prefill_avg_power_w?: number; decode_avg_power_w?: number; joules_per_input_token?: number; + prefill_joules_per_input_token?: number; + decode_joules_per_output_token?: number; // Cluster-wide GPU telemetry beyond power (temperature, utilization, memory). // Emitted by aggregate_power.py when the perfmon CSVs include the matching // sample columns. Optional because older runs (and runs without the relevant @@ -315,6 +318,9 @@ export interface InferenceData extends Partial = {}): BenchmarkRow { }; } +function rawPowerArtifact(overrides: Record = {}): Record { + return { + infmax_model_prefix: 'dsr1', + hw: 'h200-nv', + framework: 'sglang', + precision: 'fp8', + isl: 1024, + osl: 1024, + conc: 128, + disagg: true, + prefill_tp: 8, + prefill_ep: 1, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 1, + decode_num_workers: 1, + num_prefill_gpu: 8, + num_decode_gpu: 8, + median_e2el: 1.4, + median_intvty: 48, + tput_per_gpu: 100.5, + ...overrides, + }; +} + // --------------------------------------------------------------------------- // parseAvailableModelsAndSequences // --------------------------------------------------------------------------- @@ -276,3 +302,64 @@ describe('buildChartData', () => { }); }); }); + +describe('schema-v2 measured-power overlay data flow', () => { + it('normalizes an overlay artifact and exposes J/query, Wh/query, and percent TDP', () => { + const rows = normalizeArtifactRows( + [ + rawPowerArtifact({ + power_valid: '1', + power_metric_schema_version: '2', + avg_power_w: 560, + joules_per_successful_query: 1800, + }), + ], + '2026-08-12', + ); + const point = buildChartData(rows)['DeepSeek-R1-0528_1k/1k'].interactivity.data[0]; + + expect(rows[0].metrics.power_valid).toBe(1); + expect(rows[0].metrics.power_metric_schema_version).toBe(2); + expect(point.measuredJPerSuccessfulQuery?.y).toBe(1800); + expect(point.measuredWhPerSuccessfulQuery?.y).toBe(0.5); + expect(point.measuredPowerPercentTdp?.y).toBe(80); + }); + + it.each([ + { + name: 'malformed boolean verdict and junk schema', + contract: { power_valid: true, power_metric_schema_version: '2garbage' }, + }, + { + name: 'malformed string verdict', + contract: { power_valid: 'garbage', power_metric_schema_version: 2 }, + }, + { + name: 'explicit invalid verdict', + contract: { power_valid: 0, power_metric_schema_version: 2 }, + }, + ])('withholds measured values for $name', ({ contract }) => { + const rows = normalizeArtifactRows( + [ + rawPowerArtifact({ + ...contract, + avg_power_w: 560, + joules_per_successful_query: 1800, + avg_temp_c: 68.4, + workers: [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 560 }], + }), + ], + '2026-08-12', + ); + const point = buildChartData(rows)['DeepSeek-R1-0528_1k/1k'].interactivity.data[0]; + + expect(rows[0].metrics.power_valid).toBe(0); + expect(point.avg_power_w).toBeUndefined(); + expect(point.joules_per_successful_query).toBeUndefined(); + expect(point.avg_temp_c).toBeUndefined(); + expect(point.workers).toBeUndefined(); + expect(point.measuredJPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredWhPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredPowerPercentTdp).toBeUndefined(); + }); +}); diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 35f1eb7ef..ed19fc338 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -229,6 +229,20 @@ describe('rowToAggDataEntry', () => { expect(entry.joules_per_output_token).toBe(8.4); }); + it('passes through versioned whole-deployment joules per successful query', () => { + const entry = rowToAggDataEntry( + makeRow({ + metrics: { + power_valid: 1, + power_metric_schema_version: 2, + joules_per_successful_query: 1542.75, + }, + }), + ); + + expect(entry.joules_per_successful_query).toBe(1542.75); + }); + it('leaves measured power fields undefined for rows that predate the metric', () => { // Distinguishing "no measurement" from "0 W" matters: createChartDataPoint // uses typeof===number to decide whether to emit the measuredAvgPower field. @@ -240,13 +254,18 @@ describe('rowToAggDataEntry', () => { it('passes through multinode / disagg role-split power scalars when present', () => { const entry = rowToAggDataEntry( makeRow({ + disagg: true, metrics: { tput_per_gpu: 100, + power_valid: 1, + power_metric_schema_version: 2, prefill_avg_power_w: 612.3, decode_avg_power_w: 701.5, joules_per_input_token: 1.2, - // disagg: joules_per_output_token IS the per-stage decode value. joules_per_output_token: 9.7, + joules_per_total_token: 0.8, + prefill_joules_per_input_token: 0.4, + decode_joules_per_output_token: 5.1, }, }), ); @@ -254,6 +273,135 @@ describe('rowToAggDataEntry', () => { expect(entry.decode_avg_power_w).toBe(701.5); expect(entry.joules_per_input_token).toBe(1.2); expect(entry.joules_per_output_token).toBe(9.7); + expect(entry.joules_per_total_token).toBe(0.8); + expect(entry.prefill_joules_per_input_token).toBe(0.4); + expect(entry.decode_joules_per_output_token).toBe(5.1); + }); + + it('withholds ambiguous unversioned disagg joules while preserving role watts', () => { + const entry = rowToAggDataEntry( + makeRow({ + disagg: true, + metrics: { + avg_power_w: 650, + joules_per_successful_query: 1542.75, + prefill_avg_power_w: 612.3, + decode_avg_power_w: 701.5, + joules_per_input_token: 1.2, + joules_per_output_token: 9.7, + joules_per_total_token: 0.8, + }, + }), + ); + + expect(entry.avg_power_w).toBe(650); + expect(entry.prefill_avg_power_w).toBe(612.3); + expect(entry.decode_avg_power_w).toBe(701.5); + expect(entry.joules_per_input_token).toBeUndefined(); + expect(entry.joules_per_successful_query).toBeUndefined(); + expect(entry.joules_per_output_token).toBeUndefined(); + expect(entry.joules_per_total_token).toBeUndefined(); + }); + + it('withholds disagg joules stamped with an unrecognized future schema version', () => { + const entry = rowToAggDataEntry( + makeRow({ + disagg: true, + metrics: { + power_valid: 1, + power_metric_schema_version: 3, + avg_power_w: 650, + prefill_avg_power_w: 612.3, + joules_per_input_token: 1.2, + joules_per_successful_query: 42.5, + joules_per_output_token: 9.7, + joules_per_total_token: 0.8, + }, + }), + ); + + expect(entry.avg_power_w).toBe(650); + expect(entry.prefill_avg_power_w).toBe(612.3); + expect(entry.joules_per_input_token).toBeUndefined(); + expect(entry.joules_per_successful_query).toBeUndefined(); + expect(entry.joules_per_output_token).toBeUndefined(); + expect(entry.joules_per_total_token).toBeUndefined(); + }); + + it('treats explicit power_valid=0 as authoritative and scrubs all measured telemetry', () => { + const workers = [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 685.5 }]; + const row = makeRow({ + metrics: { + power_valid: 0, + power_metric_schema_version: 2, + avg_power_w: 685.5, + joules_per_successful_query: 1542.75, + joules_per_input_token: 1.2, + joules_per_output_token: 8.4, + joules_per_total_token: 0.8, + prefill_avg_power_w: 612.3, + decode_avg_power_w: 701.5, + prefill_joules_per_input_token: 0.4, + decode_joules_per_output_token: 5.1, + avg_temp_c: 68.4, + peak_temp_c: 79.2, + avg_util_pct: 88.5, + avg_mem_used_mb: 71234.5, + }, + workers, + }); + const entry = rowToAggDataEntry(row); + + expect(entry.avg_power_w).toBeUndefined(); + expect(entry.joules_per_successful_query).toBeUndefined(); + expect(entry.joules_per_input_token).toBeUndefined(); + expect(entry.joules_per_output_token).toBeUndefined(); + expect(entry.joules_per_total_token).toBeUndefined(); + expect(entry.prefill_avg_power_w).toBeUndefined(); + expect(entry.decode_avg_power_w).toBeUndefined(); + expect(entry.prefill_joules_per_input_token).toBeUndefined(); + expect(entry.decode_joules_per_output_token).toBeUndefined(); + expect(entry.avg_temp_c).toBeUndefined(); + expect(entry.peak_temp_c).toBeUndefined(); + expect(entry.avg_util_pct).toBeUndefined(); + expect(entry.avg_mem_used_mb).toBeUndefined(); + expect(entry.workers).toBeUndefined(); + + const { chartData } = transformBenchmarkRows([row]); + const point = chartData.find((data) => data.length > 0)![0]; + expect(point.measuredJPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredWhPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredPowerPercentTdp).toBeUndefined(); + }); + + it('keeps measured telemetry compatible when a legacy row omits power_valid', () => { + const workers = [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 560 }]; + const row = makeRow({ + metrics: { + avg_power_w: 560, + joules_per_successful_query: 1800, + avg_temp_c: 68.4, + peak_temp_c: 79.2, + avg_util_pct: 88.5, + avg_mem_used_mb: 71234.5, + }, + workers, + }); + const entry = rowToAggDataEntry(row); + + expect(entry.avg_power_w).toBe(560); + expect(entry.joules_per_successful_query).toBe(1800); + expect(entry.avg_temp_c).toBe(68.4); + expect(entry.peak_temp_c).toBe(79.2); + expect(entry.avg_util_pct).toBe(88.5); + expect(entry.avg_mem_used_mb).toBe(71234.5); + expect(entry.workers).toEqual(workers); + + const { chartData } = transformBenchmarkRows([row]); + const point = chartData.find((data) => data.length > 0)![0]; + expect(point.measuredJPerSuccessfulQuery?.y).toBe(1800); + expect(point.measuredWhPerSuccessfulQuery?.y).toBe(0.5); + expect(point.measuredPowerPercentTdp?.y).toBe(80); }); it('passes through per-worker measured power array intact', () => { diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index a3b6cbfdc..c1bb83d5d 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -16,6 +16,14 @@ import { getHardwareConfig } from '@/lib/constants'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import type { BenchmarkRow } from '@/lib/api'; +/** + * Producer schema version whose unprefixed `joules_per_*` fields are documented + * as whole-deployment energy. Mirrors `POWER_METRIC_SCHEMA_VERSION` in the + * runner's `utils/aggregate_power.py`; bump both together when the semantics of + * those fields change again. + */ +const WHOLE_DEPLOYMENT_ENERGY_SCHEMA_VERSION = 2; + /** * Agentic trace-replay runs (`benchmark_type === 'agentic_traces'`) emit ttft/ttlt/itl * but not the intvty/e2el/tpot keys the chart pipeline expects. Bridge them here: @@ -95,6 +103,16 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { const aggregateDpAttention = aggregateUsesPrefill ? row.prefill_dp_attention : row.decode_dp_attention; + // An explicit invalid verdict is authoritative. Legacy rows without the + // verdict remain eligible so historical single-node measurements do not + // disappear. Unversioned disaggregated joules are withheld because their + // unprefixed fields changed from role-local to whole-deployment semantics. + const measuredPowerValid = m.power_valid !== 0; + // Match the version exactly rather than `>= 2`: an open bound would silently + // admit a future schema whose semantics changed again, which is the exact + // failure that made versioning necessary in the first place. + const hasWholeDeploymentEnergySemantics = + !row.disagg || m.power_metric_schema_version === WHOLE_DEPLOYMENT_ENERGY_SCHEMA_VERSION; // Prefer the dedicated column (added in migration 004); fall back to the // legacy stash inside `metrics` for any rows ingested before that column // existed. @@ -168,26 +186,46 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { // Measured GPU telemetry (runner's aggregate_power.py). Left undefined for // rows predating the field so downstream chart code can distinguish // "no measurement" from "0 W" via createChartDataPoint's typeof guard. - avg_power_w: m.avg_power_w, - joules_per_output_token: m.joules_per_output_token, - joules_per_total_token: m.joules_per_total_token, - // Multinode / disagg-only role splits — same undefined-for-legacy pattern. - // (disagg's decode-only J/output is carried by joules_per_output_token above, - // which the runner overrides to the per-stage value — no separate _decode key.) - prefill_avg_power_w: m.prefill_avg_power_w, - decode_avg_power_w: m.decode_avg_power_w, - joules_per_input_token: m.joules_per_input_token, + power_valid: m.power_valid, + power_metric_schema_version: m.power_metric_schema_version, + avg_power_w: measuredPowerValid ? m.avg_power_w : undefined, + joules_per_successful_query: + measuredPowerValid && hasWholeDeploymentEnergySemantics + ? m.joules_per_successful_query + : undefined, + joules_per_output_token: + measuredPowerValid && hasWholeDeploymentEnergySemantics + ? m.joules_per_output_token + : undefined, + joules_per_total_token: + measuredPowerValid && hasWholeDeploymentEnergySemantics + ? m.joules_per_total_token + : undefined, + // Role power remains unambiguous across schema versions. Version 2 also + // publishes explicit role energy alongside whole-deployment joules. + prefill_avg_power_w: measuredPowerValid ? m.prefill_avg_power_w : undefined, + decode_avg_power_w: measuredPowerValid ? m.decode_avg_power_w : undefined, + joules_per_input_token: + measuredPowerValid && hasWholeDeploymentEnergySemantics + ? m.joules_per_input_token + : undefined, + prefill_joules_per_input_token: measuredPowerValid + ? m.prefill_joules_per_input_token + : undefined, + decode_joules_per_output_token: measuredPowerValid + ? m.decode_joules_per_output_token + : undefined, // Cluster-wide GPU telemetry beyond power. Emitted when the perfmon CSVs // include the corresponding sample columns; left undefined otherwise so // the chart layer can distinguish "no measurement" from a real zero. - avg_temp_c: m.avg_temp_c, - peak_temp_c: m.peak_temp_c, - avg_util_pct: m.avg_util_pct, - avg_mem_used_mb: m.avg_mem_used_mb, + avg_temp_c: measuredPowerValid ? m.avg_temp_c : undefined, + peak_temp_c: measuredPowerValid ? m.peak_temp_c : undefined, + avg_util_pct: measuredPowerValid ? m.avg_util_pct : undefined, + avg_mem_used_mb: measuredPowerValid ? m.avg_mem_used_mb : undefined, // Per-worker measured power. Surfaced on BenchmarkRow as a sibling of the // scalar `metrics` dict (see api.ts). Narrow defensively so a malformed // payload can't poison downstream consumers. - workers: Array.isArray(row.workers) ? row.workers : undefined, + workers: measuredPowerValid && Array.isArray(row.workers) ? row.workers : undefined, disagg: row.disagg, num_prefill_gpu: row.num_prefill_gpu, num_decode_gpu: row.num_decode_gpu, diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts index b5d667cba..47de35310 100644 --- a/packages/app/src/lib/chart-utils.test.ts +++ b/packages/app/src/lib/chart-utils.test.ts @@ -28,7 +28,7 @@ vi.mock('@/lib/constants', async (importOriginal) => { return { ...actual, getHardwareConfig: vi.fn(() => ({ label: 'H100', suffix: '' })), - getGpuSpecs: vi.fn(() => ({ power: 700, costh: 2.8, costn: 1.4, costr: 0.7 })), + getGpuSpecs: vi.fn(() => ({ power: 700, tdp: 700, costh: 2.8, costn: 1.4, costr: 0.7 })), }; }); @@ -1333,6 +1333,29 @@ describe('createChartDataPoint measured power fields', () => { expect(point.measuredJPerOutputToken!.y).toBe(8.4); }); + it('derives J/query, Wh/query, and percent TDP from validated source fields', () => { + const e = entry({ avg_power_w: 560, joules_per_successful_query: 1800 }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + + expect(point.measuredJPerSuccessfulQuery?.y).toBe(1800); + expect(point.measuredWhPerSuccessfulQuery?.y).toBe(0.5); + expect(point.measuredPowerPercentTdp?.y).toBe(80); + }); + + it('omits derived query and TDP axes when their inputs are absent', () => { + const point = createChartDataPoint( + '2025-01-01', + entry(), + 'median_e2el', + 'tput_per_gpu', + 'h100', + ); + + expect(point.measuredJPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredWhPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredPowerPercentTdp).toBeUndefined(); + }); + it('omits both fields when neither is on the entry', () => { // Legacy runs predating aggregate_power.py. const point = createChartDataPoint( diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts index 087b3a114..6f9653fbc 100644 --- a/packages/app/src/lib/chart-utils.ts +++ b/packages/app/src/lib/chart-utils.ts @@ -191,6 +191,9 @@ export const Y_AXIS_METRICS = [ 'y_measuredJPerOutputToken', 'y_measuredJPerTotalToken', 'y_measuredJPerInputToken', + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', + 'y_measuredPowerPercentTdp', ] as const; export type YAxisMetric = (typeof Y_AXIS_METRICS)[number]; @@ -451,26 +454,59 @@ export function createChartDataPoint( } : {}), - // Measured power / energy from runner's aggregate_power.py. Gated on the - // raw fields existing so points from runs predating the measurement land - // without these keys and the chart correctly filters them out. + ...buildMeasuredPowerChartFields(entry, specs.tdp), + }; +} + +type MeasuredPowerChartFields = Partial< + Pick< + InferenceData, + | 'measuredAvgPower' + | 'measuredPrefillAvgPower' + | 'measuredDecodeAvgPower' + | 'measuredJPerOutputToken' + | 'measuredJPerTotalToken' + | 'measuredJPerInputToken' + | 'measuredJPerSuccessfulQuery' + | 'measuredWhPerSuccessfulQuery' + | 'measuredPowerPercentTdp' + > +>; + +const measuredMetric = (y: number): { y: number; roof: boolean } => ({ y, roof: false }); + +/** Build the measured fields shared by scatter points and historical trends. */ +export function buildMeasuredPowerChartFields( + entry: AggDataEntry, + tdpWatts: number, +): MeasuredPowerChartFields { + return { ...(typeof entry.avg_power_w === 'number' - ? { measuredAvgPower: { y: entry.avg_power_w, roof: false } } + ? { measuredAvgPower: measuredMetric(entry.avg_power_w) } : {}), ...(typeof entry.prefill_avg_power_w === 'number' - ? { measuredPrefillAvgPower: { y: entry.prefill_avg_power_w, roof: false } } + ? { measuredPrefillAvgPower: measuredMetric(entry.prefill_avg_power_w) } : {}), ...(typeof entry.decode_avg_power_w === 'number' - ? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } } + ? { measuredDecodeAvgPower: measuredMetric(entry.decode_avg_power_w) } : {}), ...(typeof entry.joules_per_output_token === 'number' - ? { measuredJPerOutputToken: { y: entry.joules_per_output_token, roof: false } } + ? { measuredJPerOutputToken: measuredMetric(entry.joules_per_output_token) } : {}), ...(typeof entry.joules_per_total_token === 'number' - ? { measuredJPerTotalToken: { y: entry.joules_per_total_token, roof: false } } + ? { measuredJPerTotalToken: measuredMetric(entry.joules_per_total_token) } : {}), ...(typeof entry.joules_per_input_token === 'number' - ? { measuredJPerInputToken: { y: entry.joules_per_input_token, roof: false } } + ? { measuredJPerInputToken: measuredMetric(entry.joules_per_input_token) } + : {}), + ...(typeof entry.joules_per_successful_query === 'number' + ? { + measuredJPerSuccessfulQuery: measuredMetric(entry.joules_per_successful_query), + measuredWhPerSuccessfulQuery: measuredMetric(entry.joules_per_successful_query / 3600), + } + : {}), + ...(typeof entry.avg_power_w === 'number' && tdpWatts > 0 + ? { measuredPowerPercentTdp: measuredMetric((entry.avg_power_w / tdpWatts) * 100) } : {}), }; } diff --git a/packages/app/src/lib/constants.test.ts b/packages/app/src/lib/constants.test.ts index f561591b6..14ec264f2 100644 --- a/packages/app/src/lib/constants.test.ts +++ b/packages/app/src/lib/constants.test.ts @@ -224,6 +224,7 @@ describe('getHardwareConfig', () => { describe('getGpuSpecs', () => { it('returns specs for a base GPU key', () => { const specs = getGpuSpecs('h100'); + expect(specs.tdp).toBe(700); expect(specs.power).toBe(1.37); expect(specs.costh).toBe(1.17); expect(specs.costn).toBe(1.55); @@ -244,6 +245,7 @@ describe('getGpuSpecs', () => { it('returns zero specs for unknown GPU', () => { const specs = getGpuSpecs('nonexistent'); expect(specs.power).toBe(0); + expect(specs.tdp).toBe(0); expect(specs.costh).toBe(0); expect(specs.costn).toBe(0); expect(specs.costr).toBe(0); @@ -253,6 +255,7 @@ describe('getGpuSpecs', () => { for (const [base, entry] of Object.entries(HW_REGISTRY)) { const result = getGpuSpecs(base); expect(result.power).toBe(entry.power); + expect(result.tdp).toBe(entry.tdp); expect(result.costh).toBe(entry.costh); } }); diff --git a/packages/app/src/lib/constants.ts b/packages/app/src/lib/constants.ts index a720077d9..b0a8ebe66 100644 --- a/packages/app/src/lib/constants.ts +++ b/packages/app/src/lib/constants.ts @@ -15,13 +15,14 @@ export const TABLEAU_10 = [ ] as const; export interface GpuSpecs { + tdp: number; power: number; costh: number; costn: number; costr: number; } -const DEFAULT_SPECS: GpuSpecs = { power: 0, costh: 0, costn: 0, costr: 0 }; +const DEFAULT_SPECS: GpuSpecs = { tdp: 0, power: 0, costh: 0, costn: 0, costr: 0 }; /** * Look up power/cost specs for a hardware key by extracting the base GPU name. @@ -31,7 +32,13 @@ export function getGpuSpecs(hwKey: string): GpuSpecs { const base = hwKey.split(/[-_]/u)[0]; const entry = HW_REGISTRY[base]; if (!entry) return DEFAULT_SPECS; - return { power: entry.power, costh: entry.costh, costn: entry.costn, costr: entry.costr }; + return { + tdp: entry.tdp, + power: entry.power, + costh: entry.costh, + costn: entry.costn, + costr: entry.costr, + }; } /** Build the vendor prefix string for the `gpu` tooltip field. */ diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 94c5b6134..99165905b 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -131,26 +131,32 @@ export const METRIC_KEYS = new Set([ // server_metrics.kv_cache.gpu_usage_pct in v3) 'gpu_kv_cache_usage_pct', // measured power / energy (emitted by runner's aggregate_power.py) + // power_valid: numeric 1/0 publication verdict; explicit 0 withholds power + // power_metric_schema_version: version 2 defines every unprefixed + // joules_per_* field as whole-deployment energy // avg_power_w: mean per-GPU draw (W) during the load window + // joules_per_successful_query: whole-deployment energy / successful requests // joules_per_output_token: energy / total_output_tokens. CLUSTER-WIDE on - // single-node / non-disagg (total_system_energy); - // PER-STAGE decode_energy on disagg (decode GPUs only), - // symmetric with joules_per_input_token below. + // schema-version-2 rows, including disaggregated runs. // joules_per_total_token: total_system_energy / (total_input + total_output) // — cluster-wide; workload-shape-fair view that // doesn't treat prompt as free. + 'power_valid', + 'power_metric_schema_version', 'avg_power_w', + 'joules_per_successful_query', 'joules_per_output_token', 'joules_per_total_token', // multinode / disagg role splits (emitted only when the deployment has // distinct prefill / decode workers) // prefill_avg_power_w / decode_avg_power_w: mean per-GPU draw within each role - // joules_per_input_token: prefill_energy / total_input_tokens (prefill GPUs only). - // The disagg output counterpart is joules_per_output_token above (decode GPUs - // only) — there is no separate _decode key. + // Explicit role-local energy remains separate from the version-2 unprefixed + // whole-deployment fields. 'prefill_avg_power_w', 'decode_avg_power_w', 'joules_per_input_token', + 'prefill_joules_per_input_token', + 'decode_joules_per_output_token', // cluster-wide GPU telemetry beyond power (emitted by aggregate_power.py when // the perfmon CSVs include temperature, utilization, or memory samples). // avg_temp_c: mean per-GPU temperature (Celsius) during load window diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 1985e69de..fe1b0e154 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -127,6 +127,108 @@ describe('mapBenchmarkRow', () => { expect(result!.metrics).not.toHaveProperty('ep'); }); + it.each([ + { + name: 'boolean verdict and junk-suffixed schema', + input: { power_valid: true, power_metric_schema_version: '2garbage' }, + expectedVerdict: 0, + expectedSchema: undefined, + }, + { + name: 'garbage verdict and valid numeric schema', + input: { power_valid: 'garbage', power_metric_schema_version: 2 }, + expectedVerdict: 0, + expectedSchema: 2, + }, + { + name: 'canonical numeric strings', + input: { power_valid: '1', power_metric_schema_version: '2' }, + expectedVerdict: 1, + expectedSchema: 2, + }, + { + name: 'explicit invalid verdict', + input: { power_valid: 0, power_metric_schema_version: 2 }, + expectedVerdict: 0, + expectedSchema: 2, + }, + { + name: 'legacy row without power contract fields', + input: {}, + expectedVerdict: undefined, + expectedSchema: undefined, + }, + ])( + 'normalizes power contract discriminators: $name', + ({ input, expectedVerdict, expectedSchema }) => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow(makeV1Row(input), tracker); + + if (expectedVerdict === undefined) { + expect(result!.metrics).not.toHaveProperty('power_valid'); + } else { + expect(result!.metrics.power_valid).toBe(expectedVerdict); + } + if (expectedSchema === undefined) { + expect(result!.metrics).not.toHaveProperty('power_metric_schema_version'); + } else { + expect(result!.metrics.power_metric_schema_version).toBe(expectedSchema); + } + }, + ); + + it.each([true, false, null, 0.5, 2, Number.NaN, Number.POSITIVE_INFINITY])( + 'fails closed for explicitly malformed power_valid=%j', + (powerValid) => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow(makeV1Row({ power_valid: powerValid }), tracker); + + expect(result!.metrics.power_valid).toBe(0); + }, + ); + + it.each([ + null, + true, + false, + 0, + -1, + 1.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + '0', + '-1', + '02', + '2.0', + ' 2', + '2 ', + '2garbage', + '1e2', + String(Number.MAX_SAFE_INTEGER + 1), + ])('omits malformed power_metric_schema_version=%j', (schemaVersion) => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeV1Row({ power_metric_schema_version: schemaVersion }), + tracker, + ); + + expect(result!.metrics).not.toHaveProperty('power_metric_schema_version'); + }); + + it.each([1, 2, Number.MAX_SAFE_INTEGER, '1', '2', String(Number.MAX_SAFE_INTEGER)])( + 'accepts safe positive integer power_metric_schema_version=%j', + (schemaVersion) => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeV1Row({ power_metric_schema_version: schemaVersion }), + tracker, + ); + + expect(result!.metrics.power_metric_schema_version).toBe(Number(schemaVersion)); + }, + ); + it('preserves the KV transfer engine for fixed-sequence multinode rows', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow( diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index ee71fad25..0179a85a1 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -243,6 +243,7 @@ export function mapBenchmarkRow( // Dynamo artifacts that incorrectly emitted disagg=false. const disagg = frameworkDisagg || parallelism.decodeNumWorkers > 0; let metrics = captureNumericMetrics(row); + normalizePowerContractMetrics(row, metrics); if (isAgentic) metrics = preferFullResponseMetrics(metrics); if (!disagg) { const usePrefill = @@ -442,6 +443,42 @@ function captureNumericMetrics(row: Record): Record return metrics; } +/** + * Re-parse the power publication contract after generic metric capture. These + * two fields are semantic discriminators, so loose numeric coercion must never + * turn malformed producer output into an affirmative verdict or schema. + */ +function normalizePowerContractMetrics( + row: Record, + metrics: Record, +): void { + if (Object.hasOwn(row, 'power_valid')) { + const verdict = row.power_valid; + metrics.power_valid = verdict === 1 || verdict === '1' ? 1 : 0; + } else { + delete metrics.power_valid; + } + + if (!Object.hasOwn(row, 'power_metric_schema_version')) { + delete metrics.power_metric_schema_version; + return; + } + + const rawVersion = row.power_metric_schema_version; + const version = + typeof rawVersion === 'number' + ? rawVersion + : // Canonical decimal form: no sign, whitespace, decimal point, or zero padding. + typeof rawVersion === 'string' && /^[1-9]\d*$/u.test(rawVersion) + ? Number(rawVersion) + : undefined; + if (typeof version === 'number' && Number.isSafeInteger(version) && version > 0) { + metrics.power_metric_schema_version = version; + } else { + delete metrics.power_metric_schema_version; + } +} + /** * Narrow a raw `workers` value from the artifact JSON to `WorkerPower[]` or * undefined. Each entry must have a string `role`, a numeric `worker_idx`,