From 9e214274cdc177d585a3d5fc8c8e261e155c3143 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 12 Aug 2026 22:16:48 -0700 Subject: [PATCH] feat(power): stamp whole-deployment semantics with a schema version --- .github/AGENT_OPERATIONS.md | 2 ++ utils/aggregate_power.py | 7 +++++++ utils/aggregate_power_multinode.py | 2 ++ utils/process_result.py | 2 ++ utils/test_aggregate_power.py | 3 +++ utils/test_aggregate_power_multinode.py | 2 ++ utils/test_process_result.py | 4 ++++ 7 files changed, 22 insertions(+) diff --git a/.github/AGENT_OPERATIONS.md b/.github/AGENT_OPERATIONS.md index 0ef9acd1ea..cb90d423b5 100644 --- a/.github/AGENT_OPERATIONS.md +++ b/.github/AGENT_OPERATIONS.md @@ -94,6 +94,8 @@ Single-node fixed-sequence results may include `power_valid`, `avg_power_w`, `av Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j`, `prefill_avg_power_w`, `decode_avg_power_w`, `prefill_joules_per_input_token`, and `decode_joules_per_output_token`. Role energy covers the full formal benchmark window, not kernel-level phases, and the role watts are that energy divided by the same window and by the role's GPU count. +Every power result — valid or invalid, single-node or multinode — carries `power_metric_schema_version`. Version 2 defines each unprefixed `joules_per_*` field as whole-deployment GPU-board energy over the named denominator; role-scoped energy uses the explicit `prefill_*` / `decode_*` keys. Rows without the field predate the whole-deployment switch and their unprefixed joules are not comparable across topologies. + For srt-slurm recipes, `telemetry: {provider: dcgm-power}` enables official energy collection. `runners/launch_gb200-nv.sh` and `runners/launch_gb300-nv.sh` are the source of truth for `POWER_SRT_SLURM_PIN`. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. `utils/test_gb200_power_official_contract.py` and `utils/test_gb300_power_official_contract.py` enforce the recipe/launcher contract. Only `PRECISION=fp8` dcgm-power lanes are validated. Power audit artifacts are named `power_audit_` and contain `power_validation_.json` for single-node runs or `power_validation__*.json` for multinode runs. They are uploaded even when validation fails. diff --git a/utils/aggregate_power.py b/utils/aggregate_power.py index 24db2cd829..b1368580f0 100644 --- a/utils/aggregate_power.py +++ b/utils/aggregate_power.py @@ -46,6 +46,12 @@ "joules_per_total_token", } +# The unprefixed joules_per_* fields silently switched from role-local to +# whole-deployment energy when multinode aggregation landed, and the values +# alone cannot distinguish the two. Stamp the semantics so consumers fail +# closed on unversioned rows instead of guessing. +POWER_METRIC_SCHEMA_VERSION = 2 + @dataclass(frozen=True) class PowerIntegration: @@ -756,6 +762,7 @@ def _patch_power_result( data.pop(key, None) # Keep the canonical aggregate numeric-only for InferenceX-app's metric # auto-capture. Detailed reason codes live in the validation sidecar. + data["power_metric_schema_version"] = POWER_METRIC_SCHEMA_VERSION data["power_valid"] = int(power_valid) data.pop("power_invalid_reasons", None) if power_valid: diff --git a/utils/aggregate_power_multinode.py b/utils/aggregate_power_multinode.py index cabed59d6d..685cd4e65f 100644 --- a/utils/aggregate_power_multinode.py +++ b/utils/aggregate_power_multinode.py @@ -39,6 +39,7 @@ from pathlib import Path, PurePosixPath from aggregate_power import ( + POWER_METRIC_SCHEMA_VERSION, BenchmarkData, _append_reason, _integrate_device, @@ -1165,6 +1166,7 @@ def _patch_agg(agg_path: Path, audit: MultinodePowerAudit) -> None: data = json.loads(agg_path.read_text(encoding="utf-8")) for key in _ALL_POWER_METRIC_KEYS: data.pop(key, None) + data["power_metric_schema_version"] = POWER_METRIC_SCHEMA_VERSION data["power_valid"] = int(audit.power_valid) data.pop("power_invalid_reasons", None) if audit.power_valid: diff --git a/utils/process_result.py b/utils/process_result.py index a8bdc8cca6..9137b55384 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -67,6 +67,7 @@ def record_power_internal_error( try: from aggregate_power import ( _POWER_METRIC_KEYS, + POWER_METRIC_SCHEMA_VERSION, _empty_integration, _validation_payload, _write_json_atomic, @@ -77,6 +78,7 @@ def record_power_internal_error( agg_data.pop(key, None) for key in _MULTINODE_ROLE_METRIC_KEYS: agg_data.pop(key, None) + agg_data["power_metric_schema_version"] = POWER_METRIC_SCHEMA_VERSION agg_data["power_valid"] = 0 agg_data.pop("power_invalid_reasons", None) _write_json_atomic(agg_result, agg_data) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index 89e42034e8..1b7029a454 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -670,6 +670,7 @@ def test_run_skips_when_bench_window_missing(tmp_path: Path): assert "avg_power_w" not in patched assert patched == { "hw": "h200", + "power_metric_schema_version": 2, "power_valid": 0, } @@ -755,6 +756,7 @@ def test_run_emits_complete_whole_deployment_metric_contract(tmp_path: Path): assert exit_code == 0 patched = json.loads(agg.read_text()) + assert patched["power_metric_schema_version"] == 2 assert type(patched["power_valid"]) is int assert patched["power_valid"] == 1 assert "power_invalid_reasons" not in patched @@ -809,6 +811,7 @@ def test_run_best_effort_marks_invalid_power_and_preserves_benchmark(tmp_path: P patched = json.loads(agg.read_text()) assert patched["hw"] == "h200" assert patched["conc"] == 4 + assert patched["power_metric_schema_version"] == 2 assert type(patched["power_valid"]) is int assert patched["power_valid"] == 0 assert "power_invalid_reasons" not in patched diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index e4c78a1156..16e77db51b 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -213,6 +213,7 @@ def assert_invalid(pkg, expected_reason, **run_kwargs): """Both modes must reject: metrics withheld always, exit code differs.""" assert pkg.run(require_power=False, **run_kwargs) == 0 agg = pkg.agg() + assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 for key in apm.WHOLE_METRIC_KEYS + apm.ROLE_METRIC_KEYS: assert key not in agg @@ -229,6 +230,7 @@ def test_emits_all_metrics_exactly(self, tmp_path): assert pkg.run() == 0 agg = pkg.agg() + assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 1 assert agg["avg_power_w"] == 350.0 assert agg["avg_total_gpu_power_w"] == 1400.0 diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 4d5219010f..c84f4dc5f5 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -875,6 +875,7 @@ def test_require_power_accepts_valid_single_node_measurement( assert result.returncode == 0, result.stderr agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) + assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 1 assert agg["total_gpu_energy_j"] == pytest.approx(40_000.0) validation = json.loads( @@ -914,6 +915,7 @@ def test_internal_aggregation_error_is_always_auditable( assert result.returncode == expected_returncode agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) + assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 assert "power_invalid_reasons" not in agg validation = json.loads( @@ -1221,6 +1223,7 @@ def test_valid_package_patches_role_energy(self, tmp_path, power_env): assert result.returncode == 0, f"Script failed: {result.stderr}" agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) + assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 1 assert agg["prefill_gpu_energy_j"] == 48000.0 assert agg["decode_gpu_energy_j"] == 36000.0 @@ -1233,6 +1236,7 @@ def test_missing_package_is_best_effort(self, tmp_path, power_env): assert result.returncode == 0, f"Script failed: {result.stderr}" agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) + assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 for key in WHOLE_METRIC_KEYS + ROLE_METRIC_KEYS: assert key not in agg