From e7901c1bb912c1f31556a8239a424a055ee74597 Mon Sep 17 00:00:00 2001 From: Jaspal Singh Date: Wed, 23 Sep 2026 16:45:33 +0000 Subject: [PATCH] new power mismatch check --- .../plugins/inband/amdsmi/amdsmi_analyzer.py | 52 +++++++++++++++- .../plugins/inband/amdsmi/analyzer_args.py | 15 ++++- test/unit/plugin/test_amdsmi_analyzer.py | 61 ++++++++++++++++++- 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index da427753..6353b53d 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -41,7 +41,7 @@ Processes, XgmiMetrics, ) -from .analyzer_args import AmdSmiAnalyzerArgs +from .analyzer_args import AmdSmiAnalyzerArgs, PowerConfig from .cper import CperAnalysisTaskMixin @@ -235,6 +235,54 @@ def check_expected_max_power( console_log=True, ) + def check_power_cap_consistency( + self, + amdsmi_static_data: list[AmdSmiStatic], + power_config: PowerConfig, + ) -> None: + """Check that all GPUs have the same resolved maximum power cap.""" + if power_config.power_cap_mismatch_allowed: + return + + power_caps: dict[int, float] = {} + for gpu in amdsmi_static_data: + limit = gpu.limit + max_power_vu = limit.resolved_max_power() if limit is not None else None + if max_power_vu is None or max_power_vu.value is None: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {gpu.gpu}: power cap is not available", + priority=EventPriority.WARNING, + data={"gpu": gpu.gpu}, + console_log=True, + ) + continue + + try: + power_caps[gpu.gpu] = float(max_power_vu.value) + except (TypeError, ValueError): + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {gpu.gpu}: power cap is invalid", + priority=EventPriority.WARNING, + data={"gpu": gpu.gpu, "power_cap": max_power_vu.value}, + console_log=True, + ) + + if len(power_caps) < 2: + return + + expected_power_cap = next(iter(power_caps.values())) + for gpu, power_cap in power_caps.items(): + if power_cap != expected_power_cap: + self._log_event( + category=EventCategory.PLATFORM, + description=f"Power cap inconsistency for gpu {gpu}", + priority=EventPriority.ERROR, + data={"power_caps": list(power_caps.values())}, + console_log=True, + ) + def check_expected_driver_version( self, amdsmi_static_data: list[AmdSmiStatic], @@ -975,6 +1023,8 @@ def analyze_data( else: if args.expected_max_power: self.check_expected_max_power(data.static, args.expected_max_power) + if args.power: + self.check_power_cap_consistency(data.static, args.power) if args.expected_driver_version: self.check_expected_driver_version(data.static, args.expected_driver_version) diff --git a/nodescraper/plugins/inband/amdsmi/analyzer_args.py b/nodescraper/plugins/inband/amdsmi/analyzer_args.py index cd08bd49..938cf0b3 100644 --- a/nodescraper/plugins/inband/amdsmi/analyzer_args.py +++ b/nodescraper/plugins/inband/amdsmi/analyzer_args.py @@ -26,12 +26,21 @@ from datetime import datetime from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from nodescraper.models import AnalyzerArgs from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel +class PowerConfig(BaseModel): + """GPU power-cap consistency policy.""" + + power_cap_mismatch_allowed: bool = Field( + default=False, + description="Whether different GPU power caps are allowed.", + ) + + class AmdSmiAnalyzerArgs(AnalyzerArgs): check_static_data: bool = Field( default=False, @@ -43,6 +52,10 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): expected_max_power: Optional[int] = Field( default=None, description="Expected maximum power value (e.g. watts)." ) + power: Optional[PowerConfig] = Field( + default=None, + description="GPU power-cap consistency policy.", + ) expected_power_management: Optional[str] = Field( default=None, description=( diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 560a9ba4..5d631100 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -64,7 +64,10 @@ XgmiLinkMetrics, XgmiMetrics, ) -from nodescraper.plugins.inband.amdsmi.analyzer_args import AmdSmiAnalyzerArgs +from nodescraper.plugins.inband.amdsmi.analyzer_args import ( + AmdSmiAnalyzerArgs, + PowerConfig, +) @pytest.fixture @@ -314,6 +317,58 @@ def test_check_expected_max_power_ppt0(mock_analyzer): assert "GPU max power mismatch" in analyzer.result.events[0].description +def test_check_power_cap_consistency_success(mock_analyzer): + """Matching GPU power caps pass consistency validation.""" + analyzer = mock_analyzer + static_data = [ + create_static_gpu(0, max_power=550.0), + create_static_gpu(1, max_power=550.0), + ] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=False), + ) + + assert not analyzer.result.events + + +def test_check_power_cap_consistency_mismatch(mock_analyzer): + """Different GPU power caps generate an error.""" + analyzer = mock_analyzer + static_data = [ + create_static_gpu(0, max_power=550.0), + create_static_gpu(1, max_power=450.0), + ] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=False), + ) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].category == "PLATFORM" + assert analyzer.result.events[0].priority == EventPriority.ERROR + assert "Power cap inconsistency for gpu 1" in analyzer.result.events[0].description + assert analyzer.result.events[0].data["power_caps"] == [550.0, 450.0] + + +def test_check_power_cap_consistency_allowed(mock_analyzer): + """Power-cap mismatches are skipped when explicitly allowed.""" + analyzer = mock_analyzer + static_data = [ + create_static_gpu(0, max_power=550.0), + create_static_gpu(1, max_power=450.0), + ] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=True), + ) + + assert not analyzer.result.events + + def test_check_expected_driver_version_success(mock_analyzer): """Test check_expected_driver_version passes when all GPUs have correct driver.""" analyzer = mock_analyzer @@ -1006,6 +1061,10 @@ def test_amdsmi_analyzer_args_rejects_unknown_fields(): """Plugin config must only use declared AmdSmiAnalyzerArgs fields.""" from pydantic import ValidationError + args = AmdSmiAnalyzerArgs.model_validate({"power": {"power_cap_mismatch_allowed": False}}) + assert args.power is not None + assert args.power.power_cap_mismatch_allowed is False + with pytest.raises(ValidationError): AmdSmiAnalyzerArgs.model_validate( {"expected_power_management": "DISABLED", "not_a_field": 1}