Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 51 additions & 1 deletion nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
Processes,
XgmiMetrics,
)
from .analyzer_args import AmdSmiAnalyzerArgs
from .analyzer_args import AmdSmiAnalyzerArgs, PowerConfig
from .cper import CperAnalysisTaskMixin


Expand Down Expand Up @@ -235,6 +235,54 @@ def check_expected_max_power(
console_log=True,
)

def check_power_cap_consistency(
self,
amdsmi_static_data: list[AmdSmiStatic],
power_config: PowerConfig,
) -> None:
"""Check that all GPUs have the same resolved maximum power cap."""
if power_config.power_cap_mismatch_allowed:
return

power_caps: dict[int, float] = {}
for gpu in amdsmi_static_data:
limit = gpu.limit
max_power_vu = limit.resolved_max_power() if limit is not None else None
if max_power_vu is None or max_power_vu.value is None:
self._log_event(
category=EventCategory.PLATFORM,
description=f"GPU {gpu.gpu}: power cap is not available",
priority=EventPriority.WARNING,
data={"gpu": gpu.gpu},
console_log=True,
)
continue

try:
power_caps[gpu.gpu] = float(max_power_vu.value)
except (TypeError, ValueError):
self._log_event(
category=EventCategory.PLATFORM,
description=f"GPU {gpu.gpu}: power cap is invalid",
priority=EventPriority.WARNING,
data={"gpu": gpu.gpu, "power_cap": max_power_vu.value},
console_log=True,
)

if len(power_caps) < 2:
return

expected_power_cap = next(iter(power_caps.values()))
for gpu, power_cap in power_caps.items():
if power_cap != expected_power_cap:
self._log_event(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will log every time we have a mismatched value. If a node has i.e 8 GPUs and all 8 have different power cap values we'll get this event logged 8 times. Would be more clear if we keep track of all mismatched values and log one event at the end highlighting all mismatches.

category=EventCategory.PLATFORM,
description=f"Power cap inconsistency for gpu {gpu}",
priority=EventPriority.ERROR,
data={"power_caps": list(power_caps.values())},

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be better to pass in power_caps itself as a dict to the data, otherwise end user will just see a list of power cap values and not the corresponding GPU.

console_log=True,
)

def check_expected_driver_version(
self,
amdsmi_static_data: list[AmdSmiStatic],
Expand Down Expand Up @@ -975,6 +1023,8 @@ def analyze_data(
else:
if args.expected_max_power:
self.check_expected_max_power(data.static, args.expected_max_power)
if args.power:
self.check_power_cap_consistency(data.static, args.power)
if args.expected_driver_version:
self.check_expected_driver_version(data.static, args.expected_driver_version)

Expand Down
15 changes: 14 additions & 1 deletion nodescraper/plugins/inband/amdsmi/analyzer_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,21 @@
from datetime import datetime
from typing import Optional

from pydantic import Field
from pydantic import BaseModel, Field

from nodescraper.models import AnalyzerArgs
from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel


class PowerConfig(BaseModel):
"""GPU power-cap consistency policy."""

power_cap_mismatch_allowed: bool = Field(
default=False,
description="Whether different GPU power caps are allowed.",
)


class AmdSmiAnalyzerArgs(AnalyzerArgs):
check_static_data: bool = Field(
default=False,
Expand All @@ -43,6 +52,10 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs):
expected_max_power: Optional[int] = Field(
default=None, description="Expected maximum power value (e.g. watts)."
)
power: Optional[PowerConfig] = Field(
default=None,
description="GPU power-cap consistency policy.",
)
expected_power_management: Optional[str] = Field(
default=None,
description=(
Expand Down
61 changes: 60 additions & 1 deletion test/unit/plugin/test_amdsmi_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@
XgmiLinkMetrics,
XgmiMetrics,
)
from nodescraper.plugins.inband.amdsmi.analyzer_args import AmdSmiAnalyzerArgs
from nodescraper.plugins.inband.amdsmi.analyzer_args import (
AmdSmiAnalyzerArgs,
PowerConfig,
)


@pytest.fixture
Expand Down Expand Up @@ -314,6 +317,58 @@ def test_check_expected_max_power_ppt0(mock_analyzer):
assert "GPU max power mismatch" in analyzer.result.events[0].description


def test_check_power_cap_consistency_success(mock_analyzer):
"""Matching GPU power caps pass consistency validation."""
analyzer = mock_analyzer
static_data = [
create_static_gpu(0, max_power=550.0),
create_static_gpu(1, max_power=550.0),
]

analyzer.check_power_cap_consistency(
static_data,
PowerConfig(power_cap_mismatch_allowed=False),
)

assert not analyzer.result.events


def test_check_power_cap_consistency_mismatch(mock_analyzer):
"""Different GPU power caps generate an error."""
analyzer = mock_analyzer
static_data = [
create_static_gpu(0, max_power=550.0),
create_static_gpu(1, max_power=450.0),
]

analyzer.check_power_cap_consistency(
static_data,
PowerConfig(power_cap_mismatch_allowed=False),
)

assert len(analyzer.result.events) == 1
assert analyzer.result.events[0].category == "PLATFORM"
assert analyzer.result.events[0].priority == EventPriority.ERROR
assert "Power cap inconsistency for gpu 1" in analyzer.result.events[0].description
assert analyzer.result.events[0].data["power_caps"] == [550.0, 450.0]


def test_check_power_cap_consistency_allowed(mock_analyzer):
"""Power-cap mismatches are skipped when explicitly allowed."""
analyzer = mock_analyzer
static_data = [
create_static_gpu(0, max_power=550.0),
create_static_gpu(1, max_power=450.0),
]

analyzer.check_power_cap_consistency(
static_data,
PowerConfig(power_cap_mismatch_allowed=True),
)

assert not analyzer.result.events


def test_check_expected_driver_version_success(mock_analyzer):
"""Test check_expected_driver_version passes when all GPUs have correct driver."""
analyzer = mock_analyzer
Expand Down Expand Up @@ -1006,6 +1061,10 @@ def test_amdsmi_analyzer_args_rejects_unknown_fields():
"""Plugin config must only use declared AmdSmiAnalyzerArgs fields."""
from pydantic import ValidationError

args = AmdSmiAnalyzerArgs.model_validate({"power": {"power_cap_mismatch_allowed": False}})
assert args.power is not None
assert args.power.power_cap_mismatch_allowed is False

with pytest.raises(ValidationError):
AmdSmiAnalyzerArgs.model_validate(
{"expected_power_management": "DISABLED", "not_a_field": 1}
Expand Down
Loading