diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index da427753..817da1e3 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -37,8 +37,10 @@ AmdSmiStatic, EccData, Fw, + LinkStatusTable, Partition, Processes, + XgmiLinks, XgmiMetrics, ) from .analyzer_args import AmdSmiAnalyzerArgs @@ -441,6 +443,73 @@ def check_amdsmi_metric_ecc_totals(self, amdsmi_metric_data: list[AmdSmiMetric]) console_log=True, ) + def check_gpu_memory( + self, + amdsmi_metric_data: list[AmdSmiMetric], + minimum_available_percent: float, + ) -> None: + """Check the minimum free VRAM percentage for each GPU.""" + for metric in amdsmi_metric_data: + memory = metric.mem_usage + total_vram = memory.total_vram if memory is not None else None + free_vram = memory.free_vram if memory is not None else None + values = { + "gpu": metric.gpu, + "total_vram": total_vram.value if total_vram is not None else None, + "free_vram": free_vram.value if free_vram is not None else None, + "unit": total_vram.unit if total_vram is not None else None, + "minimum_available_percent": minimum_available_percent, + } + + if total_vram is None or free_vram is None: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} VRAM availability is not available", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + try: + total_value = float(total_vram.value) + free_value = float(free_vram.value) + except (TypeError, ValueError): + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} VRAM availability is invalid", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + if total_value <= 0: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} total VRAM is invalid", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + available_percent = free_value / total_value * 100 + if available_percent < minimum_available_percent: + self._log_event( + category=EventCategory.PLATFORM, + description=( + f"GPU {metric.gpu} free VRAM is {available_percent:.2f}% " + f"(minimum {minimum_available_percent:.2f}%)" + ), + priority=EventPriority.WARNING, + data={ + **values, + "available_percent": available_percent, + }, + console_log=True, + ) + def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): """Check ECC counts in all blocks for all GPUs @@ -933,6 +1002,71 @@ def check_expected_xgmi_link_speed( console_log=True, ) + def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) -> None: + """Check XGMI or peer-link status: U passes, SELF is ignored, D/X warn.""" + if not xgmi_links: + self._log_event( + category=EventCategory.IO, + description="XGMI/peer link data is not available and cannot be checked", + priority=EventPriority.WARNING, + data={"xgmi_links": xgmi_links}, + console_log=True, + ) + return + + down_links: list[dict[str, Any]] = [] + degraded_links: list[dict[str, Any]] = [] + healthy_link_count = 0 + for gpu_links in xgmi_links: + for link_index, status in enumerate(gpu_links.link_status): + if status == LinkStatusTable.SELF: + continue + link_data = { + "gpu": gpu_links.gpu, + "link_index": link_index, + "status": status.value, + } + if status == LinkStatusTable.DOWN: + down_links.append(link_data) + elif status == LinkStatusTable.DISABLED: + degraded_links.append(link_data) + elif status == LinkStatusTable.UP: + healthy_link_count += 1 + + if not down_links and not degraded_links: + self._log_event( + category=EventCategory.IO, + description="All XGMI/peer GPU links are working fine", + priority=EventPriority.INFO, + data={"healthy_link_count": healthy_link_count}, + console_log=True, + ) + + if down_links: + self._log_event( + category=EventCategory.IO, + description=(f"XGMI/peer links contain {len(down_links)} down/error links"), + priority=EventPriority.WARNING, + data={ + "down_links": down_links, + "link_error_count": len(down_links), + }, + console_log=True, + ) + + if degraded_links: + self._log_event( + category=EventCategory.IO, + description=( + f"XGMI/peer links contain {len(degraded_links)} " "disabled/degraded links" + ), + priority=EventPriority.WARNING, + data={ + "degraded_links": degraded_links, + }, + console_log=True, + ) + def analyze_data( self, data: AmdSmiDataModel, args: Optional[AmdSmiAnalyzerArgs] = None ) -> TaskResult: @@ -959,6 +1093,11 @@ def analyze_data( args.l0_to_recovery_count_error_threshold, args.l0_to_recovery_count_warning_threshold or 1, ) + if args.gpu_memory: + self.check_gpu_memory( + data.metric, + args.gpu_memory.minimum_available_percent, + ) self.check_amdsmi_metric_ecc_totals(data.metric) self.check_amdsmi_metric_ecc(data.metric) @@ -1020,4 +1159,7 @@ def analyze_data( data.xgmi_metric, expected_xgmi_speed=args.expected_xgmi_speed ) + if args.check_xgmi_or_peer_links_status: + self.check_xgmi_or_peer_links_status(data.xgmi_link) + return self.result diff --git a/nodescraper/plugins/inband/amdsmi/analyzer_args.py b/nodescraper/plugins/inband/amdsmi/analyzer_args.py index cd08bd49..3e2b084d 100644 --- a/nodescraper/plugins/inband/amdsmi/analyzer_args.py +++ b/nodescraper/plugins/inband/amdsmi/analyzer_args.py @@ -26,12 +26,22 @@ from datetime import datetime from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from nodescraper.models import AnalyzerArgs from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel +class GpuMemoryConfig(BaseModel): + """GPU VRAM availability threshold.""" + + minimum_available_percent: float = Field( + ge=0, + le=100, + description="Minimum free VRAM percentage required for each GPU.", + ) + + class AmdSmiAnalyzerArgs(AnalyzerArgs): check_static_data: bool = Field( default=False, @@ -63,6 +73,14 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): default=None, description="Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).", ) + gpu_memory: Optional[GpuMemoryConfig] = Field( + default=None, + description="Minimum free VRAM threshold to validate for each GPU.", + ) + check_xgmi_or_peer_links_status: bool = Field( + default=False, + description="Check XGMI or peer-link status for each GPU.", + ) l0_to_recovery_count_error_threshold: Optional[int] = Field( default=3, description="L0-to-recovery count above which an error is raised.", diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 560a9ba4..74be5099 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -42,6 +42,7 @@ EccState, Fw, FwListItem, + LinkStatusTable, MetricEccTotals, MetricPcie, Partition, @@ -62,6 +63,7 @@ StaticVram, ValueUnit, XgmiLinkMetrics, + XgmiLinks, XgmiMetrics, ) from nodescraper.plugins.inband.amdsmi.analyzer_args import AmdSmiAnalyzerArgs @@ -751,6 +753,77 @@ def test_check_expected_xgmi_link_speed_missing_bit_rate(mock_analyzer): assert "XGMI link speed not available" in analyzer.result.events[0].description +def test_check_xgmi_or_peer_links_status_accepts_up_and_self_links(mock_analyzer): + """Healthy links generate an informational event and self-links are ignored.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.UP, LinkStatusTable.SELF], + ), + XgmiLinks( + gpu=1, + bdf="0000:02:00.0", + link_status=[LinkStatusTable.UP], + ), + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.INFO + assert "All XGMI/peer GPU links are working fine" in analyzer.result.events[0].description + + +def test_check_xgmi_or_peer_links_status_reports_down_and_degraded_links(mock_analyzer): + """Down and degraded links generate IO warnings.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.DOWN, LinkStatusTable.DISABLED], + ) + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert len(analyzer.result.events) == 2 + assert all(event.category == "IO" for event in analyzer.result.events) + assert all(event.priority == EventPriority.WARNING for event in analyzer.result.events) + assert analyzer.result.events[0].data["link_error_count"] == 1 + assert analyzer.result.events[1].data["degraded_links"][0]["status"] == "X" + + +def test_check_xgmi_or_peer_links_status_reports_degraded_links(mock_analyzer): + """Disabled/degraded links generate warnings.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.DISABLED], + ) + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + + +def test_check_xgmi_or_peer_links_status_reports_missing_data(mock_analyzer): + """Configured link validation reports when XGMI data is unavailable.""" + analyzer = mock_analyzer + + analyzer.check_xgmi_or_peer_links_status(None) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + assert "XGMI/peer link data is not available" in analyzer.result.events[0].description + + def test_analyze_data_full_workflow(mock_analyzer): """Test full analyze_data workflow with various checks.""" analyzer = mock_analyzer @@ -882,10 +955,24 @@ def _minimal_amdsmi_metric( ecc: Optional[MetricEccTotals] = None, ecc_blocks: Optional[dict] = None, power_management: Optional[str] = None, + mem_usage: Optional[dict] = None, ) -> AmdSmiMetric: """Build minimal AmdSmiMetric for PCIe/ECC tests with all required fields present.""" pcie_dict = pcie.model_dump() if pcie is not None else {k: None for k in _PCIE_KEYS} ecc_dict = ecc.model_dump() if ecc is not None else {k: None for k in _ECC_TOTALS_KEYS} + mem_usage_dict = { + "total_vram": None, + "used_vram": None, + "free_vram": None, + "total_visible_vram": None, + "used_visible_vram": None, + "free_visible_vram": None, + "total_gtt": None, + "used_gtt": None, + "free_gtt": None, + } + if mem_usage is not None: + mem_usage_dict.update(mem_usage) return AmdSmiMetric.model_validate( { "gpu": gpu, @@ -917,17 +1004,7 @@ def _minimal_amdsmi_metric( "perf_level": None, "xgmi_err": None, "energy": None, - "mem_usage": { - "total_vram": None, - "used_vram": None, - "free_vram": None, - "total_visible_vram": None, - "used_visible_vram": None, - "free_visible_vram": None, - "total_gtt": None, - "used_gtt": None, - "free_gtt": None, - }, + "mem_usage": mem_usage_dict, "throttle": {}, } ) @@ -1002,10 +1079,57 @@ def test_analyze_data_expected_power_management(mock_analyzer): assert not any("power_management mismatch" in e.description for e in result.events) +def test_check_gpu_memory_meets_minimum(mock_analyzer): + """GPU VRAM availability at the configured minimum passes.""" + analyzer = mock_analyzer + metrics = [ + _minimal_amdsmi_metric( + 0, + mem_usage={ + "total_vram": {"value": 100, "unit": "B"}, + "free_vram": {"value": 95, "unit": "B"}, + }, + ) + ] + + analyzer.check_gpu_memory(metrics, 95) + + assert not analyzer.result.events + + +def test_check_gpu_memory_below_minimum_logs_warning(mock_analyzer): + """GPU VRAM availability below the configured minimum logs a warning.""" + analyzer = mock_analyzer + metrics = [ + _minimal_amdsmi_metric( + 1, + mem_usage={ + "total_vram": {"value": 100, "unit": "B"}, + "free_vram": {"value": 90, "unit": "B"}, + }, + ) + ] + + analyzer.check_gpu_memory(metrics, 95) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.WARNING + assert "GPU 1 free VRAM is 90.00%" in event.description + assert event.data["available_percent"] == 90.0 + assert event.data["minimum_available_percent"] == 95 + + def test_amdsmi_analyzer_args_rejects_unknown_fields(): """Plugin config must only use declared AmdSmiAnalyzerArgs fields.""" from pydantic import ValidationError + args = AmdSmiAnalyzerArgs.model_validate({"gpu_memory": {"minimum_available_percent": 95}}) + assert args.gpu_memory is not None + assert args.gpu_memory.minimum_available_percent == 95 + args = AmdSmiAnalyzerArgs.model_validate({"check_xgmi_or_peer_links_status": True}) + assert args.check_xgmi_or_peer_links_status is True + with pytest.raises(ValidationError): AmdSmiAnalyzerArgs.model_validate( {"expected_power_management": "DISABLED", "not_a_field": 1}