gpu xgmi/peer link checks

nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py

@@ -37,8 +37,10 @@
     AmdSmiStatic,
     EccData,
     Fw,
+    LinkStatusTable,
     Partition,
     Processes,
+    XgmiLinks,
     XgmiMetrics,
 )
 from .analyzer_args import AmdSmiAnalyzerArgs
@@ -441,6 +443,73 @@ def check_amdsmi_metric_ecc_totals(self, amdsmi_metric_data: list[AmdSmiMetric])
                         console_log=True,
                     )
 
+    def check_gpu_memory(
+        self,
+        amdsmi_metric_data: list[AmdSmiMetric],
+        minimum_available_percent: float,
+    ) -> None:
+        """Check the minimum free VRAM percentage for each GPU."""
+        for metric in amdsmi_metric_data:
+            memory = metric.mem_usage
+            total_vram = memory.total_vram if memory is not None else None
+            free_vram = memory.free_vram if memory is not None else None
+            values = {
+                "gpu": metric.gpu,
+                "total_vram": total_vram.value if total_vram is not None else None,
+                "free_vram": free_vram.value if free_vram is not None else None,
+                "unit": total_vram.unit if total_vram is not None else None,
+                "minimum_available_percent": minimum_available_percent,
+            }
+
+            if total_vram is None or free_vram is None:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU {metric.gpu} VRAM availability is not available",
+                    priority=EventPriority.WARNING,
+                    data=values,
+                    console_log=True,
+                )
+                continue
+
+            try:
+                total_value = float(total_vram.value)
+                free_value = float(free_vram.value)
+            except (TypeError, ValueError):
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU {metric.gpu} VRAM availability is invalid",
+                    priority=EventPriority.WARNING,
+                    data=values,
+                    console_log=True,
+                )
+                continue
+
+            if total_value <= 0:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU {metric.gpu} total VRAM is invalid",
+                    priority=EventPriority.WARNING,
+                    data=values,
+                    console_log=True,
+                )
+                continue
+
+            available_percent = free_value / total_value * 100
+            if available_percent < minimum_available_percent:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=(
+                        f"GPU {metric.gpu} free VRAM is {available_percent:.2f}% "
+                        f"(minimum {minimum_available_percent:.2f}%)"
+                    ),
+                    priority=EventPriority.WARNING,
+                    data={
+                        **values,
+                        "available_percent": available_percent,
+                    },
+                    console_log=True,
+                )
+
     def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]):
         """Check ECC counts in all blocks for all GPUs
 
@@ -933,6 +1002,71 @@ def check_expected_xgmi_link_speed(
                     console_log=True,
                 )
 
+    def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) -> None:
+        """Check XGMI or peer-link status: U passes, SELF is ignored, D/X warn."""
+        if not xgmi_links:
+            self._log_event(
+                category=EventCategory.IO,
+                description="XGMI/peer link data is not available and cannot be checked",
+                priority=EventPriority.WARNING,
+                data={"xgmi_links": xgmi_links},
+                console_log=True,
+            )
+            return
+
+        down_links: list[dict[str, Any]] = []
+        degraded_links: list[dict[str, Any]] = []
+        healthy_link_count = 0
+        for gpu_links in xgmi_links:
+            for link_index, status in enumerate(gpu_links.link_status):
+                if status == LinkStatusTable.SELF:
+                    continue
+                link_data = {
+                    "gpu": gpu_links.gpu,
+                    "link_index": link_index,
+                    "status": status.value,
+                }
+                if status == LinkStatusTable.DOWN:
+                    down_links.append(link_data)
+                elif status == LinkStatusTable.DISABLED:
+                    degraded_links.append(link_data)
+                elif status == LinkStatusTable.UP:
+                    healthy_link_count += 1
+
+        if not down_links and not degraded_links:
+            self._log_event(
+                category=EventCategory.IO,
+                description="All XGMI/peer GPU links are working fine",
+                priority=EventPriority.INFO,
+                data={"healthy_link_count": healthy_link_count},
+                console_log=True,
+            )
+
+        if down_links:
+            self._log_event(
+                category=EventCategory.IO,
+                description=(f"XGMI/peer links contain {len(down_links)} down/error links"),
+                priority=EventPriority.WARNING,
+                data={
+                    "down_links": down_links,
+                    "link_error_count": len(down_links),
+                },
+                console_log=True,
+            )
+
+        if degraded_links:
+            self._log_event(
+                category=EventCategory.IO,
+                description=(
+                    f"XGMI/peer links contain {len(degraded_links)} " "disabled/degraded links"
+                ),
+                priority=EventPriority.WARNING,
+                data={
+                    "degraded_links": degraded_links,
+                },
+                console_log=True,
+            )
+
     def analyze_data(
         self, data: AmdSmiDataModel, args: Optional[AmdSmiAnalyzerArgs] = None
     ) -> TaskResult:
@@ -959,6 +1093,11 @@ def analyze_data(
                     args.l0_to_recovery_count_error_threshold,
                     args.l0_to_recovery_count_warning_threshold or 1,
                 )
+            if args.gpu_memory:
+                self.check_gpu_memory(
+                    data.metric,
+                    args.gpu_memory.minimum_available_percent,
+                )
             self.check_amdsmi_metric_ecc_totals(data.metric)
             self.check_amdsmi_metric_ecc(data.metric)
 
@@ -1020,4 +1159,7 @@ def analyze_data(
                 data.xgmi_metric, expected_xgmi_speed=args.expected_xgmi_speed
             )
 
+        if args.check_xgmi_or_peer_links_status:
+            self.check_xgmi_or_peer_links_status(data.xgmi_link)
+
         return self.result

nodescraper/plugins/inband/amdsmi/analyzer_args.py

@@ -26,12 +26,22 @@
 from datetime import datetime
 from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from nodescraper.models import AnalyzerArgs
 from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel
 
 
+class GpuMemoryConfig(BaseModel):
+    """GPU VRAM availability threshold."""
+
+    minimum_available_percent: float = Field(
+        ge=0,
+        le=100,
+        description="Minimum free VRAM percentage required for each GPU.",
+    )
+
+
 class AmdSmiAnalyzerArgs(AnalyzerArgs):
     check_static_data: bool = Field(
         default=False,
@@ -63,6 +73,14 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs):
         default=None,
         description="Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).",
     )
+    gpu_memory: Optional[GpuMemoryConfig] = Field(
+        default=None,
+        description="Minimum free VRAM threshold to validate for each GPU.",
+    )
+    check_xgmi_or_peer_links_status: bool = Field(
+        default=False,
+        description="Check XGMI or peer-link status for each GPU.",
+    )
     l0_to_recovery_count_error_threshold: Optional[int] = Field(
         default=3,
         description="L0-to-recovery count above which an error is raised.",