Amdsmi Fabric Add

nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py

@@ -36,6 +36,7 @@
     AmdSmiMetric,
     AmdSmiStatic,
     EccData,
+    Fabric,
     Fw,
     Partition,
     Processes,
@@ -79,6 +80,15 @@ def _gpu_unavailable_description(
     return description, description
 
 
+def _is_all_zeros(value: Union[str, list[str]]) -> bool:
+    """Return True if every comma or space separated entry is zero."""
+    values = value if isinstance(value, list) else [value]
+    tokens = [tok for item in values for tok in str(item).replace(",", " ").split()]
+    if not tokens:
+        return False
+    return all(set(tok) == {"0"} for tok in tokens)
+
+
 def _static_mismatch_description(payload: dict[str, Any]) -> tuple[str, str]:
     """Build description/details from ``check_static_data`` per-GPU payload."""
     per_gpu = payload.get("per_gpu") or []
@@ -933,6 +943,82 @@ def check_expected_xgmi_link_speed(
                     console_log=True,
                 )
 
+    def check_fabric(
+        self,
+        fabric_data: Optional[list[Fabric]],
+        expected_accel_state: str = "ACTIVE",
+        expected_fabric_type: str = "UALOE",
+    ) -> None:
+        """Check fabric state, type, pod IDs and accelerator maps for all GPUs
+
+        Args:
+            fabric_data (Optional[list[Fabric]]): fabric data from amd-smi fabric
+            expected_accel_state (str): expected accel_state value
+            expected_fabric_type (str): expected fabric_type value
+        """
+        if not fabric_data:
+            self._log_event(
+                category=EventCategory.NETWORK,
+                description="Fabric data is not available and cannot be checked",
+                priority=EventPriority.WARNING,
+                data={"fabric": fabric_data},
+            )
+            return
+
+        expected_state = expected_accel_state.strip().upper()
+        expected_type = expected_fabric_type.strip().upper()
+        issues: list[dict[str, Any]] = []
+
+        def _add(gpu: int, field: str, expected: object, actual: object) -> None:
+            issues.append(
+                {"gpu": gpu, "field": field, "expected": str(expected), "actual": str(actual)}
+            )
+
+        for entry in fabric_data:
+            info = entry.fabric_info
+            gpu = entry.gpu
+
+            accel_state = info.accel_state if info else None
+            if accel_state is None or accel_state.strip().upper() != expected_state:
+                _add(gpu, "accel_state", expected_state, accel_state or "N/A")
+
+            fabric_type = info.fabric_type if info else None
+            if fabric_type is None or fabric_type.strip().upper() != expected_type:
+                _add(gpu, "fabric_type", expected_type, fabric_type or "N/A")
+
+            ppod_id = info.ppod_id if info else None
+            if ppod_id is None or set(ppod_id) <= {"0", "-"}:
+                _add(gpu, "ppod_id", "non-zero", ppod_id or "N/A")
+
+            for field in ("local_accelerators", "local_active_accelerators"):
+                value = getattr(info, field) if info else None
+                if value is None or _is_all_zeros(value):
+                    _add(gpu, field, "non-zero", value if value is not None else "N/A")
+
+            for field in ("ppod_size", "vpod_size"):
+                value = getattr(info, field) if info else None
+                if not value:
+                    _add(gpu, field, "non-zero", value if value is not None else "N/A")
+
+        if issues:
+            details = "; ".join(
+                f"GPU {i['gpu']} {i['field']}: expected {i['expected']}, actual {i['actual']}"
+                for i in issues
+            )
+            gpus_affected = len({i["gpu"] for i in issues})
+            self._log_event(
+                category=EventCategory.NETWORK,
+                description=f"Fabric data mismatch on {gpus_affected} GPU(s): {details}",
+                priority=EventPriority.ERROR,
+                data={
+                    "expected_accel_state": expected_state,
+                    "expected_fabric_type": expected_type,
+                    "mismatches": issues,
+                    "details": details,
+                },
+                console_log=True,
+            )
+
     def analyze_data(
         self, data: AmdSmiDataModel, args: Optional[AmdSmiAnalyzerArgs] = None
     ) -> TaskResult:
@@ -1020,4 +1106,7 @@ def analyze_data(
                 data.xgmi_metric, expected_xgmi_speed=args.expected_xgmi_speed
             )
 
+        if data.fabric:
+            self.check_fabric(data.fabric)
+
         return self.result

nodescraper/plugins/inband/amdsmi/amdsmi_collector.py

@@ -43,6 +43,7 @@
     AmdSmiVersion,
     BadPages,
     EccState,
+    Fabric,
     Fw,
     FwListItem,
     Partition,
@@ -88,20 +89,21 @@ class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs])
 
     DATA_MODEL = AmdSmiDataModel
 
-    CMD_VERSION = "version --json"
-    CMD_LIST = "list --json"
-    CMD_PROCESS = "process --json"
-    CMD_PARTITION = "partition --json"
-    CMD_FIRMWARE = "firmware --json"
-    CMD_STATIC = "static -g all --json"
-    CMD_STATIC_GPU = "static -g {gpu_id} --json"
+    CMD_VERSION = "version"
+    CMD_LIST = "list"
+    CMD_PROCESS = "process"
+    CMD_PARTITION = "partition"
+    CMD_FIRMWARE = "firmware"
+    CMD_STATIC = "static -g all"
+    CMD_STATIC_GPU = "static -g {gpu_id}"
     CMD_TOPOLOGY = "topology"
     CMD_METRIC = "metric -g all"
     CMD_BAD_PAGES = "bad-pages"
     CMD_XGMI_METRIC = "xgmi -m"
     CMD_XGMI_LINK = "xgmi -l"
     CMD_RAS = "ras --cper --folder={folder}"
     CMD_RAS_AFID = "ras --afid --cper-file {cper_file}"
+    CMD_FABRIC = "fabric"
 
     def _check_amdsmi_installed(self) -> bool:
         """Check if amd-smi is installed
@@ -449,6 +451,33 @@ def get_xgmi_data(
 
         return xgmi_metrics or [], xgmi_links or []
 
+    def get_fabric(self) -> List[Fabric]:
+        """Get fabric data from amd-smi fabric --json.
+
+        ``fabric_telemetry`` is not collected as the payload is very large.
+        """
+        ret = self._run_amd_smi_dict(self.CMD_FABRIC)
+        if ret is None:
+            return []
+        if isinstance(ret, dict) and "gpu_data" in ret:
+            ret = ret["gpu_data"]
+        data = ret if isinstance(ret, list) else [ret]
+
+        fabric_entries: List[dict] = []
+        for item in data:
+            if not isinstance(item, dict):
+                continue
+            # each entry is {"gpu": <id>, "fabric": {...}}
+            entry = item.get("fabric") if isinstance(item.get("fabric"), dict) else item
+            if not isinstance(entry, dict):
+                continue
+            entry = {k: v for k, v in entry.items() if k != "fabric_telemetry"}
+            entry.setdefault("gpu", item.get("gpu", 0))
+            fabric_entries.append(entry)
+
+        built = self._build_amdsmi_sub_data(Fabric, fabric_entries)
+        return built if isinstance(built, list) else ([built] if built else [])
+
     def _get_amdsmi_data(
         self, args: Optional[AmdSmiCollectorArgs] = None
     ) -> Optional[AmdSmiDataModel]:
@@ -468,6 +497,7 @@ def _get_amdsmi_data(
             metric = self.get_metric()
             bad_pages = self.get_bad_pages()
             xgmi_metric, xgmi_link = self.get_xgmi_data()
+            fabric = self.get_fabric()
             cper_data, cper_afids = self.get_cper_data()
         except Exception as e:
             self._log_event(
@@ -494,6 +524,7 @@ def _get_amdsmi_data(
                 bad_pages=bad_pages or [],
                 xgmi_metric=xgmi_metric or [],
                 xgmi_link=xgmi_link or [],
+                fabric=fabric or [],
                 cper_data=cper_data,
                 cper_afids=cper_afids,
                 analysis_firmware_ids=fw_ids,

nodescraper/plugins/inband/amdsmi/amdsmidata.py

@@ -1066,6 +1066,8 @@ class AccessTable(Enum):
 
 
 # XGMI
+
+
 class XgmiLink(BaseModel):
     gpu: int
     bdf: str
@@ -1144,6 +1146,63 @@ class Topo(BaseModel):
     links: list[TopoLink]
 
 
+# FABRIC
+
+
+class FabricInfo(BaseModel):
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    bdf: Optional[str] = None
+    version: Optional[int] = None
+    accelerator_id: Optional[int] = None
+    fabric_type: Optional[str] = None
+    bandwidth: Optional[ValueUnit] = None
+    latency: Optional[ValueUnit] = None
+    ppod_id: Optional[str] = None
+    ppod_size: Optional[int] = None
+    vpod_id: Optional[int] = None
+    vpod_size: Optional[int] = None
+    local_accelerators: Optional[Union[str, list[str]]] = None
+    local_active_accelerators: Optional[list[str]] = None
+    addr_mode: Optional[str] = None
+    accel_state: Optional[str] = None
+
+    na_validator = field_validator(
+        "bdf",
+        "version",
+        "accelerator_id",
+        "fabric_type",
+        "ppod_id",
+        "ppod_size",
+        "vpod_id",
+        "vpod_size",
+        "local_accelerators",
+        "local_active_accelerators",
+        "addr_mode",
+        "accel_state",
+        mode="before",
+    )(na_to_none)
+
+    value_unit_validator = field_validator("bandwidth", "latency", mode="before")(
+        coerce_value_unit_input
+    )
+
+
+class Fabric(BaseModel):
+    """Per GPU fabric data from ``amd-smi fabric``.
+
+    ``fabric_telemetry`` is intentionally not collected for now as it is very large.
+    """
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    gpu: int
+    bdf: Optional[str] = None
+    fabric_info: Optional[FabricInfo] = None
+
+    na_validator = field_validator("bdf", "fabric_info", mode="before")(na_to_none)
+
+
 class AmdSmiAnalysisRef(BaseModel):
     """Collector-filled summary for reference config"""
 
@@ -1189,6 +1248,7 @@ class AmdSmiDataModel(DataModel):
     metric: Optional[list[AmdSmiMetric]] = Field(default_factory=list)
     xgmi_metric: Optional[list[XgmiMetrics]] = Field(default_factory=list)
     xgmi_link: Optional[list[XgmiLinks]] = Field(default_factory=list)
+    fabric: Optional[list[Fabric]] = Field(default_factory=list)
     cper_data: Optional[list[FileModel]] = Field(default_factory=list)
     cper_afids: dict[str, int] = Field(default_factory=dict)
 
@@ -1244,6 +1304,15 @@ def get_bad_pages(self, gpu: int) -> Optional[BadPages]:
                 return item
         return None
 
+    def get_fabric(self, gpu: int) -> Optional[Fabric]:
+        """Get the fabric data for the given gpu id."""
+        if self.fabric is None:
+            return None
+        for item in self.fabric:
+            if item.gpu == gpu:
+                return item
+        return None
+
     def _sorted_static_gpus(self) -> list[AmdSmiStatic]:
         return sorted(self.static or [], key=lambda s: s.gpu)