From caebe1b2ec2ca083ca2e8ef28aa14ad613e6ea08 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Thu, 24 Sep 2026 10:59:15 -0700 Subject: [PATCH 1/3] amdsmi fabric --- .../plugins/inband/amdsmi/amdsmi_analyzer.py | 90 +++++++++++ .../plugins/inband/amdsmi/amdsmi_collector.py | 45 +++++- .../plugins/inband/amdsmi/amdsmidata.py | 69 +++++++++ test/unit/plugin/test_amdsmi_analyzer.py | 146 ++++++++++++++++++ test/unit/plugin/test_amdsmi_collector.py | 124 +++++++++++++++ test/unit/plugin/test_amdsmi_data.py | 145 +++++++++++++++++ 6 files changed, 612 insertions(+), 7 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index da427753..01c7700b 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -24,6 +24,7 @@ # ############################################################################### import io +import re from collections import defaultdict from typing import Any, Mapping, Optional, Union @@ -36,6 +37,7 @@ AmdSmiMetric, AmdSmiStatic, EccData, + Fabric, Fw, Partition, Processes, @@ -79,6 +81,15 @@ def _gpu_unavailable_description( return description, description +def _is_all_zeros(value: Union[str, list[str]]) -> bool: + """Return True if every numeric/hex token in the value is zero.""" + values = value if isinstance(value, list) else [value] + tokens = [tok for item in values for tok in re.split(r"[^0-9A-Za-z]+", str(item)) if tok] + if not tokens: + return False + return all(set(tok) == {"0"} for tok in tokens) + + def _static_mismatch_description(payload: dict[str, Any]) -> tuple[str, str]: """Build description/details from ``check_static_data`` per-GPU payload.""" per_gpu = payload.get("per_gpu") or [] @@ -933,6 +944,82 @@ def check_expected_xgmi_link_speed( console_log=True, ) + def check_fabric( + self, + fabric_data: Optional[list[Fabric]], + expected_accel_state: str = "ACTIVE", + expected_fabric_type: str = "UALOE", + ) -> None: + """Check fabric state, type, pod IDs and accelerator maps for all GPUs + + Args: + fabric_data (Optional[list[Fabric]]): fabric data from amd-smi fabric + expected_accel_state (str): expected accel_state value + expected_fabric_type (str): expected fabric_type value + """ + if not fabric_data: + self._log_event( + category=EventCategory.NETWORK, + description="Fabric data is not available and cannot be checked", + priority=EventPriority.WARNING, + data={"fabric": fabric_data}, + ) + return + + expected_state = expected_accel_state.strip().upper() + expected_type = expected_fabric_type.strip().upper() + issues: list[dict[str, Any]] = [] + + def _add(gpu: int, field: str, expected: object, actual: object) -> None: + issues.append( + {"gpu": gpu, "field": field, "expected": str(expected), "actual": str(actual)} + ) + + for entry in fabric_data: + info = entry.fabric_info + gpu = entry.gpu + + accel_state = info.accel_state if info else None + if accel_state is None or accel_state.strip().upper() != expected_state: + _add(gpu, "accel_state", expected_state, accel_state or "N/A") + + fabric_type = info.fabric_type if info else None + if fabric_type is None or fabric_type.strip().upper() != expected_type: + _add(gpu, "fabric_type", expected_type, fabric_type or "N/A") + + ppod_id = info.ppod_id if info else None + if ppod_id is None or _is_all_zeros(ppod_id): + _add(gpu, "ppod_id", "non-zero", ppod_id or "N/A") + + for field in ("local_accelerators", "local_active_accelerators"): + value = getattr(info, field) if info else None + if value is None or _is_all_zeros(value): + _add(gpu, field, "non-zero", value if value is not None else "N/A") + + for field in ("ppod_size", "vpod_size"): + value = getattr(info, field) if info else None + if not value: + _add(gpu, field, "non-zero", value if value is not None else "N/A") + + if issues: + details = "; ".join( + f"GPU {i['gpu']} {i['field']}: expected {i['expected']}, actual {i['actual']}" + for i in issues + ) + gpus_affected = len({i["gpu"] for i in issues}) + self._log_event( + category=EventCategory.NETWORK, + description=f"Fabric data mismatch on {gpus_affected} GPU(s): {details}", + priority=EventPriority.ERROR, + data={ + "expected_accel_state": expected_state, + "expected_fabric_type": expected_type, + "mismatches": issues, + "details": details, + }, + console_log=True, + ) + def analyze_data( self, data: AmdSmiDataModel, args: Optional[AmdSmiAnalyzerArgs] = None ) -> TaskResult: @@ -1020,4 +1107,7 @@ def analyze_data( data.xgmi_metric, expected_xgmi_speed=args.expected_xgmi_speed ) + if data.fabric: + self.check_fabric(data.fabric) + return self.result diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py index 92d42252..b072767b 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py @@ -43,6 +43,7 @@ AmdSmiVersion, BadPages, EccState, + Fabric, Fw, FwListItem, Partition, @@ -88,13 +89,13 @@ class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs]) DATA_MODEL = AmdSmiDataModel - CMD_VERSION = "version --json" - CMD_LIST = "list --json" - CMD_PROCESS = "process --json" - CMD_PARTITION = "partition --json" - CMD_FIRMWARE = "firmware --json" - CMD_STATIC = "static -g all --json" - CMD_STATIC_GPU = "static -g {gpu_id} --json" + CMD_VERSION = "version" + CMD_LIST = "list" + CMD_PROCESS = "process" + CMD_PARTITION = "partition" + CMD_FIRMWARE = "firmware" + CMD_STATIC = "static -g all" + CMD_STATIC_GPU = "static -g {gpu_id}" CMD_TOPOLOGY = "topology" CMD_METRIC = "metric -g all" CMD_BAD_PAGES = "bad-pages" @@ -102,6 +103,7 @@ class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs]) CMD_XGMI_LINK = "xgmi -l" CMD_RAS = "ras --cper --folder={folder}" CMD_RAS_AFID = "ras --afid --cper-file {cper_file}" + CMD_FABRIC = "fabric" def _check_amdsmi_installed(self) -> bool: """Check if amd-smi is installed @@ -449,6 +451,33 @@ def get_xgmi_data( return xgmi_metrics or [], xgmi_links or [] + def get_fabric(self) -> List[Fabric]: + """Get fabric data from amd-smi fabric --json. + + ``fabric_telemetry`` is not collected as the payload is very large. + """ + ret = self._run_amd_smi_dict(self.CMD_FABRIC) + if ret is None: + return [] + if isinstance(ret, dict) and "gpu_data" in ret: + ret = ret["gpu_data"] + data = ret if isinstance(ret, list) else [ret] + + fabric_entries: List[dict] = [] + for item in data: + if not isinstance(item, dict): + continue + # each entry is {"gpu": , "fabric": {...}} + entry = item.get("fabric") if isinstance(item.get("fabric"), dict) else item + if not isinstance(entry, dict): + continue + entry = {k: v for k, v in entry.items() if k != "fabric_telemetry"} + entry.setdefault("gpu", item.get("gpu", 0)) + fabric_entries.append(entry) + + built = self._build_amdsmi_sub_data(Fabric, fabric_entries) + return built if isinstance(built, list) else ([built] if built else []) + def _get_amdsmi_data( self, args: Optional[AmdSmiCollectorArgs] = None ) -> Optional[AmdSmiDataModel]: @@ -468,6 +497,7 @@ def _get_amdsmi_data( metric = self.get_metric() bad_pages = self.get_bad_pages() xgmi_metric, xgmi_link = self.get_xgmi_data() + fabric = self.get_fabric() cper_data, cper_afids = self.get_cper_data() except Exception as e: self._log_event( @@ -494,6 +524,7 @@ def _get_amdsmi_data( bad_pages=bad_pages or [], xgmi_metric=xgmi_metric or [], xgmi_link=xgmi_link or [], + fabric=fabric or [], cper_data=cper_data, cper_afids=cper_afids, analysis_firmware_ids=fw_ids, diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata.py b/nodescraper/plugins/inband/amdsmi/amdsmidata.py index 3b8aae3c..bcc9a6b9 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmidata.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata.py @@ -1066,6 +1066,8 @@ class AccessTable(Enum): # XGMI + + class XgmiLink(BaseModel): gpu: int bdf: str @@ -1144,6 +1146,63 @@ class Topo(BaseModel): links: list[TopoLink] +# FABRIC + + +class FabricInfo(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + bdf: Optional[str] = None + version: Optional[int] = None + accelerator_id: Optional[int] = None + fabric_type: Optional[str] = None + bandwidth: Optional[ValueUnit] = None + latency: Optional[ValueUnit] = None + ppod_id: Optional[str] = None + ppod_size: Optional[int] = None + vpod_id: Optional[int] = None + vpod_size: Optional[int] = None + local_accelerators: Optional[Union[str, list[str]]] = None + local_active_accelerators: Optional[list[str]] = None + addr_mode: Optional[str] = None + accel_state: Optional[str] = None + + na_validator = field_validator( + "bdf", + "version", + "accelerator_id", + "fabric_type", + "ppod_id", + "ppod_size", + "vpod_id", + "vpod_size", + "local_accelerators", + "local_active_accelerators", + "addr_mode", + "accel_state", + mode="before", + )(na_to_none) + + value_unit_validator = field_validator("bandwidth", "latency", mode="before")( + coerce_value_unit_input + ) + + +class Fabric(BaseModel): + """Per GPU fabric data from ``amd-smi fabric``. + + ``fabric_telemetry`` is intentionally not collected for now as it is very large. + """ + + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + gpu: int + bdf: Optional[str] = None + fabric_info: Optional[FabricInfo] = None + + na_validator = field_validator("bdf", "fabric_info", mode="before")(na_to_none) + + class AmdSmiAnalysisRef(BaseModel): """Collector-filled summary for reference config""" @@ -1189,6 +1248,7 @@ class AmdSmiDataModel(DataModel): metric: Optional[list[AmdSmiMetric]] = Field(default_factory=list) xgmi_metric: Optional[list[XgmiMetrics]] = Field(default_factory=list) xgmi_link: Optional[list[XgmiLinks]] = Field(default_factory=list) + fabric: Optional[list[Fabric]] = Field(default_factory=list) cper_data: Optional[list[FileModel]] = Field(default_factory=list) cper_afids: dict[str, int] = Field(default_factory=dict) @@ -1244,6 +1304,15 @@ def get_bad_pages(self, gpu: int) -> Optional[BadPages]: return item return None + def get_fabric(self, gpu: int) -> Optional[Fabric]: + """Get the fabric data for the given gpu id.""" + if self.fabric is None: + return None + for item in self.fabric: + if item.gpu == gpu: + return item + return None + def _sorted_static_gpus(self) -> list[AmdSmiStatic]: return sorted(self.static or [], key=lambda s: s.gpu) diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 560a9ba4..d2efaeba 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -40,6 +40,8 @@ AmdSmiStatic, AmdSmiVersion, EccState, + Fabric, + FabricInfo, Fw, FwListItem, MetricEccTotals, @@ -1162,3 +1164,147 @@ def test_check_amdsmi_metric_ecc_blocks(mock_analyzer): assert any("GFX" in d and "correctable" in d for d in desc) assert any("MMHUB" in d for d in desc) assert any("HDP" in d for d in desc) + + +def create_fabric( + gpu: int = 0, + fabric_type: str = "UALOE", + accel_state: str = "ACTIVE", + ppod_id: str = "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", +) -> Fabric: + """Helper function to create a healthy mock Fabric object for testing.""" + return Fabric( + gpu=gpu, + bdf=f"{gpu + 1:04d}:01:00.0", + fabric_info=FabricInfo( + bdf=f"{gpu + 1:04d}:01:00.1", + version=4294967295, + accelerator_id=7 - gpu, + fabric_type=fabric_type, + bandwidth=ValueUnit(value=400000, unit="Mb/s"), + latency=ValueUnit(value=150, unit="ns"), + ppod_id=ppod_id, + ppod_size=72, + vpod_id=1, + vpod_size=8, + local_accelerators="1, 2, 3, 4, 5, 6, 7, 8", + local_active_accelerators=["1, 2, 3, 4, 5, 6, 7, 8"], + addr_mode="DIRECT", + accel_state=accel_state, + ), + ) + + +def test_check_fabric_success(mock_analyzer): + """Healthy fabric data on all GPUs generates no events.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_fabric(gpu) for gpu in range(4)]) + + assert len(analyzer.result.events) == 0 + + +def test_check_fabric_no_data(mock_analyzer): + """Missing fabric data logs a warning rather than an error.""" + analyzer = mock_analyzer + + analyzer.check_fabric([]) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + assert "Fabric data is not available" in analyzer.result.events[0].description + + +def test_check_fabric_inactive_output(mock_analyzer): + """Real amd-smi fabric output from an inactive fabric reports every mismatch.""" + analyzer = mock_analyzer + + fabric_data = [ + Fabric.model_validate( + { + "gpu": gpu, + "bdf": f"{gpu + 1:04d}:01:00.0", + "fabric_info": { + "bdf": f"{gpu + 1:04d}:01:00.1", + "version": 4294967295, + "accelerator_id": 7 - gpu, + "fabric_type": "UALOE", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", + "ppod_size": 72, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNKNOWN", + }, + } + ) + for gpu in range(4) + ] + + analyzer.check_fabric(fabric_data) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.ERROR + assert event.category == "NETWORK" + mismatched_fields = {m["field"] for m in event.data["mismatches"]} + assert mismatched_fields == { + "accel_state", + "local_accelerators", + "local_active_accelerators", + "vpod_size", + } + assert {m["gpu"] for m in event.data["mismatches"]} == {0, 1, 2, 3} + assert "Fabric data mismatch on 4 GPU(s)" in event.description + + +def test_check_fabric_unexpected_fabric_type(mock_analyzer): + """A fabric_type other than the expected one is reported.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_fabric(0, fabric_type="XGMI")]) + + assert len(analyzer.result.events) == 1 + mismatches = analyzer.result.events[0].data["mismatches"] + assert mismatches == [{"gpu": 0, "field": "fabric_type", "expected": "UALOE", "actual": "XGMI"}] + + +def test_check_fabric_custom_expected_values(mock_analyzer): + """Expected accel_state/fabric_type are configurable and case insensitive.""" + analyzer = mock_analyzer + + analyzer.check_fabric( + [create_fabric(0, fabric_type="xgmi", accel_state="up")], + expected_accel_state="UP", + expected_fabric_type="XGMI", + ) + + assert len(analyzer.result.events) == 0 + + +def test_check_fabric_missing_fabric_info(mock_analyzer): + """A GPU with no fabric_info reports all fields as N/A.""" + analyzer = mock_analyzer + + analyzer.check_fabric([Fabric(gpu=0, bdf="0001:01:00.0", fabric_info=None)]) + + assert len(analyzer.result.events) == 1 + mismatches = analyzer.result.events[0].data["mismatches"] + assert len(mismatches) == 7 + assert all(m["actual"] == "N/A" and m["gpu"] == 0 for m in mismatches) + + +def test_check_fabric_zero_ppod_id(mock_analyzer): + """An all-zero ppod_id is treated as a mismatch.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_fabric(0, ppod_id="00000000-0000-0000-0000-000000000000")]) + + assert len(analyzer.result.events) == 1 + mismatches = analyzer.result.events[0].data["mismatches"] + assert [m["field"] for m in mismatches] == ["ppod_id"] + assert mismatches[0]["expected"] == "non-zero" diff --git a/test/unit/plugin/test_amdsmi_collector.py b/test/unit/plugin/test_amdsmi_collector.py index 74b56266..47e7b5f8 100644 --- a/test/unit/plugin/test_amdsmi_collector.py +++ b/test/unit/plugin/test_amdsmi_collector.py @@ -832,3 +832,127 @@ def mock_read_sut_file(path: str, encoding=None, strip=False, log_artifact=False assert data.cper_afids["auto1.cper"] == 11111 assert "/home/user/custom.cper" in data.cper_afids assert data.cper_afids["/home/user/custom.cper"] == 99999 + + +def fabric_json_entry(gpu: int) -> dict[str, Any]: + """Build one `amd-smi fabric --json` entry as emitted by the tool.""" + return { + "gpu": gpu, + "fabric": { + "gpu": gpu, + "bdf": f"{gpu + 1:04d}:01:00.0", + "fabric_info": { + "bdf": f"{gpu + 1:04d}:01:00.1", + "version": 4294967295, + "accelerator_id": 7 - gpu, + "fabric_type": "UALOE", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", + "ppod_size": 72, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNKNOWN", + }, + "fabric_telemetry": "N/A", + }, + } + + +def make_fabric_collector(conn_mock, system_info, monkeypatch, fabric_payload) -> AmdSmiCollector: + """Create a collector whose only mocked amd-smi command is `fabric`.""" + + def mock_run_sut_cmd(cmd: str, sudo: bool = False) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "fabric --json" in cmd: + if fabric_payload is None: + return make_cmd_result("", "fabric not supported", 1) + return make_cmd_result(make_json_response(fabric_payload)) + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + return c + + +def test_get_fabric(conn_mock, system_info, monkeypatch): + """Test fabric parsing from the nested {'gpu': n, 'fabric': {...}} output""" + payload = [fabric_json_entry(0), fabric_json_entry(1)] + c = make_fabric_collector(conn_mock, system_info, monkeypatch, payload) + + fabric = c.get_fabric() + + assert len(fabric) == 2 + assert [f.gpu for f in fabric] == [0, 1] + assert fabric[0].bdf == "0001:01:00.0" + info = fabric[0].fabric_info + assert info is not None + assert info.fabric_type == "UALOE" + assert info.accelerator_id == 7 + assert info.ppod_id == "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb" + assert info.ppod_size == 72 + assert info.bandwidth is not None and info.bandwidth.unit == "Mb/s" + assert info.latency is not None and info.latency.unit == "ns" + + +def test_get_fabric_drops_fabric_telemetry(conn_mock, system_info, monkeypatch): + """Test that the large fabric_telemetry payload is not collected""" + entry = fabric_json_entry(0) + entry["fabric"]["fabric_telemetry"] = {"links": [{"id": i} for i in range(64)]} + c = make_fabric_collector(conn_mock, system_info, monkeypatch, [entry]) + + fabric = c.get_fabric() + + assert len(fabric) == 1 + assert "fabric_telemetry" not in fabric[0].model_dump() + + +def test_get_fabric_gpu_data_wrapper(conn_mock, system_info, monkeypatch): + """Test fabric parsing when output is wrapped in a gpu_data key""" + payload = {"gpu_data": [fabric_json_entry(0), fabric_json_entry(1), fabric_json_entry(2)]} + c = make_fabric_collector(conn_mock, system_info, monkeypatch, payload) + + fabric = c.get_fabric() + + assert [f.gpu for f in fabric] == [0, 1, 2] + + +def test_get_fabric_flat_entry(conn_mock, system_info, monkeypatch): + """Test fabric parsing when entries are not nested under a fabric key""" + payload = [fabric_json_entry(0)["fabric"]] + c = make_fabric_collector(conn_mock, system_info, monkeypatch, payload) + + fabric = c.get_fabric() + + assert len(fabric) == 1 + assert fabric[0].gpu == 0 + assert fabric[0].fabric_info is not None + + +def test_get_fabric_command_failure(conn_mock, system_info, monkeypatch): + """Test fabric returns an empty list when amd-smi fabric fails""" + c = make_fabric_collector(conn_mock, system_info, monkeypatch, None) + + assert c.get_fabric() == [] + + +def test_get_fabric_na_values(conn_mock, system_info, monkeypatch): + """Test fabric handles N/A values reported by amd-smi""" + entry = fabric_json_entry(0) + entry["fabric"]["bdf"] = "N/A" + entry["fabric"]["fabric_info"] = "N/A" + c = make_fabric_collector(conn_mock, system_info, monkeypatch, [entry]) + + fabric = c.get_fabric() + + assert len(fabric) == 1 + assert fabric[0].bdf is None + assert fabric[0].fabric_info is None diff --git a/test/unit/plugin/test_amdsmi_data.py b/test/unit/plugin/test_amdsmi_data.py index f6c4f750..60120799 100644 --- a/test/unit/plugin/test_amdsmi_data.py +++ b/test/unit/plugin/test_amdsmi_data.py @@ -32,6 +32,8 @@ from nodescraper.plugins.inband.amdsmi.amdsmidata import ( AmdSmiDataModel, AmdSmiMetric, + Fabric, + FabricInfo, MetricClockData, MetricPcie, MetricPower, @@ -542,3 +544,146 @@ def test_metric_clock_per_aid_na_maps(): assert metric.clock["uclk_aid"]["AID_0"] == "N/A" assert isinstance(metric.clock["GFX_0"], MetricClockData) assert metric.pcie.lc_perf_other_end_recovery_count == 0 + + +# FABRIC + + +def dummy_fabric_entry(gpu: int = 0, **overrides: Any) -> dict[str, Any]: + """Build one ``amd-smi fabric --json`` entry as emitted by the tool.""" + fabric_info: dict[str, Any] = { + "bdf": f"{gpu + 1:04d}:01:00.1", + "version": 4294967295, + "accelerator_id": 7 - gpu, + "fabric_type": "UALOE", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", + "ppod_size": 72, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": [ + "0, 0, 0, 0, 0, 0, 0, 0", + "0, 0, 0, 0, 0, 0, 0, 0", + "0, 0, 0, 0, 0, 0, 0, 0", + "0, 0, 0, 0, 0, 0, 0, 0", + ], + "addr_mode": "UNKNOWN", + "accel_state": "UNKNOWN", + } + fabric_info.update(overrides) + return { + "gpu": gpu, + "fabric": { + "gpu": gpu, + "bdf": f"{gpu + 1:04d}:01:00.0", + "fabric_info": fabric_info, + "fabric_telemetry": "N/A", + }, + } + + +def test_fabric_model_from_amd_smi_output(): + """Fabric parses the nested ``fabric`` payload from amd-smi fabric --json.""" + fabric = Fabric.model_validate(dummy_fabric_entry(gpu=0)["fabric"]) + + assert fabric.gpu == 0 + assert fabric.bdf == "0001:01:00.0" + info = fabric.fabric_info + assert info is not None + assert info.bdf == "0001:01:00.1" + assert info.version == 4294967295 + assert info.accelerator_id == 7 + assert info.fabric_type == "UALOE" + assert info.ppod_id == "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb" + assert info.ppod_size == 72 + assert info.vpod_id == 0 + assert info.vpod_size == 0 + assert info.addr_mode == "UNKNOWN" + assert info.accel_state == "UNKNOWN" + assert len(info.local_active_accelerators) == 4 + + +def test_fabric_ignores_fabric_telemetry(): + """fabric_telemetry is not modelled and must not cause a validation error.""" + entry = dummy_fabric_entry(gpu=1)["fabric"] + entry["fabric_telemetry"] = {"huge": [1, 2, 3]} + + fabric = Fabric.model_validate(entry) + + assert fabric.gpu == 1 + assert not hasattr(fabric, "fabric_telemetry") + + +def test_fabric_info_value_unit_coercion(): + """bandwidth/latency accept dicts, bare numbers and number+unit strings.""" + info = FabricInfo.model_validate( + { + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": "150 ns", + } + ) + + assert info.bandwidth == ValueUnit(value=0, unit="Mb/s") + assert info.latency is not None + assert info.latency.value == 150 + assert info.latency.unit == "ns" + + +def test_fabric_na_fields_map_to_none(): + """'N/A' values from amd-smi are normalized to None.""" + fabric = Fabric.model_validate( + { + "gpu": 2, + "bdf": "N/A", + "fabric_info": { + "bdf": "N/A", + "fabric_type": "N/A", + "ppod_id": "N/A", + "ppod_size": "N/A", + "accel_state": "N/A", + "bandwidth": "N/A", + "latency": None, + }, + } + ) + + assert fabric.bdf is None + info = fabric.fabric_info + assert info is not None + assert info.bdf is None + assert info.fabric_type is None + assert info.ppod_id is None + assert info.ppod_size is None + assert info.accel_state is None + assert info.bandwidth is None + assert info.latency is None + + +def test_fabric_info_na_maps_to_none(): + """A whole fabric_info block reported as 'N/A' becomes None.""" + fabric = Fabric.model_validate({"gpu": 3, "bdf": "0004:01:00.0", "fabric_info": "N/A"}) + + assert fabric.fabric_info is None + + +def test_data_model_get_fabric(): + """AmdSmiDataModel.get_fabric returns the entry for the requested gpu.""" + data = AmdSmiDataModel( + fabric=[ + Fabric.model_validate(dummy_fabric_entry(gpu=0)["fabric"]), + Fabric.model_validate(dummy_fabric_entry(gpu=1)["fabric"]), + ] + ) + + fabric = data.get_fabric(1) + assert fabric is not None + assert fabric.gpu == 1 + assert fabric.bdf == "0002:01:00.0" + assert data.get_fabric(9) is None + + +def test_data_model_fabric_defaults_to_empty_list(): + """fabric defaults to an empty list when amd-smi fabric is unavailable.""" + assert AmdSmiDataModel().fabric == [] From 16fe66b6be1d2fff8359eeb25cc54ff83500c5d3 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Thu, 24 Sep 2026 11:49:16 -0700 Subject: [PATCH 2/3] logic simplify --- nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index 01c7700b..10473814 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -24,7 +24,6 @@ # ############################################################################### import io -import re from collections import defaultdict from typing import Any, Mapping, Optional, Union @@ -82,9 +81,9 @@ def _gpu_unavailable_description( def _is_all_zeros(value: Union[str, list[str]]) -> bool: - """Return True if every numeric/hex token in the value is zero.""" + """Return True if every comma or space separated entry is zero.""" values = value if isinstance(value, list) else [value] - tokens = [tok for item in values for tok in re.split(r"[^0-9A-Za-z]+", str(item)) if tok] + tokens = [tok for item in values for tok in str(item).replace(",", " ").split()] if not tokens: return False return all(set(tok) == {"0"} for tok in tokens) @@ -988,7 +987,7 @@ def _add(gpu: int, field: str, expected: object, actual: object) -> None: _add(gpu, "fabric_type", expected_type, fabric_type or "N/A") ppod_id = info.ppod_id if info else None - if ppod_id is None or _is_all_zeros(ppod_id): + if ppod_id is None or set(ppod_id) <= {"0", "-"}: _add(gpu, "ppod_id", "non-zero", ppod_id or "N/A") for field in ("local_accelerators", "local_active_accelerators"): From 4216a3306dfd958822940e9fb3c88aac7cce7d8e Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Mon, 28 Sep 2026 16:39:23 -0700 Subject: [PATCH 3/3] move fabric check to args --- nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py | 6 +++++- nodescraper/plugins/inband/amdsmi/analyzer_args.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index 10473814..714bd10d 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -1107,6 +1107,10 @@ def analyze_data( ) if data.fabric: - self.check_fabric(data.fabric) + self.check_fabric( + data.fabric, + expected_accel_state=args.expected_accel_state, + expected_fabric_type=args.expected_fabric_type, + ) return self.result diff --git a/nodescraper/plugins/inband/amdsmi/analyzer_args.py b/nodescraper/plugins/inband/amdsmi/analyzer_args.py index cd08bd49..019fa24e 100644 --- a/nodescraper/plugins/inband/amdsmi/analyzer_args.py +++ b/nodescraper/plugins/inband/amdsmi/analyzer_args.py @@ -83,6 +83,12 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): expected_xgmi_speed: Optional[list[float]] = Field( default=None, description="Expected xGMI speed value(s) (e.g. link rate)." ) + expected_accel_state: str = Field( + default="ACTIVE", description="Expected amd-smi fabric accel_state value." + ) + expected_fabric_type: str = Field( + default="UALOE", description="Expected amd-smi fabric fabric_type value." + ) analysis_range_start: Optional[datetime] = Field( default=None, description="Start of time range for time-windowed analysis." )