From 6a76321951cd49ff5740280db67bda11606324d8 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Thu, 10 Sep 2026 23:36:31 +0000 Subject: [PATCH 01/12] Add ESXi support to platform in-band collectors Enable os, bios, dimm, kernel, storage, and device_enumeration collectors on ESXi via esxcli/smbiosDump. kernel reuses the existing `uname -a` path (ESXi reports the release in the same field). dimm shares a _parse_dmi_sizes helper across dmidecode (Linux) and smbiosDump (ESXi). device_enumeration counts GPU PF/VF by device ID, adding devid_ep/devid_ep_vf to SystemInfo. Validated on ESXi 9.1.0 and Linux; Linux/Windows paths unchanged. --- nodescraper/models/systeminfo.py | 2 + .../plugins/inband/bios/bios_collector.py | 8 ++++ .../device_enumeration_collector.py | 28 +++++++++++ .../plugins/inband/dimm/dimm_collector.py | 47 ++++++++++--------- .../plugins/inband/kernel/kernel_collector.py | 4 ++ nodescraper/plugins/inband/os/os_collector.py | 29 ++++++++++++ .../inband/storage/storage_collector.py | 20 ++++++++ 7 files changed, 115 insertions(+), 23 deletions(-) diff --git a/nodescraper/models/systeminfo.py b/nodescraper/models/systeminfo.py index d91a68cf..d593a9a0 100644 --- a/nodescraper/models/systeminfo.py +++ b/nodescraper/models/systeminfo.py @@ -44,3 +44,5 @@ class SystemInfo(BaseModel): metadata: Optional[dict] = Field(default_factory=dict) location: Optional[SystemLocation] = SystemLocation.LOCAL vendorid_ep: int = 0x1002 + devid_ep: Optional[int] = None + devid_ep_vf: Optional[int] = None diff --git a/nodescraper/plugins/inband/bios/bios_collector.py b/nodescraper/plugins/inband/bios/bios_collector.py index e0ab1011..e94242ef 100644 --- a/nodescraper/plugins/inband/bios/bios_collector.py +++ b/nodescraper/plugins/inband/bios/bios_collector.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.base import InBandDataCollector @@ -35,9 +36,11 @@ class BiosCollector(InBandDataCollector[BiosDataModel, None]): """Collect BIOS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = BiosDataModel CMD_WINDOWS = "wmic bios get SMBIOSBIOSVersion /Value" CMD = "sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'" + CMD_ESXI = "smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1" def collect_data( self, @@ -57,6 +60,11 @@ def collect_data( bios = [line for line in res.stdout.splitlines() if "SMBIOSBIOSVersion=" in line][ 0 ].split("=")[1] + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + match = re.search(r'Version:\s*"?([^"]+)"?', res.stdout) + bios = match.group(1).strip() if match else res.stdout.strip() else: res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 9b0dc295..9f579672 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -36,6 +36,7 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, None]): """Collect CPU and GPU count""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DeviceEnumerationDataModel CMD_GPU_COUNT_LINUX = ( @@ -55,6 +56,12 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, 'powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count"' ) + # ESXi busybox `lspci -d` dumps hex instead of filtering, so use esxcli. GPUs are + # counted by exact device ID (PF vs VF), anchored on "Device ID:" to avoid also + # matching "SubDevice ID:". + CMD_CPU_COUNT_ESXI = "esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'" + CMD_PCI_COUNT_ESXI = "esxcli hardware pci list | grep -E '^ *Device ID: 0x{device_id}' | wc -l" + def _warning( self, description: str, @@ -72,10 +79,20 @@ def _warning( priority=EventPriority.WARNING, ) + def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: + """Count PCI devices on ESXi whose Device ID matches ``device_id`` (as hex). + + A None id produces an unmatched pattern (count 0) so the caller still gets a + valid CommandArtifact to parse. + """ + hex_id = format(device_id, "x") if device_id is not None else "__unset__" + return self._run_sut_cmd(self.CMD_PCI_COUNT_ESXI.format(device_id=hex_id)) + def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: """ Read CPU and GPU count On Linux, use lscpu and lspci + On ESXi, use esxcli On Windows, use WMI and hyper-v cmdlets """ if self.system_info.os_family == OSFamily.LINUX: @@ -92,6 +109,17 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio # Collect lshw output lshw_res = self._run_sut_cmd(self.CMD_LSHW_LINUX, sudo=True, log_artifact=False) + elif self.system_info.os_family == OSFamily.ESXI: + cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_ESXI) + if self.system_info.devid_ep is None: + self._log_event( + category=EventCategory.PLATFORM, + description="devid_ep not set; cannot count GPUs/VFs on ESXi by device ID", + priority=EventPriority.WARNING, + ) + # PFs and (SR-IOV) VFs are distinguished by device ID on ESXi. + gpu_count_res = self._esxi_device_count(self.system_info.devid_ep) + vf_count_res = self._esxi_device_count(self.system_info.devid_ep_vf) else: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_WINDOWS) gpu_count_res = self._run_sut_cmd(self.CMD_GPU_COUNT_WINDOWS) diff --git a/nodescraper/plugins/inband/dimm/dimm_collector.py b/nodescraper/plugins/inband/dimm/dimm_collector.py index b6b91987..c9b4dd8f 100644 --- a/nodescraper/plugins/inband/dimm/dimm_collector.py +++ b/nodescraper/plugins/inband/dimm/dimm_collector.py @@ -38,12 +38,31 @@ class DimmCollector(InBandDataCollector[DimmDataModel, DimmCollectorArgs]): """Collect data on installed DIMMs""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DimmDataModel CMD_WINDOWS = "wmic memorychip get Capacity" CMD = """sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\\|None\\|Module" | grep Size' 2>/dev/null""" + CMD_ESXI = "smbiosDump | grep -A15 'Memory Device (Type 17)' | grep 'Size:'" CMD_DMIDECODE_FULL = "dmidecode" + def _parse_dmi_sizes(self, stdout: str) -> str: + """Build the DIMM summary from 'Size: ' lines (dmidecode on Linux, + smbiosDump on ESXi — both emit the same field format).""" + total = 0 + topology: dict[str, int] = {} + size = "" + dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") + for num, unit in dimm_size_pattern.findall(stdout): + size = unit + total += int(num) + key = num + unit + topology[key] = topology.get(key, 0) + 1 + if total == 0: + return "0 GB" + dimm_entries = [f"{v} x {k}" for k, v in topology.items()] + return f"{total}{size} @ {' '.join(dimm_entries)}" + def collect_data( self, args: Optional[DimmCollectorArgs] = None, @@ -70,6 +89,10 @@ def collect_data( dimm_str = f"{total / 1024 / 1024:.2f}GB @ " for capacity, count in capacities.items(): dimm_str += f"{count} x {capacity / 1024 / 1024:.2f}GB " + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + dimm_str = self._parse_dmi_sizes(res.stdout) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" @@ -96,29 +119,7 @@ def collect_data( res = self._run_sut_cmd(self.CMD, sudo=True) if res.exit_code == 0: - total = 0 - topology = {} - size = "" - dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") - matches = dimm_size_pattern.findall(res.stdout) - if matches: - for match in matches: - size = match[1] - total += int(match[0]) - key = match[0] + match[1] - if not topology.get(key, None): - topology[key] = 1 - else: - topology[key] += 1 - topology["total"] = total - topology["size"] = size - total_gb = topology.pop("total") - size = topology.pop("size") - if total_gb == 0: - dimm_str = "0 GB" - else: - dimm_entries = [f"{v} x {k}" for k, v in topology.items()] - dimm_str = f"{total_gb}{size} @ {' '.join(dimm_entries)}" + dimm_str = self._parse_dmi_sizes(res.stdout) if res.exit_code != 0: self._log_event( category=EventCategory.OS, diff --git a/nodescraper/plugins/inband/kernel/kernel_collector.py b/nodescraper/plugins/inband/kernel/kernel_collector.py index 6b188940..d93fc215 100644 --- a/nodescraper/plugins/inband/kernel/kernel_collector.py +++ b/nodescraper/plugins/inband/kernel/kernel_collector.py @@ -36,6 +36,7 @@ class KernelCollector(InBandDataCollector[KernelDataModel, None]): """Read kernel version""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = KernelDataModel CMD_WINDOWS = "wmic os get Version /Value" CMD = "sh -c 'uname -a'" @@ -88,6 +89,9 @@ def collect_data( "=" )[1] else: + # Non-Windows (Linux and ESXi). ESXi `uname -a` yields the release in the + # same field the Linux parser reads (verified: "9.1.0"); numa_balancing has + # no ESXi equivalent and its command fails gracefully, leaving None. res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: kernel_info = res.stdout diff --git a/nodescraper/plugins/inband/os/os_collector.py b/nodescraper/plugins/inband/os/os_collector.py index 42e435a9..2fc46ab5 100644 --- a/nodescraper/plugins/inband/os/os_collector.py +++ b/nodescraper/plugins/inband/os/os_collector.py @@ -36,10 +36,13 @@ class OsCollector(InBandDataCollector[OsDataModel, None]): """Collect OS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = OsDataModel CMD_VERSION_WINDOWS = "wmic os get Version /value" CMD_VERSION = "cat /etc/*release | grep VERSION_ID" + CMD_VERSION_ESXI = "esxcli system version get" CMD_WINDOWS = "wmic os get Caption /Value" + CMD_ESXI = "vmware -v" PRETTY_STR = "PRETTY_NAME" # noqa: N806 CMD = f"sh -c '( lsb_release -ds || (cat /etc/*release | grep {PRETTY_STR}) || uname -om ) 2>/dev/null | head -n1'" @@ -60,6 +63,22 @@ def collect_version(self) -> str: priority=EventPriority.ERROR, ) os_version = "" + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_VERSION_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + if "Version:" in line: + os_version = line.split(":", 1)[1].strip() + break + else: + os_version = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS version not found", + priority=EventPriority.ERROR, + ) + os_version = "" else: res = self._run_sut_cmd(self.CMD_VERSION) if res.exit_code == 0: @@ -86,6 +105,16 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[OsDataModel]]: res = self._run_sut_cmd(self.CMD_WINDOWS) if res.exit_code == 0: os_name = re.search(r"Caption=([\w\s]+)", res.stdout).group(1) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + os_name = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS name not found", + priority=EventPriority.ERROR, + ) else: res = self._run_sut_cmd(self.CMD) # search for PRETTY_NAME in res diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index e5373ebc..a97aefac 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -37,9 +37,11 @@ class StorageCollector(InBandDataCollector[StorageDataModel, None]): """Collect disk usage details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = StorageDataModel CMD_WINDOWS = """wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace""" CMD = """sh -c 'df -lH -B1 | grep -v 'boot''""" + CMD_ESXI = "esxcli storage filesystem list" def collect_data( self, args: Optional[StorageCollectorArgs] = None @@ -61,6 +63,24 @@ def collect_data( used=int(size) - int(free_space), percent=round((int(size) - int(free_space)) / int(size) * 100, 2), ) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + # esxcli columns (fixed order): [0] Mount Point [1] Volume Name + # [2] UUID [3] Mounted [4] Type [5] Size [6] Free + fields = re.split(r"\s{2,}", line.strip()) + if len(fields) >= 7 and fields[5].isdigit() and fields[6].isdigit(): + device_id = fields[0] + total_bytes = int(fields[5]) + free_bytes = int(fields[6]) + used_bytes = total_bytes - free_bytes + storage_data[device_id] = DeviceStorageData( + total=total_bytes, + free=free_bytes, + used=used_bytes, + percent=round(used_bytes / total_bytes * 100, 2) if total_bytes else 0.0, + ) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" From 83bcf0680469a386f06358ac6c2e1430e2ef7f70 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 03:58:15 +0000 Subject: [PATCH 02/12] Add ESXi support to PcieCollector ESXi busybox lspci lacks per-device (-s) and bus-path (-PP) options, so dump all extended config space once via `lspci -e`, split by BDF, and select the GPU/VF BDFs resolved from `esxcli hardware pci list` by SKU device ID (system_info.devid_ep/_vf). Extract a shared _cfg_space_from_hex parser (used by the Linux per-BDF path too). Upstream-bridge traversal is skipped on ESXi (GPU + VF only). Depends on the SystemInfo devid_ep/devid_ep_vf fields. --- .../plugins/inband/pcie/pcie_collector.py | 157 ++++++++++++++++-- 1 file changed, 145 insertions(+), 12 deletions(-) diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 624122ec..7259d085 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -80,7 +80,10 @@ class PcieCollector(InBandDataCollector[PcieDataModel, None]): """ - SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX, OSFamily.ESXI} + + # A bare BDF line that begins an esxcli/lspci device block, e.g. "0000:05:00.0". + _BDF_LINE = re.compile(r"^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]+", re.IGNORECASE) DATA_MODEL = PcieDataModel @@ -518,17 +521,8 @@ def get_cap_cfg( return cap_structure # type: ignore[return-value] - def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: - """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" - hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) - if hex_data_raw is None: - self._log_event( - category=EventCategory.IO, - description="Failed to get hex data for BDF.", - data={"bdf": bdf}, - priority=EventPriority.ERROR, - ) - return PcieCfgSpace() + def _cfg_space_from_hex(self, hex_data_raw: str, bdf: str) -> PcieCfgSpace: + """Parse a raw lspci hex dump (Linux ``-xxxx`` or ESXi ``-e``) into a PcieCfgSpace.""" hex_data: List[int] = self.parse_hex_dump(hex_data_raw) if len(hex_data) < 64: # Expect at least 256 bytes of data, for the first 256 bytes of the PCIe config space @@ -542,6 +536,19 @@ def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: cap_data, ecap_data = self.discover_capability_structure(hex_data) return self.get_pcie_cfg(hex_data, cap_data, ecap_data) + def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: + """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" + hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) + if hex_data_raw is None: + self._log_event( + category=EventCategory.IO, + description="Failed to get hex data for BDF.", + data={"bdf": bdf}, + priority=EventPriority.ERROR, + ) + return PcieCfgSpace() + return self._cfg_space_from_hex(hex_data_raw, bdf) + def get_pcie_cfg( self, config_data: List[int], @@ -595,6 +602,129 @@ def _log_pcie_artifacts( if data is not None: self.result.artifacts.append(TextFileArtifact(filename=name, contents=data)) + def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: + """Return (pf_bdfs, vf_bdfs) for the GPUs on an ESXi host via esxcli. + + ESXi busybox lspci has no device filter, so GPU/VF BDFs are resolved from + ``esxcli hardware pci list`` by matching the SKU's PF/VF device IDs + (system_info.devid_ep / devid_ep_vf). Each device block starts with a bare + BDF line followed by indented fields incl. "Device ID". + """ + pf_bdfs: List[str] = [] + vf_bdfs: List[str] = [] + pf_devid = ( + format(self.system_info.devid_ep, "x") + if self.system_info.devid_ep is not None + else "" + ) + vf_devid = ( + format(self.system_info.devid_ep_vf, "x") + if self.system_info.devid_ep_vf is not None + else "" + ) + if not pf_devid and not vf_devid: + return pf_bdfs, vf_bdfs + + out = self._run_os_cmd("esxcli hardware pci list", sudo=False) + if not out: + return pf_bdfs, vf_bdfs + + current_bdf: Optional[str] = None + for line in out.splitlines(): + stripped = line.strip() + if self._BDF_LINE.match(stripped) and ":" in stripped and " " not in stripped: + # Bare BDF header line (anchors the block). + current_bdf = stripped + elif current_bdf and stripped.lower().startswith("device id:"): + devid = stripped.split(":", 1)[1].strip().lower().removeprefix("0x") + if pf_devid and devid == pf_devid: + pf_bdfs.append(current_bdf) + elif vf_devid and devid == vf_devid: + vf_bdfs.append(current_bdf) + return pf_bdfs, vf_bdfs + + def _get_all_cfg_space_esxi(self) -> Dict[str, str]: + """Return {bdf: hex_dump_text} for every device from a single ``lspci -e``. + + ESXi has no per-device dump; ``lspci -e`` emits the full extended (4096-byte) + config space for all devices in one blob. Each device section starts with a + header line " " followed by "NN: .." hex lines. + """ + blob = self._run_os_cmd("lspci -e", sudo=False) + if not blob: + return {} + self.result.artifacts.append(TextFileArtifact(filename="lspci_e.txt", contents=blob)) + sections: Dict[str, List[str]] = {} + current_bdf: Optional[str] = None + for line in blob.splitlines(): + header = self._BDF_LINE.match(line) + if header and " " in line: + # Device header line: " ". + current_bdf = line.split(" ", 1)[0] + sections[current_bdf] = [] + elif current_bdf is not None: + sections[current_bdf].append(line) + return {bdf: "\n".join(lines) for bdf, lines in sections.items()} + + def _get_pcie_data_esxi(self) -> Optional[PcieDataModel]: + """Collect GPU + VF PCIe config space on ESXi. + + ESXi busybox lspci lacks ``-s`` (per-device) and ``-PP`` (bus-path), so dump + all extended config space once via ``lspci -e``, split it by BDF, and select the + GPU/VF BDFs resolved from esxcli. Upstream-bridge traversal is not available on + ESXi and is intentionally skipped (GPU + VF only). + """ + pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi() + if not pf_bdfs and not vf_bdfs: + self._log_event( + category=EventCategory.IO, + description="No GPU/VF BDFs found on ESXi host for this SKU.", + data={ + "devid_ep": self.system_info.devid_ep, + "devid_ep_vf": self.system_info.devid_ep_vf, + }, + priority=EventPriority.WARNING, + ) + return None + + cfg_by_bdf = self._get_all_cfg_space_esxi() + if not cfg_by_bdf: + self.result.status = ExecutionStatus.ERROR + return None + + self._log_event( + category=EventCategory.IO, + description=( + "Upstream-bridge PCIe collection is not supported on ESXi; " + "collecting GPU + VF only." + ), + priority=EventPriority.INFO, + ) + + try: + pcie_cfg_dict: Dict[str, PcieCfgSpace] = {} + for bdf in pf_bdfs: + if bdf in cfg_by_bdf: + pcie_cfg_dict[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + vf_pcie_cfg_data: Dict[str, PcieCfgSpace] = {} + for bdf in vf_bdfs: + if bdf in cfg_by_bdf: + vf_pcie_cfg_data[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + pcie_data = PcieDataModel( + pcie_cfg_space=pcie_cfg_dict, + vf_pcie_cfg_space=vf_pcie_cfg_data, + ) + except ValidationError as e: + self._log_event( + category=EventCategory.OS, + description="Failed to build model for PCIe data", + data=get_exception_details(e), + priority=EventPriority.ERROR, + ) + self.result.status = ExecutionStatus.ERROR + return None + return pcie_data + def _get_pcie_data( self, upstream_steps_to_collect: Optional[int] = None ) -> Optional[PcieDataModel]: @@ -605,6 +735,9 @@ def _get_pcie_data( Optional[PcieDataModel] The data in a PcieDataModel object or None on failure """ + if self.system_info.os_family == OSFamily.ESXI: + return self._get_pcie_data_esxi() + minimum_system_interaction_level_required_for_sudo = SystemInteractionLevel.INTERACTIVE try: From 5bd26fe3dc0d0fb968dae83da6c9eab2609c55ee Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:08:00 +0000 Subject: [PATCH 03/12] Add ESXi support to DmesgCollector ESXi has no dmesg ring buffer; read the kernel log from /var/log/vmkernel.log (and vmkernel.[.gz] rotations) instead of `dmesg`. Validated on ESXi: reads vmkernel.log; Linux path unchanged. --- .../plugins/inband/dmesg/dmesg_collector.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index c280d7d2..0d2894c6 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -38,24 +38,33 @@ class DmesgCollector(InBandDataCollector[DmesgData, DmesgCollectorArgs]): """Read dmesg log""" - SUPPORTED_OS_FAMILY = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY = {OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DmesgData CMD = "dmesg --time-format iso -x" + # ESXi has no dmesg ring buffer; the kernel log is the vmkernel.log file. + CMD_ESXI = "cat /var/log/vmkernel.log" CMD_LOGS = ( r"ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true" ) + # ESXi rotates vmkernel.log to vmkernel. / vmkernel..gz. + CMD_LOGS_ESXI = ( + r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" + ) def _collect_dmesg_rotations(self): - """Collect dmesg logs""" - list_res = self._run_sut_cmd(self.CMD_LOGS, sudo=True) + """Collect dmesg (Linux) / vmkernel.log (ESXi) rotated logs""" + is_esxi = self.system_info.os_family == OSFamily.ESXI + log_label = "vmkernel" if is_esxi else "dmesg" + cmd_logs = self.CMD_LOGS_ESXI if is_esxi else self.CMD_LOGS + list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: self._log_event( category=EventCategory.OS, - description="No /var/log/dmesg files found (including rotations).", + description=f"No rotated {log_label} log files found.", data={"list_exit_code": list_res.exit_code}, priority=EventPriority.WARNING, ) @@ -68,7 +77,7 @@ def _collect_dmesg_rotations(self): cmd = f"gzip -dc {qp} 2>/dev/null || zcat {qp} 2>/dev/null" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -84,7 +93,7 @@ def _collect_dmesg_rotations(self): cmd = f"cat {qp}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -121,8 +130,10 @@ def _get_dmesg_content(self) -> str: str: dmesg output """ - self.logger.info("Running dmesg command on system") - res = self._run_sut_cmd(self.CMD, sudo=True, log_artifact=False) + is_esxi = self.system_info.os_family == OSFamily.ESXI + cmd = self.CMD_ESXI if is_esxi else self.CMD + self.logger.info("Reading kernel log from system") + res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code != 0: self._log_event( category=EventCategory.OS, From fb75f5b30b616fb7deff79966f8bad1df47dd0e8 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:25:22 +0000 Subject: [PATCH 04/12] Add ESXi support to DmesgAnalyzer Make the dmesg analyzer format-aware so it handles ESXi vmkernel.log as well as Linux dmesg: - Extract ESXi ISO8601 dot-ms/Z timestamps (e.g. 2026-08-20T09:35:58.380Z) via ESXI_TIMESTAMP_PATTERN, set on __init__ so event grouping and date-range filtering both use it; Linux keeps the base comma-form pattern. - filter_dmesg is now an instance method and reuses the base timestamp extractor, so a single code path honors whichever pattern is active. - Add ESXi mxGPU (gim/amdgpuv) RAS ERROR_REGEX entries (Block-capitalized correctable/uncorrectable, ECC Fatal Error, Whole GPU reset); these are inert on Linux logs. - Unknown-error detection keys off the driver-internal severity in the message body ("gim/amdgpuv error/warn") on ESXi, where the vmkernel -ALERT/-INFO tokens are unreliable; Linux keeps the "kern :err:" form. Validated on real ESXi vmkernel.log (7.2 MB) and Linux dmesg (no regression), plus synthetic RAS lines confirming per-OS phrasing is discriminated correctly. --- .../plugins/inband/dmesg/dmesg_analyzer.py | 79 ++++++++++++++++--- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 5ae53f77..40e8cdef 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -30,7 +30,7 @@ from nodescraper.base.match_ignore import parse_ignore_match_rules from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer from nodescraper.connection.inband import TextFileArtifact -from nodescraper.enums import EventCategory, EventPriority +from nodescraper.enums import EventCategory, EventPriority, OSFamily from nodescraper.models import Event, TaskResult from .analyzer_args import DmesgAnalyzerArgs @@ -47,10 +47,26 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): - """Check dmesg for errors""" + """Check dmesg (Linux) or vmkernel.log (ESXi) for errors""" DATA_MODEL = DmesgData + # ESXi vmkernel.log timestamp, e.g. "2026-08-05T19:53:35.178Z" (ISO8601 dot-ms + Z). + # Linux uses the base RegexAnalyzer.TIMESTAMP_PATTERN (comma-form). + ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile( + r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" + ) + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # On ESXi, extract vmkernel.log timestamps so event grouping and date-range + # filtering both work; Linux keeps the base comma-form pattern. + if self._is_esxi(): + self.TIMESTAMP_PATTERN = self.ESXI_TIMESTAMP_PATTERN + + def _is_esxi(self) -> bool: + return self.system_info.os_family == OSFamily.ESXI + ERROR_REGEX: list[ErrorRegex] = [ ErrorRegex( regex=re.compile(r"(?:oom_kill_process.*)|(?:Out of memory.*)"), @@ -273,6 +289,30 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): message="RAS Deferred Error", event_category=EventCategory.RAS, ), + # ESXi mxGPU (gim/amdgpuv) RAS phrasing differs from Linux: the block name is + # capitalized ("... detected in MMHUB Block."), there is no "in total", and no + # "kern :err:" prefix. These match the ESXi host-driver forms and are inert on + # Linux logs (which use the lowercase "in total in block" phrasing above). + ErrorRegex( + regex=re.compile(r"(\d+ new uncorrectable hardware errors detected in \w+ Block.*)"), + message="RAS Uncorrectable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(\d+ new correctable hardware errors detected in \w+ Block.*)"), + message="RAS Correctable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(GPU detected ECC Fatal Error\.)"), + message="RAS ECC Fatal Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(Issuing Whole GPU reset\.)"), + message="GPU Reset", + event_category=EventCategory.RAS, + ), ErrorRegex( regex=re.compile( r"((?:\[Hardware Error\]:\s+)?event severity: corrected.*)" @@ -463,14 +503,13 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): ), ] - @classmethod def filter_dmesg( - cls, + self, dmesg_content: str, analysis_range_start: Optional[datetime.datetime] = None, analysis_range_end: Optional[datetime.datetime] = None, ) -> str: - """Filter a dmesg log by date + """Filter a dmesg (Linux) or vmkernel.log (ESXi) log by date Args: dmesg_content (str): unfiltered dmesg log @@ -482,9 +521,16 @@ def filter_dmesg( filtered_dmesg = "" found_start = False if analysis_range_start else True for line in dmesg_content.splitlines(): - date = re.search(r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)", line) - if date is not None: - date = datetime.datetime.fromisoformat(f"{date.group(1)}.{date.group(2)}") + # Reuse the base extractor so the active TIMESTAMP_PATTERN (ESXi dot-Z form + # when on ESXi, else Linux comma-form) is honored in exactly one place. + date_str = self._extract_timestamp_from_match_position(line, 0) + if date_str is not None: + # Linux uses a comma before fractional seconds; normalize to "." so + # fromisoformat() accepts it (no-op for the ESXi "...Z" form). + try: + date = datetime.datetime.fromisoformat(date_str.replace(",", ".")) + except ValueError: + continue # show date in UTC now date = date.astimezone(datetime.timezone.utc) if analysis_range_start and not found_start and date >= analysis_range_start: @@ -743,11 +789,22 @@ def analyze_data( self.result.events += known_err_events if args.check_unknown_dmesg_errors: + if self._is_esxi(): + # ESXi vmkernel severity tokens are unreliable (-ALERT is used for benign + # boot notices; -ERROR/-CRIT are never emitted). The reliable error signal + # is the driver-internal severity in the message body: "gim error/warning", + # "amdgpuv error/warning", or the bracket form "[amdgpuv warn]". + unknown_error_regex = re.compile( + r"(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning) [^:]*:\s*(.*)" + r"|\[(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning)\]:?\s*(.*)" + ) + else: + unknown_error_regex = re.compile( + r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" + ) unknown_dmesg_error_regexes = [ ErrorRegex( - regex=re.compile( - r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" - ), + regex=unknown_error_regex, message="Unknown dmesg error", event_category=EventCategory.UNKNOWN, event_priority=EventPriority.WARNING, From 3416c9eb54383670dd22b896a15222a99b481ff0 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:34:55 +0000 Subject: [PATCH 05/12] Fix dmesg ESXi unit-test regressions - filter_dmesg: keep it a classmethod (public API used as DmesgAnalyzer.filter_dmesg(content, ...) in tests). Recognize both Linux comma-form and ESXi dot-ms/Z timestamps via a combined pattern instead of the instance TIMESTAMP_PATTERN; normalize the trailing Z so fromisoformat accepts it on Python < 3.11. - Collector: restore the exact Linux "No /var/log/dmesg files found (including rotations)." wording for the no-rotations event and add an ESXi-specific vmkernel.log variant, rather than a generic reword. Restores test_dmesg_filter and test_collect_rotations_no_files; full dmesg collector+analyzer suite (57 tests) green. --- .../plugins/inband/dmesg/dmesg_analyzer.py | 32 ++++++++++++------- .../plugins/inband/dmesg/dmesg_collector.py | 6 +++- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 40e8cdef..56bf6934 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -503,8 +503,18 @@ def _is_esxi(self) -> bool: ), ] + # Date-range filtering must recognize both Linux dmesg comma-form timestamps + # (2024-10-01T05:00:00,000000-05:00) and ESXi vmkernel.log dot-ms/Z timestamps + # (2026-08-20T09:35:58.380Z). filter_dmesg stays a classmethod (public API), so it + # carries its own combined pattern rather than the instance TIMESTAMP_PATTERN. + _FILTER_TIMESTAMP_PATTERN: re.Pattern = re.compile( + r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)" + r"|(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" + ) + + @classmethod def filter_dmesg( - self, + cls, dmesg_content: str, analysis_range_start: Optional[datetime.datetime] = None, analysis_range_end: Optional[datetime.datetime] = None, @@ -521,16 +531,16 @@ def filter_dmesg( filtered_dmesg = "" found_start = False if analysis_range_start else True for line in dmesg_content.splitlines(): - # Reuse the base extractor so the active TIMESTAMP_PATTERN (ESXi dot-Z form - # when on ESXi, else Linux comma-form) is honored in exactly one place. - date_str = self._extract_timestamp_from_match_position(line, 0) - if date_str is not None: - # Linux uses a comma before fractional seconds; normalize to "." so - # fromisoformat() accepts it (no-op for the ESXi "...Z" form). - try: - date = datetime.datetime.fromisoformat(date_str.replace(",", ".")) - except ValueError: - continue + match = cls._FILTER_TIMESTAMP_PATTERN.search(line) + if match is not None: + if match.group(1) is not None: + # Linux comma-form: swap the comma for a dot so fromisoformat accepts it + iso = f"{match.group(1)}.{match.group(2)}" + else: + # ESXi dot-Z form: normalize the trailing Z so fromisoformat accepts it + # on Python < 3.11 as well + iso = match.group(3).replace("Z", "+00:00") + date = datetime.datetime.fromisoformat(iso) # show date in UTC now date = date.astimezone(datetime.timezone.utc) if analysis_range_start and not found_start and date >= analysis_range_start: diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index 0d2894c6..4c20420a 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -62,9 +62,13 @@ def _collect_dmesg_rotations(self): list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: + if is_esxi: + description = "No /var/log/vmkernel.log files found (including rotations)." + else: + description = "No /var/log/dmesg files found (including rotations)." self._log_event( category=EventCategory.OS, - description=f"No rotated {log_label} log files found.", + description=description, data={"list_exit_code": list_res.exit_code}, priority=EventPriority.WARNING, ) From ed0bdec8adf1b7f74834a62aff4e1cb9b35a6ce1 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:42:58 +0000 Subject: [PATCH 06/12] Satisfy black/ruff pre-commit on ESXi collectors Pre-commit black (line-length 100) flagged formatting in the ESXi branches. Expand the branch-selection ternaries to explicit if/else (pcie devid resolve, storage percent, dmesg cmd/log-label selection) and let black normalize the two long single-line constants (dmesg CMD_LOGS_ESXI, analyzer ESXI_TIMESTAMP_PATTERN). No behavior change; black --check and ruff clean, dmesg+storage suites green. --- .../plugins/inband/dmesg/dmesg_analyzer.py | 4 +--- .../plugins/inband/dmesg/dmesg_collector.py | 17 +++++++++++------ .../plugins/inband/pcie/pcie_collector.py | 18 ++++++++---------- .../inband/storage/storage_collector.py | 6 +++++- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 56bf6934..5bf93c36 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -53,9 +53,7 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): # ESXi vmkernel.log timestamp, e.g. "2026-08-05T19:53:35.178Z" (ISO8601 dot-ms + Z). # Linux uses the base RegexAnalyzer.TIMESTAMP_PATTERN (comma-form). - ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile( - r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" - ) + ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)") def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index 4c20420a..4fcbc4b7 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -50,15 +50,17 @@ class DmesgCollector(InBandDataCollector[DmesgData, DmesgCollectorArgs]): r"ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true" ) # ESXi rotates vmkernel.log to vmkernel. / vmkernel..gz. - CMD_LOGS_ESXI = ( - r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" - ) + CMD_LOGS_ESXI = r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" def _collect_dmesg_rotations(self): """Collect dmesg (Linux) / vmkernel.log (ESXi) rotated logs""" is_esxi = self.system_info.os_family == OSFamily.ESXI - log_label = "vmkernel" if is_esxi else "dmesg" - cmd_logs = self.CMD_LOGS_ESXI if is_esxi else self.CMD_LOGS + if is_esxi: + log_label = "vmkernel" + cmd_logs = self.CMD_LOGS_ESXI + else: + log_label = "dmesg" + cmd_logs = self.CMD_LOGS list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: @@ -135,7 +137,10 @@ def _get_dmesg_content(self) -> str: """ is_esxi = self.system_info.os_family == OSFamily.ESXI - cmd = self.CMD_ESXI if is_esxi else self.CMD + if is_esxi: + cmd = self.CMD_ESXI + else: + cmd = self.CMD self.logger.info("Reading kernel log from system") res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code != 0: diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 7259d085..2f46c19f 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -612,16 +612,14 @@ def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: """ pf_bdfs: List[str] = [] vf_bdfs: List[str] = [] - pf_devid = ( - format(self.system_info.devid_ep, "x") - if self.system_info.devid_ep is not None - else "" - ) - vf_devid = ( - format(self.system_info.devid_ep_vf, "x") - if self.system_info.devid_ep_vf is not None - else "" - ) + if self.system_info.devid_ep is not None: + pf_devid = format(self.system_info.devid_ep, "x") + else: + pf_devid = "" + if self.system_info.devid_ep_vf is not None: + vf_devid = format(self.system_info.devid_ep_vf, "x") + else: + vf_devid = "" if not pf_devid and not vf_devid: return pf_bdfs, vf_bdfs diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index a97aefac..bb9d4c32 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -75,11 +75,15 @@ def collect_data( total_bytes = int(fields[5]) free_bytes = int(fields[6]) used_bytes = total_bytes - free_bytes + if total_bytes: + percent = round(used_bytes / total_bytes * 100, 2) + else: + percent = 0.0 storage_data[device_id] = DeviceStorageData( total=total_bytes, free=free_bytes, used=used_bytes, - percent=round(used_bytes / total_bytes * 100, 2) if total_bytes else 0.0, + percent=percent, ) else: if args.skip_sudo: From 6e9d3d26be0b3eebe450b3a86ab718e53df9bbac Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:49:48 +0000 Subject: [PATCH 07/12] Fix mypy name collision in storage ESXi branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hoisting the percent computation reused the name "percent", which the Linux branch later binds to a str from split() before re.sub()/float() — mypy flagged the float-vs-str conflict. Rename the ESXi-branch value to usage_percent. --- nodescraper/plugins/inband/storage/storage_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index bb9d4c32..7b096b52 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -76,14 +76,14 @@ def collect_data( free_bytes = int(fields[6]) used_bytes = total_bytes - free_bytes if total_bytes: - percent = round(used_bytes / total_bytes * 100, 2) + usage_percent = round(used_bytes / total_bytes * 100, 2) else: - percent = 0.0 + usage_percent = 0.0 storage_data[device_id] = DeviceStorageData( total=total_bytes, free=free_bytes, used=used_bytes, - percent=percent, + percent=usage_percent, ) else: if args.skip_sudo: From 545661e195d62a3500879c51f59b94a9cb84ed31 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Mon, 14 Sep 2026 18:26:24 +0000 Subject: [PATCH 08/12] Address review: robust device-ID matching + safe count parsing (ESXi) Per review feedback on the ESXi device-ID handling: - pcie: compare the esxcli "Device ID" to the expected PF/VF id by integer value instead of an exact lowercase-string match, so uppercase ("0x744C") and zero-padded ("0x0000744c") ids are matched. - device_enumeration: make the PCI-count grep case-insensitive and zero-pad tolerant ("0x0*"), with a trailing [^0-9a-f]/$ guard so a shorter id does not match a longer one (744c vs 744cd). - device_enumeration: parse the CPU/GPU/VF counts defensively (guard non-zero exit and non-numeric stdout) instead of int()-ing command output directly, so an unexpected esxcli/awk result warns rather than raising. Validated on ESXi 9.1 (8 GPUs matched; counts parsed) and Linux (no regression); padded/uppercase ids confirmed against busybox grep and the int compare. --- .../device_enumeration_collector.py | 59 +++++++++++++------ .../plugins/inband/pcie/pcie_collector.py | 24 ++++---- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 9f579672..48853473 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -57,10 +57,15 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, ) # ESXi busybox `lspci -d` dumps hex instead of filtering, so use esxcli. GPUs are - # counted by exact device ID (PF vs VF), anchored on "Device ID:" to avoid also - # matching "SubDevice ID:". + # counted by device ID (PF vs VF), anchored on "Device ID:" to avoid also matching + # "SubDevice ID:". The match is case-insensitive and tolerates zero-padding + # ("0x744C" / "0x0000744c"); the trailing [^0-9a-f]/$ guard stops a shorter ID from + # matching a longer one (e.g. 744c vs 744cd). CMD_CPU_COUNT_ESXI = "esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'" - CMD_PCI_COUNT_ESXI = "esxcli hardware pci list | grep -E '^ *Device ID: 0x{device_id}' | wc -l" + CMD_PCI_COUNT_ESXI = ( + "esxcli hardware pci list | " + "grep -iE '^ *Device ID: 0x0*{device_id}([^0-9a-f]|$)' | wc -l" + ) def _warning( self, @@ -79,6 +84,27 @@ def _warning( priority=EventPriority.WARNING, ) + def _parse_count( + self, + res: CommandArtifact, + description: str, + category: EventCategory = EventCategory.PLATFORM, + ) -> Optional[int]: + """Parse a numeric count from command stdout, warning (not raising) on a + non-zero exit or non-numeric output (e.g. an unexpected esxcli/awk result).""" + if res.exit_code != 0: + self._warning(description=description, command=res, category=category) + return None + text = (res.stdout or "").strip() + if not text.isdigit(): + self._warning( + description=f"{description} (non-numeric output: {text!r})", + command=res, + category=category, + ) + return None + return int(text) + def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: """Count PCI devices on ESXi whose Device ID matches ``device_id`` (as hex). @@ -149,24 +175,19 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio else: self._warning(description="Cannot collect lscpu output", command=lscpu_res) else: - if cpu_count_res.exit_code == 0: - device_enum.cpu_count = int(cpu_count_res.stdout) - else: - self._warning(description="Cannot determine CPU count", command=cpu_count_res) + cpu_count = self._parse_count(cpu_count_res, "Cannot determine CPU count") + if cpu_count is not None: + device_enum.cpu_count = cpu_count - if gpu_count_res.exit_code == 0: - device_enum.gpu_count = int(gpu_count_res.stdout) - else: - self._warning(description="Cannot determine GPU count", command=gpu_count_res) + gpu_count = self._parse_count(gpu_count_res, "Cannot determine GPU count") + if gpu_count is not None: + device_enum.gpu_count = gpu_count - if vf_count_res.exit_code == 0: - device_enum.vf_count = int(vf_count_res.stdout) - else: - self._warning( - description="Cannot determine VF count", - command=vf_count_res, - category=EventCategory.SW_DRIVER, - ) + vf_count = self._parse_count( + vf_count_res, "Cannot determine VF count", category=EventCategory.SW_DRIVER + ) + if vf_count is not None: + device_enum.vf_count = vf_count # Collect lshw output on Linux if self.system_info.os_family == OSFamily.LINUX: diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 2f46c19f..f690aa55 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -612,15 +612,9 @@ def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: """ pf_bdfs: List[str] = [] vf_bdfs: List[str] = [] - if self.system_info.devid_ep is not None: - pf_devid = format(self.system_info.devid_ep, "x") - else: - pf_devid = "" - if self.system_info.devid_ep_vf is not None: - vf_devid = format(self.system_info.devid_ep_vf, "x") - else: - vf_devid = "" - if not pf_devid and not vf_devid: + pf_devid = self.system_info.devid_ep + vf_devid = self.system_info.devid_ep_vf + if pf_devid is None and vf_devid is None: return pf_bdfs, vf_bdfs out = self._run_os_cmd("esxcli hardware pci list", sudo=False) @@ -634,10 +628,16 @@ def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: # Bare BDF header line (anchors the block). current_bdf = stripped elif current_bdf and stripped.lower().startswith("device id:"): - devid = stripped.split(":", 1)[1].strip().lower().removeprefix("0x") - if pf_devid and devid == pf_devid: + # Compare by integer value so case ("0x744C") and zero-padding + # ("0x0000744c") both match the expected device ID. + raw = stripped.split(":", 1)[1].strip() + try: + devid = int(raw, 16) + except ValueError: + continue + if pf_devid is not None and devid == pf_devid: pf_bdfs.append(current_bdf) - elif vf_devid and devid == vf_devid: + elif vf_devid is not None and devid == vf_devid: vf_bdfs.append(current_bdf) return pf_bdfs, vf_bdfs From 053b734ee24dde63abd6038b139fe05682bd6517 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Tue, 15 Sep 2026 03:24:56 +0000 Subject: [PATCH 09/12] Address review: move devid_ep/devid_ep_vf to collector args The expected GPU PF/VF PCI device IDs were SystemInfo fields that nothing populated upstream, so the ESXi GPU/VF resolution in device_enumeration and pcie was always inert. Move them to per-collector args (DeviceEnumerationCollectorArgs / PcieCollectorArgs), user-populated, matching how amd-smi takes them via args; read from args instead of SystemInfo and drop the unused SystemInfo fields. Validated on ESXi (8 GPU PF BDFs / gpu_count 8 via args) and Linux (no regression). --- nodescraper/models/systeminfo.py | 2 - .../device_enumeration/collector_args.py | 23 +++++++++ .../device_enumeration_collector.py | 19 ++++--- .../device_enumeration_plugin.py | 9 +++- .../plugins/inband/pcie/collector_args.py | 24 +++++++++ .../plugins/inband/pcie/pcie_collector.py | 49 +++++++++++-------- .../plugins/inband/pcie/pcie_plugin.py | 5 +- 7 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 nodescraper/plugins/inband/device_enumeration/collector_args.py create mode 100644 nodescraper/plugins/inband/pcie/collector_args.py diff --git a/nodescraper/models/systeminfo.py b/nodescraper/models/systeminfo.py index d593a9a0..d91a68cf 100644 --- a/nodescraper/models/systeminfo.py +++ b/nodescraper/models/systeminfo.py @@ -44,5 +44,3 @@ class SystemInfo(BaseModel): metadata: Optional[dict] = Field(default_factory=dict) location: Optional[SystemLocation] = SystemLocation.LOCAL vendorid_ep: int = 0x1002 - devid_ep: Optional[int] = None - devid_ep_vf: Optional[int] = None diff --git a/nodescraper/plugins/inband/device_enumeration/collector_args.py b/nodescraper/plugins/inband/device_enumeration/collector_args.py new file mode 100644 index 00000000..c2d7b35c --- /dev/null +++ b/nodescraper/plugins/inband/device_enumeration/collector_args.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class DeviceEnumerationCollectorArgs(CollectorArgs): + """Collector args for device enumeration. + + On ESXi, GPUs and their SR-IOV VFs are counted by PCI device ID (esxcli has no + device filter). Provide the expected PF/VF device IDs here; when unset the ESXi + GPU/VF counts are skipped. The caller populates these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi device counting.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF counting.", + ) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 48853473..78c54ad0 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -30,10 +30,13 @@ from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult +from .collector_args import DeviceEnumerationCollectorArgs from .deviceenumdata import DeviceEnumerationDataModel -class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, None]): +class DeviceEnumerationCollector( + InBandDataCollector[DeviceEnumerationDataModel, DeviceEnumerationCollectorArgs] +): """Collect CPU and GPU count""" SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} @@ -114,13 +117,17 @@ def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: hex_id = format(device_id, "x") if device_id is not None else "__unset__" return self._run_sut_cmd(self.CMD_PCI_COUNT_ESXI.format(device_id=hex_id)) - def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: + def collect_data( + self, args: Optional[DeviceEnumerationCollectorArgs] = None + ) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: """ Read CPU and GPU count On Linux, use lscpu and lspci - On ESXi, use esxcli + On ESXi, use esxcli (GPU/VF counts need args.devid_ep / devid_ep_vf) On Windows, use WMI and hyper-v cmdlets """ + if args is None: + args = DeviceEnumerationCollectorArgs() if self.system_info.os_family == OSFamily.LINUX: lscpu_res = self._run_sut_cmd(self.CMD_LSCPU_LINUX, log_artifact=False) @@ -137,15 +144,15 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio lshw_res = self._run_sut_cmd(self.CMD_LSHW_LINUX, sudo=True, log_artifact=False) elif self.system_info.os_family == OSFamily.ESXI: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_ESXI) - if self.system_info.devid_ep is None: + if args.devid_ep is None: self._log_event( category=EventCategory.PLATFORM, description="devid_ep not set; cannot count GPUs/VFs on ESXi by device ID", priority=EventPriority.WARNING, ) # PFs and (SR-IOV) VFs are distinguished by device ID on ESXi. - gpu_count_res = self._esxi_device_count(self.system_info.devid_ep) - vf_count_res = self._esxi_device_count(self.system_info.devid_ep_vf) + gpu_count_res = self._esxi_device_count(args.devid_ep) + vf_count_res = self._esxi_device_count(args.devid_ep_vf) else: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_WINDOWS) gpu_count_res = self._run_sut_cmd(self.CMD_GPU_COUNT_WINDOWS) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py index baf2aa2d..cff51210 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py @@ -26,13 +26,18 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import DeviceEnumerationAnalyzerArgs +from .collector_args import DeviceEnumerationCollectorArgs from .device_enumeration_analyzer import DeviceEnumerationAnalyzer from .device_enumeration_collector import DeviceEnumerationCollector from .deviceenumdata import DeviceEnumerationDataModel class DeviceEnumerationPlugin( - InBandDataPlugin[DeviceEnumerationDataModel, None, DeviceEnumerationAnalyzerArgs] + InBandDataPlugin[ + DeviceEnumerationDataModel, + DeviceEnumerationCollectorArgs, + DeviceEnumerationAnalyzerArgs, + ] ): """Plugin for collection and analysis of BIOS data""" @@ -40,6 +45,8 @@ class DeviceEnumerationPlugin( COLLECTOR = DeviceEnumerationCollector + COLLECTOR_ARGS = DeviceEnumerationCollectorArgs + ANALYZER = DeviceEnumerationAnalyzer ANALYZER_ARGS = DeviceEnumerationAnalyzerArgs diff --git a/nodescraper/plugins/inband/pcie/collector_args.py b/nodescraper/plugins/inband/pcie/collector_args.py new file mode 100644 index 00000000..21260fe2 --- /dev/null +++ b/nodescraper/plugins/inband/pcie/collector_args.py @@ -0,0 +1,24 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class PcieCollectorArgs(CollectorArgs): + """Collector args for PCIe data. + + On ESXi, GPU/VF BDFs are resolved from ``esxcli hardware pci list`` by matching + the expected PF/VF PCI device IDs (esxcli has no device filter). Provide them + here; when both are unset no ESXi GPU BDFs are resolved. The caller populates + these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi BDF resolution.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF BDF resolution.", + ) diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index f690aa55..1085bf2a 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -41,6 +41,7 @@ from nodescraper.models import TaskResult from nodescraper.utils import get_all_subclasses, get_exception_details +from .collector_args import PcieCollectorArgs from .pcie_data import ( MAX_CAP_ID, MAX_ECAP_ID, @@ -54,7 +55,7 @@ ) -class PcieCollector(InBandDataCollector[PcieDataModel, None]): +class PcieCollector(InBandDataCollector[PcieDataModel, PcieCollectorArgs]): """class for collection of PCIe data only supports Linux OS type. This class collects the PCIE config space using the lspci hex dump and then parses the hex dump to get the @@ -602,18 +603,18 @@ def _log_pcie_artifacts( if data is not None: self.result.artifacts.append(TextFileArtifact(filename=name, contents=data)) - def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: + def _get_gpu_vf_bdfs_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Tuple[List[str], List[str]]: """Return (pf_bdfs, vf_bdfs) for the GPUs on an ESXi host via esxcli. ESXi busybox lspci has no device filter, so GPU/VF BDFs are resolved from - ``esxcli hardware pci list`` by matching the SKU's PF/VF device IDs - (system_info.devid_ep / devid_ep_vf). Each device block starts with a bare - BDF line followed by indented fields incl. "Device ID". + ``esxcli hardware pci list`` by matching the expected PF/VF device IDs + (``pf_devid`` / ``vf_devid``, from the collector args). Each device block + starts with a bare BDF line followed by indented fields incl. "Device ID". """ pf_bdfs: List[str] = [] vf_bdfs: List[str] = [] - pf_devid = self.system_info.devid_ep - vf_devid = self.system_info.devid_ep_vf if pf_devid is None and vf_devid is None: return pf_bdfs, vf_bdfs @@ -664,23 +665,23 @@ def _get_all_cfg_space_esxi(self) -> Dict[str, str]: sections[current_bdf].append(line) return {bdf: "\n".join(lines) for bdf, lines in sections.items()} - def _get_pcie_data_esxi(self) -> Optional[PcieDataModel]: + def _get_pcie_data_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Optional[PcieDataModel]: """Collect GPU + VF PCIe config space on ESXi. ESXi busybox lspci lacks ``-s`` (per-device) and ``-PP`` (bus-path), so dump all extended config space once via ``lspci -e``, split it by BDF, and select the - GPU/VF BDFs resolved from esxcli. Upstream-bridge traversal is not available on - ESXi and is intentionally skipped (GPU + VF only). + GPU/VF BDFs resolved from esxcli (matching ``pf_devid`` / ``vf_devid`` from the + collector args). Upstream-bridge traversal is not available on ESXi and is + intentionally skipped (GPU + VF only). """ - pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi() + pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi(pf_devid, vf_devid) if not pf_bdfs and not vf_bdfs: self._log_event( category=EventCategory.IO, description="No GPU/VF BDFs found on ESXi host for this SKU.", - data={ - "devid_ep": self.system_info.devid_ep, - "devid_ep_vf": self.system_info.devid_ep_vf, - }, + data={"devid_ep": pf_devid, "devid_ep_vf": vf_devid}, priority=EventPriority.WARNING, ) return None @@ -724,7 +725,10 @@ def _get_pcie_data_esxi(self) -> Optional[PcieDataModel]: return pcie_data def _get_pcie_data( - self, upstream_steps_to_collect: Optional[int] = None + self, + upstream_steps_to_collect: Optional[int] = None, + pf_devid: Optional[int] = None, + vf_devid: Optional[int] = None, ) -> Optional[PcieDataModel]: """Will return all PCIe data in a PcieDataModel object. @@ -734,7 +738,7 @@ def _get_pcie_data( The data in a PcieDataModel object or None on failure """ if self.system_info.os_family == OSFamily.ESXI: - return self._get_pcie_data_esxi() + return self._get_pcie_data_esxi(pf_devid, vf_devid) minimum_system_interaction_level_required_for_sudo = SystemInteractionLevel.INTERACTIVE @@ -833,19 +837,24 @@ def discover_capability_structure( return cap, ecap def collect_data( - self, args=None, upstream_steps_to_collect: Optional[int] = None, **kwargs + self, + args: Optional[PcieCollectorArgs] = None, + upstream_steps_to_collect: Optional[int] = None, + **kwargs, ) -> Tuple[TaskResult, Optional[PcieDataModel]]: """Read PCIe data. Args: - args: Optional collector arguments (not used) + args: Optional collector arguments (devid_ep / devid_ep_vf for ESXi GPU BDF resolution) upstream_steps_to_collect: Number of upstream devices to collect **kwargs: Additional keyword arguments Returns: Tuple[TaskResult, Optional[PcieDataModel]]: tuple containing the result of the task and the PCIe data if available """ - pcie_data = self._get_pcie_data(upstream_steps_to_collect) + if args is None: + args = PcieCollectorArgs() + pcie_data = self._get_pcie_data(upstream_steps_to_collect, args.devid_ep, args.devid_ep_vf) if pcie_data: self._log_event( category=EventCategory.IO, diff --git a/nodescraper/plugins/inband/pcie/pcie_plugin.py b/nodescraper/plugins/inband/pcie/pcie_plugin.py index 0e4f3eb0..9d894ade 100644 --- a/nodescraper/plugins/inband/pcie/pcie_plugin.py +++ b/nodescraper/plugins/inband/pcie/pcie_plugin.py @@ -26,18 +26,21 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import PcieAnalyzerArgs +from .collector_args import PcieCollectorArgs from .pcie_analyzer import PcieAnalyzer from .pcie_collector import PcieCollector from .pcie_data import PcieDataModel -class PciePlugin(InBandDataPlugin[PcieDataModel, None, PcieAnalyzerArgs]): +class PciePlugin(InBandDataPlugin[PcieDataModel, PcieCollectorArgs, PcieAnalyzerArgs]): """Plugin for collection and analysis of PCIe data""" DATA_MODEL = PcieDataModel COLLECTOR = PcieCollector + COLLECTOR_ARGS = PcieCollectorArgs + ANALYZER = PcieAnalyzer ANALYZER_ARGS = PcieAnalyzerArgs From 3ef99285d0b63f0469659e650e7d1f596bd8dab5 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Thu, 17 Sep 2026 20:17:51 +0000 Subject: [PATCH 10/12] Address review: unit tests for the ESXi collector/analyzer additions Cover the ESXi functionality added across the platform collectors: - pcie: _get_gpu_vf_bdfs_esxi (device-ID match incl. case/zero-padding, both- unset short-circuit, unparseable id), _get_all_cfg_space_esxi (lspci -e split), _get_pcie_data_esxi (no-BDF warn / empty-cfg error / model build), and collect_data threading devid_ep/devid_ep_vf from the collector args. - device_enumeration: ESXi collect path, the new _parse_count guard (valid / non-numeric / bad-exit), _esxi_device_count id formatting, and the collector args (devid unset -> warning). - os / bios / storage / kernel / dmesg collectors: an ESXi case each for the new esxcli/smbiosDump/vmkernel branches. - dmesg analyzer: ESXi timestamp pattern selection, dual-form filter_dmesg, the ESXi mxGPU RAS phrasing, and the driver-body unknown-error signal. - New test_dimm_collector.py: _parse_dmi_sizes plus the ESXi smbiosDump path. Full unit suite green (+31 tests). --- test/unit/plugin/test_bios_collector.py | 16 +++ .../test_device_enumeration_collector.py | 102 +++++++++++++++ test/unit/plugin/test_dimm_collector.py | 59 +++++++++ test/unit/plugin/test_dmesg_analyzer.py | 73 +++++++++++ test/unit/plugin/test_dmesg_collector.py | 26 ++++ test/unit/plugin/test_kernel_collector.py | 20 +++ test/unit/plugin/test_os_collector.py | 23 ++++ test/unit/plugin/test_pcie_collector.py | 120 ++++++++++++++++++ test/unit/plugin/test_storage_collector.py | 23 ++++ 9 files changed, 462 insertions(+) create mode 100644 test/unit/plugin/test_dimm_collector.py diff --git a/test/unit/plugin/test_bios_collector.py b/test/unit/plugin/test_bios_collector.py index 48dda07a..a2455e1c 100644 --- a/test/unit/plugin/test_bios_collector.py +++ b/test/unit/plugin/test_bios_collector.py @@ -78,6 +78,22 @@ def test_task_body_linux(system_info, bios_collector): assert data == exp_data +def test_task_body_esxi(system_info, bios_collector): + """ESXi: BIOS version is parsed from the smbiosDump 'Version:' line.""" + system_info.os_family = OSFamily.ESXI + + bios_collector._run_sut_cmd = MagicMock( + return_value=MagicMock( + exit_code=0, + stdout=' Version: "1.8"', + ) + ) + + res, data = bios_collector.collect_data() + assert res.status == ExecutionStatus.OK + assert data == BiosDataModel(bios_version="1.8") + + def test_task_body_error(system_info, bios_collector): """Test the _task_body method when an error occurs.""" system_info.os_family = OSFamily.LINUX diff --git a/test/unit/plugin/test_device_enumeration_collector.py b/test/unit/plugin/test_device_enumeration_collector.py index 50335f1f..3577e402 100644 --- a/test/unit/plugin/test_device_enumeration_collector.py +++ b/test/unit/plugin/test_device_enumeration_collector.py @@ -27,9 +27,13 @@ import pytest +from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.device_enumeration.collector_args import ( + DeviceEnumerationCollectorArgs, +) from nodescraper.plugins.inband.device_enumeration.device_enumeration_collector import ( DeviceEnumerationCollector, ) @@ -170,3 +174,101 @@ def test_collect_error(system_info, device_enumeration_collector): result, data = device_enumeration_collector.collect_data() assert result.status == ExecutionStatus.EXECUTION_FAILURE assert data is None + + +def test_collect_esxi(system_info, device_enumeration_collector): + """ESXi counts CPUs via esxcli and GPUs/VFs by device ID from collector args.""" + system_info.os_family = OSFamily.ESXI + + device_enumeration_collector._run_sut_cmd = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="2\n", stderr="", command="cpu"), + MagicMock(exit_code=0, stdout="8\n", stderr="", command="gpu"), + MagicMock(exit_code=0, stdout="0\n", stderr="", command="vf"), + ] + ) + + args = DeviceEnumerationCollectorArgs(devid_ep=0x75A3, devid_ep_vf=0x75B3) + result, data = device_enumeration_collector.collect_data(args) + + assert result.status == ExecutionStatus.OK + assert data == DeviceEnumerationDataModel(cpu_count=2, gpu_count=8, vf_count=0) + # GPU count command must carry the PF device id (hex, unpadded) into the grep. + gpu_cmd = device_enumeration_collector._run_sut_cmd.call_args_list[1].args[0] + assert "0x0*75a3" in gpu_cmd + vf_cmd = device_enumeration_collector._run_sut_cmd.call_args_list[2].args[0] + assert "0x0*75b3" in vf_cmd + + +def test_collect_esxi_no_devid(system_info, device_enumeration_collector): + """Without a devid_ep the ESXi run warns and does not match GPUs by device id.""" + system_info.os_family = OSFamily.ESXI + + device_enumeration_collector._run_sut_cmd = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="2\n", stderr="", command="cpu"), + MagicMock(exit_code=0, stdout="0\n", stderr="", command="gpu"), + MagicMock(exit_code=0, stdout="0\n", stderr="", command="vf"), + ] + ) + + result, data = device_enumeration_collector.collect_data(DeviceEnumerationCollectorArgs()) + + assert result.status == ExecutionStatus.OK + assert data == DeviceEnumerationDataModel(cpu_count=2, gpu_count=0, vf_count=0) + assert any("devid_ep not set" in e.description for e in result.events) + # The unset device id must not be formatted as hex into the grep (no match). + gpu_cmd = device_enumeration_collector._run_sut_cmd.call_args_list[1].args[0] + assert "__unset__" in gpu_cmd + + +def test_collect_esxi_none_args_defaults(system_info, device_enumeration_collector): + """collect_data(None) on ESXi falls back to default args (devid_ep unset).""" + system_info.os_family = OSFamily.ESXI + device_enumeration_collector._run_sut_cmd = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="2", stderr="", command="cpu"), + MagicMock(exit_code=0, stdout="0", stderr="", command="gpu"), + MagicMock(exit_code=0, stdout="0", stderr="", command="vf"), + ] + ) + result, data = device_enumeration_collector.collect_data(None) + assert result.status == ExecutionStatus.OK + assert data.cpu_count == 2 + + +def test_esxi_device_count_formats_device_id(device_enumeration_collector): + """_esxi_device_count renders the id as bare lowercase hex; None -> unmatched sentinel.""" + device_enumeration_collector._run_sut_cmd = MagicMock( + return_value=MagicMock(exit_code=0, stdout="1", stderr="", command="x") + ) + device_enumeration_collector._esxi_device_count(0x744C) + assert "0x0*744c" in device_enumeration_collector._run_sut_cmd.call_args.args[0] + + device_enumeration_collector._run_sut_cmd.reset_mock() + device_enumeration_collector._esxi_device_count(None) + assert "__unset__" in device_enumeration_collector._run_sut_cmd.call_args.args[0] + + +def test_parse_count_valid(device_enumeration_collector): + """A clean numeric stdout parses to int (with surrounding whitespace stripped).""" + res = MagicMock(exit_code=0, stdout=" 8 \n", stderr="", command="c") + assert device_enumeration_collector._parse_count(res, "count") == 8 + + +def test_parse_count_non_numeric(device_enumeration_collector): + """Non-numeric stdout returns None and warns instead of raising ValueError.""" + res = MagicMock(exit_code=0, stdout="N/A", stderr="", command="c") + assert device_enumeration_collector._parse_count(res, "count") is None + assert any( + e.priority == EventPriority.WARNING for e in device_enumeration_collector.result.events + ) + + +def test_parse_count_bad_exit(device_enumeration_collector): + """A non-zero exit returns None and warns.""" + res = MagicMock(exit_code=1, stdout="", stderr="boom", command="c") + assert device_enumeration_collector._parse_count(res, "count") is None + assert any( + e.priority == EventPriority.WARNING for e in device_enumeration_collector.result.events + ) diff --git a/test/unit/plugin/test_dimm_collector.py b/test/unit/plugin/test_dimm_collector.py new file mode 100644 index 00000000..ed84bcdb --- /dev/null +++ b/test/unit/plugin/test_dimm_collector.py @@ -0,0 +1,59 @@ +from unittest.mock import MagicMock + +import pytest + +from nodescraper.enums.executionstatus import ExecutionStatus +from nodescraper.enums.systeminteraction import SystemInteractionLevel +from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.dimm.dimm_collector import DimmCollector +from nodescraper.plugins.inband.dimm.dimmdata import DimmDataModel + + +@pytest.fixture +def dimm_collector(system_info, conn_mock): + return DimmCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + + +def test_parse_dmi_sizes_totals_and_topology(dimm_collector): + """'Size: ' lines (dmidecode/smbiosDump) sum to a total + per-size counts.""" + stdout = " Size: 128 GB\n Size: 128 GB\n Size: 64 GB\n" + assert dimm_collector._parse_dmi_sizes(stdout) == "320GB @ 2 x 128GB 1 x 64GB" + + +def test_parse_dmi_sizes_empty(dimm_collector): + """No parseable Size lines yields the zero summary.""" + assert dimm_collector._parse_dmi_sizes("No Module Installed\n") == "0 GB" + + +def test_collect_esxi(system_info, dimm_collector): + """ESXi parses DIMM sizes from smbiosDump (same 'Size:' field as dmidecode).""" + system_info.os_family = OSFamily.ESXI + dimm_collector._run_sut_cmd = MagicMock( + return_value=MagicMock( + exit_code=0, + stdout=" Size: 128 GB\n Size: 128 GB\n", + stderr="", + command=DimmCollector.CMD_ESXI, + ) + ) + + result, data = dimm_collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == DimmDataModel(dimms="256GB @ 2 x 128GB") + + +def test_collect_esxi_error(system_info, dimm_collector): + """A failed esxcli/smbiosDump run yields no DIMM data.""" + system_info.os_family = OSFamily.ESXI + dimm_collector._run_sut_cmd = MagicMock( + return_value=MagicMock( + exit_code=1, stdout="", stderr="boom", command=DimmCollector.CMD_ESXI + ) + ) + + _, data = dimm_collector.collect_data() + assert data is None diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 784b5453..d0baa4e6 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -31,6 +31,7 @@ from nodescraper.enums.eventcategory import EventCategory from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus +from nodescraper.models.systeminfo import OSFamily from nodescraper.plugins.inband.dmesg.analyzer_args import DmesgAnalyzerArgs from nodescraper.plugins.inband.dmesg.dmesg_analyzer import DmesgAnalyzer from nodescraper.plugins.inband.dmesg.dmesgdata import DmesgData @@ -1435,3 +1436,75 @@ def test_mce_match_content_is_single_status_line(system_info): assert "CPU:29" in match_content assert "CPU:8" not in match_content assert "\n" not in match_content + + +# --- ESXi ------------------------------------------------------------------ + + +def test_esxi_analyzer_uses_esxi_timestamp_pattern(system_info): + """On ESXi the analyzer swaps in the vmkernel.log (dot-ms/Z) timestamp pattern.""" + system_info.os_family = OSFamily.ESXI + analyzer = DmesgAnalyzer(system_info=system_info) + assert analyzer._is_esxi() is True + assert analyzer.TIMESTAMP_PATTERN is DmesgAnalyzer.ESXI_TIMESTAMP_PATTERN + + system_info.os_family = OSFamily.LINUX + linux_analyzer = DmesgAnalyzer(system_info=system_info) + assert linux_analyzer._is_esxi() is False + assert linux_analyzer.TIMESTAMP_PATTERN is not DmesgAnalyzer.ESXI_TIMESTAMP_PATTERN + + +def test_filter_dmesg_handles_esxi_and_linux_timestamps(): + """filter_dmesg accepts both ESXi dot-ms/Z and Linux comma-form timestamps.""" + esxi_log = ( + "2026-08-20T08:00:00.100Z -INFO vmkernel - line A\n" + "2026-08-20T09:00:00.100Z -INFO vmkernel - line B\n" + "2026-08-20T10:00:00.100Z -INFO vmkernel - line C\n" + ) + start = datetime.datetime.fromisoformat("2026-08-20T08:30:00+00:00") + end = datetime.datetime.fromisoformat("2026-08-20T09:30:00+00:00") + filtered = DmesgAnalyzer.filter_dmesg(esxi_log, start, end) + assert filtered.strip() == "2026-08-20T09:00:00.100Z -INFO vmkernel - line B" + + # Linux comma-form still filters (no regression). + linux_log = "2024-10-01T07:00:00,000000-05:00 log1\n" "2024-10-01T09:00:00,000000-05:00 log2\n" + l_start = datetime.datetime.fromisoformat("2024-10-01T08:00:00-05:00") + assert "log2" in DmesgAnalyzer.filter_dmesg(linux_log, l_start) + assert "log1" not in DmesgAnalyzer.filter_dmesg(linux_log, l_start) + + +def test_esxi_ras_regex_phrasing(system_info): + """ESXi mxGPU RAS phrasing ('... detected in Block.') is flagged as RAS.""" + system_info.os_family = OSFamily.ESXI + analyzer = DmesgAnalyzer(system_info=system_info) + data = DmesgData( + dmesg_content=( + "2026-08-20T09:35:58.380Z -ALERT vmkernel - " + "3 new uncorrectable hardware errors detected in MMHUB Block.\n" + "2026-08-20T09:35:59.000Z -ALERT vmkernel - GPU detected ECC Fatal Error.\n" + ), + skip_log_file=True, + ) + res = analyzer.analyze_data(data) + by_desc = {e.description: e for e in res.events} + assert "RAS Uncorrectable Error" in by_desc + assert "RAS ECC Fatal Error" in by_desc + assert by_desc["RAS Uncorrectable Error"].category == EventCategory.RAS.value + + +def test_esxi_unknown_error_uses_driver_body_severity(system_info): + """On ESXi the unknown-error signal is the driver-body severity (gim/amdgpuv), + not the unreliable vmkernel -ALERT/-INFO token.""" + system_info.os_family = OSFamily.ESXI + analyzer = DmesgAnalyzer(system_info=system_info) + data = DmesgData( + dmesg_content=( + "2026-08-20T09:35:58.380Z -INFO vmkernel - amdgpuv error [0:65:0]: unexpected thing\n" + "2026-08-20T09:35:59.000Z -ALERT vmkernel - benign boot notice, no driver marker\n" + ), + skip_log_file=True, + ) + res = analyzer.analyze_data(data) + unknown = [e for e in res.events if e.description == "Unknown dmesg error"] + assert len(unknown) == 1 + assert "unexpected thing" in str(unknown[0].data["match_content"]) diff --git a/test/unit/plugin/test_dmesg_collector.py b/test/unit/plugin/test_dmesg_collector.py index 4202c0f9..4dbccfbd 100644 --- a/test/unit/plugin/test_dmesg_collector.py +++ b/test/unit/plugin/test_dmesg_collector.py @@ -101,6 +101,32 @@ def test_dmesg_collection(system_info, conn_mock): assert data.dmesg_content == dmesg +def test_dmesg_collection_esxi(system_info, conn_mock): + """ESXi has no dmesg ring buffer; the kernel log is read from vmkernel.log.""" + system_info.os_family = OSFamily.ESXI + collector = DmesgCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.INTERACTIVE, + connection=conn_mock, + ) + + vmkernel = ( + "2026-08-20T09:35:58.380Z -INFO vmkernel - boot line\n" + "2026-08-20T09:36:00.000Z -WARNING vmkernel - a warning\n" + ) + conn_mock.run_command.return_value = CommandArtifact( + exit_code=0, + stdout=vmkernel, + stderr="", + command="cat /var/log/vmkernel.log", + ) + + res, data = collector.collect_data() + assert res.status == ExecutionStatus.OK + assert data is not None + assert data.dmesg_content == vmkernel + + def test_bad_exit_code(conn_mock, system_info): conn_mock.run_command.return_value = CommandArtifact( diff --git a/test/unit/plugin/test_kernel_collector.py b/test/unit/plugin/test_kernel_collector.py index 3b370783..fa1420e9 100644 --- a/test/unit/plugin/test_kernel_collector.py +++ b/test/unit/plugin/test_kernel_collector.py @@ -88,6 +88,26 @@ def test_run_linux(collector, conn_mock): assert result.status == ExecutionStatus.OK +def test_run_esxi(collector, conn_mock): + """ESXi reuses the `uname -a` path (release in the same field); numa_balancing + has no ESXi equivalent and its command fails gracefully -> None.""" + collector.system_info.os_family = OSFamily.ESXI + uname = "VMkernel host 9.1.0 #1 SMP Release build-25166133 Jan 14 2026 x86_64" + conn_mock.run_command.side_effect = [ + CommandArtifact(exit_code=0, stdout=uname, stderr="", command="sh -c 'uname -a'"), + CommandArtifact( + exit_code=1, + stdout="", + stderr="not found", + command="sh -c 'cat /proc/sys/kernel/numa_balancing'", + ), + ] + + result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == KernelDataModel(kernel_info=uname, kernel_version="9.1.0", numa_balancing=None) + + def test_run_error(collector, conn_mock): collector.system_info.os_family = OSFamily.LINUX conn_mock.run_command.return_value = CommandArtifact( diff --git a/test/unit/plugin/test_os_collector.py b/test/unit/plugin/test_os_collector.py index 480e76da..8ac921dc 100644 --- a/test/unit/plugin/test_os_collector.py +++ b/test/unit/plugin/test_os_collector.py @@ -128,3 +128,26 @@ def test_os_collector_error(collector, conn_mock, system_info): _, data = collector.collect_data() assert data is None + + +def test_os_collector_esxi(collector, conn_mock, system_info): + """ESXi: os_name from `vmware -v`, os_version from the esxcli 'Version:' field.""" + system_info.os_family = OSFamily.ESXI + conn_mock.run_command.side_effect = [ + CommandArtifact( + exit_code=0, + stdout="VMware ESXi 9.1.0 build-25166133", + stderr="", + command="vmware -v", + ), + CommandArtifact( + exit_code=0, + stdout=" Product: VMware ESXi\n Version: 9.1.0\n Build: Releasebuild-25166133", + stderr="", + command="esxcli system version get", + ), + ] + + result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == OsDataModel(os_name="VMware ESXi 9.1.0 build-25166133", os_version="9.1.0") diff --git a/test/unit/plugin/test_pcie_collector.py b/test/unit/plugin/test_pcie_collector.py index 6aabc5c0..8bae8d7e 100644 --- a/test/unit/plugin/test_pcie_collector.py +++ b/test/unit/plugin/test_pcie_collector.py @@ -27,8 +27,11 @@ import pytest +from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel +from nodescraper.plugins.inband.pcie.collector_args import PcieCollectorArgs from nodescraper.plugins.inband.pcie.pcie_collector import PcieCollector +from nodescraper.plugins.inband.pcie.pcie_data import PcieCfgSpace @pytest.fixture @@ -109,3 +112,120 @@ def test_log_pcie_artifacts_includes_lspci_pp_d(collector): artifact for artifact in collector.result.artifacts if artifact.filename == "lspci_pp_d.txt" ) assert lspci_pp_d.contents == "0001:00:01.1/0001:00:02.0/0001:00:03.0" + + +# --- ESXi ------------------------------------------------------------------- + +# esxcli hardware pci list: bare BDF header lines, then indented fields. +ESXCLI_PCI_LIST = ( + "0000:05:00.0\n" + " Device ID: 0x75a3\n" # PF + "0000:15:00.0\n" + " Device ID: 0x0000744C\n" # padded + uppercase -> 0x744c + "0000:05:02.0\n" + " Device ID: 0x75b3\n" # VF + "0000:99:00.0\n" + " SubDevice ID: 0x75a3\n" # must NOT be treated as a Device ID + " Device ID: 0xdead\n" # no match +) + +# lspci -e: " " header lines, then hex rows. +LSPCI_E_BLOB = ( + "0000:05:00.0 Processing accelerators: AMD\n" + "00: 12 34 56 78\n" + "10: 9a bc de f0\n" + "0000:05:02.0 Processing accelerators: AMD VF\n" + "00: aa bb cc dd\n" +) + + +def test_get_gpu_vf_bdfs_esxi_matches_pf_and_vf(collector): + """PF/VF BDFs are selected by matching the expected device IDs.""" + collector._run_os_cmd = MagicMock(return_value=ESXCLI_PCI_LIST) + pf, vf = collector._get_gpu_vf_bdfs_esxi(0x75A3, 0x75B3) + assert pf == ["0000:05:00.0"] + assert vf == ["0000:05:02.0"] + + +def test_get_gpu_vf_bdfs_esxi_case_and_padding(collector): + """A padded/uppercase Device ID ("0x0000744C") matches the expected 0x744c.""" + collector._run_os_cmd = MagicMock(return_value=ESXCLI_PCI_LIST) + pf, vf = collector._get_gpu_vf_bdfs_esxi(0x744C, None) + assert pf == ["0000:15:00.0"] + assert vf == [] + + +def test_get_gpu_vf_bdfs_esxi_both_none_short_circuits(collector): + """With no expected device IDs, esxcli is not even queried.""" + collector._run_os_cmd = MagicMock(return_value=ESXCLI_PCI_LIST) + pf, vf = collector._get_gpu_vf_bdfs_esxi(None, None) + assert (pf, vf) == ([], []) + collector._run_os_cmd.assert_not_called() + + +def test_get_gpu_vf_bdfs_esxi_ignores_unparseable_id(collector): + """A non-hex Device ID value is skipped rather than raising.""" + collector._run_os_cmd = MagicMock( + return_value="0000:05:00.0\n Device ID: N/A\n0000:05:02.0\n Device ID: 0x75a3\n" + ) + pf, vf = collector._get_gpu_vf_bdfs_esxi(0x75A3, None) + assert pf == ["0000:05:02.0"] + + +def test_get_all_cfg_space_esxi_splits_by_bdf(collector): + """lspci -e is split into {bdf: hex-rows} and saved as an artifact.""" + collector._run_os_cmd = MagicMock(return_value=LSPCI_E_BLOB) + cfg = collector._get_all_cfg_space_esxi() + assert set(cfg) == {"0000:05:00.0", "0000:05:02.0"} + assert cfg["0000:05:00.0"] == "00: 12 34 56 78\n10: 9a bc de f0" + assert any(a.filename == "lspci_e.txt" for a in collector.result.artifacts) + + +def test_get_all_cfg_space_esxi_empty(collector): + """No lspci output -> empty mapping.""" + collector._run_os_cmd = MagicMock(return_value="") + assert collector._get_all_cfg_space_esxi() == {} + + +def test_get_pcie_data_esxi_no_bdfs_warns(collector): + """When no GPU/VF BDFs match, a warning is logged and None returned.""" + collector._get_gpu_vf_bdfs_esxi = MagicMock(return_value=([], [])) + assert collector._get_pcie_data_esxi(0x75A3, None) is None + assert any("No GPU/VF BDFs" in e.description for e in collector.result.events) + + +def test_get_pcie_data_esxi_empty_cfg_errors(collector): + """BDFs found but no config space dump -> ERROR status, None.""" + collector._get_gpu_vf_bdfs_esxi = MagicMock(return_value=(["0000:05:00.0"], [])) + collector._get_all_cfg_space_esxi = MagicMock(return_value={}) + assert collector._get_pcie_data_esxi(0x75A3, None) is None + assert collector.result.status == ExecutionStatus.ERROR + + +def test_get_pcie_data_esxi_builds_model(collector): + """PF/VF BDFs present in the cfg dump are parsed into the PcieDataModel.""" + collector._get_gpu_vf_bdfs_esxi = MagicMock(return_value=(["0000:05:00.0"], ["0000:05:02.0"])) + collector._get_all_cfg_space_esxi = MagicMock( + return_value={"0000:05:00.0": "pf-hex", "0000:05:02.0": "vf-hex"} + ) + collector._cfg_space_from_hex = MagicMock(return_value=PcieCfgSpace()) + + data = collector._get_pcie_data_esxi(0x75A3, 0x75B3) + assert list(data.pcie_cfg_space) == ["0000:05:00.0"] + assert list(data.vf_pcie_cfg_space) == ["0000:05:02.0"] + collector._cfg_space_from_hex.assert_any_call("pf-hex", "0000:05:00.0") + collector._cfg_space_from_hex.assert_any_call("vf-hex", "0000:05:02.0") + + +def test_cfg_space_from_hex_too_short_logs_error(collector): + """Fewer than 64 parsed bytes logs an error (short/truncated dump).""" + collector._cfg_space_from_hex("00: 12 34 56 78", "0000:05:00.0") + assert any("not the expected length" in e.description for e in collector.result.events) + + +def test_collect_data_threads_devid_from_args(collector): + """collect_data forwards args.devid_ep / devid_ep_vf into the ESXi resolver.""" + collector._get_pcie_data = MagicMock(return_value=None) + args = PcieCollectorArgs(devid_ep=0x75A3, devid_ep_vf=0x75B3) + collector.collect_data(args) + collector._get_pcie_data.assert_called_once_with(None, 0x75A3, 0x75B3) diff --git a/test/unit/plugin/test_storage_collector.py b/test/unit/plugin/test_storage_collector.py index 02a96c4a..31688aca 100644 --- a/test/unit/plugin/test_storage_collector.py +++ b/test/unit/plugin/test_storage_collector.py @@ -76,6 +76,29 @@ def test_run_linux(collector, conn_mock): ) +def test_run_esxi(collector, conn_mock): + """ESXi parses fixed esxcli filesystem-list columns (Size/Free by position).""" + collector.system_info.os_family = OSFamily.ESXI + conn_mock.run_command.return_value = CommandArtifact( + exit_code=0, + stdout=( + "Mount Point Volume Name UUID Mounted Type Size Free\n" + "------------------- ----------- -------- ------- ------ ---- ----\n" + "/vmfs/volumes/abc datastore1 uuid-1 true VMFS-6 2000 500" + ), + stderr="", + command="esxcli storage filesystem list", + ) + + result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == StorageDataModel( + storage_data={ + "/vmfs/volumes/abc": DeviceStorageData(total=2000, free=500, used=1500, percent=75.0) + } + ) + + def test_run_windows(system_info, conn_mock): system_info.os_family = OSFamily.WINDOWS collector = StorageCollector( From 218c11d52ae6f9e817ab557a315f443465dd15ae Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 18 Sep 2026 15:46:29 +0000 Subject: [PATCH 11/12] Add amd-smi driver-flavor collection to the amd-smi plugin Adds host (mxGPU gim/amdgpuv incl. ESXi), guest-VF, and bare-metal support alongside the existing guest/bare-metal AmdSmiCollector: - HostDriver*/Guest* model modules and AmdSmiFlavorDataModel, which widens the base AmdSmiDataModel field unions to also accept the host/VF model variants - AmdSmiFlavorCollector: detects the loaded driver flavor and issues the flavor-appropriate amd-smi commands, building the matching model variants - AmdSmiFlavorPlugin wires the flavor collector + data model together; the base AmdSmiPlugin (guest/bare-metal only) is left unchanged - Unit tests for flavor detection, model building, and flavor command dispatch --- nodescraper/plugins/inband/amdsmi/__init__.py | 10 +- .../inband/amdsmi/amdsmi_flavor_collector.py | 579 ++++++++++++++++++ .../inband/amdsmi/amdsmi_flavor_plugin.py | 18 + .../inband/amdsmi/amdsmidata_flavor.py | 61 ++ .../plugins/inband/amdsmi/amdsmidata_guest.py | 71 +++ .../plugins/inband/amdsmi/amdsmidata_host.py | 502 +++++++++++++++ .../plugin/test_amdsmi_flavor_collector.py | 213 +++++++ 7 files changed, 1453 insertions(+), 1 deletion(-) create mode 100644 nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py create mode 100644 nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py create mode 100644 nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py create mode 100644 nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py create mode 100644 nodescraper/plugins/inband/amdsmi/amdsmidata_host.py create mode 100644 test/unit/plugin/test_amdsmi_flavor_collector.py diff --git a/nodescraper/plugins/inband/amdsmi/__init__.py b/nodescraper/plugins/inband/amdsmi/__init__.py index f117a9fd..89f4d2c1 100644 --- a/nodescraper/plugins/inband/amdsmi/__init__.py +++ b/nodescraper/plugins/inband/amdsmi/__init__.py @@ -23,6 +23,14 @@ # SOFTWARE. # ############################################################################### +from .amdsmi_flavor_collector import AmdSmiFlavorCollector +from .amdsmi_flavor_plugin import AmdSmiFlavorPlugin from .amdsmi_plugin import AmdSmiPlugin +from .amdsmidata_flavor import AmdSmiFlavorDataModel -__all__ = ["AmdSmiPlugin"] +__all__ = [ + "AmdSmiPlugin", + "AmdSmiFlavorPlugin", + "AmdSmiFlavorCollector", + "AmdSmiFlavorDataModel", +] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py new file mode 100644 index 00000000..364788a2 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py @@ -0,0 +1,579 @@ +"""amd-smi collector adding mxGPU host-driver (gim/amdgpuv, incl. ESXi) and guest-VF +support on top of the base guest/bare-metal-only AmdSmiCollector. + +The amd-smi CLI syntax and JSON schema depend on the loaded driver "flavor": + - host - mxGPU host driver (gim on Linux, amdgpuv on ESXi) -> HostDriver* models + - guest - amdgpu on a virtual function inside a VM -> Guest* models + - bare-metal - amdgpu on physical hardware -> base AmdSmi* models +This collector detects the flavor, issues the flavor-appropriate commands, and builds +the matching model variants into AmdSmiFlavorDataModel. The base AmdSmiCollector is +left unchanged for callers that do not need host/VF support. +""" + +from __future__ import annotations + +import io +import json +import re +from tarfile import TarFile +from typing import Any, Optional, Union + +from pydantic import BaseModel, ValidationError + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily +from nodescraper.models import TaskResult +from nodescraper.models.datamodel import FileModel +from nodescraper.plugins.inband.amdsmi.amdsmi_collector import AmdSmiCollector +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiListItem, + AmdSmiMetric, + AmdSmiStatic, + AmdSmiVersion, + BadPages, + Fw, + Partition, + Processes, + Topo, + XgmiLinks, + XgmiMetrics, +) +from nodescraper.plugins.inband.amdsmi.collector_args import AmdSmiCollectorArgs +from nodescraper.utils import get_exception_details, get_exception_traceback + +from .amdsmidata_flavor import AmdSmiFlavorDataModel +from .amdsmidata_guest import GuestAmdSmiMetric, GuestAmdSmiStatic +from .amdsmidata_host import ( + HostDriverAmdSmiListItem, + HostDriverAmdSmiMetric, + HostDriverAmdSmiStatic, + HostDriverAmdSmiVersion, + HostDriverBadPages, + HostDriverTopo, + HostDriverXgmiLinks, + HostDriverXgmiMetrics, +) + +AMD_SMI_CPER_FOLDER = "/tmp/amd_smi_cper" + + +class AmdSmiFlavorCollector(AmdSmiCollector): + """amd-smi collector with host/guest driver-flavor support (incl. ESXi mxGPU).""" + + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.LINUX, OSFamily.ESXI} + DATA_MODEL = AmdSmiFlavorDataModel # type: ignore[assignment] + + # amd-smi runs with any of these drivers; the driver sets the CLI/JSON flavor. + _HOST_DRIVER_MODULES = ("gim", "amdgpuv") + _GUEST_DRIVER_MODULES = ("amdgpu",) + + # ---- driver-flavor detection -------------------------------------------------- + + def _get_loaded_gpu_driver(self) -> Optional[str]: + """Loaded AMD GPU driver module name (gim/amdgpuv/amdgpu), or None. Cached.""" + cached = getattr(self, "_loaded_gpu_driver_cache", None) + if cached is not None: + return cached + if self.system_info.os_family == OSFamily.ESXI: + cmd_ret = self._run_sut_cmd("vmkload_mod -l") + else: + cmd_ret = self._run_sut_cmd("lsmod", sudo=True) + if cmd_ret.exit_code != 0: + return None + known = self._HOST_DRIVER_MODULES + self._GUEST_DRIVER_MODULES + driver: Optional[str] = None + for line in cmd_ret.stdout.splitlines(): + module = line.strip().split()[0] if line.strip() else "" + if module in known: + driver = module + break + self._loaded_gpu_driver_cache = driver + return driver + + def _check_gpu_driver_loaded(self) -> bool: + """A driver is required for all amd-smi commands (including help/version).""" + return self._get_loaded_gpu_driver() is not None + + @property + def is_host_driver(self) -> bool: + """True for the mxGPU host driver (gim on Linux, amdgpuv on ESXi).""" + return self._get_loaded_gpu_driver() in self._HOST_DRIVER_MODULES + + def _is_virtualized(self) -> bool: + """True inside a virtualized guest (VM), via the CPU hypervisor flag. Cached.""" + cached = getattr(self, "_is_virtualized_cache", None) + if cached is not None: + return cached + if self.system_info.os_family == OSFamily.ESXI: + self._is_virtualized_cache = False + return False + cmd_ret = self._run_sut_cmd("grep -c hypervisor /proc/cpuinfo") + try: + value = int(cmd_ret.stdout.strip()) > 0 + except (ValueError, AttributeError): + value = False + self._is_virtualized_cache = value + return value + + @property + def is_guest_driver(self) -> bool: + """True for a guest-VF amdgpu (amdgpu inside a VM). Bare-metal amdgpu is not guest.""" + return not self.is_host_driver and self._is_virtualized() + + @property + def _amd_smi_needs_sudo(self) -> bool: + """amd-smi needs sudo on Linux (guest + mxGPU host); not on ESXi (no sudo binary).""" + return self.system_info.os_family != OSFamily.ESXI + + # ---- command execution -------------------------------------------------------- + + def _run_amd_smi(self, cmd: str, sudo: Optional[bool] = None) -> Optional[str]: + """Run amd-smi, elevating on Linux by default (ESXi needs no sudo).""" + if sudo is None: + sudo = self._amd_smi_needs_sudo + cmd_ret = self._run_sut_cmd(f"{self.AMD_SMI_EXE} {cmd}", sudo=sudo) + # Host amd-smi reports a benign "no data" status on stderr with rc=0 (e.g. + # bad-pages when none exist) — a valid empty result, not an error. + if cmd_ret.exit_code == 0 and "No data was found" in cmd_ret.stderr: + self._log_event( + category=EventCategory.APPLICATION, + description="amd-smi returned no data for command", + data={"command": cmd, "stderr": cmd_ret.stderr}, + priority=EventPriority.INFO, + ) + return None + if cmd_ret.stderr != "" or cmd_ret.exit_code != 0: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi command", + data={"command": cmd, "exit_code": cmd_ret.exit_code, "stderr": cmd_ret.stderr}, + priority=EventPriority.ERROR, + console_log=True, + ) + return None + return cmd_ret.stdout + + def _run_amd_smi_dict( + self, + cmd: str, + sudo: Optional[bool] = None, + raise_event: bool = True, + *, + patch_invalid_json: bool = False, + ) -> Union[dict, list, None]: + """Run an amd-smi command with --json and parse the output.""" + cmd += " --json" + cmd_ret = self._run_amd_smi(cmd, sudo=sudo) + if not cmd_ret: + return None + try: + if patch_invalid_json: + # amd-smi has been observed to emit invalid JSON ("]\n[") between records. + cmd_ret = cmd_ret.replace("]\n[", ",") + return json.loads(cmd_ret) + except json.JSONDecodeError as e: + if raise_event: + self._log_event( + category=EventCategory.APPLICATION, + description=f"Error parsing command: `{cmd}` json data", + data={"cmd": cmd, "exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + return None + + def _check_command_supported(self, command: str) -> bool: + """Log an INFO event when amd-smi help does not list the command.""" + if command not in getattr(self, "amd_smi_commands", set()): + self._log_event( + category=EventCategory.APPLICATION, + description=f"amd-smi does not support command: `{command}`", + priority=EventPriority.INFO, + ) + return False + return True + + def detect_amdsmi_commands(self) -> set[str]: + """Parse `amd-smi help` (host) / `amd-smi -h` (guest) for supported commands.""" + command_pattern = re.compile(r"^\s{4}([\w\-]+)\s", re.MULTILINE) + help_flag = "help" if self.is_host_driver else "-h" + help_output = self._run_amd_smi(help_flag) + if help_output is None: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi help command", + priority=EventPriority.ERROR, + console_log=True, + ) + return set() + return set(command_pattern.findall(help_output)) + + # ---- model builder ------------------------------------------------------------ + + def _build_flavor_model( + self, + model_class: type[BaseModel], + json_data: Union[dict, list, None], + *, + return_first: bool = False, + keep_key: Optional[str] = None, + ) -> Any: + """Validate raw amd-smi JSON into ``model_class`` (per-flavor class chosen by caller). + + keep_key drops rows missing that key (host list/topology enumerate non-GPU rows); + return_first collapses to a single record (e.g. version). + """ + if json_data is None: + self._log_event( + category=EventCategory.APPLICATION, + description=f"No data to build model: {model_class.__name__}", + priority=EventPriority.ERROR, + ) + return None + if isinstance(json_data, dict): + try: + return model_class.model_validate(json_data) + except ValidationError as e: + self._log_event( + category=EventCategory.APPLICATION, + description=f"Failed to build amd-smi model: {model_class.__name__}", + data=get_exception_details(e), + priority=EventPriority.WARNING, + ) + return None + + # Build per row so one unparseable entry doesn't discard the whole list. A + # guest amd-smi can enumerate a VF's secondary PCI functions (bdf .1-.7) as + # all-N/A rows alongside the real GPUs; skip those but keep the real ones. + validated: list = [] + skipped: list = [] + for item in json_data: + if not isinstance(item, dict): + self._log_event( + category=EventCategory.APPLICATION, + description=f"Invalid data type for amd-smi model: {model_class.__name__}", + data={"data_type": type(item).__name__}, + priority=EventPriority.WARNING, + ) + return None + if keep_key is not None and keep_key not in item: + continue + try: + validated.append(model_class.model_validate(item)) + except ValidationError as e: + skipped.append(get_exception_details(e)) + if skipped: + self._log_event( + category=EventCategory.APPLICATION, + description=( + f"Skipped {len(skipped)} unparseable {model_class.__name__} " + f"row(s); kept {len(validated)}" + ), + data={"first_error": skipped[0]}, + priority=EventPriority.WARNING, + ) + if return_first: + return validated[0] if validated else None + return validated + + # ---- raw-JSON getters (flavor-aware commands) -------------------------------- + + def _get_amdsmi_version(self) -> Union[AmdSmiVersion, HostDriverAmdSmiVersion, None]: # type: ignore[override] + """Pick the version model by flavor (host nests fields under a "version" key).""" + ret = self._run_amd_smi_dict("version") + model = HostDriverAmdSmiVersion if self.is_host_driver else AmdSmiVersion + return self._build_flavor_model(model, ret, return_first=True) + + def get_gpu_list(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("list"): + return None + return self._run_amd_smi_dict("list", patch_invalid_json=True) + + def get_process(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("process"): + return None + return self._run_amd_smi_dict("process") + + _PARTITION_COLUMNS = ( + "gpu_id", + "memory", + "accelerator_type", + "accelerator_profile_index", + "partition_id", + ) + + def get_partition(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("partition"): + return None + # Host-driver amd-smi has no --json for partition; parse the text table. + if self.is_host_driver: + return self._get_partition_host() + return self._run_amd_smi_dict("partition") + + def _get_partition_host(self) -> Optional[dict]: + """Parse host `partition -c` text (--json unsupported on gim/amdgpuv).""" + raw = self._run_amd_smi("partition -c") + if raw is None: + return None + lines = raw.splitlines() + header_index = None + for index, line in enumerate(lines): + columns = line.split() + if columns and columns[0] == "GPU": + header_index = index + break + if header_index is None: + return None + current_partitions = [] + for line in lines[header_index + 1 :]: + columns = line.split() + is_data_row = len(columns) >= len(self._PARTITION_COLUMNS) and columns[0].isdigit() + if not is_data_row: + break + partition: dict[str, object] = dict(zip(self._PARTITION_COLUMNS, columns)) + partition["gpu_id"] = int(str(partition["gpu_id"])) + current_partitions.append(partition) + if not current_partitions: + return None + return {"current_partition": current_partitions} + + def get_topology(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("topology"): + return None + return self._run_amd_smi_dict("topology") + + def get_static(self) -> Union[list, None]: + if not self._check_command_supported("static"): + return None + # Host-driver amd-smi does not accept "-g all". + static_sub_cmd = "static" if self.is_host_driver else "static -g all" + static_data = self._run_amd_smi_dict(static_sub_cmd) + if static_data is None: + return None + if isinstance(static_data, dict) and "gpu_data" in static_data: + static_data = static_data["gpu_data"] + return [s for s in static_data if isinstance(s, dict) and "gpu" in s] + + def get_metric(self) -> Union[list, None]: + if not self._check_command_supported("metric"): + return None + metric_sub_cmd = "metric" if self.is_host_driver else "metric -g all" + metric_data = self._run_amd_smi_dict(metric_sub_cmd) + if metric_data is None: + return None + if isinstance(metric_data, dict) and "gpu_data" in metric_data: + metric_data = metric_data["gpu_data"] + return [m for m in metric_data if isinstance(m, dict) and "gpu" in m] + + def get_firmware(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("firmware"): + return None + return self._run_amd_smi_dict("firmware") + + def get_bad_pages(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("bad-pages"): + return None + return self._run_amd_smi_dict("bad-pages") + + def get_xgmi_data_metric(self) -> Optional[dict]: + """Fetch xgmi metric + link data (host uses --metric/--link-status vs guest -m/-l).""" + if not self._check_command_supported("xgmi"): + return None + metric_flag = "--metric" if self.is_host_driver else "-m" + xgmi_metric_data = self._run_amd_smi_dict(f"xgmi {metric_flag}") + if xgmi_metric_data is None: + xgmi_metric_data = [] + elif isinstance(xgmi_metric_data, dict) and "xgmi_metric" in xgmi_metric_data: + xgmi_metric_data = xgmi_metric_data["xgmi_metric"] + if isinstance(xgmi_metric_data, list) and len(xgmi_metric_data) == 1: + xgmi_metric_data = xgmi_metric_data[0] + + link_flag = "--link-status" if self.is_host_driver else "-l" + xgmi_link_data = self._run_amd_smi_dict(f"xgmi {link_flag}", raise_event=False) + if isinstance(xgmi_link_data, dict) and "link_status" in xgmi_link_data: + xgmi_link_data = xgmi_link_data["link_status"] + if xgmi_link_data is None: + xgmi_link_data = [] + return {"metric": xgmi_metric_data, "link": xgmi_link_data} + + def get_cper_data(self) -> tuple[list[FileModel], dict[str, int]]: + """Collect CPER files. Host amd-smi requires --severity; guest/bare-metal do not.""" + if not self._check_command_supported("ras"): + return [], {} + self._run_sut_cmd( + f"mkdir -p {AMD_SMI_CPER_FOLDER} && rm -f {AMD_SMI_CPER_FOLDER}/*.cper " + f"&& rm -f {AMD_SMI_CPER_FOLDER}/*.json", + sudo=False, + ) + if self.is_host_driver: + cper_sub_cmd = f"ras --cper --severity=all --folder={AMD_SMI_CPER_FOLDER}" + else: + cper_sub_cmd = f"ras --cper --folder={AMD_SMI_CPER_FOLDER}" + cper_out = self._run_amd_smi(cper_sub_cmd) + if cper_out is None or not re.findall(r"(\w+\.cper)", cper_out): + return [], {} + self._run_sut_cmd( + f"tar -czf {AMD_SMI_CPER_FOLDER}.tar.gz -C {AMD_SMI_CPER_FOLDER} .", + sudo=self._amd_smi_needs_sudo, + ) + cper_zip = self._read_sut_file( + f"{AMD_SMI_CPER_FOLDER}.tar.gz", encoding=None, strip=False, log_artifact=True + ) + if not hasattr(cper_zip, "contents"): + return [], {} + io_bytes = io.BytesIO(cper_zip.contents) # type: ignore[attr-defined] + cper_data: list[FileModel] = [] + cper_afids: dict[str, int] = {} + try: + with TarFile.open(fileobj=io_bytes, mode="r:gz") as tar_file: + for member in tar_file.getmembers(): + if not (member.isfile() and member.name.endswith(".cper")): + continue + content = tar_file.extractfile(member) + file_bytes = content.read() if content is not None else b"" + cper_data.append(FileModel(file_contents=file_bytes, file_name=member.name)) + afid = self._get_cper_afid(f"{AMD_SMI_CPER_FOLDER}/{member.name}") + if afid is not None: + cper_afids[member.name] = afid + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Error extracting cper data", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + return [], {} + if cper_data: + self._log_event( + category=EventCategory.APPLICATION, + description="CPER data has been extracted from amd-smi", + data={"cper_count": len(cper_data), "afid_count": len(cper_afids)}, + priority=EventPriority.INFO, + ) + return cper_data, cper_afids + + # ---- orchestration ------------------------------------------------------------ + + def _get_amdsmi_data( + self, args: Optional[AmdSmiCollectorArgs] = None + ) -> Optional[AmdSmiFlavorDataModel]: + """Fetch all amd-smi sub-data and build the model with per-flavor variants.""" + try: + version = self._get_amdsmi_version() + gpu_list = self.get_gpu_list() + processes = self.get_process() + partition = self.get_partition() + firmware = self.get_firmware() + topology = self.get_topology() + amdsmi_static = self.get_static() + amdsmi_metric = self.get_metric() + bad_pages = self.get_bad_pages() + xgmi = self.get_xgmi_data_metric() or {"metric": [], "link": []} + cper_data, cper_afids = self.get_cper_data() + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi sub commands", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return None + + host = self.is_host_driver + guest = self.is_guest_driver + + # list / topology / bad-pages / xgmi split host vs non-host. + if host: + list_item_cls: type[BaseModel] = HostDriverAmdSmiListItem + topo_cls: type[BaseModel] = HostDriverTopo + bad_pages_cls: type[BaseModel] = HostDriverBadPages + xgmi_metric_cls: type[BaseModel] = HostDriverXgmiMetrics + xgmi_link_cls: type[BaseModel] = HostDriverXgmiLinks + else: + list_item_cls = AmdSmiListItem + topo_cls = Topo + bad_pages_cls = BadPages + xgmi_metric_cls = XgmiMetrics + xgmi_link_cls = XgmiLinks + + # static / metric are three-way (a guest VF omits physical fields). + if host: + static_cls: type[BaseModel] = HostDriverAmdSmiStatic + metric_cls: type[BaseModel] = HostDriverAmdSmiMetric + elif guest: + static_cls = GuestAmdSmiStatic + metric_cls = GuestAmdSmiMetric + else: + static_cls = AmdSmiStatic + metric_cls = AmdSmiMetric + + gpu_list_model = self._build_flavor_model(list_item_cls, gpu_list, keep_key="gpu") + topo_model = self._build_flavor_model(topo_cls, topology, keep_key="gpu") + bad_pages_model = self._build_flavor_model(bad_pages_cls, bad_pages) if bad_pages else [] + partition_model = self._build_flavor_model(Partition, partition) + # Host-driver amd-smi has no `process` command; avoid a spurious error. + process_model = self._build_flavor_model(Processes, processes) if processes else [] + firmware_model = self._build_flavor_model(Fw, firmware) + static_model = self._build_flavor_model(static_cls, amdsmi_static) + metric_model = self._build_flavor_model(metric_cls, amdsmi_metric) + xgmi_metric_model = self._build_flavor_model(xgmi_metric_cls, xgmi["metric"]) + xgmi_link_model = self._build_flavor_model(xgmi_link_cls, xgmi["link"]) + + try: + fw_ids = args.analysis_firmware_ids if args and args.analysis_firmware_ids else None + base = AmdSmiFlavorDataModel( + version=version, + gpu_list=gpu_list_model or [], + process=process_model or [], + partition=partition_model, + firmware=firmware_model or [], + static=static_model or [], + topology=topo_model or [], + metric=metric_model or [], + bad_pages=bad_pages_model or [], + xgmi_metric=xgmi_metric_model or [], + xgmi_link=xgmi_link_model or [], + cper_data=cper_data, + cper_afids=cper_afids, + analysis_firmware_ids=fw_ids, + analysis_ref=None, + ) + return base.model_copy(update={"analysis_ref": base.build_analysis_ref()}) + except ValidationError as err: + self._log_event( + category=EventCategory.APPLICATION, + description="Failed to build AmdSmiFlavorDataModel", + data={"errors": err.errors(include_url=False)}, + priority=EventPriority.ERROR, + ) + return None + + def collect_data( + self, args: Optional[AmdSmiCollectorArgs] = None + ) -> tuple[TaskResult, Optional[AmdSmiFlavorDataModel]]: + """Collect amd-smi data across driver flavors (guest/bare-metal/host, incl. ESXi).""" + if not self._check_amdsmi_installed() or not self._check_gpu_driver_loaded(): + self._log_event( + category=EventCategory.APPLICATION, + description="amd-smi not installed or no AMD GPU driver loaded", + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.NOT_RAN + return self.result, None + try: + self.amd_smi_commands = self.detect_amdsmi_commands() + amd_smi_data = self._get_amdsmi_data(args) + return self.result, amd_smi_data + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi collector", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result, None diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py new file mode 100644 index 00000000..18d46235 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py @@ -0,0 +1,18 @@ +from .amdsmi_flavor_collector import AmdSmiFlavorCollector +from .amdsmi_plugin import AmdSmiPlugin +from .amdsmidata_flavor import AmdSmiFlavorDataModel + + +class AmdSmiFlavorPlugin(AmdSmiPlugin): + """amd-smi plugin variant with driver-flavor collection. + + Detects the loaded driver flavor and issues flavor-appropriate amd-smi commands: + - mxGPU host driver (gim on Linux, amdgpuv on ESXi) -> HostDriver* models + - guest amdgpu on a virtual function inside a VM -> Guest* models + - bare-metal amdgpu on physical hardware -> base AmdSmi* models + Data is built into AmdSmiFlavorDataModel. The base AmdSmiPlugin (guest/bare-metal + only) is left unchanged for callers that do not need host/VF support. + """ + + DATA_MODEL = AmdSmiFlavorDataModel # type: ignore[assignment] + COLLECTOR = AmdSmiFlavorCollector diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py new file mode 100644 index 00000000..82a68f5d --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py @@ -0,0 +1,61 @@ +"""amd-smi data model that widens the base (guest/bare-metal only) ``AmdSmiDataModel`` +field types to also accept the mxGPU host-driver and guest-VF model variants. The +collector picks the concrete model per driver flavor; this model only relaxes the +annotations so those instances are accepted and serialized. The base ``AmdSmiDataModel`` +and its consumers are unchanged. +""" + +from __future__ import annotations + +from typing import Optional, Union + +from pydantic import Field + +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiDataModel, + AmdSmiListItem, + AmdSmiMetric, + AmdSmiStatic, + AmdSmiVersion, + BadPages, + Topo, + XgmiLinks, + XgmiMetrics, +) + +from .amdsmidata_guest import GuestAmdSmiMetric, GuestAmdSmiStatic +from .amdsmidata_host import ( + HostDriverAmdSmiListItem, + HostDriverAmdSmiMetric, + HostDriverAmdSmiStatic, + HostDriverAmdSmiVersion, + HostDriverBadPages, + HostDriverTopo, + HostDriverXgmiLinks, + HostDriverXgmiMetrics, +) + + +class AmdSmiFlavorDataModel(AmdSmiDataModel): + """AmdSmiDataModel with host/guest driver-flavor model variants allowed.""" + + # Base (guest/bare-metal) type is listed first; the collector builds the right + # concrete instance per flavor, so widening the annotation is sufficient. + # Widening the base annotations is an intentional LSP override (invariant list), + # so the type: ignore[assignment] markers below are expected. + version: Optional[Union[AmdSmiVersion, HostDriverAmdSmiVersion]] = None # type: ignore[assignment] + gpu_list: Optional[list[Union[AmdSmiListItem, HostDriverAmdSmiListItem]]] = Field( # type: ignore[assignment] + default_factory=list + ) + topology: Optional[list[Union[Topo, HostDriverTopo]]] = Field(default_factory=list) # type: ignore[assignment] + bad_pages: Optional[list[Union[BadPages, HostDriverBadPages]]] = Field(default_factory=list) # type: ignore[assignment] + static: Optional[list[Union[AmdSmiStatic, GuestAmdSmiStatic, HostDriverAmdSmiStatic]]] = Field( # type: ignore[assignment] + default_factory=list + ) + metric: Optional[list[Union[AmdSmiMetric, GuestAmdSmiMetric, HostDriverAmdSmiMetric]]] = Field( # type: ignore[assignment] + default_factory=list + ) + xgmi_metric: Optional[list[Union[XgmiMetrics, HostDriverXgmiMetrics]]] = Field( # type: ignore[assignment] + default_factory=list + ) + xgmi_link: Optional[list[Union[XgmiLinks, HostDriverXgmiLinks]]] = Field(default_factory=list) # type: ignore[assignment] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py new file mode 100644 index 00000000..033726f2 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py @@ -0,0 +1,71 @@ +"""Guest-VF (virtualized amdgpu) Pydantic models for amd-smi data. + +A guest VM sees a single amdgpu virtual function (VF). Its amd-smi build shares +the bare-metal amdgpu JSON schema EXCEPT that a VF does not expose physical / +host-owned fields: fan control, SoC power state, power/thermal limits, NUMA +topology, XGMI power-down policy, and energy/throttle/voltage-curve/perf-level +counters. The bare-metal ``AmdSmi*`` models mark those fields as required, so a +guest payload fails to build against them; these subclasses relax exactly those +fields to optional and inherit all other structure unchanged. + +amd-smi was not exercised on the guest before the host/guest driver-flavor work, +so the base ``AmdSmi*`` models had implicitly assumed bare-metal amdgpu. +""" + +from __future__ import annotations + +from pydantic import ConfigDict + +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiMetric, + AmdSmiStatic, + MetricEnergy, + MetricFan, + MetricMemUsage, + MetricThrottle, + MetricVoltageCurve, + StaticBus, + StaticLimit, + StaticNuma, + StaticSocPstate, + StaticVbios, + StaticXgmiPlpd, +) + + +class GuestStaticBus(StaticBus): + """Guest amd-smi (ROCm 7.14+) adds a per-link ``pcie_levels`` block to the bus + section that the base ``StaticBus`` (``extra="forbid"``) rejects. Ignore unknown + bus fields so a new field doesn't drop the whole static payload, matching the + host models' schema-drift resilience.""" + + model_config = ConfigDict(extra="ignore") + + +class GuestAmdSmiStatic(AmdSmiStatic): + """Bare-metal static schema minus the physical fields a guest VF omits.""" + + # Base bus (StaticBus) forbids extras; the guest build adds pcie_levels. + bus: GuestStaticBus # type: ignore[assignment] + soc_pstate: StaticSocPstate | None = None + xgmi_plpd: StaticXgmiPlpd | None = None + numa: StaticNuma | None = None # type: ignore[assignment] + limit: StaticLimit | None = None + # A guest VF's static payload carries an ``ifwi`` firmware block (name, + # build_date, part_number, version) that the bare-metal schema omits; its + # shape matches StaticVbios. + ifwi: StaticVbios | None = None + + +class GuestAmdSmiMetric(AmdSmiMetric): + """Bare-metal metric schema minus the physical fields a guest VF omits.""" + + # These physical fields are required on the bare-metal base model; a guest VF + # omits them, so relax to Optional (intentional LSP-widening override). + fan: MetricFan | None = None # type: ignore[assignment] + voltage_curve: MetricVoltageCurve | None = None + perf_level: str | dict | None = None + xgmi_err: str | dict | None = None + energy: MetricEnergy | None = None + throttle: MetricThrottle | None = None # type: ignore[assignment] + mem_usage: MetricMemUsage | None = None # type: ignore[assignment] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py new file mode 100644 index 00000000..ce0a91be --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py @@ -0,0 +1,502 @@ +"""Host-driver (mxGPU) Pydantic models for amd-smi data. + +The mxGPU host driver ships an amd-smi build whose JSON schema differs from the +guest amdgpu build. The same host schema is used by both `gim` (Linux host) and +`amdgpuv` (ESXi host), so these models serve both; the base AmdSmi* models serve +the guest amdgpu build. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import ( + AliasPath, + BaseModel, + ConfigDict, + Field, + computed_field, + field_validator, +) + +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiBaseModel, + EccData, + EccState, + ValueUnit, +) + + +def _host_na_to_none(value: Any) -> Any: + """Convert amd-smi N/A markers to None. + + Newer amd-smi (e.g. 37.0.5) emits N/A both as a scalar ("N/A") and as a + ``{"value": "N/A", "unit": "N/A"}`` dict for ValueUnit fields; the base + ValueUnit coercion does not collapse the dict form inside a ``ValueUnit | None`` + union, so normalize both to None before field validation. + """ + if isinstance(value, str) and value.strip().upper() in ("N/A", "NA", ""): + return None + if isinstance(value, dict): + inner = value.get("value") + if isinstance(inner, str) and inner.strip().upper() in ("N/A", "NA", ""): + return None + return value + + +class HostDriverAmdSmiVersion(BaseModel): + """mxGPU host `amd-smi version --json` nests its fields under a top-level + "version" key: ``{"version": {"tool_name": ..., "tool_version": ...}}``. + + AliasPath reads each field from that nested dict, so the collector can build + this model directly from the raw item - no unwrapping needed. (Guest/bare-metal + amdgpu instead put a scalar "version" string in a flat dict; that is the + separate AmdSmiVersion model.) + """ + + tool_name: str | None = Field(default=None, validation_alias=AliasPath("version", "tool_name")) + tool_version: str | None = Field( + default=None, validation_alias=AliasPath("version", "tool_version") + ) + lib_version: str | None = Field( + default=None, validation_alias=AliasPath("version", "lib_version") + ) + driver_version: str | None = Field( + default=None, validation_alias=AliasPath("version", "driver_version") + ) + # mxGPU host amd-smi does not report a ROCm version; kept here (always None) so + # version consumers - e.g. the amdsmitst ROCm-version gate - can read it uniformly. + rocm_version: str | None = None + + +class HostDriverAmdSmiListItem(BaseModel): + """mxGPU host amd-smi list has gpu/bdf/uuid/vfs — no kfd_id/node_id/partition_id.""" + + model_config = ConfigDict(extra="ignore") + + gpu: int + bdf: str + uuid: str + + +class HostDriverStaticAsic(BaseModel): + # Expose num_compute_units (matching the guest model + AmdSmiAnalyzer) while + # accepting the host driver's JSON key "num_of_compute_units" via alias. + model_config = ConfigDict(populate_by_name=True) + + market_name: str + vendor_id: str + vendor_name: str + subvendor_id: str + device_id: str + subsystem_id: str + rev_id: str + asic_serial: str + oam_id: int | str + num_compute_units: int | str = Field(alias="num_of_compute_units") + # Host driver amd-smi does not report target_graphics_version; keep optional + # so AmdSmiAnalyzer.static_consistancy_check can read it uniformly. + target_graphics_version: str | None = None + + +class HostDriverStaticIfwi(BaseModel): + name: str + build_date: str + part_number: str + version: str + boot_firmware: str | None = None + + +class HostDriverStaticBus(AmdSmiBaseModel): + bdf: str + max_pcie_width: ValueUnit + max_pcie_speed: ValueUnit + pcie_interface_version: str + slot_type: str + max_pcie_interface_version: str | None = None + + +class HostDriverStaticPpt(AmdSmiBaseModel): + """Host per-PPT power caps (amd-smi 37.0.5+ nests these under limit.ppt0/ppt1). + + The host build keys these ``max_power``/``min_power``/``socket_power`` — unlike + the base ``StaticPowerLimit`` (guest/bare-metal ROCm 7+) which uses the + ``*_power_limit`` key names. + """ + + model_config = ConfigDict(extra="ignore") + + max_power: ValueUnit | None = None + min_power: ValueUnit | None = None + socket_power: ValueUnit | None = None + na_validator = field_validator( + "max_power", + "min_power", + "socket_power", + mode="before", + )(_host_na_to_none) + + +class HostDriverStaticLimit(AmdSmiBaseModel): + # Host amd-smi adds per-partition power fields (ppt0/ppt1, ...) across tool + # versions; ignore unknown fields so a new field doesn't drop all static data. + model_config = ConfigDict(extra="ignore") + + max_power: ValueUnit | None = None + min_power: ValueUnit | None = None + socket_power: ValueUnit | None = None + slowdown_edge_temperature: ValueUnit | None = None + slowdown_hotspot_temperature: ValueUnit | None = None + slowdown_mem_temperature: ValueUnit | None = None + shutdown_edge_temperature: ValueUnit | None = None + shutdown_hotspot_temperature: ValueUnit | None = None + shutdown_mem_temperature: ValueUnit | None = None + # amd-smi 37.0.5 moved the flat max/min/socket power caps under ppt0 (active + # profile) / ppt1 (secondary, often N/A); older builds used the flat fields. + ppt0: HostDriverStaticPpt | None = None + ppt1: HostDriverStaticPpt | None = None + na_validator = field_validator( + "max_power", + "min_power", + "socket_power", + "slowdown_edge_temperature", + "slowdown_hotspot_temperature", + "slowdown_mem_temperature", + "shutdown_edge_temperature", + "shutdown_hotspot_temperature", + "shutdown_mem_temperature", + "ppt0", + "ppt1", + mode="before", + )(_host_na_to_none) + + def resolved_max_power(self) -> ValueUnit | None: + """Match the base StaticLimit contract so the shared analyzer / data-model + max-power helpers work on host data. Older host builds expose a flat + max_power; amd-smi 37.0.5 moved it under limit.ppt0.""" + if self.max_power is not None: + return self.max_power + if self.ppt0 is not None and self.ppt0.max_power is not None: + return self.ppt0.max_power + return None + + +class HostDriverStaticDriver(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + version: str + date: str | None = None + model: str | None = None + + +class HostDriverStaticBoard(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + # "model_" is a pydantic-reserved namespace; expose as amdsmi_model_number + # (matching the base StaticBoard) while reading the JSON "model_number" key. + amdsmi_model_number: str = Field(alias="model_number") + product_serial: str + fru_id: str + product_name: str + manufacturer_name: str + + +class HostDriverStaticRas(BaseModel): + model_config = ConfigDict(extra="ignore") + + eeprom_version: str + parity_schema: EccState + single_bit_schema: EccState + double_bit_schema: EccState + poison_schema: EccState + block_state: dict[str, EccState] | str + + +class HostDriverStaticFbInfo(AmdSmiBaseModel): + total_fb_size: ValueUnit | None = None + pf_fb_reserved: ValueUnit | None = None + pf_fb_offset: ValueUnit | None = None + fb_alignment: ValueUnit | None = None + max_vf_fb_usable: ValueUnit | None = None + min_vf_fb_usable: ValueUnit | None = None + + +class HostDriverStaticNumVf(BaseModel): + supported: int + enabled: int + + +class HostDriverStaticVram(AmdSmiBaseModel): + type: str + vendor: str | None = None + size: ValueUnit | None = None + bit_width: ValueUnit | int | None = None + max_bandwidth: ValueUnit | None = None + na_validator = field_validator("vendor", "size", "bit_width", "max_bandwidth", mode="before")( + _host_na_to_none + ) + + +class HostDriverStaticNuma(BaseModel): + # gim (Linux mxGPU host) reports cpu_affinity as a dict ({"cpu_list": [...]}); + # amdgpuv (ESXi mxGPU host) reports it as int/str. Accept both. + node: int | str | dict | None = None + cpu_affinity: int | str | dict | None = None + socket_affinity: int | str | dict | None = None + na_validator = field_validator("node", "cpu_affinity", "socket_affinity", mode="before")( + _host_na_to_none + ) + + +class HostDriverStaticPartition(BaseModel): + model_config = ConfigDict(extra="ignore") + + accelerator_partition: str + memory_partition: str + partition_id: list[int] | int + + +class HostDriverStaticXgmiPlpd(BaseModel): + num_supported: int + current_id: int + policies: list[dict] = Field(default_factory=list) + + +class HostDriverAmdSmiStatic(BaseModel): + """mxGPU host static model — uses ifwi instead of vbios, has fb_info/num-vf, etc.""" + + model_config = ConfigDict(extra="ignore") + + gpu: int + asic: HostDriverStaticAsic + bus: HostDriverStaticBus + ifwi: HostDriverStaticIfwi | None = None + limit: HostDriverStaticLimit | None = None + driver: HostDriverStaticDriver + board: HostDriverStaticBoard + ras: HostDriverStaticRas + fb_info: HostDriverStaticFbInfo | None = None + num_vf: HostDriverStaticNumVf | None = Field(default=None, alias="num-vf") + vram: HostDriverStaticVram + cache_info: list[dict] = Field(default_factory=list) + xgmi_plpd: HostDriverStaticXgmiPlpd | None = Field(default=None, alias="xgmi-plpd") + partition: HostDriverStaticPartition | None = None + numa: HostDriverStaticNuma + na_validator = field_validator("limit", "ifwi", "xgmi_plpd", mode="before")(_host_na_to_none) + + +# --- Metric models --- + + +class HostDriverMetricUsage(BaseModel): + gfx_activity: ValueUnit | None = None + umc_activity: ValueUnit | None = None + mm_activity: ValueUnit | None = None + vcn_activity: list[ValueUnit | str | None] = Field(default_factory=list) + jpeg_activity: list[ValueUnit | str | None] = Field(default_factory=list) + na_validator = field_validator("gfx_activity", "umc_activity", "mm_activity", mode="before")( + _host_na_to_none + ) + + +class HostDriverMetricPower(BaseModel): + socket_power: ValueUnit | None = None + gfx_voltage: ValueUnit | None = None + soc_voltage: ValueUnit | None = None + mem_voltage: ValueUnit | None = None + power_management: str | None = None + na_validator = field_validator( + "socket_power", + "gfx_voltage", + "soc_voltage", + "mem_voltage", + "power_management", + mode="before", + )(_host_na_to_none) + + +class HostDriverMetricClockData(BaseModel): + clk: ValueUnit | None = None + min_clk: ValueUnit | None = None + max_clk: ValueUnit | None = None + clk_locked: int | str | dict | None = None + deep_sleep: int | str | dict | None = None + na_validator = field_validator( + "clk", "min_clk", "max_clk", "clk_locked", "deep_sleep", mode="before" + )(_host_na_to_none) + + +class HostDriverMetricTemperature(BaseModel): + edge: ValueUnit | None = None + hotspot: ValueUnit | None = None + mem: ValueUnit | None = None + na_validator = field_validator("edge", "hotspot", "mem", mode="before")(_host_na_to_none) + + +class HostDriverMetricPcie(BaseModel): + width: int | None = None + speed: ValueUnit | None = None + bandwidth: ValueUnit | None = None + replay_count: int | None = None + l0_to_recovery_count: int | None = None + replay_roll_over_count: int | None = None + nak_sent_count: int | None = None + nak_received_count: int | None = None + na_validator = field_validator( + "width", + "speed", + "bandwidth", + "replay_count", + "l0_to_recovery_count", + "replay_roll_over_count", + "nak_sent_count", + "nak_received_count", + mode="before", + )(_host_na_to_none) + + +class HostDriverMetricEccTotals(BaseModel): + total_correctable_count: int | None = None + total_uncorrectable_count: int | None = None + total_deferred_count: int | None = None + cache_correctable_count: int | None = None + cache_uncorrectable_count: int | None = None + na_validator = field_validator( + "total_correctable_count", + "total_uncorrectable_count", + "total_deferred_count", + "cache_correctable_count", + "cache_uncorrectable_count", + mode="before", + )(_host_na_to_none) + + +class HostDriverMetricEnergy(BaseModel): + total_energy_consumption: ValueUnit | None = None + na_validator = field_validator("total_energy_consumption", mode="before")(_host_na_to_none) + + +class HostDriverMetricGpuBoard(BaseModel): + model_config = ConfigDict(extra="allow") + + +class HostDriverAmdSmiMetric(BaseModel): + """mxGPU host metric model — no fan/voltage_curve/perf_level/xgmi_err/mem_usage/throttle, + has gpuboard instead.""" + + model_config = ConfigDict(extra="ignore") + + gpu: int + usage: HostDriverMetricUsage | str + power: HostDriverMetricPower + clock: dict[str, HostDriverMetricClockData | dict] + temperature: HostDriverMetricTemperature + pcie: HostDriverMetricPcie + ecc: HostDriverMetricEccTotals + ecc_blocks: dict[str, EccData] | str = Field(default_factory=dict) + energy: HostDriverMetricEnergy | None = None + gpuboard: HostDriverMetricGpuBoard | None = None + + @field_validator("ecc_blocks", mode="before") + @classmethod + def validate_ecc_blocks(cls, value): + if isinstance(value, str): + return {} + return value + + @field_validator("energy", mode="before") + @classmethod + def validate_energy(cls, value): + if value == "N/A" or value is None: + return None + return value + + +# --- Topology --- + + +class HostDriverTopoLink(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + gpu: int + bdf: str + weight: int + link_type: str + num_hops: int + bandwidth: str = "" + fb_sharing: str | None = None + coherent: str | None = None + atomics: str | None = None + dma: str | None = None + bi_dir: str | None = Field(default=None, alias="bi-dir") + + @computed_field + def bandwidth_from(self) -> int | None: + bw_split = self.bandwidth.split("-") + return int(bw_split[0]) if len(bw_split) == 2 else None + + @computed_field + def bandwidth_to(self) -> int | None: + bw_split = self.bandwidth.split("-") + return int(bw_split[1]) if len(bw_split) == 2 else None + + +class HostDriverTopo(BaseModel): + gpu: int + bdf: str + links: list[HostDriverTopoLink] + + +# --- Bad pages --- + + +class HostDriverBadPageEntry(BaseModel): + model_config = ConfigDict(extra="ignore") + + bad_page: int + retired_bad_page: str + timestamp: str | None = None + mem_channel: int | None = None + mcumc_id: int | None = None + + +class HostDriverBadPages(BaseModel): + gpu: int + bad_pages: list[HostDriverBadPageEntry] = Field(default_factory=list) + + +# --- XGMI --- + + +class HostDriverXgmiLink(BaseModel): + gpu: int + bdf: str + read: ValueUnit | None = None + write: ValueUnit | None = None + na_validator = field_validator("read", "write", mode="before")(_host_na_to_none) + + +class HostDriverXgmiLinkMetrics(BaseModel): + bit_rate: ValueUnit | None = None + max_bandwidth: ValueUnit | None = None + links: list[HostDriverXgmiLink] = Field(default_factory=list) + na_validator = field_validator("max_bandwidth", "bit_rate", mode="before")(_host_na_to_none) + + +class HostDriverXgmiMetrics(BaseModel): + gpu: int + bdf: str + link_metrics: HostDriverXgmiLinkMetrics + + +class HostDriverXgmiLinkStatus(BaseModel): + gpu: int + bdf: str + status: str + + +class HostDriverXgmiLinks(BaseModel): + gpu: int + bdf: str + link_status: list[HostDriverXgmiLinkStatus] diff --git a/test/unit/plugin/test_amdsmi_flavor_collector.py b/test/unit/plugin/test_amdsmi_flavor_collector.py new file mode 100644 index 00000000..49c2013a --- /dev/null +++ b/test/unit/plugin/test_amdsmi_flavor_collector.py @@ -0,0 +1,213 @@ +from unittest.mock import MagicMock + +import pytest +from pydantic import BaseModel + +from nodescraper.enums import ExecutionStatus, OSFamily +from nodescraper.enums.systeminteraction import SystemInteractionLevel +from nodescraper.plugins.inband.amdsmi.amdsmi_flavor_collector import ( + AmdSmiFlavorCollector, +) +from nodescraper.plugins.inband.amdsmi.amdsmidata_host import HostDriverAmdSmiVersion + + +class _Tiny(BaseModel): + gpu: int + + +@pytest.fixture +def collector(system_info, conn_mock): + return AmdSmiFlavorCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + + +def _cmd(exit_code=0, stdout="", stderr=""): + return MagicMock(exit_code=exit_code, stdout=stdout, stderr=stderr, command="x") + + +# ---- driver-flavor detection -------------------------------------------------- + + +def test_loaded_driver_host_linux(collector): + """gim in lsmod -> host driver flavor.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="gim 12345 0\namdxcp 1 0")) + assert collector._get_loaded_gpu_driver() == "gim" + assert collector.is_host_driver is True + assert collector.is_guest_driver is False + + +def test_loaded_driver_guest_linux(collector): + """amdgpu + hypervisor flag -> guest-VF flavor.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(side_effect=[_cmd(stdout="amdgpu 999 0"), _cmd(stdout="1")]) + assert collector.is_host_driver is False + assert collector.is_guest_driver is True + + +def test_loaded_driver_baremetal_linux(collector): + """amdgpu with no hypervisor flag -> bare-metal (neither host nor guest).""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(side_effect=[_cmd(stdout="amdgpu 999 0"), _cmd(stdout="0")]) + assert collector.is_host_driver is False + assert collector.is_guest_driver is False + + +def test_loaded_driver_esxi_host(collector): + """ESXi reads modules via vmkload_mod; amdgpuv -> host flavor, never virtualized.""" + collector.system_info.os_family = OSFamily.ESXI + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="vmklinux\namdgpuv\nvmkapi")) + assert collector._get_loaded_gpu_driver() == "amdgpuv" + assert collector.is_host_driver is True + assert collector._is_virtualized() is False + + +def test_loaded_driver_none_means_not_loaded(collector): + """No known AMD GPU module -> None -> driver not loaded.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="ext4\nnvme")) + assert collector._get_loaded_gpu_driver() is None + assert collector._check_gpu_driver_loaded() is False + + +def test_loaded_driver_cached(collector): + """The driver lookup is cached after the first probe.""" + collector.system_info.os_family = OSFamily.LINUX + run = MagicMock(return_value=_cmd(stdout="gim 1 0")) + collector._run_sut_cmd = run + collector._get_loaded_gpu_driver() + collector._get_loaded_gpu_driver() + assert run.call_count == 1 + + +# ---- _build_flavor_model ------------------------------------------------------ + + +def test_build_flavor_model_dict(collector): + out = collector._build_flavor_model(_Tiny, {"gpu": 3}) + assert isinstance(out, _Tiny) and out.gpu == 3 + + +def test_build_flavor_model_none_logs_error(collector): + assert collector._build_flavor_model(_Tiny, None) is None + assert any(e.priority.name == "ERROR" for e in collector.result.events) + + +def test_build_flavor_model_keep_key_drops_missing(collector): + """Rows without keep_key are dropped silently (non-GPU enumeration rows).""" + out = collector._build_flavor_model( + _Tiny, [{"gpu": 0}, {"other": 1}, {"gpu": 2}], keep_key="gpu" + ) + assert [m.gpu for m in out] == [0, 2] + + +def test_build_flavor_model_skips_unparseable_row(collector): + """A present-but-invalid row is skipped and a warning is logged; valid rows kept.""" + out = collector._build_flavor_model(_Tiny, [{"gpu": 0}, {"gpu": "NA"}]) + assert [m.gpu for m in out] == [0] + assert any(e.priority.name == "WARNING" for e in collector.result.events) + + +def test_build_flavor_model_return_first(collector): + out = collector._build_flavor_model(_Tiny, [{"gpu": 7}, {"gpu": 8}], return_first=True) + assert isinstance(out, _Tiny) and out.gpu == 7 + + +# ---- flavor-aware command dispatch ------------------------------------------- + + +def test_detect_commands_host_uses_help_flag(collector): + collector._loaded_gpu_driver_cache = "gim" # force host + collector._run_amd_smi = MagicMock(return_value=" static Foo\n metric Bar\n") + cmds = collector.detect_amdsmi_commands() + collector._run_amd_smi.assert_called_once_with("help") + assert {"static", "metric"} <= cmds + + +def test_detect_commands_guest_uses_dash_h(collector): + collector._loaded_gpu_driver_cache = "amdgpu" # not host + collector._run_amd_smi = MagicMock(return_value=" list Foo\n") + collector.detect_amdsmi_commands() + collector._run_amd_smi.assert_called_once_with("-h") + + +def test_get_static_host_vs_nonhost_subcommand(collector): + collector.amd_smi_commands = {"static"} + collector._run_amd_smi_dict = MagicMock(return_value=[{"gpu": 0}, {"no_gpu": 1}]) + collector._loaded_gpu_driver_cache = "gim" # host + assert collector.get_static() == [{"gpu": 0}] + assert collector._run_amd_smi_dict.call_args.args[0] == "static" + + collector._run_amd_smi_dict.reset_mock() + collector._loaded_gpu_driver_cache = "amdgpu" # non-host + collector.get_static() + assert collector._run_amd_smi_dict.call_args.args[0] == "static -g all" + + +def test_get_static_unsupported_returns_none(collector): + collector.amd_smi_commands = set() + assert collector.get_static() is None + + +def test_get_metric_unwraps_gpu_data(collector): + collector.amd_smi_commands = {"metric"} + collector._loaded_gpu_driver_cache = "gim" + collector._run_amd_smi_dict = MagicMock(return_value={"gpu_data": [{"gpu": 0}, {"x": 1}]}) + assert collector.get_metric() == [{"gpu": 0}] + + +def test_get_partition_host_parses_text_table(collector): + collector.amd_smi_commands = {"partition"} + collector._loaded_gpu_driver_cache = "gim" # host: no --json, parse text + table = "GPU memory accelerator_type accelerator_profile_index partition_id\n0 NPS1 SPX 0 0\n" + collector._run_amd_smi = MagicMock(return_value=table) + out = collector.get_partition() + assert out == { + "current_partition": [ + { + "gpu_id": 0, + "memory": "NPS1", + "accelerator_type": "SPX", + "accelerator_profile_index": "0", + "partition_id": "0", + } + ] + } + + +def test_get_xgmi_flags_host_vs_guest(collector): + collector.amd_smi_commands = {"xgmi"} + collector._loaded_gpu_driver_cache = "gim" # host + collector._run_amd_smi_dict = MagicMock(return_value=[]) + collector.get_xgmi_data_metric() + flags = [c.args[0] for c in collector._run_amd_smi_dict.call_args_list] + assert "xgmi --metric" in flags and "xgmi --link-status" in flags + + collector._run_amd_smi_dict.reset_mock() + collector._loaded_gpu_driver_cache = "amdgpu" # guest/bare-metal + collector.get_xgmi_data_metric() + flags = [c.args[0] for c in collector._run_amd_smi_dict.call_args_list] + assert "xgmi -m" in flags and "xgmi -l" in flags + + +def test_version_model_selected_by_flavor(collector): + collector._loaded_gpu_driver_cache = "gim" # host + collector._run_amd_smi_dict = MagicMock( + return_value={"version": "1.0", "amdsmi_library_version": "37.0.5"} + ) + out = collector._get_amdsmi_version() + assert isinstance(out, HostDriverAmdSmiVersion) + + +# ---- collect_data guard ------------------------------------------------------- + + +def test_collect_data_no_driver_not_ran(collector): + collector._check_amdsmi_installed = MagicMock(return_value=True) + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="ext4")) # no driver + result, data = collector.collect_data() + assert result.status == ExecutionStatus.NOT_RAN + assert data is None From dba4aa814c39364bf084183f3b0a2577e0d1d82d Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 18 Sep 2026 21:31:31 +0000 Subject: [PATCH 12/12] Use Optional/Union instead of PEP 604 unions in amd-smi host/guest models node-scraper targets Python 3.9, where pydantic cannot evaluate `X | Y` string annotations at model-build time (no eval_type_backport dependency). Convert the migrated host/guest model field annotations and the two computed_field return types to typing.Optional/Union, matching the rest of the codebase. --- .../plugins/inband/amdsmi/amdsmidata_guest.py | 26 +-- .../plugins/inband/amdsmi/amdsmidata_host.py | 202 +++++++++--------- 2 files changed, 116 insertions(+), 112 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py index 033726f2..ef2eb658 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py @@ -14,6 +14,8 @@ from __future__ import annotations +from typing import Optional, Union + from pydantic import ConfigDict from nodescraper.plugins.inband.amdsmi.amdsmidata import ( @@ -47,14 +49,14 @@ class GuestAmdSmiStatic(AmdSmiStatic): # Base bus (StaticBus) forbids extras; the guest build adds pcie_levels. bus: GuestStaticBus # type: ignore[assignment] - soc_pstate: StaticSocPstate | None = None - xgmi_plpd: StaticXgmiPlpd | None = None - numa: StaticNuma | None = None # type: ignore[assignment] - limit: StaticLimit | None = None + soc_pstate: Optional[StaticSocPstate] = None + xgmi_plpd: Optional[StaticXgmiPlpd] = None + numa: Optional[StaticNuma] = None # type: ignore[assignment] + limit: Optional[StaticLimit] = None # A guest VF's static payload carries an ``ifwi`` firmware block (name, # build_date, part_number, version) that the bare-metal schema omits; its # shape matches StaticVbios. - ifwi: StaticVbios | None = None + ifwi: Optional[StaticVbios] = None class GuestAmdSmiMetric(AmdSmiMetric): @@ -62,10 +64,10 @@ class GuestAmdSmiMetric(AmdSmiMetric): # These physical fields are required on the bare-metal base model; a guest VF # omits them, so relax to Optional (intentional LSP-widening override). - fan: MetricFan | None = None # type: ignore[assignment] - voltage_curve: MetricVoltageCurve | None = None - perf_level: str | dict | None = None - xgmi_err: str | dict | None = None - energy: MetricEnergy | None = None - throttle: MetricThrottle | None = None # type: ignore[assignment] - mem_usage: MetricMemUsage | None = None # type: ignore[assignment] + fan: Optional[MetricFan] = None # type: ignore[assignment] + voltage_curve: Optional[MetricVoltageCurve] = None + perf_level: Optional[Union[str, dict]] = None + xgmi_err: Optional[Union[str, dict]] = None + energy: Optional[MetricEnergy] = None + throttle: Optional[MetricThrottle] = None # type: ignore[assignment] + mem_usage: Optional[MetricMemUsage] = None # type: ignore[assignment] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py index ce0a91be..6f53e6fc 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py @@ -8,7 +8,7 @@ from __future__ import annotations -from typing import Any +from typing import Any, Optional, Union from pydantic import ( AliasPath, @@ -54,19 +54,21 @@ class HostDriverAmdSmiVersion(BaseModel): separate AmdSmiVersion model.) """ - tool_name: str | None = Field(default=None, validation_alias=AliasPath("version", "tool_name")) - tool_version: str | None = Field( + tool_name: Optional[str] = Field( + default=None, validation_alias=AliasPath("version", "tool_name") + ) + tool_version: Optional[str] = Field( default=None, validation_alias=AliasPath("version", "tool_version") ) - lib_version: str | None = Field( + lib_version: Optional[str] = Field( default=None, validation_alias=AliasPath("version", "lib_version") ) - driver_version: str | None = Field( + driver_version: Optional[str] = Field( default=None, validation_alias=AliasPath("version", "driver_version") ) # mxGPU host amd-smi does not report a ROCm version; kept here (always None) so # version consumers - e.g. the amdsmitst ROCm-version gate - can read it uniformly. - rocm_version: str | None = None + rocm_version: Optional[str] = None class HostDriverAmdSmiListItem(BaseModel): @@ -92,11 +94,11 @@ class HostDriverStaticAsic(BaseModel): subsystem_id: str rev_id: str asic_serial: str - oam_id: int | str - num_compute_units: int | str = Field(alias="num_of_compute_units") + oam_id: Union[int, str] + num_compute_units: Union[int, str] = Field(alias="num_of_compute_units") # Host driver amd-smi does not report target_graphics_version; keep optional # so AmdSmiAnalyzer.static_consistancy_check can read it uniformly. - target_graphics_version: str | None = None + target_graphics_version: Optional[str] = None class HostDriverStaticIfwi(BaseModel): @@ -104,7 +106,7 @@ class HostDriverStaticIfwi(BaseModel): build_date: str part_number: str version: str - boot_firmware: str | None = None + boot_firmware: Optional[str] = None class HostDriverStaticBus(AmdSmiBaseModel): @@ -113,7 +115,7 @@ class HostDriverStaticBus(AmdSmiBaseModel): max_pcie_speed: ValueUnit pcie_interface_version: str slot_type: str - max_pcie_interface_version: str | None = None + max_pcie_interface_version: Optional[str] = None class HostDriverStaticPpt(AmdSmiBaseModel): @@ -126,9 +128,9 @@ class HostDriverStaticPpt(AmdSmiBaseModel): model_config = ConfigDict(extra="ignore") - max_power: ValueUnit | None = None - min_power: ValueUnit | None = None - socket_power: ValueUnit | None = None + max_power: Optional[ValueUnit] = None + min_power: Optional[ValueUnit] = None + socket_power: Optional[ValueUnit] = None na_validator = field_validator( "max_power", "min_power", @@ -142,19 +144,19 @@ class HostDriverStaticLimit(AmdSmiBaseModel): # versions; ignore unknown fields so a new field doesn't drop all static data. model_config = ConfigDict(extra="ignore") - max_power: ValueUnit | None = None - min_power: ValueUnit | None = None - socket_power: ValueUnit | None = None - slowdown_edge_temperature: ValueUnit | None = None - slowdown_hotspot_temperature: ValueUnit | None = None - slowdown_mem_temperature: ValueUnit | None = None - shutdown_edge_temperature: ValueUnit | None = None - shutdown_hotspot_temperature: ValueUnit | None = None - shutdown_mem_temperature: ValueUnit | None = None + max_power: Optional[ValueUnit] = None + min_power: Optional[ValueUnit] = None + socket_power: Optional[ValueUnit] = None + slowdown_edge_temperature: Optional[ValueUnit] = None + slowdown_hotspot_temperature: Optional[ValueUnit] = None + slowdown_mem_temperature: Optional[ValueUnit] = None + shutdown_edge_temperature: Optional[ValueUnit] = None + shutdown_hotspot_temperature: Optional[ValueUnit] = None + shutdown_mem_temperature: Optional[ValueUnit] = None # amd-smi 37.0.5 moved the flat max/min/socket power caps under ppt0 (active # profile) / ppt1 (secondary, often N/A); older builds used the flat fields. - ppt0: HostDriverStaticPpt | None = None - ppt1: HostDriverStaticPpt | None = None + ppt0: Optional[HostDriverStaticPpt] = None + ppt1: Optional[HostDriverStaticPpt] = None na_validator = field_validator( "max_power", "min_power", @@ -170,7 +172,7 @@ class HostDriverStaticLimit(AmdSmiBaseModel): mode="before", )(_host_na_to_none) - def resolved_max_power(self) -> ValueUnit | None: + def resolved_max_power(self) -> Optional[ValueUnit]: """Match the base StaticLimit contract so the shared analyzer / data-model max-power helpers work on host data. Older host builds expose a flat max_power; amd-smi 37.0.5 moved it under limit.ppt0.""" @@ -186,8 +188,8 @@ class HostDriverStaticDriver(BaseModel): name: str version: str - date: str | None = None - model: str | None = None + date: Optional[str] = None + model: Optional[str] = None class HostDriverStaticBoard(BaseModel): @@ -210,16 +212,16 @@ class HostDriverStaticRas(BaseModel): single_bit_schema: EccState double_bit_schema: EccState poison_schema: EccState - block_state: dict[str, EccState] | str + block_state: Union[dict[str, EccState], str] class HostDriverStaticFbInfo(AmdSmiBaseModel): - total_fb_size: ValueUnit | None = None - pf_fb_reserved: ValueUnit | None = None - pf_fb_offset: ValueUnit | None = None - fb_alignment: ValueUnit | None = None - max_vf_fb_usable: ValueUnit | None = None - min_vf_fb_usable: ValueUnit | None = None + total_fb_size: Optional[ValueUnit] = None + pf_fb_reserved: Optional[ValueUnit] = None + pf_fb_offset: Optional[ValueUnit] = None + fb_alignment: Optional[ValueUnit] = None + max_vf_fb_usable: Optional[ValueUnit] = None + min_vf_fb_usable: Optional[ValueUnit] = None class HostDriverStaticNumVf(BaseModel): @@ -229,10 +231,10 @@ class HostDriverStaticNumVf(BaseModel): class HostDriverStaticVram(AmdSmiBaseModel): type: str - vendor: str | None = None - size: ValueUnit | None = None - bit_width: ValueUnit | int | None = None - max_bandwidth: ValueUnit | None = None + vendor: Optional[str] = None + size: Optional[ValueUnit] = None + bit_width: Optional[Union[ValueUnit, int]] = None + max_bandwidth: Optional[ValueUnit] = None na_validator = field_validator("vendor", "size", "bit_width", "max_bandwidth", mode="before")( _host_na_to_none ) @@ -241,9 +243,9 @@ class HostDriverStaticVram(AmdSmiBaseModel): class HostDriverStaticNuma(BaseModel): # gim (Linux mxGPU host) reports cpu_affinity as a dict ({"cpu_list": [...]}); # amdgpuv (ESXi mxGPU host) reports it as int/str. Accept both. - node: int | str | dict | None = None - cpu_affinity: int | str | dict | None = None - socket_affinity: int | str | dict | None = None + node: Optional[Union[int, str, dict]] = None + cpu_affinity: Optional[Union[int, str, dict]] = None + socket_affinity: Optional[Union[int, str, dict]] = None na_validator = field_validator("node", "cpu_affinity", "socket_affinity", mode="before")( _host_na_to_none ) @@ -254,7 +256,7 @@ class HostDriverStaticPartition(BaseModel): accelerator_partition: str memory_partition: str - partition_id: list[int] | int + partition_id: Union[list[int], int] class HostDriverStaticXgmiPlpd(BaseModel): @@ -271,17 +273,17 @@ class HostDriverAmdSmiStatic(BaseModel): gpu: int asic: HostDriverStaticAsic bus: HostDriverStaticBus - ifwi: HostDriverStaticIfwi | None = None - limit: HostDriverStaticLimit | None = None + ifwi: Optional[HostDriverStaticIfwi] = None + limit: Optional[HostDriverStaticLimit] = None driver: HostDriverStaticDriver board: HostDriverStaticBoard ras: HostDriverStaticRas - fb_info: HostDriverStaticFbInfo | None = None - num_vf: HostDriverStaticNumVf | None = Field(default=None, alias="num-vf") + fb_info: Optional[HostDriverStaticFbInfo] = None + num_vf: Optional[HostDriverStaticNumVf] = Field(default=None, alias="num-vf") vram: HostDriverStaticVram cache_info: list[dict] = Field(default_factory=list) - xgmi_plpd: HostDriverStaticXgmiPlpd | None = Field(default=None, alias="xgmi-plpd") - partition: HostDriverStaticPartition | None = None + xgmi_plpd: Optional[HostDriverStaticXgmiPlpd] = Field(default=None, alias="xgmi-plpd") + partition: Optional[HostDriverStaticPartition] = None numa: HostDriverStaticNuma na_validator = field_validator("limit", "ifwi", "xgmi_plpd", mode="before")(_host_na_to_none) @@ -290,22 +292,22 @@ class HostDriverAmdSmiStatic(BaseModel): class HostDriverMetricUsage(BaseModel): - gfx_activity: ValueUnit | None = None - umc_activity: ValueUnit | None = None - mm_activity: ValueUnit | None = None - vcn_activity: list[ValueUnit | str | None] = Field(default_factory=list) - jpeg_activity: list[ValueUnit | str | None] = Field(default_factory=list) + gfx_activity: Optional[ValueUnit] = None + umc_activity: Optional[ValueUnit] = None + mm_activity: Optional[ValueUnit] = None + vcn_activity: list[Optional[Union[ValueUnit, str]]] = Field(default_factory=list) + jpeg_activity: list[Optional[Union[ValueUnit, str]]] = Field(default_factory=list) na_validator = field_validator("gfx_activity", "umc_activity", "mm_activity", mode="before")( _host_na_to_none ) class HostDriverMetricPower(BaseModel): - socket_power: ValueUnit | None = None - gfx_voltage: ValueUnit | None = None - soc_voltage: ValueUnit | None = None - mem_voltage: ValueUnit | None = None - power_management: str | None = None + socket_power: Optional[ValueUnit] = None + gfx_voltage: Optional[ValueUnit] = None + soc_voltage: Optional[ValueUnit] = None + mem_voltage: Optional[ValueUnit] = None + power_management: Optional[str] = None na_validator = field_validator( "socket_power", "gfx_voltage", @@ -317,32 +319,32 @@ class HostDriverMetricPower(BaseModel): class HostDriverMetricClockData(BaseModel): - clk: ValueUnit | None = None - min_clk: ValueUnit | None = None - max_clk: ValueUnit | None = None - clk_locked: int | str | dict | None = None - deep_sleep: int | str | dict | None = None + clk: Optional[ValueUnit] = None + min_clk: Optional[ValueUnit] = None + max_clk: Optional[ValueUnit] = None + clk_locked: Optional[Union[int, str, dict]] = None + deep_sleep: Optional[Union[int, str, dict]] = None na_validator = field_validator( "clk", "min_clk", "max_clk", "clk_locked", "deep_sleep", mode="before" )(_host_na_to_none) class HostDriverMetricTemperature(BaseModel): - edge: ValueUnit | None = None - hotspot: ValueUnit | None = None - mem: ValueUnit | None = None + edge: Optional[ValueUnit] = None + hotspot: Optional[ValueUnit] = None + mem: Optional[ValueUnit] = None na_validator = field_validator("edge", "hotspot", "mem", mode="before")(_host_na_to_none) class HostDriverMetricPcie(BaseModel): - width: int | None = None - speed: ValueUnit | None = None - bandwidth: ValueUnit | None = None - replay_count: int | None = None - l0_to_recovery_count: int | None = None - replay_roll_over_count: int | None = None - nak_sent_count: int | None = None - nak_received_count: int | None = None + width: Optional[int] = None + speed: Optional[ValueUnit] = None + bandwidth: Optional[ValueUnit] = None + replay_count: Optional[int] = None + l0_to_recovery_count: Optional[int] = None + replay_roll_over_count: Optional[int] = None + nak_sent_count: Optional[int] = None + nak_received_count: Optional[int] = None na_validator = field_validator( "width", "speed", @@ -357,11 +359,11 @@ class HostDriverMetricPcie(BaseModel): class HostDriverMetricEccTotals(BaseModel): - total_correctable_count: int | None = None - total_uncorrectable_count: int | None = None - total_deferred_count: int | None = None - cache_correctable_count: int | None = None - cache_uncorrectable_count: int | None = None + total_correctable_count: Optional[int] = None + total_uncorrectable_count: Optional[int] = None + total_deferred_count: Optional[int] = None + cache_correctable_count: Optional[int] = None + cache_uncorrectable_count: Optional[int] = None na_validator = field_validator( "total_correctable_count", "total_uncorrectable_count", @@ -373,7 +375,7 @@ class HostDriverMetricEccTotals(BaseModel): class HostDriverMetricEnergy(BaseModel): - total_energy_consumption: ValueUnit | None = None + total_energy_consumption: Optional[ValueUnit] = None na_validator = field_validator("total_energy_consumption", mode="before")(_host_na_to_none) @@ -388,15 +390,15 @@ class HostDriverAmdSmiMetric(BaseModel): model_config = ConfigDict(extra="ignore") gpu: int - usage: HostDriverMetricUsage | str + usage: Union[HostDriverMetricUsage, str] power: HostDriverMetricPower - clock: dict[str, HostDriverMetricClockData | dict] + clock: dict[str, Union[HostDriverMetricClockData, dict]] temperature: HostDriverMetricTemperature pcie: HostDriverMetricPcie ecc: HostDriverMetricEccTotals - ecc_blocks: dict[str, EccData] | str = Field(default_factory=dict) - energy: HostDriverMetricEnergy | None = None - gpuboard: HostDriverMetricGpuBoard | None = None + ecc_blocks: Union[dict[str, EccData], str] = Field(default_factory=dict) + energy: Optional[HostDriverMetricEnergy] = None + gpuboard: Optional[HostDriverMetricGpuBoard] = None @field_validator("ecc_blocks", mode="before") @classmethod @@ -425,19 +427,19 @@ class HostDriverTopoLink(BaseModel): link_type: str num_hops: int bandwidth: str = "" - fb_sharing: str | None = None - coherent: str | None = None - atomics: str | None = None - dma: str | None = None - bi_dir: str | None = Field(default=None, alias="bi-dir") + fb_sharing: Optional[str] = None + coherent: Optional[str] = None + atomics: Optional[str] = None + dma: Optional[str] = None + bi_dir: Optional[str] = Field(default=None, alias="bi-dir") @computed_field - def bandwidth_from(self) -> int | None: + def bandwidth_from(self) -> Optional[int]: bw_split = self.bandwidth.split("-") return int(bw_split[0]) if len(bw_split) == 2 else None @computed_field - def bandwidth_to(self) -> int | None: + def bandwidth_to(self) -> Optional[int]: bw_split = self.bandwidth.split("-") return int(bw_split[1]) if len(bw_split) == 2 else None @@ -456,9 +458,9 @@ class HostDriverBadPageEntry(BaseModel): bad_page: int retired_bad_page: str - timestamp: str | None = None - mem_channel: int | None = None - mcumc_id: int | None = None + timestamp: Optional[str] = None + mem_channel: Optional[int] = None + mcumc_id: Optional[int] = None class HostDriverBadPages(BaseModel): @@ -472,14 +474,14 @@ class HostDriverBadPages(BaseModel): class HostDriverXgmiLink(BaseModel): gpu: int bdf: str - read: ValueUnit | None = None - write: ValueUnit | None = None + read: Optional[ValueUnit] = None + write: Optional[ValueUnit] = None na_validator = field_validator("read", "write", mode="before")(_host_na_to_none) class HostDriverXgmiLinkMetrics(BaseModel): - bit_rate: ValueUnit | None = None - max_bandwidth: ValueUnit | None = None + bit_rate: Optional[ValueUnit] = None + max_bandwidth: Optional[ValueUnit] = None links: list[HostDriverXgmiLink] = Field(default_factory=list) na_validator = field_validator("max_bandwidth", "bit_rate", mode="before")(_host_na_to_none)