diff --git a/gpustack_runtime/deployer/cdi/__utils__.py b/gpustack_runtime/deployer/cdi/__utils__.py index ce34a80..d56c2e5 100644 --- a/gpustack_runtime/deployer/cdi/__utils__.py +++ b/gpustack_runtime/deployer/cdi/__utils__.py @@ -143,6 +143,56 @@ def device_to_cdi_device_node( ) +def path_to_cdi_device_nodes( + path: str, + permission: str = "rw", + no_user: bool = False, +) -> list[ConfigDeviceNode]: + """ + Convert a device path, or a directory holding device paths, to ConfigDeviceNodes. + + A generation may expose a bus as a directory of device nodes rather than as + a single node -- Ascend's UB does, which is why the operator enumerates it, + see addUBDevicesFromDir in + https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/cdi/devnode.go. + Both shapes are accepted so that a path which is a plain device node on one + driver and a directory on another needs no caller-side branch. + + Args: + path: + Path to the device, or to a directory of devices, on the host. + permission: + Permissions for the devices. + no_user: + Whether to omit user and group information. + + Returns: + The ConfigDeviceNode objects, empty if the path holds no device. + + """ + p = Path(path) + if not p.is_dir(): + cdn = device_to_cdi_device_node( + path=path, + permission=permission, + no_user=no_user, + ) + return [cdn] if cdn else [] + + return [ + cdn + for entry in sorted(p.iterdir()) + if not entry.is_dir() + and ( + cdn := device_to_cdi_device_node( + path=str(entry), + permission=permission, + no_user=no_user, + ) + ) + ] + + def path_to_cdi_mount( path: str, container_path: str | None = None, @@ -181,3 +231,34 @@ def path_to_cdi_mount( container_path=container_path, options=options, ) + + +def glob_to_cdi_mounts( + pattern: str, + options: list[str] | None = None, +) -> list[ConfigMount]: + """ + Convert every path matching a glob pattern to ConfigMounts. + + A user-space library is versioned in its file name, so the set of files to + mount cannot be spelled out ahead of time -- the operator's mount profile + lists them as patterns for the same reason. + + Args: + pattern: + Path on the host whose last segment may carry a wildcard; the + directory part is taken literally, so a wildcard there matches + nothing. A pattern without a wildcard resolves to that path alone. + options: + Mount options. + + Returns: + The ConfigMount objects, empty if nothing matches. + + """ + p = Path(pattern) + return [ + cm + for path in sorted(p.parent.glob(p.name)) + if (cm := path_to_cdi_mount(path=str(path), options=options)) + ] diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index 40b1ac1..b171387 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -1,11 +1,14 @@ from __future__ import annotations as __future_annotations__ +import logging + from ...detector import ( Devices, ManufacturerEnum, detect_devices, filter_devices_by_manufacturer, ) +from ...detector.ascend import get_ascend_cann_variant from .__types__ import ( Config, ConfigContainerEdits, @@ -14,7 +17,44 @@ manufacturer_to_cdi_kind, manufacturer_to_runtime_env, ) -from .__utils__ import device_to_cdi_device_node, path_to_cdi_mount +from .__utils__ import ( + device_to_cdi_device_node, + glob_to_cdi_mounts, + path_to_cdi_device_nodes, + path_to_cdi_mount, +) + +logger = logging.getLogger(__name__) + +_A5_CANN_VARIANT = "950" +""" +The CANN variant of the A5 generation, whose UB fabric replaces what the +earlier generations reach through the shared memory device. +""" + +_A5_UB_MOUNT_PATTERNS = [ + "/usr/lib64/libummu*", + "/usr/lib64/liburma*", + "/usr/lib64/urma", + "/usr/lib64/libnl*", + "/usr/bin/urma_admin", + "/usr/bin/urma_perftest", + "/usr/bin/urma_ping", +] +""" +The UB user-space libraries an A5 container needs, mirroring the operator's +mount profile for the Ascend950 generation, see +https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/cdi/mount/profile.go. + +These are mounted for the A5 generation only. Some of them -- libnl above all +-- are ordinary system libraries present on any host, so mounting them +unconditionally would shadow what an earlier generation's container ships with +its own image. + +The /usr/lib64 prefix is the operator's, and holds on the openEuler and CentOS +hosts an A5 ships on; a Debian-derived host uses /usr/lib/, where none +of these match -- hence the log line when a pattern finds nothing. +""" class AscendGenerator(Generator): @@ -73,16 +113,14 @@ def generate( break for p in [ "/dev/dvpp_cmdlist", + # UB exposes a directory of device nodes rather than a single one, + # so every entry below it has to be injected. "/dev/uburma", "/dev/ummu", "/dev/devmm_svm", "/dev/hisi_hdc", ]: - cdn = device_to_cdi_device_node( - path=p, - ) - if cdn: - common_device_nodes.append(cdn) + common_device_nodes.extend(path_to_cdi_device_nodes(path=p)) if not common_device_nodes: return None @@ -102,6 +140,25 @@ def generate( if cm: common_mounts.append(cm) + # Device.appendix defaults to None and callers pass their own devices + # in, so every read goes through `or {}`. + if any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ): + for pattern in _A5_UB_MOUNT_PATTERNS: + ub_mounts = glob_to_cdi_mounts(pattern=pattern) + if not ub_mounts: + # Otherwise this surfaces later as a container that cannot + # reach the UB fabric. + logger.debug( + "No UB library matched %s, an A5 container will lack it", + pattern, + ) + common_mounts.extend(ub_mounts) + cdi_devices: list[ConfigDevice] = [] all_device_nodes = [] @@ -119,7 +176,7 @@ def generate( # addressed by the index, which would resolve to another NPU's # node. The detector already drops such a device; this guards the # devices a caller passes in. - cdn_number = dev.appendix.get("physical_id") + cdn_number = (dev.appendix or {}).get("physical_id") if cdn_number is None: continue cdn_path = f"/dev/davinci{cdn_number}" diff --git a/gpustack_runtime/detector/ascend.py b/gpustack_runtime/detector/ascend.py index 0f6208f..8981416 100644 --- a/gpustack_runtime/detector/ascend.py +++ b/gpustack_runtime/detector/ascend.py @@ -76,8 +76,29 @@ def is_supported() -> bool: try: pydcmi.dcmi_init() supported = True - except Exception: - debug_log_exception(logger, "Failed to initialize DCMI") + except Exception as v1_error: + # A V2-only driver refuses the V1 entry point, so this is a probe + # result, not a verdict on the hardware -- as the operator's + # DetectDcmiApiVersion treats it, see + # https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/devmanager/devmanager_common.go. + # Logged only if V2 fails too: on an A5 host a traceback here would + # sit right above the V2 success line. + try: + pydcmi.dcmiv2_init() + supported = True + logger.info( + "Initialized DCMI through the V2 API, the V1 API being unavailable", + ) + except Exception: + # A card on the PCI bus but both APIs refused: the library was + # found and rejected, not missing -- so name it. + debug_log_exception( + logger, + "Failed to initialize DCMI through either API," + " loaded from %s, the V1 API having reported: %s", + pydcmi.dcmi_library_path() or "nothing", + v1_error, + ) return supported @@ -108,6 +129,9 @@ def detect_info(self) -> Devices | None: if not self.is_supported(): return None + if pydcmi.dcmi_api_version() == 2: + return self._detect_info_v2() + ret: Devices = [] try: @@ -233,6 +257,128 @@ def detect_info(self) -> Devices | None: return ret + def _detect_info_v2(self) -> Devices: + """ + Detect Ascend NPUs' inventory through the DCMI V2 API. + + V2 enumerates devices flat, indexed by the logic id that V1 reports + through a separate call, so there is no card to walk here. + + Returns: + A list of detected Ascend NPU devices. + + Raises: + If there is an error during detection. + + """ + ret: Devices = [] + + try: + sys_runtime_ver_original = _get_toolkit_version() + sys_runtime_ver = get_brief_version(sys_runtime_ver_original) + + # V2 declares no driver version call, only the DCMI library's own + # -- a different number, so it goes to the appendix under its own + # name instead of standing in for driver_version. + sys_dcmi_ver = None + with contextlib.suppress(pydcmi.DCMIError): + sys_dcmi_ver = pydcmi.dcmiv2_get_dcmi_version() + + for dev_index in pydcmi.dcmiv2_get_device_list(): + if not _is_npu_device_v2(dev_index): + continue + + dev_chip_info = pydcmi.dcmiv2_get_device_chip_info(dev_index) + dev_cores_aicore = dev_chip_info.aicore_cnt + dev_name = dev_chip_info.chip_name + + # V2 has no non-HBM call to fall back to, so a refusal is final + # for this device -- dropped like an unreadable physical id + # below, rather than hiding every other device. + try: + dev_mem, _ = _get_device_memory_info_v2(dev_index) + except pydcmi.DCMIError: + debug_log_warning( + logger, + "Failed to fetch memory of device %d, skipping it", + dev_index, + ) + continue + dev_mem_status = _get_device_memory_status_v2(dev_index) + + # As on the V1 path, a device whose physical id cannot be read + # cannot be addressed: /dev/davinciN is numbered by it. + try: + dev_physical_id = pydcmi.dcmiv2_get_chip_phy_id_by_dev_id( + dev_index, + ) + except pydcmi.DCMIError: + debug_log_warning( + logger, + "Failed to fetch physical id of device %d, skipping it", + dev_index, + ) + continue + + dev_bdf = pydcmi.dcmiv2_get_device_bdf(dev_index) + + dev_uuid = _get_device_die_v2(dev_index, dev_bdf) + + dev_numa = get_numa_node_by_bdf(dev_bdf) + if not dev_numa: + with contextlib.suppress(pydcmi.DCMIError): + dev_cpu_affinity = ( + pydcmi.dcmiv2_get_affinity_cpu_info_by_dev_id( + dev_index, + ) + ) + dev_numa = map_cpu_affinity_to_numa_node(dev_cpu_affinity) + + # No card_id/device_id here: V2 has no card level to report, + # and the consumers that matter -- CDI's device node, the + # deployer's CANN variant -- read the physical id and the + # arch family instead. + # + # No roce_* either, by design: V2 declares no IP call, this + # generation addressing devices by URMA EID over UnifiedBus + # instead. A consumer grouping by IP needs a UB-aware path. + dev_appendix = { + "arch_family": _guess_soc_name_from_dev_name(dev_name), + "bdf": dev_bdf, + "physical_id": dev_physical_id, + } + if dev_numa: + dev_appendix["numa"] = dev_numa + if sys_dcmi_ver: + dev_appendix["dcmi_version"] = sys_dcmi_ver + + ret.append( + Device( + manufacturer=self.manufacturer, + index=dev_index, + name=dev_name, + uuid=dev_uuid.upper(), + driver_version=None, + runtime_version=sys_runtime_ver, + runtime_version_original=sys_runtime_ver_original, + cores=dev_cores_aicore, + memory=dev_mem, + memory_status=dev_mem_status, + appendix=dev_appendix, + ), + ) + except pydcmi.DCMIError: + debug_log_exception(logger, "Failed to fetch devices through the V2 API") + raise + except Exception: + debug_log_exception( + logger, + "Failed to process devices fetching through the V2 API", + ) + raise + + return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: """ Fetch Ascend NPUs' usage using pydcmi. @@ -257,6 +403,9 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: if not devices: return devices + if pydcmi.dcmi_api_version() == 2: + return self._detect_usage_v2(devices) + usages: Devices = [] try: @@ -337,6 +486,99 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: return merge_devices_usage(devices, usages) + def _detect_usage_v2(self, devices: Devices) -> Devices: + """ + Fetch Ascend NPUs' usage through the DCMI V2 API. + + Args: + devices: + The devices to refresh, matched by UUID. + + Returns: + The devices carrying usage. + + Raises: + If there is an error during detection. + + """ + usages: Devices = [] + + # The uuid a device was detected under is what the usage merges on, and + # it may have fallen back to the address, so it is read off the device + # rather than derived a second time -- deriving it twice is how the two + # sides come to disagree. + uuid_by_index = {dev.index: dev.uuid for dev in devices if dev} + + try: + for dev_index in pydcmi.dcmiv2_get_device_list(): + dev_uuid = uuid_by_index.get(dev_index) + if not dev_uuid: + continue + + # Polled path: merge_devices_usage joins by uuid, so a device + # left out keeps its last figures. Propagating would clear + # every device's metrics over one transient error. + try: + dev_mem, dev_mem_used = _get_device_memory_info_v2(dev_index) + except pydcmi.DCMIError: + debug_log_warning( + logger, + "Failed to get device %d memory usage, skipping it this round", + dev_index, + ) + continue + dev_mem_status = _get_device_memory_status_v2(dev_index) + + dev_util_aicore = None + with contextlib.suppress(pydcmi.DCMIError): + dev_util_aicore = pydcmi.dcmiv2_get_device_utilization_rate( + dev_index, + pydcmi.DCMI_INPUT_TYPE_AICORE, + ) + if dev_util_aicore is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_index, + ) + dev_util_aicore = 0 + + dev_temp = None + with contextlib.suppress(pydcmi.DCMIError): + dev_temp = pydcmi.dcmiv2_get_device_temperature(dev_index) + + dev_power_used = None + with contextlib.suppress(pydcmi.DCMIError): + dev_power_used = pydcmi.dcmiv2_get_device_power_info(dev_index) + if dev_power_used: + dev_power_used = dev_power_used / 10 # 0.1W to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_util_aicore, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pydcmi.DCMIError: + debug_log_exception( + logger, + "Failed to fetch devices usage through the V2 API", + ) + raise + except Exception: + debug_log_exception( + logger, + "Failed to process devices usage fetching through the V2 API", + ) + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between Ascend NPUs. @@ -350,6 +592,13 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: A Topology object, or None if not supported. """ + # detect_topologies() hands the devices in directly, skipping + # detect_info() and with it the call that resolves the API version -- + # leaving the V1 dcmi_init() below to raise on a V2-only driver. + # Cached, so asking again is free. + if not self.is_supported(): + return None + if devices is None: devices = self.detect_info() if devices is None: @@ -360,8 +609,17 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: devices_count=len(devices), ) + # V2 declares no topology call at all, so the distances stay unknown + # there. The NUMA and CPU affinities come from the appendix and are + # reported either way. + distances_available = pydcmi.dcmi_api_version() == 1 + try: - pydcmi.dcmi_init() + # V2 needs no call here at all: the affinities are read off the + # appendix and the distances are unavailable, so nothing has to be + # initialized to report what can be reported. + if distances_available: + pydcmi.dcmi_init() for i, dev_i in enumerate(devices): dev_i_card_id = dev_i.appendix.get("card_id", i) @@ -374,6 +632,8 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: ) # Get distances to other devices. + if not distances_available: + continue for j, dev_j in enumerate(devices): if dev_i.index == dev_j.index or ret.devices_distances[i][j] != 0: continue @@ -447,6 +707,117 @@ def _is_npu_device(dev_card_id, dev_device_id) -> bool: return True +def _is_npu_device_v2(dev_id) -> bool: + """ + Report whether the given device is an NPU, through the V2 API. + + As on the V1 path, a device whose type cannot be read is kept: only a + reading that succeeds and says something other than NPU disqualifies it. + + Args: + dev_id: + The device ID of the device. + + Returns: + True if the device is an NPU, or its type is unreadable. + + """ + dev_type = None + with contextlib.suppress(pydcmi.DCMIError): + dev_type = pydcmi.dcmiv2_get_device_type(dev_id) + + if dev_type is not None and dev_type != pydcmi.DCMI_UNIT_TYPE_NPU: + slogger.debug("Skipping non-NPU device %d, type %d", dev_id, dev_type) + return False + + return True + + +def _get_device_die_v2(dev_id, dev_bdf: str) -> str: + """ + Get the device's SoC die through the V2 API, falling back to its address. + + The A5 driver reports neither die type -- both answer NOT_SUPPORT -- and a + die is not what the uuid is needed for: it only has to tell one device from + another, which is what the usage pass merges on and what the inventory is + keyed by. The PCI address does that on a machine whose cards have not + moved, where dropping the device would leave the NPU unusable outright. + + Args: + dev_id: + The device ID of the device. + dev_bdf: + The device's PCI address, used when no die can be read. + + Returns: + The die as a string, or the PCI address. + + """ + for dev_die_type in (pydcmi.DCMI_DIE_TYPE_VDIE, pydcmi.DCMI_DIE_TYPE_NDIE): + with contextlib.suppress(pydcmi.DCMIError): + return pydcmi.dcmiv2_get_device_die_id(dev_id, dev_die_type) + + debug_log_warning( + logger, + "Failed to fetch die of device %d, identifying it by its address %s", + dev_id, + dev_bdf, + ) + return dev_bdf + + +def _get_device_memory_info_v2(dev_id) -> tuple[int, int]: + """ + Get device memory information through the V2 API. + + V2 declares the HBM call alone, which this generation carries anyway. The + V1 helper's `memory_size > 0` guard reroutes a non-HBM device; with nowhere + to reroute, repeating it here would only silence a readable zero. + + Args: + dev_id: + The device ID of the device. + + Returns: + A tuple containing total memory and used memory in MiB. + + Raises: + pydcmi.DCMIError: If the driver refuses the HBM query. + + """ + dev_hbm_info = pydcmi.dcmiv2_get_device_hbm_info(dev_id) + return dev_hbm_info.memory_size, dev_hbm_info.memory_usage + + +def _get_device_memory_status_v2(dev_id) -> DeviceMemoryStatusEnum: + """ + Get device memory ECC status through the V2 API. + + Args: + dev_id: + The device ID of the device. + + Returns: + DeviceMemoryStatusEnum indicating the ECC status. + + """ + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + for dev_mem_type in [pydcmi.DCMI_DEVICE_TYPE_HBM, pydcmi.DCMI_DEVICE_TYPE_DDR]: + with contextlib.suppress(pydcmi.DCMIError): + dev_ecc_info = pydcmi.dcmiv2_get_device_ecc_info( + dev_id, + dev_mem_type, + ) + if dev_ecc_info.enable_flag and ( + dev_ecc_info.single_bit_error_cnt > 0 + or dev_ecc_info.double_bit_error_cnt > 0 + ): + return DeviceMemoryStatusEnum.UNHEALTHY + return DeviceMemoryStatusEnum.HEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_device_die(dev_card_id, dev_device_id) -> str: """ Get the device's SoC die, which identifies it. @@ -745,6 +1116,7 @@ def _get_toolkit_version() -> str | None: "Ascend910_9579": 260, "Ascend910_95": 260, "Ascend950": 260, + "Ascend950PR": 260, } @@ -752,6 +1124,14 @@ def _get_toolkit_version() -> str | None: _910B_REGEX = re.compile(r"^(910B\d|A2G\d)") _310P_REGEX = re.compile(r"^(310P\d?|I2\d?)") +# An A5 chip names itself "Ascend950XX" -- keeping the "Ascend" prefix the +# earlier generations drop -- so the operator matches it by prefix instead of +# by an exact name. See api.Ascend910A5Prefix in +# https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/api/default_name_v2.go, +# used by +# https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-docker-runtime/runtime/process/process.go. +_950_PREFIX = "Ascend950" + def _guess_soc_name_from_dev_name(dev_name: str) -> str | None: """ @@ -773,6 +1153,12 @@ def _guess_soc_name_from_dev_name(dev_name: str) -> str | None: return soc_name # https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/devmanager/common/utils.go#L159-L176 + # + # The A5 prefix is matched first: a name the mapping does not carry yet, + # like a later 950 variant, still belongs to the generation, and none of + # the regexes below would claim it. + if soc_name.startswith(_950_PREFIX): + return "Ascend950" if _310P_REGEX.match(dev_name): return "Ascend310P1" if "310B" in dev_name: diff --git a/gpustack_runtime/detector/pydcmi/__init__.py b/gpustack_runtime/detector/pydcmi/__init__.py index 7c35e68..4ce0cd4 100644 --- a/gpustack_runtime/detector/pydcmi/__init__.py +++ b/gpustack_runtime/detector/pydcmi/__init__.py @@ -171,6 +171,7 @@ libLoadLock = threading.Lock() _libInitialized = False _libInitializedException = None +_apiVersion = 1 ## Error Checking ## @@ -796,6 +797,21 @@ def _LoadDcmiLibrary(): libLoadLock.release() +def dcmi_library_path() -> str | None: + """ + Report which candidate the DCMI library was loaded from. + + Initialization can fail after the library loads, and the candidates differ + in where they come from -- the linker's search path, the driver, or the + DCMI package -- so which one answered narrows down the failure. + + Returns: + The name the library was loaded by, or None if it is not loaded yet. + + """ + return getattr(dcmiLib, "_name", None) if dcmiLib is not None else None + + ## C function wrappers ## def dcmi_init(): _LoadDcmiLibrary() @@ -826,8 +842,41 @@ def dcmi_init(): _libInitialized = True +def dcmiv2_init(): + _LoadDcmiLibrary() + + global _libInitialized, _libInitializedException, _apiVersion + + # Short-circuit as dcmi_init() does. Only a prior V2 init counts: getting + # here after a V1 init means a caller is probing V2 anyway. + if _libInitialized and _apiVersion == 2: + return + + fn = _dcmiGetFunctionPointer("dcmiv2_init") + ret = fn() + _dcmiCheckReturn(ret) + + # The library is usable once V2 answers, so the V1 entry point's refusal is + # no longer the final word: clear it, or every later dcmi_init() would + # re-raise it and no call could be made at all. + with libLoadLock: + _libInitialized = True + _libInitializedException = None + _apiVersion = 2 + + +def dcmi_api_version() -> int: + """ + Report which API version initialized the library: 1 or 2. + + The two do not interoperate -- a driver serving V2 refuses every V1 entry + point -- so a caller has to keep to whichever one answered. + """ + return _apiVersion + + def dcmi_shutdown(): - global _libInitialized, _libInitializedException + global _libInitialized, _libInitializedException, _apiVersion with libLoadLock: if not _libInitialized: @@ -835,6 +884,8 @@ def dcmi_shutdown(): _libInitialized = False _libInitializedException = None + # Or a re-init on a V1 host would keep answering 2. + _apiVersion = 1 def dcmi_get_card_list(): @@ -1295,3 +1346,131 @@ def dcmi_get_affinity_cpu_info_by_device_id(card_id, device_id): ret = fn(card_id, device_id, c_cpu_info, byref(c_cpu_info_len)) _dcmiCheckReturn(ret) return c_cpu_info.value + + +## DCMI V2 API wrappers ## +# A driver serving the V2 API answers every V1 entry point with +# DCMI_ERROR_NOT_SUPPORT -- initialization included -- so a caller that had to +# reach for V2 has to stay on it throughout. +# +# The structs are the V1 ones: dcmi_interface_api_v2.h declares no type of its +# own. What differs is the index, a single device id in place of the +# card/device pair, and that device id is the logic id the V1 API enumerates +# separately. +def dcmiv2_get_device_list(list_len=64): + c_device_list = (c_int * list_len)() + c_device_num = c_int() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_list") + ret = fn(c_device_list, byref(c_device_num), list_len) + _dcmiCheckReturn(ret) + return list(c_device_list[: c_device_num.value]) + + +def dcmiv2_get_device_type(dev_id): + c_device_type = c_uint() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_type") + ret = fn(dev_id, byref(c_device_type)) + _dcmiCheckReturn(ret) + return c_device_type.value + + +def dcmiv2_get_device_chip_info(dev_id): + c_chip_info = c_dcmi_chip_info_v2() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_chip_info") + ret = fn(dev_id, byref(c_chip_info)) + _dcmiCheckReturn(ret) + return c_chip_info + + +def dcmiv2_get_device_hbm_info(dev_id): + c_hbm_info = c_dcmi_hbm_info() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_hbm_info") + ret = fn(dev_id, byref(c_hbm_info)) + _dcmiCheckReturn(ret) + return c_hbm_info + + +def dcmiv2_get_device_die_id(dev_id, input_type): + c_die_id = c_dcmi_die_id() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_die_id") + ret = fn(dev_id, input_type, byref(c_die_id)) + _dcmiCheckReturn(ret) + return " ".join([hex(i)[2:] for i in c_die_id.soc_die]) + + +def dcmiv2_get_chip_phy_id_by_dev_id(dev_id): + c_phyid = c_uint() + fn = _dcmiGetFunctionPointer("dcmiv2_get_chip_phy_id_by_dev_id") + ret = fn(dev_id, byref(c_phyid)) + _dcmiCheckReturn(ret) + return c_phyid.value + + +def dcmiv2_get_device_pcie_info(dev_id): + c_pcie_info = c_dcmi_pcie_info_all() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_pcie_info") + ret = fn(dev_id, byref(c_pcie_info)) + _dcmiCheckReturn(ret) + return c_pcie_info + + +def dcmiv2_get_device_bdf(dev_id): + c_pcie_info = dcmiv2_get_device_pcie_info(dev_id) + + domain = c_pcie_info.domain + bus = c_pcie_info.bdf_busid + device = c_pcie_info.bdf_deviceid + function = c_pcie_info.bdf_funcid + return f"{domain:04x}:{bus:02x}:{device:02x}.{function:x}" + + +def dcmiv2_get_device_ecc_info(dev_id, device_type): + c_device_ecc_info = c_dcmi_ecc_info() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_ecc_info") + ret = fn(dev_id, device_type, byref(c_device_ecc_info)) + _dcmiCheckReturn(ret) + return c_device_ecc_info + + +def dcmiv2_get_device_utilization_rate(dev_id, input_type): + c_utilization_rate = c_uint() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_utilization_rate") + ret = fn(dev_id, input_type, byref(c_utilization_rate)) + _dcmiCheckReturn(ret) + return c_utilization_rate.value + + +def dcmiv2_get_device_temperature(dev_id): + c_temperature = c_int() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_temperature") + ret = fn(dev_id, byref(c_temperature)) + _dcmiCheckReturn(ret) + return c_temperature.value + + +def dcmiv2_get_device_power_info(dev_id): + c_power = c_int() + fn = _dcmiGetFunctionPointer("dcmiv2_get_device_power_info") + ret = fn(dev_id, byref(c_power)) + _dcmiCheckReturn(ret) + return c_power.value + + +@convertStrBytes +def dcmiv2_get_affinity_cpu_info_by_dev_id(dev_id): + c_cpu_info = create_string_buffer(TOPO_INFO_MAX_LENGTH) + c_cpu_info_len = c_int() + fn = _dcmiGetFunctionPointer("dcmiv2_get_affinity_cpu_info_by_dev_id") + ret = fn(dev_id, c_cpu_info, byref(c_cpu_info_len)) + _dcmiCheckReturn(ret) + return c_cpu_info.value + + +@convertStrBytes +def dcmiv2_get_dcmi_version(): + # V2 exposes no driver version call at all, only the DCMI library's own. + c_dcmi_ver = create_string_buffer(32) + fn = _dcmiGetFunctionPointer("dcmiv2_get_dcmi_version") + ret = fn(c_dcmi_ver, c_int(32)) + _dcmiCheckReturn(ret) + return c_dcmi_ver.value diff --git a/tests/gpustack_runtime/detector/test_ascend.py b/tests/gpustack_runtime/detector/test_ascend.py index 1851f7a..6c335b3 100644 --- a/tests/gpustack_runtime/detector/test_ascend.py +++ b/tests/gpustack_runtime/detector/test_ascend.py @@ -6,6 +6,7 @@ import pytest from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import __utils__ as cdi_utils from gpustack_runtime.deployer.cdi import ascend as cdi_ascend from gpustack_runtime.deployer.cdi.ascend import AscendGenerator from gpustack_runtime.detector import ( @@ -202,6 +203,12 @@ class _FakeDCMI: v2_size: int = 16384 v2_utiliza: int = 25 + def dcmi_api_version(self): + # Defined on the class rather than dispatched through __getattr__, so + # that it stays out of the call log the usage/information split is + # asserted against. This fake serves the V1 API. + return 1 + def __getattr__(self, name: str): handler = { "dcmi_init": self._init, @@ -733,3 +740,707 @@ def _fake_device_node(path, **_kwargs): # The device without one is skipped: Device.index is the logic id, so # standing it in for the physical id would address another NPU's node. assert "/dev/davinci1" not in seen_paths + + +# --------------------------------------------------------------------------- # +# SoC naming: the A5 generation names itself apart from its predecessors. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "dev_name, soc_name, variant", + [ + # An A5 chip keeps the "Ascend" prefix that the earlier generations + # drop, and is reported either way round. + ("Ascend950PR", "Ascend950PR", "950"), + ("950PR", "Ascend950PR", "950"), + # A 950 variant the mapping does not carry yet still belongs to the + # generation rather than falling through to nothing. + ("Ascend950DT", "Ascend950", "950"), + # The generations that already worked, both where the mapping answers + # directly and where the regexes have to: the A5 prefix must claim + # neither. + ("910B4", "Ascend910B4", "910b"), + ("910B9", "Ascend910B1", "910b"), + ("310P3", "Ascend310P3", "310p"), + ("310P9", "Ascend310P1", "310p"), + ("910_9391", "Ascend910_9391", "a3"), + ("910A", "Ascend910A", "910"), + ], +) +def test_guess_soc_name_carries_the_generation(dev_name, soc_name, variant): + assert ascend._guess_soc_name_from_dev_name(dev_name) == soc_name # noqa: SLF001 + assert ascend.get_ascend_cann_variant(soc_name) == variant + + +def test_guess_soc_name_still_yields_nothing_for_an_unknown_chip(): + assert ascend._guess_soc_name_from_dev_name("NotAnAscendChip") is None # noqa: SLF001 + assert ascend.get_ascend_cann_variant(None) is None + + +# --------------------------------------------------------------------------- # +# CDI: the A5 UB fabric, which the earlier generations have no part of. # +# --------------------------------------------------------------------------- # + + +def test_cdi_enumerates_a_device_directory(tmp_path, monkeypatch): + ub_dir = tmp_path / "uburma" + ub_dir.mkdir() + (ub_dir / "uburma0").touch() + (ub_dir / "uburma1").touch() + (ub_dir / "nested").mkdir() + + monkeypatch.setattr( + cdi_utils, + "device_to_cdi_device_node", + lambda path, **_kwargs: {"path": path}, + ) + + nodes = cdi_utils.path_to_cdi_device_nodes(path=str(ub_dir)) + + assert [n["path"] for n in nodes] == [ + str(ub_dir / "uburma0"), + str(ub_dir / "uburma1"), + ] + + +def test_cdi_still_accepts_a_plain_device_node(tmp_path, monkeypatch): + dev = tmp_path / "hisi_hdc" + dev.touch() + + monkeypatch.setattr( + cdi_utils, + "device_to_cdi_device_node", + lambda path, **_kwargs: {"path": path}, + ) + + nodes = cdi_utils.path_to_cdi_device_nodes(path=str(dev)) + + assert [n["path"] for n in nodes] == [str(dev)] + + +def test_cdi_holds_the_ub_mounts_back_from_an_earlier_generation(monkeypatch): + patterns = _collect_ub_mount_patterns(monkeypatch, arch_family="Ascend910B4") + + # libnl and friends are ordinary system libraries: mounting them here would + # shadow what a 910B container ships with its own image. + assert patterns == [] + + +def test_cdi_mounts_the_ub_libraries_for_the_a5_generation(monkeypatch): + patterns = _collect_ub_mount_patterns(monkeypatch, arch_family="Ascend950PR") + + assert any("liburma" in p for p in patterns) + assert any("libummu" in p for p in patterns) + assert any("libnl" in p for p in patterns) + + +def test_cdi_holds_the_ub_mounts_back_without_an_arch_family(monkeypatch): + # A device detected by an older runtime carries no arch_family at all, + # which must not be read as the A5 generation. + patterns = _collect_ub_mount_patterns(monkeypatch, arch_family=None) + + assert patterns == [] + + +def test_cdi_survives_a_device_carrying_no_appendix(monkeypatch): + # Device.appendix defaults to None, so a bare Device would crash on .get() + # before reaching the guard meant to skip it. + monkeypatch.setattr( + cdi_ascend, + "device_to_cdi_device_node", + lambda path, **_kwargs: {"path": path}, + ) + monkeypatch.setattr( + cdi_ascend, + "path_to_cdi_device_nodes", + lambda path, **_kwargs: [{"path": path}], + ) + monkeypatch.setattr(cdi_ascend, "path_to_cdi_mount", lambda **_kwargs: None) + monkeypatch.setattr(cdi_ascend, "glob_to_cdi_mounts", lambda **_kwargs: []) + + devices = [ + Device( + manufacturer=ManufacturerEnum.ASCEND, + index=0, + name="chip", + uuid="DIE-0", + memory=65536, + ), + ] + + # No physical id, so nothing is generated -- the guard, not a crash. + assert AscendGenerator().generate(devices) is None + + +def _collect_ub_mount_patterns(monkeypatch, arch_family: str | None) -> list[str]: + """ + Run the CDI generator over one device and report the globbed mount patterns. + """ + seen: list[str] = [] + + monkeypatch.setattr( + cdi_ascend, + "device_to_cdi_device_node", + lambda path, **_kwargs: {"path": path}, + ) + monkeypatch.setattr( + cdi_ascend, + "path_to_cdi_device_nodes", + lambda path, **_kwargs: [{"path": path}], + ) + monkeypatch.setattr(cdi_ascend, "path_to_cdi_mount", lambda **_kwargs: None) + monkeypatch.setattr( + cdi_ascend, + "glob_to_cdi_mounts", + lambda pattern, **_kwargs: seen.append(pattern) or [], + ) + + appendix = {"card_id": 0, "device_id": 0, "physical_id": 0} + if arch_family is not None: + appendix["arch_family"] = arch_family + + devices = [ + Device( + manufacturer=ManufacturerEnum.ASCEND, + index=0, + name="chip", + uuid="DIE-0", + memory=65536, + appendix=appendix, + ), + ] + + assert AscendGenerator().generate(devices) is not None + + return seen + + +# --------------------------------------------------------------------------- # +# is_supported: a driver serving the V2 API only, as the A5 generation does. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _FakeInitOnlyDCMI: + """ + A stand-in exposing only the initialization entry points, recording which + of them the detector reaches for. + """ + + v1_error: Exception | None = None + v2_error: Exception | None = None + calls: list[str] = field(default_factory=list) + + DCMIError = pydcmi.DCMIError + + def dcmi_init(self): + self.calls.append("dcmi_init") + if self.v1_error is not None: + raise self.v1_error + + def dcmiv2_init(self): + self.calls.append("dcmiv2_init") + if self.v2_error is not None: + raise self.v2_error + + def dcmi_library_path(self): + return "/usr/local/dcmi/libdcmi.so" + + +def _install_init_only_dcmi(monkeypatch, **kwargs) -> _FakeInitOnlyDCMI: + fake = _FakeInitOnlyDCMI(**kwargs) + monkeypatch.setattr(ascend, "pydcmi", fake) + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_PCI_CHECK", True) + return fake + + +def test_is_supported_falls_back_to_the_v2_api(monkeypatch): + # -8255 is what a V2-only driver answers the V1 entry point with: a + # statement about the API, not about the hardware. + fake = _install_init_only_dcmi( + monkeypatch, + v1_error=pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT), + ) + + assert AscendDetector.is_supported() is True + assert fake.calls == ["dcmi_init", "dcmiv2_init"] + + +def test_is_supported_leaves_the_v2_api_alone_when_v1_answers(monkeypatch): + # The generations that already worked must not pay for the fallback. + fake = _install_init_only_dcmi(monkeypatch) + + assert AscendDetector.is_supported() is True + assert fake.calls == ["dcmi_init"] + + +def test_is_supported_stays_false_when_neither_api_initializes(monkeypatch): + fake = _install_init_only_dcmi( + monkeypatch, + v1_error=pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT), + v2_error=pydcmi.DCMIError(pydcmi.DCMI_ERROR_FUNCTION_NOT_FOUND), + ) + + assert AscendDetector.is_supported() is False + assert fake.calls == ["dcmi_init", "dcmiv2_init"] + + +# --------------------------------------------------------------------------- # +# A fake driver serving the V2 API only, as an A5 host's does. # +# # +# Every V1 entry point refuses with DCMI_ERROR_NOT_SUPPORT, which is what the # +# 950PR host reported for dcmi_init and dcmi_get_driver_version alike -- so a # +# V2 path that slips back into a V1 call fails here, not on hardware. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _UnitV2: + """ + One device as the V2 API reports it: flat, indexed by its logic id. + """ + + dev_id: int = 0 + unit_type: int = pydcmi.DCMI_UNIT_TYPE_NPU + unit_type_readable: bool = True + phy_id: int = 0 + phy_id_readable: bool = True + chip_name: str = "Ascend950PR" + aicore_cnt: int = 32 + hbm_size: int = 131072 + hbm_usage: int = 5225 + cores_utilization: int = 17 + temperature: int = 51 + power_deciwatts: int = 2012 + ecc_errors: int = 0 + # Which die types the driver answers for; the rest raise. A real 950PR + # answers neither, which the die-fallback tests set explicitly. + die_types: tuple[int, ...] = (pydcmi.DCMI_DIE_TYPE_VDIE,) + + @property + def die_id(self) -> str: + return f"5a 6b 7c 8d {self.dev_id:x}" + + @property + def bdf(self) -> str: + return f"0000:{0x01 + self.dev_id * 0x10:02x}:00.0" + + +@dataclass +class _FakeDCMIV2: + """ + A stand-in for the pydcmi binding on a V2-only driver. + """ + + units: list[_UnitV2] = field(default_factory=lambda: [_UnitV2()]) + calls: list[str] = field(default_factory=list) + dcmi_version: str | None = "25.7.rc1.6" + # As the real binding: 1 until dcmiv2_init() succeeds. Hardcoding 2 would + # make "caller never initialized" untestable. + api_version: int = 1 + + DCMIError = pydcmi.DCMIError + DCMI_UNIT_TYPE_NPU = pydcmi.DCMI_UNIT_TYPE_NPU + DCMI_UNIT_TYPE_MCU = pydcmi.DCMI_UNIT_TYPE_MCU + DCMI_DIE_TYPE_VDIE = pydcmi.DCMI_DIE_TYPE_VDIE + DCMI_DIE_TYPE_NDIE = pydcmi.DCMI_DIE_TYPE_NDIE + DCMI_DEVICE_TYPE_HBM = pydcmi.DCMI_DEVICE_TYPE_HBM + DCMI_DEVICE_TYPE_DDR = pydcmi.DCMI_DEVICE_TYPE_DDR + DCMI_INPUT_TYPE_AICORE = pydcmi.DCMI_INPUT_TYPE_AICORE + DCMI_ERROR_NOT_SUPPORT = pydcmi.DCMI_ERROR_NOT_SUPPORT + DCMI_ERROR_FUNCTION_NOT_FOUND = pydcmi.DCMI_ERROR_FUNCTION_NOT_FOUND + + def dcmi_api_version(self): + return self.api_version + + def dcmi_library_path(self): + return "libdcmi.so" + + def _unit(self, dev_id: int) -> _UnitV2: + for u in self.units: + if u.dev_id == dev_id: + return u + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_INVALID_DEVICE_ID) + + def __getattr__(self, name: str): + handler = { + "dcmiv2_init": self._init, + "dcmiv2_get_device_list": self._get_device_list, + "dcmiv2_get_device_type": self._get_device_type, + "dcmiv2_get_device_chip_info": self._get_chip_info, + "dcmiv2_get_device_hbm_info": self._get_hbm_info, + "dcmiv2_get_device_ecc_info": self._get_ecc_info, + "dcmiv2_get_device_die_id": self._get_die_id, + "dcmiv2_get_chip_phy_id_by_dev_id": self._get_phy_id, + "dcmiv2_get_device_bdf": self._get_bdf, + "dcmiv2_get_device_utilization_rate": self._get_utilization_rate, + "dcmiv2_get_device_temperature": self._get_temperature, + "dcmiv2_get_device_power_info": self._get_power_info, + "dcmiv2_get_affinity_cpu_info_by_dev_id": self._get_affinity, + "dcmiv2_get_dcmi_version": self._get_dcmi_version, + }.get(name) + if handler is not None: + return handler + + if name.startswith("dcmi_"): + # A V2 driver refuses the V1 API wholesale. + def _refuse(*_args, **_kwargs): + self.calls.append(name) + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + + return _refuse + + raise AttributeError(name) + + def _init(self): + self.calls.append("dcmiv2_init") + self.api_version = 2 + + def _get_device_list(self): + self.calls.append("dcmiv2_get_device_list") + return [u.dev_id for u in self.units] + + def _get_device_type(self, dev_id): + self.calls.append("dcmiv2_get_device_type") + u = self._unit(dev_id) + if not u.unit_type_readable: + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + return u.unit_type + + def _get_chip_info(self, dev_id): + self.calls.append("dcmiv2_get_device_chip_info") + u = self._unit(dev_id) + return _FakeStruct(chip_name=u.chip_name, aicore_cnt=u.aicore_cnt) + + def _get_hbm_info(self, dev_id): + self.calls.append("dcmiv2_get_device_hbm_info") + u = self._unit(dev_id) + # A None size stands for a driver refusing the only memory call V2 has. + if u.hbm_size is None: + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + return _FakeStruct(memory_size=u.hbm_size, memory_usage=u.hbm_usage) + + def _get_ecc_info(self, dev_id, device_type): + self.calls.append("dcmiv2_get_device_ecc_info") + u = self._unit(dev_id) + return _FakeStruct( + enable_flag=1, + single_bit_error_cnt=u.ecc_errors, + double_bit_error_cnt=0, + ) + + def _get_die_id(self, dev_id, input_type): + self.calls.append("dcmiv2_get_device_die_id") + u = self._unit(dev_id) + if input_type not in u.die_types: + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + return u.die_id + + def _get_phy_id(self, dev_id): + self.calls.append("dcmiv2_get_chip_phy_id_by_dev_id") + u = self._unit(dev_id) + if not u.phy_id_readable: + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + return u.phy_id + + def _get_bdf(self, dev_id): + self.calls.append("dcmiv2_get_device_bdf") + return self._unit(dev_id).bdf + + def _get_utilization_rate(self, dev_id, input_type): + self.calls.append("dcmiv2_get_device_utilization_rate") + assert input_type == pydcmi.DCMI_INPUT_TYPE_AICORE + return self._unit(dev_id).cores_utilization + + def _get_temperature(self, dev_id): + self.calls.append("dcmiv2_get_device_temperature") + return self._unit(dev_id).temperature + + def _get_power_info(self, dev_id): + self.calls.append("dcmiv2_get_device_power_info") + return self._unit(dev_id).power_deciwatts + + def _get_affinity(self, dev_id): + self.calls.append("dcmiv2_get_affinity_cpu_info_by_dev_id") + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + + def _get_dcmi_version(self): + self.calls.append("dcmiv2_get_dcmi_version") + if self.dcmi_version is None: + raise pydcmi.DCMIError(pydcmi.DCMI_ERROR_NOT_SUPPORT) + return self.dcmi_version + + +@pytest.fixture +def fake_pydcmi_v2(monkeypatch): + def _install(units: list[_UnitV2] | None = None, **kwargs) -> _FakeDCMIV2: + fake = _FakeDCMIV2(units=units if units is not None else [_UnitV2()], **kwargs) + monkeypatch.setattr(ascend, "pydcmi", fake) + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_PCI_CHECK", True) + return fake + + return _install + + +def test_detect_info_v2_enumerates_devices_flat(fake_pydcmi_v2): + fake_pydcmi_v2( + [ + _UnitV2(dev_id=0, phy_id=3), + _UnitV2(dev_id=1, phy_id=7), + ], + ) + + devices = AscendDetector().detect_info() + + assert [d.index for d in devices] == [0, 1] + # The device node is numbered by the physical id, which V2 reports through + # a call of its own rather than deriving from the index. + assert [d.appendix["physical_id"] for d in devices] == [3, 7] + assert [d.name for d in devices] == ["Ascend950PR", "Ascend950PR"] + assert [d.memory for d in devices] == [131072, 131072] + assert devices[0].uuid == "5A 6B 7C 8D 0" + # V2 has no card level, so nothing pretends there is one. + assert "card_id" not in devices[0].appendix + assert "device_id" not in devices[0].appendix + # The whole point: an A5 chip resolves to its generation. + assert devices[0].appendix["arch_family"] == "Ascend950PR" + assert ascend.get_ascend_cann_variant(devices[0].appendix["arch_family"]) == "950" + + +def test_detect_info_v2_touches_no_v1_call(fake_pydcmi_v2): + fake = fake_pydcmi_v2() + + AscendDetector().detect_info() + + # dcmi_init aside -- is_supported has to try it before falling back -- no + # V1 entry point may be reached: this driver refuses every one of them. + assert [c for c in fake.calls if c.startswith("dcmi_")] == ["dcmi_init"] + + +def test_detect_info_v2_skips_a_non_npu_device(fake_pydcmi_v2): + fake_pydcmi_v2( + [ + _UnitV2(dev_id=0), + _UnitV2(dev_id=1, unit_type=pydcmi.DCMI_UNIT_TYPE_MCU), + ], + ) + + devices = AscendDetector().detect_info() + + assert [d.index for d in devices] == [0] + + +def test_detect_info_v2_keeps_a_device_whose_type_is_unreadable(fake_pydcmi_v2): + fake_pydcmi_v2([_UnitV2(dev_id=0, unit_type_readable=False)]) + + devices = AscendDetector().detect_info() + + assert [d.index for d in devices] == [0] + + +def test_detect_info_v2_skips_a_device_without_a_readable_physical_id(fake_pydcmi_v2): + fake_pydcmi_v2( + [ + _UnitV2(dev_id=0, phy_id_readable=False), + _UnitV2(dev_id=1, phy_id=1), + ], + ) + + devices = AscendDetector().detect_info() + + # Standing the index in for an unreadable physical id would hand a + # container another NPU's device node. + assert [d.index for d in devices] == [1] + + +def test_detect_info_v2_reports_the_dcmi_version_as_its_own_field(fake_pydcmi_v2): + # The DCMI library version is not the driver version, so driver_version + # stays empty rather than carrying a different number under its name. + fake_pydcmi_v2() + + devices = AscendDetector().detect_info() + + assert len(devices) == 1 + assert devices[0].driver_version is None + assert devices[0].appendix["dcmi_version"] == "25.7.rc1.6" + + +def test_detect_info_v2_yields_devices_without_a_dcmi_version(fake_pydcmi_v2): + # The DCMI version call may fail too, and a device is still addressable + # without it. + fake_pydcmi_v2(dcmi_version=None) + + devices = AscendDetector().detect_info() + + assert len(devices) == 1 + assert devices[0].driver_version is None + assert "dcmi_version" not in devices[0].appendix + + +def test_detect_usage_v2_merges_the_usage_fields_by_uuid(fake_pydcmi_v2): + fake_pydcmi_v2([_UnitV2(dev_id=0, phy_id=0)]) + + devices = AscendDetector().detect(usage=True) + + assert len(devices) == 1 + dev = devices[0] + assert dev.cores_utilization == 17 + assert dev.memory_used == 5225 + assert dev.temperature == 51 + # 0.1W as the driver reports it, W as the detector does. + assert dev.power_used == _UnitV2().power_deciwatts / 10 + assert dev.memory_utilization == get_utilization(5225, 131072) + + +def test_detect_v2_reports_an_uncorrectable_ecc_error(fake_pydcmi_v2, monkeypatch): + # The ECC read costs a driver call per device, so the health check is off + # by default and nothing else here reaches this path. + fake_pydcmi_v2([_UnitV2(dev_id=0, ecc_errors=3)]) + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + + devices = AscendDetector().detect_info() + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + # merge_devices_usage overwrites memory_status, so the usage pass has to + # produce it too or the verdict is lost. + AscendDetector().detect_usage(devices) + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +def test_detect_v2_reports_a_healthy_device(fake_pydcmi_v2, monkeypatch): + fake = fake_pydcmi_v2([_UnitV2(dev_id=0)]) + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + + devices = AscendDetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + # The verdict must come from the driver, not from a skipped health check + # -- both produce the same enum. + assert "dcmiv2_get_device_ecc_info" in fake.calls + + +def test_detect_info_v2_skips_a_device_whose_memory_is_unreadable(fake_pydcmi_v2): + # The device is dropped, but only that one: the refusal must not take the + # whole detection with it. + fake_pydcmi_v2( + [ + _UnitV2(dev_id=0, hbm_size=None), + _UnitV2(dev_id=1, phy_id=1), + ], + ) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [1] + + +def test_detect_usage_v2_keeps_the_other_devices_when_one_memory_read_fails( + fake_pydcmi_v2, +): + # A device left out of a round keeps its last figures, rather than every + # device's metrics being cleared. + fake = fake_pydcmi_v2( + [ + _UnitV2(dev_id=0), + _UnitV2(dev_id=1, phy_id=1), + ], + ) + + devices = AscendDetector().detect_info() + AscendDetector().detect_usage(devices) + assert devices[0].cores_utilization == _UnitV2().cores_utilization + + # The driver starts refusing the memory call, for device 0 alone. + fake.units[0].hbm_size = None + fake.units[1].cores_utilization = 42 + AscendDetector().detect_usage(devices) + + assert devices[0].cores_utilization == _UnitV2().cores_utilization + assert devices[1].cores_utilization == 42 + + +def test_get_topology_v2_reports_no_distances(fake_pydcmi_v2): + fake = fake_pydcmi_v2([_UnitV2(dev_id=0, phy_id=0), _UnitV2(dev_id=1, phy_id=1)]) + + topo = AscendDetector().get_topology() + + assert topo is not None + assert len(topo.devices_distances) == 2 + # V2 declares no topology call, so nothing is guessed at and no V1 call is + # made behind the scenes -- not even the initialization. + assert [c for c in fake.calls if c.startswith("dcmi_")] == ["dcmi_init"] + assert "dcmi_get_topo_info_by_device_id" not in fake.calls + + +def test_get_topology_v2_resolves_the_api_version_when_handed_devices(fake_pydcmi_v2): + # detect_topologies() passes the devices straight in, so detect_info() + # never runs. Left unresolved, the API version reads 1 and the V1 + # dcmi_init() raises. + fake = fake_pydcmi_v2([_UnitV2(dev_id=0), _UnitV2(dev_id=1, phy_id=1)]) + devices = AscendDetector().detect_info() + + # Back to a process that has not probed anything yet. + AscendDetector.is_supported.cache_clear() + fake.api_version = 1 + fake.calls.clear() + + topo = AscendDetector().get_topology(devices) + + assert topo is not None + assert len(topo.devices_distances) == 2 + assert "dcmi_get_topo_info_by_device_id" not in fake.calls + + +def test_detect_info_v2_falls_back_to_the_ndie(fake_pydcmi_v2): + fake_pydcmi_v2([_UnitV2(dev_id=0, die_types=(pydcmi.DCMI_DIE_TYPE_NDIE,))]) + + devices = AscendDetector().detect_info() + + assert [d.uuid for d in devices] == ["5A 6B 7C 8D 0"] + + +def test_detect_info_v2_identifies_a_dieless_device_by_its_address(fake_pydcmi_v2): + # What the 950PR driver does: both die types answer NOT_SUPPORT. Dropping + # the device over it would leave eight usable NPUs invisible. + fake_pydcmi_v2([_UnitV2(dev_id=0, die_types=())]) + + devices = AscendDetector().detect_info() + + assert len(devices) == 1 + assert devices[0].uuid == "0000:01:00.0" + + +def test_detect_usage_v2_merges_when_the_uuid_fell_back_to_the_address(fake_pydcmi_v2): + # The usage pass merges on the uuid, so it has to arrive at the same one + # the inventory did -- including when that was the address. + fake_pydcmi_v2([_UnitV2(dev_id=0, die_types=())]) + + devices = AscendDetector().detect(usage=True) + + assert len(devices) == 1 + assert devices[0].uuid == "0000:01:00.0" + assert devices[0].cores_utilization == _UnitV2().cores_utilization + assert devices[0].memory_used == _UnitV2().hbm_usage + + +def test_detect_reproduces_the_950pr_host(fake_pydcmi_v2): + # The 950PR host exactly as probed: eight NPUs, no die of either type, + # 28 AI cores and 128GiB of HBM each, on the buses npu-smi reported. + fake_pydcmi_v2( + [_UnitV2(dev_id=i, phy_id=i, aicore_cnt=28, die_types=()) for i in range(8)], + ) + + devices = AscendDetector().detect() + + assert len(devices) == 8 + assert {d.name for d in devices} == {"Ascend950PR"} + assert {d.appendix["arch_family"] for d in devices} == {"Ascend950PR"} + assert { + ascend.get_ascend_cann_variant(d.appendix["arch_family"]) for d in devices + } == {"950"} + assert all(d.memory == 131072 for d in devices) + assert all(d.cores == 28 for d in devices) + # Every card has to be told apart, which the address fallback still manages. + assert len({d.uuid for d in devices}) == 8 + assert [d.appendix["physical_id"] for d in devices] == list(range(8))