From 0865ff802134d375929b90488ea5fecb6d755e3d Mon Sep 17 00:00:00 2001 From: xuexu6666 Date: Thu, 16 Jul 2026 21:21:46 -0500 Subject: [PATCH 1/2] fix: skip fabric manager on arm64 (Grace-Blackwell GB200/GB300) arm64 cuda nodes are the Grace-Blackwell SKUs (GB200/GB300), which have no node-local fabric manager -- the NVLink fabric is managed out-of-band via IMEX. The driver container should neither install nor bundle fabricmanager there. Today device_init() runs the fabricmanager package installer for every cuda node. On arm64 it computes NVIDIA_FM_ARCH from $ARCH (uname -m = "aarch64") and then only remaps to "sbsa" when the value equals "arm64" -- which never matches -- so it looks for /opt/gpu/fabricmanager-linux-aarch64-/... which does not exist (the tarball download.sh bundles is named "sbsa"). The installer exits 127; installGPUDriverImage retries the whole container until CSE hits its ~780s timeout and exits 84, so ensureKubelet never runs and the node never joins the cluster. Surfaced by the GB300 managed-driver (--gpu-driver Install) e2e. Fix: gate fabric manager to x86_64 only. - install.sh: run fm_run_package_installer.sh only when ARCH == x86_64; drop the dead arm64->sbsa remap. - download.sh: download/bundle the fabricmanager tarball only when TARGETARCH == amd64, so the arm64 image no longer ships an unused tarball. --- download.sh | 5 +++-- install.sh | 13 +++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/download.sh b/download.sh index b8e1bb0..906ced8 100755 --- a/download.sh +++ b/download.sh @@ -48,8 +48,9 @@ install_fabric_manager () { mv /opt/gpu/fm_run_package_installer.sh /opt/gpu/fabricmanager-linux-${NVIDIA_FM_ARCH}-${DRIVER_VERSION}/sbin/fm_run_package_installer.sh } -if [[ "${DRIVER_KIND}" == "cuda" ]]; then - # download fabricmanager for nvlink based systems, e.g. multi instance gpu vms. +# download fabricmanager for nvlink based systems, but skip it on arm64: +# arm64 = Grace-Blackwell (GB200/GB300), which uses IMEX, not a node-local FM. +if [[ "${DRIVER_KIND}" == "cuda" && "${TARGETARCH}" != "arm64" ]]; then install_fabric_manager fi diff --git a/install.sh b/install.sh index d9917a6..229ea34 100644 --- a/install.sh +++ b/install.sh @@ -159,14 +159,11 @@ device_init() { cp -r /usr/bin/lib64/lib64/* "/usr/lib/${ARCH}-linux-gnu/" nvidia-smi - # install fabricmanager for nvlink based systems - if [[ "${DRIVER_KIND}" == "cuda" ]]; then - NVIDIA_FM_ARCH=$ARCH - if [ "$NVIDIA_FM_ARCH" = "arm64" ]; then - # NVIDIA uses the name "SBSA" for ARM64 platforms for the fabric manager. See https://en.wikipedia.org/wiki/Server_Base_System_Architecture - NVIDIA_FM_ARCH="sbsa" - fi - bash /opt/gpu/fabricmanager-linux-${NVIDIA_FM_ARCH}-${DRIVER_VERSION}/sbin/fm_run_package_installer.sh + # install fabricmanager for nvlink based systems, but skip it on arm64: + # arm64 = Grace-Blackwell (GB200/GB300), which uses IMEX, not a node-local FM. + # (uname -m reports "aarch64" for arm64.) + if [[ "${DRIVER_KIND}" == "cuda" && "${ARCH}" != "aarch64" ]]; then + bash /opt/gpu/fabricmanager-linux-${ARCH}-${DRIVER_VERSION}/sbin/fm_run_package_installer.sh fi mkdir -p /etc/containerd/config.d From f3a3752cf8f4a194748241cf335ea481ab2f6062 Mon Sep 17 00:00:00 2001 From: xuexu6666 Date: Fri, 17 Jul 2026 19:48:16 -0500 Subject: [PATCH 2/2] feat: install nvidia-imex on arm64 (Grace-Blackwell GB200/GB300) GB uses IMEX -- not a node-local fabric manager -- to coordinate cross-node NVLink (MNNVL / ComputeDomains). The nvidia-imex daemon ships only as a separate deb in the CUDA repo (not in the driver .run or the fabric-manager redist), so: - download.sh bundles the version-matched nvidia-imex arm64 deb into the image, resolving the exact filename from the repo Packages index (the revision suffix varies by version). Skipped on x86 (which uses fabric manager instead). - install.sh installs it at node boot via dpkg -i --force-depends (the deb declares nvidia-modprobe, provided by the runfile driver rather than a deb), and leaves the node-wide nvidia-imex.service OFF -- IMEX is orchestrated per ComputeDomain by the NVIDIA DRA driver, not run cluster-wide from the host. The host binary must be present so the DRA driver can CDI-inject it into its per-workload IMEX daemon pods. Pairs with the fabric-manager arm64 skip: on GB, skip FM and install IMEX in the same code path where x86 installs FM. --- download.sh | 28 ++++++++++++++++++++++++++++ install.sh | 12 ++++++++++++ 2 files changed, 40 insertions(+) diff --git a/download.sh b/download.sh index 906ced8..61c80ee 100755 --- a/download.sh +++ b/download.sh @@ -48,12 +48,40 @@ install_fabric_manager () { mv /opt/gpu/fm_run_package_installer.sh /opt/gpu/fabricmanager-linux-${NVIDIA_FM_ARCH}-${DRIVER_VERSION}/sbin/fm_run_package_installer.sh } +install_imex () { + # nvidia-imex is the cross-node NVLink (MNNVL / ComputeDomains) coordinator for + # Grace-Blackwell. It is NOT in the driver .run or the fabric-manager redist -- it + # ships only as a separate deb in the CUDA repo. Bundle the version-matched deb into + # the image; it is installed at node boot (see install.sh device_init). The exact deb + # revision suffix (e.g. -1ubuntu1) varies by version, so resolve the filename from the + # repo Packages index rather than hard-coding it. + local repo="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${VERSION_ID//./}/sbsa" + local deb + # NB: awk must read to EOF (no early `exit`) -- exiting mid-stream closes the pipe and + # SIGPIPEs curl, which `set -o pipefail` would turn into a build failure. Gate on a flag + # to keep only the first match instead. + deb="$(curl -fsSL "${repo}/Packages" \ + | awk -v p="nvidia-imex_${DRIVER_VERSION}-" '$1=="Filename:" && index($2,p) && !f{print $2; f=1}')" + if [[ -z "${deb}" ]]; then + echo "nvidia-imex ${DRIVER_VERSION} not found in ${repo}" + exit 1 + fi + curl -fsSLO "${repo}/${deb#./}" + mv "$(basename "${deb}")" /opt/gpu/ +} + # download fabricmanager for nvlink based systems, but skip it on arm64: # arm64 = Grace-Blackwell (GB200/GB300), which uses IMEX, not a node-local FM. if [[ "${DRIVER_KIND}" == "cuda" && "${TARGETARCH}" != "arm64" ]]; then install_fabric_manager fi +# download nvidia-imex for nvlink based arm64 systems (Grace-Blackwell GB200/GB300): +# GB uses IMEX (not a node-local fabric manager) to coordinate cross-node NVLink. +if [[ "${DRIVER_KIND}" == "cuda" && "${TARGETARCH}" == "arm64" ]]; then + install_imex +fi + # configure nvidia apt repo to cache packages curl -fsSLO https://nvidia.github.io/libnvidia-container/gpgkey diff --git a/install.sh b/install.sh index 229ea34..2e044d1 100644 --- a/install.sh +++ b/install.sh @@ -166,6 +166,18 @@ device_init() { bash /opt/gpu/fabricmanager-linux-${ARCH}-${DRIVER_VERSION}/sbin/fm_run_package_installer.sh fi + # install nvidia-imex on arm64 (Grace-Blackwell): the cross-node NVLink (MNNVL) + # coordinator that GB uses in place of a node-local fabric manager. The binary must be + # present on the host so the NVIDIA DRA driver (ComputeDomains) can inject it into its + # per-workload IMEX daemon pods. --force-depends: the deb declares nvidia-modprobe, + # which is provided by the runfile driver (present on disk) rather than as a deb. + # The node-wide nvidia-imex.service stays OFF -- IMEX is orchestrated per ComputeDomain + # by DRA, not run cluster-wide from the host. + if [[ "${DRIVER_KIND}" == "cuda" && "${ARCH}" == "aarch64" ]]; then + dpkg -i --force-depends /opt/gpu/nvidia-imex_*_arm64.deb + systemctl disable --now nvidia-imex.service 2>/dev/null || true + fi + mkdir -p /etc/containerd/config.d cp /opt/gpu/10-nvidia-runtime.toml /etc/containerd/config.d/10-nvidia-runtime.toml