From e614a286395b01d058725bd41c7009ada590da79 Mon Sep 17 00:00:00 2001 From: mrL <1417558386@qq.com> Date: Sat, 1 Aug 2026 23:35:44 +0800 Subject: [PATCH 1/4] ci(cuda): build only for CI runner GPU arch and raise parallelism The CUDA CI built every .cu file for 7 GPU architectures (60/70/75/80/86/89/90) with a hardcoded -j4, so the Configure & Build step took ~33 min even with a warm ccache. - Pin CMAKE_CUDA_ARCHITECTURES=70: the CI GPU pool is Tesla V100 (sm_70, per nvidia-smi in the run logs and the '16V100' Slurm partition in .ci/slurm/config.ini). This cuts nvcc work by ~7x. - Build with -j $(nproc) instead of -j4; with the arch list reduced, the higher parallelism is memory-safe. Expected: Configure & Build ~33 min -> ~10 min on a cache-cold run. --- .github/workflows/cuda.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 2216ab9873..f03d6f1af5 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -46,8 +46,14 @@ jobs: nvidia-smi source toolchain/install/setup rm -rf build - cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON - cmake --build build -j4 + # Build only for the GPU architecture of the CI runner instead of the + # default 7 architectures (60/70/75/80/86/89/90). The CI GPU pool is + # Tesla V100 (sm_70, see the nvidia-smi output above); update this if + # the runner GPU model changes (V100=70, A100=80, H100=90, T4=75). + # This cuts nvcc work by ~7x and allows safe -j. + cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON \ + -DCMAKE_CUDA_ARCHITECTURES=70 + cmake --build build -j "$(nproc)" cmake --install build - name: Module_LCAO CUDA Unittests From 167904f9e0cfbd9d758bd42fb01fe2a246dde980 Mon Sep 17 00:00:00 2001 From: mrL <1417558386@qq.com> Date: Sun, 2 Aug 2026 17:02:51 +0800 Subject: [PATCH 2/4] ci: auto-detect GPU arch via nvidia-smi at CMake configure time Sister commit to e614a2863. The previous commit hardcoded -DCMAKE_CUDA_ARCHITECTURES=70 assuming the CI pool is Tesla V100. Reviewers (Stardust0831 + chenmohan) pointed out this couples the workflow to a specific GPU model and breaks if the runner pool is heterogeneous or upgraded. This commit moves the arch selection into CMakeLists.txt: - When CMAKE_CUDA_ARCHITECTURES is unset and nvidia-smi is available, query --query-gpu=compute_cap and set the arch from the result. Map '7.0' -> 70, '8.0' -> 80, '8.9' -> 89, '9.0' -> 90, etc. - Falls back to the historical multi-arch default if nvidia-smi is not present (CPU-only build host) or returns an unrecognized value. - User-provided CMAKE_CUDA_ARCHITECTURES still takes precedence. Workflow file: removed the -DCMAKE_CUDA_ARCHITECTURES=70 line; the Configure step now relies on the CMake-side detection. Verified the plain -DCMAKE_CUDA_ARCHITECTURES=70 still produces a 38 min cold Build on the existing V100 runner. --- .github/workflows/cuda.yml | 13 ++++++------- CMakeLists.txt | 32 +++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f03d6f1af5..9f6c6bb933 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -46,13 +46,12 @@ jobs: nvidia-smi source toolchain/install/setup rm -rf build - # Build only for the GPU architecture of the CI runner instead of the - # default 7 architectures (60/70/75/80/86/89/90). The CI GPU pool is - # Tesla V100 (sm_70, see the nvidia-smi output above); update this if - # the runner GPU model changes (V100=70, A100=80, H100=90, T4=75). - # This cuts nvcc work by ~7x and allows safe -j. - cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON \ - -DCMAKE_CUDA_ARCHITECTURES=70 + # The GPU architecture is detected automatically by CMakeLists.txt at + # configure time (via `nvidia-smi --query-gpu=compute_cap`), so no + # explicit -DCMAKE_CUDA_ARCHITECTURES is needed here. Building for a + # single architecture instead of the default 7 (60/70/75/80/86/89/90) + # cuts nvcc work by ~7x and allows safe -j on the build host. + cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON cmake --build build -j "$(nproc)" cmake --install build diff --git a/CMakeLists.txt b/CMakeLists.txt index 5556be319e..baf0dfa93b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -435,7 +435,37 @@ if(USE_CUDA) # check # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Modules/Internal/CMakeCUDAArchitecturesAll.cmake # for available architectures in different CUDA versions - + + # Auto-detect GPU compute capability at configure time when the user has + # not explicitly set CMAKE_CUDA_ARCHITECTURES and the build host has an + # NVIDIA GPU. This lets CI builds (and any other single-GPU host) compile + # for the GPU they actually have, instead of hardcoding an architecture. + # Falls back to the historical multi-arch default if nvidia-smi is not + # present (e.g., CPU-only build host) or returns an unrecognized value. + # User-provided CMAKE_CUDA_ARCHITECTURES still takes precedence. + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES AND USE_CUDA AND COMMAND nvidia-smi) + execute_process( + COMMAND nvidia-smi --query-gpu=compute_cap --format=csv,noheader + OUTPUT_VARIABLE ABACUS_DETECTED_GPU_CAP + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE ABACUS_DETECTED_GPU_CAP_RC) + if(ABACUS_DETECTED_GPU_CAP_RC EQUAL 0 AND NOT "${ABACUS_DETECTED_GPU_CAP}" STREQUAL "") + # Map "7.0" -> 70, "8.0" -> 80, "8.9" -> 89, "9.0" -> 90, etc. + string(REPLACE "." "" ABACUS_DETECTED_ARCH "${ABACUS_DETECTED_GPU_CAP}") + # Accept known CUDA architectures (50..99 covers all current generations). + if("${ABACUS_DETECTED_ARCH}" MATCHES "^[5-9][0-9]$") + set(CMAKE_CUDA_ARCHITECTURES "${ABACUS_DETECTED_ARCH}" + CACHE STRING "GPU architectures to compile for") + message(STATUS "ABACUS: detected GPU compute capability " + "${ABACUS_DETECTED_GPU_CAP}; building for sm_${ABACUS_DETECTED_ARCH}") + else() + message(STATUS "ABACUS: detected GPU compute capability " + "'${ABACUS_DETECTED_GPU_CAP}' looks unrecognized; " + "falling back to default arch list") + endif() + endif() + endif() + # CUDA 13.0+ dropped support for architectures below 75 if(CUDAToolkit_VERSION VERSION_LESS "13.0") set(CMAKE_CUDA_ARCHITECTURES From 2f429be8d04d96e6e881a34abf02556d83e9a7ab Mon Sep 17 00:00:00 2001 From: mrL <1417558386@qq.com> Date: Sun, 2 Aug 2026 19:03:02 +0800 Subject: [PATCH 3/4] ci(cuda): revert broken auto-detect; use explicit sm_70 pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit (167904f9e) tried to make the workflow adapt to whatever GPU the runner has by auto-detecting compute capability at CMake configure time. Reviewers (Stardust0831 + 张笑扬) correctly flagged two independent defects and a deeper design issue. Defects in the previous commit: 1. if(COMMAND nvidia-smi) tests for a CMake command, not an executable on PATH. It was always false, so execute_process never ran. The correct check is find_program(NVIDIA_SMI_EXECUTABLE nvidia-smi). 2. The detection block and the historical default list were both inside the same if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) outer guard. The historical default uses plain set() which shadows the cache entry; all 7 archs were appended regardless of detection. 3. AND USE_CUDA inside the detection block was redundant; the surrounding if(USE_CUDA) at line 419 already guards it. Design issue: Auto-detection is the wrong default for a cluster codebase. ABACUS is configured on whichever host runs cmake (often a login node on an HPC system) but the resulting binary may run on a different compute node. Silent auto-detection produces a binary that fails at run time with 'no kernel image is available for execution on the device' and no warning at configure time. HPC convention is to build on the compute node, where the workflow's hardcoded -D matches the hardware. Furthermore, heterogeneous auto-detection fragments the ccache key per node, which defeats cache_warmer (#7756). An explicit uniform pin across CI is exactly what cache_warmer relies on. This commit: - Reverts the CMakeLists.txt auto-detect block. - Restores -DCMAKE_CUDA_ARCHITECTURES=70 in cuda.yml, with a comment noting that the value assumes a homogeneous V100 pool and should be updated if the pool changes. - Keeps -j $(nproc), which is the real win in e614a2863 (~2x parallelism). Note on the measured 33 min -> 18 min result: the reviewer is correct that the savings probably come almost entirely from -j4 -> -j $(nproc), not from the arch cut (7 -> 1 arch). With nvcc being fast and C++ TUs dominating build time, doubling the build parallelism is enough to halve the wall time; the arch reduction's contribution, if any, is small. The arch pin still avoids fatbin bloat and uniform ccache keys, but the headline number in the PR body should not over-claim it. --- .github/workflows/cuda.yml | 23 +++++++++++++++++------ CMakeLists.txt | 32 +------------------------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 9f6c6bb933..103c831aef 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -46,12 +46,23 @@ jobs: nvidia-smi source toolchain/install/setup rm -rf build - # The GPU architecture is detected automatically by CMakeLists.txt at - # configure time (via `nvidia-smi --query-gpu=compute_cap`), so no - # explicit -DCMAKE_CUDA_ARCHITECTURES is needed here. Building for a - # single architecture instead of the default 7 (60/70/75/80/86/89/90) - # cuts nvcc work by ~7x and allows safe -j on the build host. - cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON + # Pin the CUDA architecture to sm_70 (Tesla V100) instead of building + # for the default 7 architectures (60/70/75/80/86/89/90). The CI GPU + # pool is a homogeneous set of V100 pods (verified by sampling 5+ + # runner IDs, all on the gpu-runner-6qqd8-* K8s deployment with + # identical test step timings across runs). Building for a single + # arch cuts nvcc work to ~1/7 and keeps the ccache key uniform so + # cache_warmer (#7756) can populate every runner with the same + # entries. + # + # NOTE: this is an explicit pin rather than auto-detection. On a + # cluster, configuring on a login node whose GPU differs from the + # compute node would silently produce a binary that fails with + # "no kernel image is available" at run time; HPC convention is to + # build on the compute node, where the workflow's hardcoded -D + # matches the actual hardware. + cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON \ + -DCMAKE_CUDA_ARCHITECTURES=70 cmake --build build -j "$(nproc)" cmake --install build diff --git a/CMakeLists.txt b/CMakeLists.txt index baf0dfa93b..5556be319e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -435,37 +435,7 @@ if(USE_CUDA) # check # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Modules/Internal/CMakeCUDAArchitecturesAll.cmake # for available architectures in different CUDA versions - - # Auto-detect GPU compute capability at configure time when the user has - # not explicitly set CMAKE_CUDA_ARCHITECTURES and the build host has an - # NVIDIA GPU. This lets CI builds (and any other single-GPU host) compile - # for the GPU they actually have, instead of hardcoding an architecture. - # Falls back to the historical multi-arch default if nvidia-smi is not - # present (e.g., CPU-only build host) or returns an unrecognized value. - # User-provided CMAKE_CUDA_ARCHITECTURES still takes precedence. - if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES AND USE_CUDA AND COMMAND nvidia-smi) - execute_process( - COMMAND nvidia-smi --query-gpu=compute_cap --format=csv,noheader - OUTPUT_VARIABLE ABACUS_DETECTED_GPU_CAP - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE ABACUS_DETECTED_GPU_CAP_RC) - if(ABACUS_DETECTED_GPU_CAP_RC EQUAL 0 AND NOT "${ABACUS_DETECTED_GPU_CAP}" STREQUAL "") - # Map "7.0" -> 70, "8.0" -> 80, "8.9" -> 89, "9.0" -> 90, etc. - string(REPLACE "." "" ABACUS_DETECTED_ARCH "${ABACUS_DETECTED_GPU_CAP}") - # Accept known CUDA architectures (50..99 covers all current generations). - if("${ABACUS_DETECTED_ARCH}" MATCHES "^[5-9][0-9]$") - set(CMAKE_CUDA_ARCHITECTURES "${ABACUS_DETECTED_ARCH}" - CACHE STRING "GPU architectures to compile for") - message(STATUS "ABACUS: detected GPU compute capability " - "${ABACUS_DETECTED_GPU_CAP}; building for sm_${ABACUS_DETECTED_ARCH}") - else() - message(STATUS "ABACUS: detected GPU compute capability " - "'${ABACUS_DETECTED_GPU_CAP}' looks unrecognized; " - "falling back to default arch list") - endif() - endif() - endif() - + # CUDA 13.0+ dropped support for architectures below 75 if(CUDAToolkit_VERSION VERSION_LESS "13.0") set(CMAKE_CUDA_ARCHITECTURES From 2cf9466f48363a1240930f8e77606562446f5922 Mon Sep 17 00:00:00 2001 From: MrLi000001 <77618365+MrLi000001@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:05:51 +0800 Subject: [PATCH 4/4] Clean up comments in CUDA workflow Removed comments explaining the CUDA architecture pinning process. --- .github/workflows/cuda.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 103c831aef..5f700d8232 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -46,21 +46,6 @@ jobs: nvidia-smi source toolchain/install/setup rm -rf build - # Pin the CUDA architecture to sm_70 (Tesla V100) instead of building - # for the default 7 architectures (60/70/75/80/86/89/90). The CI GPU - # pool is a homogeneous set of V100 pods (verified by sampling 5+ - # runner IDs, all on the gpu-runner-6qqd8-* K8s deployment with - # identical test step timings across runs). Building for a single - # arch cuts nvcc work to ~1/7 and keeps the ccache key uniform so - # cache_warmer (#7756) can populate every runner with the same - # entries. - # - # NOTE: this is an explicit pin rather than auto-detection. On a - # cluster, configuring on a login node whose GPU differs from the - # compute node would silently produce a binary that fails with - # "no kernel image is available" at run time; HPC convention is to - # build on the compute node, where the workflow's hardcoded -D - # matches the actual hardware. cmake -B build -G Ninja -DUSE_CUDA=ON -DBUILD_TESTING=ON -DENABLE_FLOAT_FFTW=ON \ -DCMAKE_CUDA_ARCHITECTURES=70 cmake --build build -j "$(nproc)"