From 99ed3cd149cf6fcc5cddc67cf1a0d147b6f3266c Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Tue, 15 Sep 2026 11:44:24 +0800 Subject: [PATCH 01/13] ci: allow reusing generated PyTorch fixtures Agent: ChatGPT Model: GPT-5.6 Sol --- source/install/test_cc_local.sh | 212 ++++++++++++++++++-------------- 1 file changed, 122 insertions(+), 90 deletions(-) diff --git a/source/install/test_cc_local.sh b/source/install/test_cc_local.sh index 17b4f2f593..57004c0d22 100755 --- a/source/install/test_cc_local.sh +++ b/source/install/test_cc_local.sh @@ -55,102 +55,134 @@ else: shutil.copy2(str(so), str(dst)) print(f"Installed {so} -> {dst}") ' - # When the build uses -fsanitize=leak, the custom op .so requires the LSAN - # runtime to be preloaded (otherwise dlopen fails). We disable leak detection - # in the gen scripts to avoid false reports from torch/paddle internals. INFER_SCRIPT_PATH=${SCRIPT_PATH}/../tests/infer - # Remove stale generated model files so they can't be accidentally reused - # if gen scripts change format or the code version changes. - rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte - _GEN_ENV="" - if echo "${CXXFLAGS:-}" | grep -q fsanitize=leak; then - _LSAN_LIB=$(gcc -print-file-name=liblsan.so 2>/dev/null || true) - if [ -n "${_LSAN_LIB}" ] && [ -f "${_LSAN_LIB}" ]; then - # DP_GEN_UNDER_SANITIZER: explicit signal for gen scripts that need - # to skip sanitizer-incompatible sections (e.g. gen_dpa2.py's - # AOTInductor graph .pt2 eval, which can SEGV under the LSAN - # runtime). Sniffing LD_PRELOAD inside the gen script is NOT - # reliable: the sanitizer runtime removes its own entry from the - # process environment during startup. - _GEN_ENV="LD_PRELOAD=${_LSAN_LIB} LSAN_OPTIONS=detect_leaks=0 DP_GEN_UNDER_SANITIZER=lsan" + _FIXTURE_MANIFEST=${INFER_SCRIPT_PATH}/.pt-fixture-cache-manifest + + # CI may restore a complete, content-addressed fixture set. The manifest + # prevents a partial/stale cache from silently turning missing coverage into + # GTEST_SKIP or making the C++ tests consume mismatched reference values. + if [ "${DP_REUSE_GENERATED_PT_MODELS:-0}" = "1" ]; then + if [ ! -s "${_FIXTURE_MANIFEST}" ]; then + echo "ERROR: PT fixture cache hit without a non-empty manifest" >&2 + exit 1 fi - fi - # Run gen scripts in parallel for faster model generation. - # Wait on each PID separately so any failure is caught by set -e. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_sea.py & - PID1=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1.py & - PID2=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa2.py & - PID3=$! - wait $PID1 - wait $PID2 - wait $PID3 + while IFS= read -r fixture; do + [ -n "${fixture}" ] || continue + if [ ! -f "${INFER_SCRIPT_PATH}/${fixture}" ]; then + echo "ERROR: cached PT fixture is missing: ${fixture}" >&2 + exit 1 + fi + done < "${_FIXTURE_MANIFEST}" + echo "Reusing $(wc -l < "${_FIXTURE_MANIFEST}") cached PT fixture files" + else + # Remove stale generated files so they can't be accidentally reused if + # generation scripts or formats change. *.expected sidecars are consumed + # by the C++ tests and therefore belong to the same cache unit as PT2/PTE. + rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte ${INFER_SCRIPT_PATH}/*.expected "${_FIXTURE_MANIFEST}" - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa3.py & - PID4=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & - PID5=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_model_devi.py & - PID6=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_chg_spin.py & - PID9=$! - wait $PID4 - wait $PID5 - wait $PID6 - wait $PID9 + # When the build uses -fsanitize=leak, the custom op .so requires the LSAN + # runtime to be preloaded (otherwise dlopen fails). We disable leak detection + # in the gen scripts to avoid false reports from torch/paddle internals. + _GEN_ENV="" + if echo "${CXXFLAGS:-}" | grep -q fsanitize=leak; then + _LSAN_LIB=$(gcc -print-file-name=liblsan.so 2>/dev/null || true) + if [ -n "${_LSAN_LIB}" ] && [ -f "${_LSAN_LIB}" ]; then + # DP_GEN_UNDER_SANITIZER: explicit signal for gen scripts that need + # to skip sanitizer-incompatible sections (e.g. gen_dpa2.py's + # AOTInductor graph .pt2 eval, which can SEGV under the LSAN + # runtime). Sniffing LD_PRELOAD inside the gen script is NOT + # reliable: the sanitizer runtime removes its own entry from the + # process environment during startup. + _GEN_ENV="LD_PRELOAD=${_LSAN_LIB} LSAN_OPTIONS=detect_leaks=0 DP_GEN_UNDER_SANITIZER=lsan" + fi + fi + # Run gen scripts in parallel for faster model generation. + # Wait on each PID separately so any failure is caught by set -e. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_sea.py & + PID1=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1.py & + PID2=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa2.py & + PID3=$! + wait $PID1 + wait $PID2 + wait $PID3 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4.py & - PID9=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py & - PID10=$! - wait $PID9 - wait $PID10 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa3.py & + PID4=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & + PID5=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_model_devi.py & + PID6=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_chg_spin.py & + PID9=$! + wait $PID4 + wait $PID5 + wait $PID6 + wait $PID9 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin.py & - PID7=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & - PID8=$! - wait $PID7 - wait $PID8 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4.py & + PID9=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py & + PID10=$! + wait $PID9 + wait $PID10 - # Native-spin DPA4 graph archives (baseline + model-level pair - # exclusion). Without this the whole native-spin graph C++ suite - # GTEST_SKIPs on the missing fixture, which is how a dead - # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & - PID11=$! - # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph - # lower, which no other C++ fixture covers. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & - PID12=$! - # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both - # is_spin=true and dim_chg_spin>0, i.e. the only one on which the - # runtime charge_spin argument of DeepSpin::compute is not inert. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & - PID13=$! - # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear - # composition, and the only fixture pinning that a bridged model - # exports NO with-comm artifact (single-rank only by construction). - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & - PID14=$! - # Native-spin DPA4C: the archives above all ship a with-comm artifact - # and are therefore served by DeepSpinPTExpt, which leaves - # NativeSpinPTExpt -- the class the dispatch selects for a native-spin - # archive without one -- uncovered. DPA4C keeps its messaging local and - # so reaches it. The compiled graph calls CUDA-only operators, so the - # fixture is generated only where the C++ suite has a GPU; the tests - # built on it skip when it is absent. - if [ "${DP_VARIANT}" = "cuda" ]; then - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & - PID15=$! - fi - wait $PID11 - wait $PID12 - wait $PID13 - wait $PID14 - if [ "${DP_VARIANT}" = "cuda" ]; then - wait $PID15 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin.py & + PID7=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & + PID8=$! + wait $PID7 + wait $PID8 + + # Native-spin DPA4 graph archives (baseline + model-level pair + # exclusion). Without this the whole native-spin graph C++ suite + # GTEST_SKIPs on the missing fixture, which is how a dead + # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & + PID11=$! + # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph + # lower, which no other C++ fixture covers. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & + PID12=$! + # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both + # is_spin=true and dim_chg_spin>0, i.e. the only one on which the + # runtime charge_spin argument of DeepSpin::compute is not inert. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & + PID13=$! + # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear + # composition, and the only fixture pinning that a bridged model + # exports NO with-comm artifact (single-rank only by construction). + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & + PID14=$! + # Native-spin DPA4C: the archives above all ship a with-comm artifact + # and are therefore served by DeepSpinPTExpt, which leaves + # NativeSpinPTExpt -- the class the dispatch selects for a native-spin + # archive without one -- uncovered. DPA4C keeps its messaging local and + # so reaches it. The compiled graph calls CUDA-only operators, so the + # fixture is generated only where the C++ suite has a GPU; the tests + # built on it skip when it is absent. + if [ "${DP_VARIANT}" = "cuda" ]; then + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & + PID15=$! + fi + wait $PID11 + wait $PID12 + wait $PID13 + wait $PID14 + if [ "${DP_VARIANT}" = "cuda" ]; then + wait $PID15 + fi + + # Record the exact generated fixture set. A warm cache is accepted only + # when every entry in this manifest was restored. + for fixture in "${INFER_SCRIPT_PATH}"/*.pt2 "${INFER_SCRIPT_PATH}"/*.pte "${INFER_SCRIPT_PATH}"/*.expected; do + [ -f "${fixture}" ] && basename "${fixture}" + done | LC_ALL=C sort > "${_FIXTURE_MANIFEST}" + if [ ! -s "${_FIXTURE_MANIFEST}" ]; then + echo "ERROR: PT fixture generation produced no cacheable files" >&2 + exit 1 + fi fi fi if [ "${ENABLE_PADDLE:-TRUE}" == "TRUE" ]; then From 8e8ced2aa755b686e327d3c153eb498e9c5dd4a1 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Tue, 15 Sep 2026 11:44:43 +0800 Subject: [PATCH 02/13] ci: cache generated PyTorch fixtures Agent: ChatGPT Model: GPT-5.6 Sol --- .github/workflows/test_cc.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 13e3e51a32..3b33c6f699 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -43,6 +43,25 @@ jobs: enable_paddle: true steps: - uses: actions/checkout@v7 + - name: Cache generated PyTorch C++ fixtures + id: pt-fixture-cache + if: matrix.enable_pytorch + uses: actions/cache@v6 + with: + path: | + source/tests/infer/*.pt2 + source/tests/infer/*.pte + source/tests/infer/*.expected + source/tests/infer/.pt-fixture-cache-manifest + key: pt-cxx-fixtures-v1-${{ runner.os }}-${{ runner.arch }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ hashFiles('uv.lock', 'pyproject.toml', 'source/tests/infer/**', 'deepmd/**', 'source/op/pt/**') }} + - name: Cache TorchInductor + if: matrix.enable_pytorch + uses: actions/cache@v6 + with: + path: .cache/torchinductor + key: torchinductor-v1-${{ runner.os }}-${{ runner.arch }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}-${{ hashFiles('source/tests/infer/**', 'deepmd/**', 'source/op/pt/**') }} + restore-keys: | + torchinductor-v1-${{ runner.os }}-${{ runner.arch }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}- - name: Install GCC run: | sudo apt-get update @@ -80,6 +99,10 @@ jobs: ENABLE_TENSORFLOW: ${{ matrix.enable_tensorflow && 'TRUE' || 'FALSE' }} ENABLE_PYTORCH: ${{ matrix.enable_pytorch && 'TRUE' || 'FALSE' }} ENABLE_PADDLE: ${{ matrix.enable_paddle && 'TRUE' || 'FALSE' }} + DP_REUSE_GENERATED_PT_MODELS: ${{ steps.pt-fixture-cache.outputs.cache-hit == 'true' && '1' || '0' }} + TORCHINDUCTOR_CACHE_DIR: ${{ github.workspace }}/.cache/torchinductor + TORCHINDUCTOR_FX_GRAPH_CACHE: 1 + TORCHINDUCTOR_AUTOGRAD_CACHE: 1 # test lammps - run: | cp ${{ github.workspace }}/source/build_tests/paddle_inference_install_dir/paddle/lib/*.so ${{ github.workspace }}/dp_test/lib/ From e3e539f46377df8a08a07475723337bb8ff906e7 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Tue, 15 Sep 2026 11:45:30 +0800 Subject: [PATCH 03/13] ci: keep fixture-cache trial diff focused Agent: ChatGPT Model: GPT-5.6 Sol --- source/install/test_cc_local.sh | 208 ++++++++++++++++---------------- 1 file changed, 102 insertions(+), 106 deletions(-) diff --git a/source/install/test_cc_local.sh b/source/install/test_cc_local.sh index 57004c0d22..26bc9203f4 100755 --- a/source/install/test_cc_local.sh +++ b/source/install/test_cc_local.sh @@ -57,10 +57,8 @@ else: ' INFER_SCRIPT_PATH=${SCRIPT_PATH}/../tests/infer _FIXTURE_MANIFEST=${INFER_SCRIPT_PATH}/.pt-fixture-cache-manifest - - # CI may restore a complete, content-addressed fixture set. The manifest - # prevents a partial/stale cache from silently turning missing coverage into - # GTEST_SKIP or making the C++ tests consume mismatched reference values. + # CI may restore a complete, content-addressed fixture set. The manifest + # prevents a partial cache from silently reducing test coverage. if [ "${DP_REUSE_GENERATED_PT_MODELS:-0}" = "1" ]; then if [ ! -s "${_FIXTURE_MANIFEST}" ]; then echo "ERROR: PT fixture cache hit without a non-empty manifest" >&2 @@ -75,114 +73,112 @@ else: done < "${_FIXTURE_MANIFEST}" echo "Reusing $(wc -l < "${_FIXTURE_MANIFEST}") cached PT fixture files" else - # Remove stale generated files so they can't be accidentally reused if - # generation scripts or formats change. *.expected sidecars are consumed - # by the C++ tests and therefore belong to the same cache unit as PT2/PTE. - rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte ${INFER_SCRIPT_PATH}/*.expected "${_FIXTURE_MANIFEST}" - - # When the build uses -fsanitize=leak, the custom op .so requires the LSAN - # runtime to be preloaded (otherwise dlopen fails). We disable leak detection - # in the gen scripts to avoid false reports from torch/paddle internals. - _GEN_ENV="" - if echo "${CXXFLAGS:-}" | grep -q fsanitize=leak; then - _LSAN_LIB=$(gcc -print-file-name=liblsan.so 2>/dev/null || true) - if [ -n "${_LSAN_LIB}" ] && [ -f "${_LSAN_LIB}" ]; then - # DP_GEN_UNDER_SANITIZER: explicit signal for gen scripts that need - # to skip sanitizer-incompatible sections (e.g. gen_dpa2.py's - # AOTInductor graph .pt2 eval, which can SEGV under the LSAN - # runtime). Sniffing LD_PRELOAD inside the gen script is NOT - # reliable: the sanitizer runtime removes its own entry from the - # process environment during startup. - _GEN_ENV="LD_PRELOAD=${_LSAN_LIB} LSAN_OPTIONS=detect_leaks=0 DP_GEN_UNDER_SANITIZER=lsan" - fi + # Remove stale generated model files so they can't be accidentally reused + # if gen scripts change format or the code version changes. *.expected files + # are generated reference values consumed by the same C++ tests. + rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte ${INFER_SCRIPT_PATH}/*.expected "${_FIXTURE_MANIFEST}" + # When the build uses -fsanitize=leak, the custom op .so requires the LSAN + # runtime to be preloaded (otherwise dlopen fails). We disable leak detection + # in the gen scripts to avoid false reports from torch/paddle internals. + _GEN_ENV="" + if echo "${CXXFLAGS:-}" | grep -q fsanitize=leak; then + _LSAN_LIB=$(gcc -print-file-name=liblsan.so 2>/dev/null || true) + if [ -n "${_LSAN_LIB}" ] && [ -f "${_LSAN_LIB}" ]; then + # DP_GEN_UNDER_SANITIZER: explicit signal for gen scripts that need + # to skip sanitizer-incompatible sections (e.g. gen_dpa2.py's + # AOTInductor graph .pt2 eval, which can SEGV under the LSAN + # runtime). Sniffing LD_PRELOAD inside the gen script is NOT + # reliable: the sanitizer runtime removes its own entry from the + # process environment during startup. + _GEN_ENV="LD_PRELOAD=${_LSAN_LIB} LSAN_OPTIONS=detect_leaks=0 DP_GEN_UNDER_SANITIZER=lsan" fi - # Run gen scripts in parallel for faster model generation. - # Wait on each PID separately so any failure is caught by set -e. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_sea.py & - PID1=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1.py & - PID2=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa2.py & - PID3=$! - wait $PID1 - wait $PID2 - wait $PID3 + fi + # Run gen scripts in parallel for faster model generation. + # Wait on each PID separately so any failure is caught by set -e. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_sea.py & + PID1=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1.py & + PID2=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa2.py & + PID3=$! + wait $PID1 + wait $PID2 + wait $PID3 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa3.py & - PID4=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & - PID5=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_model_devi.py & - PID6=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_chg_spin.py & - PID9=$! - wait $PID4 - wait $PID5 - wait $PID6 - wait $PID9 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa3.py & + PID4=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & + PID5=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_model_devi.py & + PID6=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_chg_spin.py & + PID9=$! + wait $PID4 + wait $PID5 + wait $PID6 + wait $PID9 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4.py & - PID9=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py & - PID10=$! - wait $PID9 - wait $PID10 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4.py & + PID9=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py & + PID10=$! + wait $PID9 + wait $PID10 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin.py & - PID7=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & - PID8=$! - wait $PID7 - wait $PID8 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin.py & + PID7=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & + PID8=$! + wait $PID7 + wait $PID8 - # Native-spin DPA4 graph archives (baseline + model-level pair - # exclusion). Without this the whole native-spin graph C++ suite - # GTEST_SKIPs on the missing fixture, which is how a dead - # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & - PID11=$! - # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph - # lower, which no other C++ fixture covers. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & - PID12=$! - # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both - # is_spin=true and dim_chg_spin>0, i.e. the only one on which the - # runtime charge_spin argument of DeepSpin::compute is not inert. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & - PID13=$! - # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear - # composition, and the only fixture pinning that a bridged model - # exports NO with-comm artifact (single-rank only by construction). - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & - PID14=$! - # Native-spin DPA4C: the archives above all ship a with-comm artifact - # and are therefore served by DeepSpinPTExpt, which leaves - # NativeSpinPTExpt -- the class the dispatch selects for a native-spin - # archive without one -- uncovered. DPA4C keeps its messaging local and - # so reaches it. The compiled graph calls CUDA-only operators, so the - # fixture is generated only where the C++ suite has a GPU; the tests - # built on it skip when it is absent. - if [ "${DP_VARIANT}" = "cuda" ]; then - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & - PID15=$! - fi - wait $PID11 - wait $PID12 - wait $PID13 - wait $PID14 - if [ "${DP_VARIANT}" = "cuda" ]; then - wait $PID15 - fi + # Native-spin DPA4 graph archives (baseline + model-level pair + # exclusion). Without this the whole native-spin graph C++ suite + # GTEST_SKIPs on the missing fixture, which is how a dead + # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & + PID11=$! + # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph + # lower, which no other C++ fixture covers. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & + PID12=$! + # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both + # is_spin=true and dim_chg_spin>0, i.e. the only one on which the + # runtime charge_spin argument of DeepSpin::compute is not inert. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & + PID13=$! + # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear + # composition, and the only fixture pinning that a bridged model + # exports NO with-comm artifact (single-rank only by construction). + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & + PID14=$! + # Native-spin DPA4C: the archives above all ship a with-comm artifact + # and are therefore served by DeepSpinPTExpt, which leaves + # NativeSpinPTExpt -- the class the dispatch selects for a native-spin + # archive without one -- uncovered. DPA4C keeps its messaging local and + # so reaches it. The compiled graph calls CUDA-only operators, so the + # fixture is generated only where the C++ suite has a GPU; the tests + # built on it skip when it is absent. + if [ "${DP_VARIANT}" = "cuda" ]; then + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & + PID15=$! + fi + wait $PID11 + wait $PID12 + wait $PID13 + wait $PID14 + if [ "${DP_VARIANT}" = "cuda" ]; then + wait $PID15 + fi - # Record the exact generated fixture set. A warm cache is accepted only - # when every entry in this manifest was restored. - for fixture in "${INFER_SCRIPT_PATH}"/*.pt2 "${INFER_SCRIPT_PATH}"/*.pte "${INFER_SCRIPT_PATH}"/*.expected; do - [ -f "${fixture}" ] && basename "${fixture}" - done | LC_ALL=C sort > "${_FIXTURE_MANIFEST}" - if [ ! -s "${_FIXTURE_MANIFEST}" ]; then - echo "ERROR: PT fixture generation produced no cacheable files" >&2 - exit 1 - fi + # Record the exact generated fixture set for cache-hit validation. + for fixture in "${INFER_SCRIPT_PATH}"/*.pt2 "${INFER_SCRIPT_PATH}"/*.pte "${INFER_SCRIPT_PATH}"/*.expected; do + [ -f "${fixture}" ] && basename "${fixture}" + done | LC_ALL=C sort > "${_FIXTURE_MANIFEST}" + if [ ! -s "${_FIXTURE_MANIFEST}" ]; then + echo "ERROR: PT fixture generation produced no cacheable files" >&2 + exit 1 + fi fi fi if [ "${ENABLE_PADDLE:-TRUE}" == "TRUE" ]; then From 3e27447ea1feb30d5ac490715108ef8b452aa986 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Tue, 15 Sep 2026 18:15:59 +0800 Subject: [PATCH 04/13] ci: make PyTorch fixture cache safe Cache all generated PyTorch fixture formats, including .pth files required when generator execution is skipped on a cache hit. Scope PT2 and TorchInductor native-code caches to the runner CPU/toolchain fingerprint and remove unsafe prefix fallback reuse. Agent: ChatGPT (GPT-5.6 Sol) --- .github/workflows/test_cc.yml | 53 +++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 3b33c6f699..d443758fcb 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -43,30 +43,52 @@ jobs: enable_paddle: true steps: - uses: actions/checkout@v7 + - name: Install GCC + run: | + sudo apt-get update + sudo apt-get install --yes build-essential + if: ${{ matrix.check_memleak == false && github.repository_owner == 'deepmodeling' }} + # PT2 fixtures and TorchInductor caches contain native code. PyTorch + # documents AOTInductor packages as non-portable, so scope cache reuse + # to the CPU ISA, libc, and compiler of the current runner. + - name: Compute native cache fingerprint + id: native-cache + if: matrix.enable_pytorch + shell: bash + run: | + { + echo "arch=$(uname -m)" + lscpu | sed -n -e 's/^Model name:[[:space:]]*/model=/p' -e 's/^Flags:[[:space:]]*/flags=/p' -e 's/^Features:[[:space:]]*/features=/p' + echo "glibc=$(getconf GNU_LIBC_VERSION 2>/dev/null || ldd --version | head -n1)" + if command -v c++ >/dev/null 2>&1; then + echo "cxx=$(c++ -dumpfullversion -dumpversion 2>/dev/null || c++ --version | head -n1)" + else + echo "cxx=missing" + fi + } > "${RUNNER_TEMP}/native-cache-env.txt" + cat "${RUNNER_TEMP}/native-cache-env.txt" + fingerprint=$(sha256sum "${RUNNER_TEMP}/native-cache-env.txt" | cut -c1-16) + source_fingerprint=$(git ls-files -s -- uv.lock pyproject.toml source deepmd | sha256sum | cut -c1-16) + echo "fingerprint=${fingerprint}" >> "${GITHUB_OUTPUT}" + echo "source_fingerprint=${source_fingerprint}" >> "${GITHUB_OUTPUT}" - name: Cache generated PyTorch C++ fixtures id: pt-fixture-cache if: matrix.enable_pytorch uses: actions/cache@v6 with: path: | + source/tests/infer/*.pth source/tests/infer/*.pt2 source/tests/infer/*.pte source/tests/infer/*.expected source/tests/infer/.pt-fixture-cache-manifest - key: pt-cxx-fixtures-v1-${{ runner.os }}-${{ runner.arch }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ hashFiles('uv.lock', 'pyproject.toml', 'source/tests/infer/**', 'deepmd/**', 'source/op/pt/**') }} + key: pt-cxx-fixtures-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} - name: Cache TorchInductor if: matrix.enable_pytorch uses: actions/cache@v6 with: path: .cache/torchinductor - key: torchinductor-v1-${{ runner.os }}-${{ runner.arch }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}-${{ hashFiles('source/tests/infer/**', 'deepmd/**', 'source/op/pt/**') }} - restore-keys: | - torchinductor-v1-${{ runner.os }}-${{ runner.arch }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}- - - name: Install GCC - run: | - sudo apt-get update - sudo apt-get install --yes build-essential - if: ${{ matrix.check_memleak == false && github.repository_owner == 'deepmodeling' }} + key: torchinductor-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} - uses: actions/setup-python@v7 with: python-version: "3.11" @@ -103,6 +125,19 @@ jobs: TORCHINDUCTOR_CACHE_DIR: ${{ github.workspace }}/.cache/torchinductor TORCHINDUCTOR_FX_GRAPH_CACHE: 1 TORCHINDUCTOR_AUTOGRAD_CACHE: 1 + # test_cc_local.sh creates the base manifest after generation. Add the + # TorchScript files too before actions/cache saves the cold-run cache; + # otherwise a cache hit skips gen_spin.py with deeppot_dpa_spin.pth absent. + - name: Complete PyTorch fixture cache manifest + if: ${{ matrix.enable_pytorch && steps.pt-fixture-cache.outputs.cache-hit != 'true' }} + shell: bash + run: | + manifest=source/tests/infer/.pt-fixture-cache-manifest + for fixture in source/tests/infer/*.pth; do + [ -f "${fixture}" ] && basename "${fixture}" + done >> "${manifest}" + LC_ALL=C sort -u -o "${manifest}" "${manifest}" + echo "Cached fixture manifest now contains $(wc -l < "${manifest}") files" # test lammps - run: | cp ${{ github.workspace }}/source/build_tests/paddle_inference_install_dir/paddle/lib/*.so ${{ github.workspace }}/dp_test/lib/ From dd2292d62129a18394b1293c8364ee3eb3a14072 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Tue, 15 Sep 2026 23:11:19 +0800 Subject: [PATCH 05/13] ci: key native cache by ISA instead of CPU model --- .github/workflows/test_cc.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index d443758fcb..8e382707e0 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -50,7 +50,9 @@ jobs: if: ${{ matrix.check_memleak == false && github.repository_owner == 'deepmodeling' }} # PT2 fixtures and TorchInductor caches contain native code. PyTorch # documents AOTInductor packages as non-portable, so scope cache reuse - # to the CPU ISA, libc, and compiler of the current runner. + # to the CPU ISA, libc, and compiler of the current runner. Do not key + # on CPU model name: hosted runners may expose the same ISA on different + # processor models, which would otherwise prevent safe warm-cache reuse. - name: Compute native cache fingerprint id: native-cache if: matrix.enable_pytorch @@ -58,7 +60,7 @@ jobs: run: | { echo "arch=$(uname -m)" - lscpu | sed -n -e 's/^Model name:[[:space:]]*/model=/p' -e 's/^Flags:[[:space:]]*/flags=/p' -e 's/^Features:[[:space:]]*/features=/p' + lscpu | sed -n -e 's/^Flags:[[:space:]]*/flags=/p' -e 's/^Features:[[:space:]]*/features=/p' echo "glibc=$(getconf GNU_LIBC_VERSION 2>/dev/null || ldd --version | head -n1)" if command -v c++ >/dev/null 2>&1; then echo "cxx=$(c++ -dumpfullversion -dumpversion 2>/dev/null || c++ --version | head -n1)" @@ -82,13 +84,13 @@ jobs: source/tests/infer/*.pte source/tests/infer/*.expected source/tests/infer/.pt-fixture-cache-manifest - key: pt-cxx-fixtures-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} + key: pt-cxx-fixtures-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} - name: Cache TorchInductor if: matrix.enable_pytorch uses: actions/cache@v6 with: path: .cache/torchinductor - key: torchinductor-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} + key: torchinductor-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} - uses: actions/setup-python@v7 with: python-version: "3.11" From 3173b48c5cce3017daa9db52278f2ac8ee5d023b Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Wed, 16 Sep 2026 07:52:44 +0800 Subject: [PATCH 06/13] ci: cache generated JAX SavedModels --- .github/workflows/test_cc.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 8e382707e0..70ba08ee5a 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -83,14 +83,28 @@ jobs: source/tests/infer/*.pt2 source/tests/infer/*.pte source/tests/infer/*.expected + source/tests/infer/*.savedmodel source/tests/infer/.pt-fixture-cache-manifest - key: pt-cxx-fixtures-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} + key: pt-cxx-fixtures-v4-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} - name: Cache TorchInductor if: matrix.enable_pytorch uses: actions/cache@v6 with: path: .cache/torchinductor key: torchinductor-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} + - name: Validate cached multi-backend fixtures + if: ${{ matrix.enable_pytorch && steps.pt-fixture-cache.outputs.cache-hit == 'true' }} + shell: bash + run: | + for fixture in \ + source/tests/infer/fparam_aparam.savedmodel \ + source/tests/infer/fparam_aparam_default.savedmodel \ + source/tests/infer/chg_spin.savedmodel; do + if [ ! -d "${fixture}" ]; then + echo "ERROR: cached generated fixture is missing: ${fixture}" >&2 + exit 1 + fi + done - uses: actions/setup-python@v7 with: python-version: "3.11" From 7cc51271c8e4349bc613ffb558b95383912bb022 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Wed, 16 Sep 2026 17:46:59 +0800 Subject: [PATCH 07/13] ci: narrow generated fixture cache fingerprint Avoid invalidating the expensive generated PT/JAX C++ fixtures when unrelated source/tests/infer or backend code changes. Fingerprint only the generator scripts and committed inputs plus the Python/backend/custom-op code that can affect those generated artifacts. Agent: ChatGPT Model: GPT-5.6 Sol --- .github/workflows/test_cc.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 70ba08ee5a..0872957628 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -70,7 +70,19 @@ jobs: } > "${RUNNER_TEMP}/native-cache-env.txt" cat "${RUNNER_TEMP}/native-cache-env.txt" fingerprint=$(sha256sum "${RUNNER_TEMP}/native-cache-env.txt" | cut -c1-16) - source_fingerprint=$(git ls-files -s -- uv.lock pyproject.toml source deepmd | sha256sum | cut -c1-16) + # Hash only inputs that can affect the generated PT/PT2/JAX fixtures. + # In particular, do not hash the whole source/ tree: unrelated C++ + # tests or backend changes should not invalidate these artifacts. + source_fingerprint=$(git ls-files -s -- \ + uv.lock pyproject.toml \ + 'source/tests/infer/gen_*.py' \ + 'source/tests/infer/*.yaml' \ + 'source/tests/infer/*.pth' \ + 'deepmd/*.py' \ + deepmd/dpmodel deepmd/pt deepmd/pt_expt deepmd/jax \ + deepmd/infer deepmd/utils deepmd/backend \ + source/op/pt \ + | sha256sum | cut -c1-16) echo "fingerprint=${fingerprint}" >> "${GITHUB_OUTPUT}" echo "source_fingerprint=${source_fingerprint}" >> "${GITHUB_OUTPUT}" - name: Cache generated PyTorch C++ fixtures From 60d83dd7ab00634118143340efb18b8b08fc2f23 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:57:31 +0000 Subject: [PATCH 08/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- source/install/test_cc_local.sh | 204 ++++++++++++++++---------------- 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/source/install/test_cc_local.sh b/source/install/test_cc_local.sh index 26bc9203f4..33d75c56ed 100755 --- a/source/install/test_cc_local.sh +++ b/source/install/test_cc_local.sh @@ -70,115 +70,115 @@ else: echo "ERROR: cached PT fixture is missing: ${fixture}" >&2 exit 1 fi - done < "${_FIXTURE_MANIFEST}" - echo "Reusing $(wc -l < "${_FIXTURE_MANIFEST}") cached PT fixture files" + done <"${_FIXTURE_MANIFEST}" + echo "Reusing $(wc -l <"${_FIXTURE_MANIFEST}") cached PT fixture files" else - # Remove stale generated model files so they can't be accidentally reused - # if gen scripts change format or the code version changes. *.expected files - # are generated reference values consumed by the same C++ tests. - rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte ${INFER_SCRIPT_PATH}/*.expected "${_FIXTURE_MANIFEST}" - # When the build uses -fsanitize=leak, the custom op .so requires the LSAN - # runtime to be preloaded (otherwise dlopen fails). We disable leak detection - # in the gen scripts to avoid false reports from torch/paddle internals. - _GEN_ENV="" - if echo "${CXXFLAGS:-}" | grep -q fsanitize=leak; then - _LSAN_LIB=$(gcc -print-file-name=liblsan.so 2>/dev/null || true) - if [ -n "${_LSAN_LIB}" ] && [ -f "${_LSAN_LIB}" ]; then - # DP_GEN_UNDER_SANITIZER: explicit signal for gen scripts that need - # to skip sanitizer-incompatible sections (e.g. gen_dpa2.py's - # AOTInductor graph .pt2 eval, which can SEGV under the LSAN - # runtime). Sniffing LD_PRELOAD inside the gen script is NOT - # reliable: the sanitizer runtime removes its own entry from the - # process environment during startup. - _GEN_ENV="LD_PRELOAD=${_LSAN_LIB} LSAN_OPTIONS=detect_leaks=0 DP_GEN_UNDER_SANITIZER=lsan" + # Remove stale generated model files so they can't be accidentally reused + # if gen scripts change format or the code version changes. *.expected files + # are generated reference values consumed by the same C++ tests. + rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte ${INFER_SCRIPT_PATH}/*.expected "${_FIXTURE_MANIFEST}" + # When the build uses -fsanitize=leak, the custom op .so requires the LSAN + # runtime to be preloaded (otherwise dlopen fails). We disable leak detection + # in the gen scripts to avoid false reports from torch/paddle internals. + _GEN_ENV="" + if echo "${CXXFLAGS:-}" | grep -q fsanitize=leak; then + _LSAN_LIB=$(gcc -print-file-name=liblsan.so 2>/dev/null || true) + if [ -n "${_LSAN_LIB}" ] && [ -f "${_LSAN_LIB}" ]; then + # DP_GEN_UNDER_SANITIZER: explicit signal for gen scripts that need + # to skip sanitizer-incompatible sections (e.g. gen_dpa2.py's + # AOTInductor graph .pt2 eval, which can SEGV under the LSAN + # runtime). Sniffing LD_PRELOAD inside the gen script is NOT + # reliable: the sanitizer runtime removes its own entry from the + # process environment during startup. + _GEN_ENV="LD_PRELOAD=${_LSAN_LIB} LSAN_OPTIONS=detect_leaks=0 DP_GEN_UNDER_SANITIZER=lsan" + fi fi - fi - # Run gen scripts in parallel for faster model generation. - # Wait on each PID separately so any failure is caught by set -e. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_sea.py & - PID1=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1.py & - PID2=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa2.py & - PID3=$! - wait $PID1 - wait $PID2 - wait $PID3 + # Run gen scripts in parallel for faster model generation. + # Wait on each PID separately so any failure is caught by set -e. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_sea.py & + PID1=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1.py & + PID2=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa2.py & + PID3=$! + wait $PID1 + wait $PID2 + wait $PID3 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa3.py & - PID4=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & - PID5=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_model_devi.py & - PID6=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_chg_spin.py & - PID9=$! - wait $PID4 - wait $PID5 - wait $PID6 - wait $PID9 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa3.py & + PID4=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & + PID5=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_model_devi.py & + PID6=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_chg_spin.py & + PID9=$! + wait $PID4 + wait $PID5 + wait $PID6 + wait $PID9 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4.py & - PID9=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py & - PID10=$! - wait $PID9 - wait $PID10 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4.py & + PID9=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py & + PID10=$! + wait $PID9 + wait $PID10 - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin.py & - PID7=$! - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & - PID8=$! - wait $PID7 - wait $PID8 + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin.py & + PID7=$! + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & + PID8=$! + wait $PID7 + wait $PID8 - # Native-spin DPA4 graph archives (baseline + model-level pair - # exclusion). Without this the whole native-spin graph C++ suite - # GTEST_SKIPs on the missing fixture, which is how a dead - # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & - PID11=$! - # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph - # lower, which no other C++ fixture covers. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & - PID12=$! - # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both - # is_spin=true and dim_chg_spin>0, i.e. the only one on which the - # runtime charge_spin argument of DeepSpin::compute is not inert. - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & - PID13=$! - # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear - # composition, and the only fixture pinning that a bridged model - # exports NO with-comm artifact (single-rank only by construction). - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & - PID14=$! - # Native-spin DPA4C: the archives above all ship a with-comm artifact - # and are therefore served by DeepSpinPTExpt, which leaves - # NativeSpinPTExpt -- the class the dispatch selects for a native-spin - # archive without one -- uncovered. DPA4C keeps its messaging local and - # so reaches it. The compiled graph calls CUDA-only operators, so the - # fixture is generated only where the C++ suite has a GPU; the tests - # built on it skip when it is absent. - if [ "${DP_VARIANT}" = "cuda" ]; then - env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & - PID15=$! - fi - wait $PID11 - wait $PID12 - wait $PID13 - wait $PID14 - if [ "${DP_VARIANT}" = "cuda" ]; then - wait $PID15 - fi + # Native-spin DPA4 graph archives (baseline + model-level pair + # exclusion). Without this the whole native-spin graph C++ suite + # GTEST_SKIPs on the missing fixture, which is how a dead + # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & + PID11=$! + # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph + # lower, which no other C++ fixture covers. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & + PID12=$! + # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both + # is_spin=true and dim_chg_spin>0, i.e. the only one on which the + # runtime charge_spin argument of DeepSpin::compute is not inert. + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & + PID13=$! + # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear + # composition, and the only fixture pinning that a bridged model + # exports NO with-comm artifact (single-rank only by construction). + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & + PID14=$! + # Native-spin DPA4C: the archives above all ship a with-comm artifact + # and are therefore served by DeepSpinPTExpt, which leaves + # NativeSpinPTExpt -- the class the dispatch selects for a native-spin + # archive without one -- uncovered. DPA4C keeps its messaging local and + # so reaches it. The compiled graph calls CUDA-only operators, so the + # fixture is generated only where the C++ suite has a GPU; the tests + # built on it skip when it is absent. + if [ "${DP_VARIANT}" = "cuda" ]; then + env ${_GEN_ENV} python ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & + PID15=$! + fi + wait $PID11 + wait $PID12 + wait $PID13 + wait $PID14 + if [ "${DP_VARIANT}" = "cuda" ]; then + wait $PID15 + fi - # Record the exact generated fixture set for cache-hit validation. - for fixture in "${INFER_SCRIPT_PATH}"/*.pt2 "${INFER_SCRIPT_PATH}"/*.pte "${INFER_SCRIPT_PATH}"/*.expected; do - [ -f "${fixture}" ] && basename "${fixture}" - done | LC_ALL=C sort > "${_FIXTURE_MANIFEST}" - if [ ! -s "${_FIXTURE_MANIFEST}" ]; then - echo "ERROR: PT fixture generation produced no cacheable files" >&2 - exit 1 - fi + # Record the exact generated fixture set for cache-hit validation. + for fixture in "${INFER_SCRIPT_PATH}"/*.pt2 "${INFER_SCRIPT_PATH}"/*.pte "${INFER_SCRIPT_PATH}"/*.expected; do + [ -f "${fixture}" ] && basename "${fixture}" + done | LC_ALL=C sort >"${_FIXTURE_MANIFEST}" + if [ ! -s "${_FIXTURE_MANIFEST}" ]; then + echo "ERROR: PT fixture generation produced no cacheable files" >&2 + exit 1 + fi fi fi if [ "${ENABLE_PADDLE:-TRUE}" == "TRUE" ]; then From bedaf3a3e322ad22822f569a5de23655303ac1ab Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Wed, 16 Sep 2026 23:47:41 +0800 Subject: [PATCH 09/13] ci: include fixture orchestrator in cache key --- .github/workflows/test_cc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 0872957628..5724b19c9a 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -75,6 +75,7 @@ jobs: # tests or backend changes should not invalidate these artifacts. source_fingerprint=$(git ls-files -s -- \ uv.lock pyproject.toml \ + source/install/test_cc_local.sh \ 'source/tests/infer/gen_*.py' \ 'source/tests/infer/*.yaml' \ 'source/tests/infer/*.pth' \ From e9db2a7ae2f86195f902d41a911ed7608df724a6 Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Wed, 28 Oct 2026 13:05:00 +0800 Subject: [PATCH 10/13] fix(ci): restore C++ fixture test script executable mode Coding-Agent: Codex Codex-Version: codex-cli 0.164.0 Model: gpt-6-astra Reasoning-Effort: xhigh --- source/install/test_cc_local.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 source/install/test_cc_local.sh diff --git a/source/install/test_cc_local.sh b/source/install/test_cc_local.sh old mode 100644 new mode 100755 From af6cb328f62f5f3e688a4f269078216135555db0 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Wed, 4 Nov 2026 02:15:43 +0800 Subject: [PATCH 11/13] fix(ci): include shared DPA4 fixture helper in native cache fingerprint Agent: ChatGPT Model: GPT-6 --- .github/workflows/test_cc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 4c8cb2bb9d..df9deadc36 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -76,6 +76,7 @@ jobs: source_fingerprint=$(git ls-files -s -- \ uv.lock pyproject.toml \ source/install/test_cc_local.sh \ + source/tests/dpa4_fixtures.py \ 'source/tests/infer/gen_*.py' \ 'source/tests/infer/*.yaml' \ 'source/tests/infer/*.pth' \ From 54fc7dae2dc9971918c9e04b1cdc2ed35f95b581 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Wed, 4 Nov 2026 02:16:13 +0800 Subject: [PATCH 12/13] test(ci): catch helper-only changes to C++ fixture cache inputs Agent: ChatGPT Model: GPT-6 --- .../test_cc_fixture_cache_fingerprint.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 source/tests/common/test_cc_fixture_cache_fingerprint.py diff --git a/source/tests/common/test_cc_fixture_cache_fingerprint.py b/source/tests/common/test_cc_fixture_cache_fingerprint.py new file mode 100644 index 0000000000..6831e3cbc5 --- /dev/null +++ b/source/tests/common/test_cc_fixture_cache_fingerprint.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +"""The fixture cache key must change when imported generator helpers change.""" + +import hashlib +import shlex +import subprocess +import tempfile +import unittest +from pathlib import ( + Path, +) + + +class TestCCFixtureCacheFingerprint(unittest.TestCase): + def test_shared_helper_change_invalidates_fingerprint(self): + workflow = ( + Path(__file__).resolve().parents[3] + / ".github" + / "workflows" + / "test_cc.yml" + ).read_text(encoding="utf-8") + command = workflow.split("source_fingerprint=$(git ls-files -s --", 1)[ + 1 + ].split("| sha256sum", 1)[0] + pathspecs = shlex.split(command.replace("\\\n", " ")) + self.assertIn("source/tests/dpa4_fixtures.py", pathspecs) + + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + helper = repo / "source" / "tests" / "dpa4_fixtures.py" + generator = repo / "source" / "tests" / "infer" / "gen_dpa4_spin.py" + helper.parent.mkdir(parents=True) + generator.parent.mkdir(parents=True) + helper.write_text("jitter = 0.05\n", encoding="utf-8") + generator.write_text("from dpa4_fixtures import jitter\n", encoding="utf-8") + + def git(*args): + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + capture_output=True, + ).stdout + + def fingerprint(): + return hashlib.sha256( + git("ls-files", "-s", "--", *pathspecs) + ).hexdigest() + + git("init", "-q") + git("add", "--", "source/tests") + before = fingerprint() + # The generator is unchanged; only its imported helper changes. + helper.write_text("jitter = 0.10\n", encoding="utf-8") + git("add", "--", "source/tests/dpa4_fixtures.py") + self.assertNotEqual(before, fingerprint()) + + +if __name__ == "__main__": + unittest.main() From fe3b596e67f3e076069004387141dbc992ef2f49 Mon Sep 17 00:00:00 2001 From: "A bot of @njzjz" Date: Wed, 4 Nov 2026 02:47:32 +0800 Subject: [PATCH 13/13] fix(ci): fingerprint installed fixture runtime dependencies Install dependencies before computing the native cache key. Include the Python runtime and deterministically sorted installed distribution versions, excluding DeePMD's commit-derived editable version because its relevant inputs are already source-addressed. Retain exact-match restores. Validation: execute the exact Bash fingerprint step against real temporary metadata; dependency-version changes invalidate, editable commit-version changes do not, and discovery order/repeated runs are stable. YAML ordering, cache regression, shell syntax and whitespace checks pass. Agent: ChatGPT Model: GPT-6 Astra Pro --- .github/workflows/test_cc.yml | 50 ++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 961049b73a..3f420a0aa3 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -48,10 +48,26 @@ jobs: sudo apt-get update sudo apt-get install --yes build-essential if: ${{ matrix.check_memleak == false && github.repository_owner == 'deepmodeling' }} + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + cache: "pip" + - uses: lukka/get-cmake@latest + - run: python -m pip install uv + - name: Install Python dependencies + run: | + source/install/uv_with_retry.sh pip install --system --group pin_tensorflow_cpu --group pin_pytorch_cpu --group pin_jax_cpu --torch-backend cpu + export TENSORFLOW_ROOT=$(python -c 'import importlib.util,pathlib;print(pathlib.Path(importlib.util.find_spec("tensorflow").origin).parent)') + export PYTORCH_ROOT=$(python -c 'import torch;print(torch.__path__[0])') + source/install/uv_with_retry.sh pip install --system -e .[cpu,test,lmp,jax,torch] mpi4py mpich + env: + DP_ENABLE_PYTORCH: 1 # PT2 fixtures and TorchInductor caches contain native code. PyTorch # documents AOTInductor packages as non-portable, so scope cache reuse - # to the CPU ISA, libc, and compiler of the current runner. Do not key - # on CPU model name: hosted runners may expose the same ISA on different + # to the CPU ISA, libc, compiler, Python, and installed dependencies. + # Fingerprint after installation because uv pip does not use uv.lock + # as a frozen resolution. Do not key on CPU model name: hosted runners + # may expose the same ISA on different # processor models, which would otherwise prevent safe warm-cache reuse. - name: Compute native cache fingerprint id: native-cache @@ -67,6 +83,22 @@ jobs: else echo "cxx=missing" fi + python - <<'PYTHON' + import importlib.metadata + import platform + import re + + print(f"python={platform.python_implementation()}-{platform.python_version()}") + distributions = { + (re.sub(r"[-_.]+", "-", dist.metadata["Name"]).lower(), dist.version) + for dist in importlib.metadata.distributions() + } + # The editable project's commit-derived version also changes for + # unrelated edits; its fixture inputs are covered by the source hash. + for name, version in sorted(distributions): + if name != "deepmd-kit": + print(f"distribution={name}=={version}") + PYTHON } > "${RUNNER_TEMP}/native-cache-env.txt" cat "${RUNNER_TEMP}/native-cache-env.txt" fingerprint=$(sha256sum "${RUNNER_TEMP}/native-cache-env.txt" | cut -c1-16) @@ -121,20 +153,6 @@ jobs: exit 1 fi done - - uses: actions/setup-python@v7 - with: - python-version: "3.11" - cache: "pip" - - uses: lukka/get-cmake@latest - - run: python -m pip install uv - - name: Install Python dependencies - run: | - source/install/uv_with_retry.sh pip install --system --group pin_tensorflow_cpu --group pin_pytorch_cpu --group pin_jax_cpu --torch-backend cpu - export TENSORFLOW_ROOT=$(python -c 'import importlib.util,pathlib;print(pathlib.Path(importlib.util.find_spec("tensorflow").origin).parent)') - export PYTORCH_ROOT=$(python -c 'import torch;print(torch.__path__[0])') - source/install/uv_with_retry.sh pip install --system -e .[cpu,test,lmp,jax,torch] mpi4py mpich - env: - DP_ENABLE_PYTORCH: 1 - name: Convert models run: source/tests/infer/convert-models.sh # https://github.com/actions/runner-images/issues/9491