diff --git a/.github/workflows/test_cc.yml b/.github/workflows/test_cc.yml index 233cb2f649..5a161a6e3d 100644 --- a/.github/workflows/test_cc.yml +++ b/.github/workflows/test_cc.yml @@ -62,6 +62,97 @@ jobs: source/install/uv_with_retry.sh pip install --system -e .[cpu,test,lmp,jax,torch] mpi4py mpich env: DP_ENABLE_PYTORCH: 1 + # PT2 fixtures and TorchInductor caches contain native code. PyTorch + # documents AOTInductor packages as non-portable, so scope cache reuse + # to the CPU ISA, libc, compiler, Python, and installed dependencies. + # Fingerprint after installation because uv pip does not use uv.lock + # as a frozen resolution. Do not key on CPU model name: hosted runners + # may expose the same ISA on different + # processor models, which would otherwise prevent safe warm-cache reuse. + - name: Compute native cache fingerprint + id: native-cache + if: matrix.enable_pytorch + shell: bash + run: | + { + echo "arch=$(uname -m)" + lscpu | sed -n -e 's/^Flags:[[:space:]]*/flags=/p' -e 's/^Features:[[:space:]]*/features=/p' + echo "glibc=$(getconf GNU_LIBC_VERSION 2>/dev/null || ldd --version | head -n1)" + if command -v c++ >/dev/null 2>&1; then + echo "cxx=$(c++ -dumpfullversion -dumpversion 2>/dev/null || c++ --version | head -n1)" + else + echo "cxx=missing" + fi + python - <<'PYTHON' + import importlib.metadata + import platform + import re + + print(f"python={platform.python_implementation()}-{platform.python_version()}") + distributions = { + (re.sub(r"[-_.]+", "-", dist.metadata["Name"]).lower(), dist.version) + for dist in importlib.metadata.distributions() + } + # The editable project's commit-derived version also changes for + # unrelated edits; its fixture inputs are covered by the source hash. + for name, version in sorted(distributions): + if name != "deepmd-kit": + print(f"distribution={name}=={version}") + PYTHON + } > "${RUNNER_TEMP}/native-cache-env.txt" + cat "${RUNNER_TEMP}/native-cache-env.txt" + fingerprint=$(sha256sum "${RUNNER_TEMP}/native-cache-env.txt" | cut -c1-16) + # Include the generators, their Python dependencies, and the native + # libraries they load while exporting models and reference values. + # Keep unrelated C++ tests outside this dependency fingerprint. + source_fingerprint=$(git ls-files -s -- \ + uv.lock pyproject.toml \ + backend source/CMakeLists.txt source/cmake source/config \ + source/install/test_cc_local.sh \ + source/tests/dpa4_fixtures.py \ + 'source/tests/infer/gen_*.py' \ + 'source/tests/infer/*.yaml' \ + 'source/tests/infer/*.pth' \ + 'deepmd/*.py' \ + deepmd/dpmodel deepmd/pt deepmd/pt_expt deepmd/jax \ + deepmd/infer deepmd/utils deepmd/backend \ + source/op/pt \ + source/lib/CMakeLists.txt source/lib/include source/lib/src \ + | sha256sum | cut -c1-16) + echo "fingerprint=${fingerprint}" >> "${GITHUB_OUTPUT}" + echo "source_fingerprint=${source_fingerprint}" >> "${GITHUB_OUTPUT}" + - name: Cache generated PyTorch C++ fixtures + id: pt-fixture-cache + if: matrix.enable_pytorch + uses: actions/cache@v6 + with: + path: | + source/tests/infer/*.pth + source/tests/infer/*.pt2 + source/tests/infer/*.pte + source/tests/infer/*.expected + source/tests/infer/*.savedmodel + source/tests/infer/.pt-fixture-cache-manifest + key: pt-cxx-fixtures-v4-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} + - name: Cache TorchInductor + if: matrix.enable_pytorch + uses: actions/cache@v6 + with: + path: .cache/torchinductor + key: torchinductor-v3-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache.outputs.fingerprint }}-py311-${{ matrix.check_memleak && 'lsan' || 'regular' }}-${{ steps.native-cache.outputs.source_fingerprint }} + - name: Validate cached multi-backend fixtures + if: ${{ matrix.enable_pytorch && steps.pt-fixture-cache.outputs.cache-hit == 'true' }} + shell: bash + run: | + for fixture in \ + source/tests/infer/fparam_aparam.savedmodel \ + source/tests/infer/fparam_aparam_default.savedmodel \ + source/tests/infer/chg_spin.savedmodel; do + if [ ! -d "${fixture}" ]; then + echo "ERROR: cached generated fixture is missing: ${fixture}" >&2 + exit 1 + fi + done - name: Convert models run: source/tests/infer/convert-models.sh # https://github.com/actions/runner-images/issues/9491 @@ -80,6 +171,23 @@ jobs: ENABLE_TENSORFLOW: ${{ matrix.enable_tensorflow && 'TRUE' || 'FALSE' }} ENABLE_PYTORCH: ${{ matrix.enable_pytorch && 'TRUE' || 'FALSE' }} ENABLE_PADDLE: ${{ matrix.enable_paddle && 'TRUE' || 'FALSE' }} + DP_REUSE_GENERATED_PT_MODELS: ${{ steps.pt-fixture-cache.outputs.cache-hit == 'true' && '1' || '0' }} + TORCHINDUCTOR_CACHE_DIR: ${{ github.workspace }}/.cache/torchinductor + TORCHINDUCTOR_FX_GRAPH_CACHE: 1 + TORCHINDUCTOR_AUTOGRAD_CACHE: 1 + # test_cc_local.sh creates the base manifest after generation. Add the + # TorchScript files too before actions/cache saves the cold-run cache; + # otherwise a cache hit skips gen_spin.py with deeppot_dpa_spin.pth absent. + - name: Complete PyTorch fixture cache manifest + if: ${{ matrix.enable_pytorch && steps.pt-fixture-cache.outputs.cache-hit != 'true' }} + shell: bash + run: | + manifest=source/tests/infer/.pt-fixture-cache-manifest + for fixture in source/tests/infer/*.pth; do + [ -f "${fixture}" ] && basename "${fixture}" + done >> "${manifest}" + LC_ALL=C sort -u -o "${manifest}" "${manifest}" + echo "Cached fixture manifest now contains $(wc -l < "${manifest}") files" # test lammps - run: | cp ${{ github.workspace }}/source/build_tests/paddle_inference_install_dir/paddle/lib/*.so ${{ github.workspace }}/dp_test/lib/ diff --git a/source/install/test_cc_local.sh b/source/install/test_cc_local.sh index 0c9c07178c..c1570eec92 100755 --- a/source/install/test_cc_local.sh +++ b/source/install/test_cc_local.sh @@ -131,6 +131,8 @@ if [ "${ENABLE_PYTORCH:-TRUE}" == "TRUE" ]; then fi fi if [ "$MODE" = "--models-only=kokkos" ]; then + # Targeted regeneration invalidates the previous full fixture manifest. + rm -f "${INFER_SCRIPT_PATH}/.pt-fixture-cache-manifest" # Leave the other C++ fixtures alone when reusing an existing build. rm -f ${INFER_SCRIPT_PATH}/deeppot_dpa4c_long_range*.pt2 \ ${INFER_SCRIPT_PATH}/deeppot_dpa4c_long_range*.pte \ @@ -138,120 +140,146 @@ if [ "${ENABLE_PYTORCH:-TRUE}" == "TRUE" ]; then run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_long_range.py --compressed-only exit 0 fi - # Remove stale generated model files so they can't be accidentally reused - # if gen scripts change format or the code version changes. - rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte - # Refill the first three generator slots instead of leaving two idle - # behind the long DPA2 export. DPA3 and DPA4 have no input dependency - # on these generators and write distinct fixtures. Keep at most three - # processes in this phase; do not add both on top of the initial batch. - # Wait on each PID separately so any failure is caught by set -e. - run_generator ${INFER_SCRIPT_PATH}/gen_sea.py & - PID1=$! - run_generator ${INFER_SCRIPT_PATH}/gen_dpa1.py & - PID2=$! - run_generator ${INFER_SCRIPT_PATH}/gen_dpa2.py & - PID3=$! - wait $PID1 - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4.py & - PID9=$! - wait $PID2 - run_generator ${INFER_SCRIPT_PATH}/gen_dpa3.py & - PID4=$! - wait $PID3 - wait $PID9 - wait $PID4 + _FIXTURE_MANIFEST="${INFER_SCRIPT_PATH}/.pt-fixture-cache-manifest" + # Reuse only the complete, content-addressed fixture set restored by CI. + if [ "${DP_REUSE_GENERATED_PT_MODELS:-0}" = "1" ]; then + if [ ! -s "${_FIXTURE_MANIFEST}" ]; then + echo "ERROR: PT fixture cache hit without a non-empty manifest" >&2 + exit 1 + fi + while IFS= read -r fixture || [ -n "${fixture}" ]; do + [ -n "${fixture}" ] || continue + if [ ! -f "${INFER_SCRIPT_PATH}/${fixture}" ]; then + echo "ERROR: cached PT fixture is missing: ${fixture}" >&2 + exit 1 + fi + done <"${_FIXTURE_MANIFEST}" + echo "Reusing $(wc -l <"${_FIXTURE_MANIFEST}") cached PT fixture files" + else + # Remove stale generated model files so they can't be accidentally reused + # if gen scripts change format or the code version changes. + rm -f ${INFER_SCRIPT_PATH}/*.pt2 ${INFER_SCRIPT_PATH}/*.pte \ + ${INFER_SCRIPT_PATH}/*.expected "${_FIXTURE_MANIFEST}" + # Refill the first three generator slots instead of leaving two idle + # behind the long DPA2 export. DPA3 and DPA4 have no input dependency + # on these generators and write distinct fixtures. Keep at most three + # processes in this phase; do not add both on top of the initial batch. + # Wait on each PID separately so any failure is caught by set -e. + run_generator ${INFER_SCRIPT_PATH}/gen_sea.py & + PID1=$! + run_generator ${INFER_SCRIPT_PATH}/gen_dpa1.py & + PID2=$! + run_generator ${INFER_SCRIPT_PATH}/gen_dpa2.py & + PID3=$! + wait $PID1 + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4.py & + PID9=$! + wait $PID2 + run_generator ${INFER_SCRIPT_PATH}/gen_dpa3.py & + PID4=$! + wait $PID3 + wait $PID9 + wait $PID4 - run_generator ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & - PID5=$! - run_generator ${INFER_SCRIPT_PATH}/gen_model_devi.py & - PID6=$! - run_generator ${INFER_SCRIPT_PATH}/gen_chg_spin.py & - PID9=$! - # DPA1 completed above; its archive-reuse consumer needs no other - # producer. Fill the fourth slot freed by moving DPA3 into the first phase. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py --reuse-dpa1 & - PID10=$! - wait $PID5 - wait $PID6 - wait $PID9 - wait $PID10 + run_generator ${INFER_SCRIPT_PATH}/gen_fparam_aparam.py & + PID5=$! + run_generator ${INFER_SCRIPT_PATH}/gen_model_devi.py & + PID6=$! + run_generator ${INFER_SCRIPT_PATH}/gen_chg_spin.py & + PID9=$! + # DPA1 completed above; its archive-reuse consumer needs no other + # producer. Fill the fourth slot freed by moving DPA3 into the first phase. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa1_pairexcl.py --reuse-dpa1 & + PID10=$! + wait $PID5 + wait $PID6 + wait $PID9 + wait $PID10 - # Keep both producers parallel, but compile the shared DPA1 spin model in - # model-devi's shorter branch, rather than repeating it on gen_spin's - # critical path. Finalize only after both producers succeed. Distinct - # report names retain both phases without overwriting their timings. - run_generator_named gen_spin_dpa3 ${INFER_SCRIPT_PATH}/gen_spin.py --dpa3-only & - PID7=$! - run_generator ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & - PID8=$! - wait $PID7 - wait $PID8 - run_generator_named gen_spin_reuse ${INFER_SCRIPT_PATH}/gen_spin.py --reuse-model-devi - # Copies of the multi-rank DPA3 archives of gen_dpa3.py and gen_spin.py - # whose nested with-comm artifact is corrupt; without them the load-failure - # cases of the pt_expt backends (test_with_comm_load_failure_ptexpt.cc) - # skip. - run_generator ${INFER_SCRIPT_PATH}/gen_corrupt_with_comm.py + # Keep both producers parallel, but compile the shared DPA1 spin model in + # model-devi's shorter branch, rather than repeating it on gen_spin's + # critical path. Finalize only after both producers succeed. Distinct + # report names retain both phases without overwriting their timings. + run_generator_named gen_spin_dpa3 ${INFER_SCRIPT_PATH}/gen_spin.py --dpa3-only & + PID7=$! + run_generator ${INFER_SCRIPT_PATH}/gen_spin_model_devi.py & + PID8=$! + wait $PID7 + wait $PID8 + run_generator_named gen_spin_reuse ${INFER_SCRIPT_PATH}/gen_spin.py --reuse-model-devi + # Copies of the multi-rank DPA3 archives of gen_dpa3.py and gen_spin.py + # whose nested with-comm artifact is corrupt; without them the load-failure + # cases of the pt_expt backends (test_with_comm_load_failure_ptexpt.cc) + # skip. + run_generator ${INFER_SCRIPT_PATH}/gen_corrupt_with_comm.py - # Native-spin DPA4 graph archives (baseline + model-level pair - # exclusion). Without this the whole native-spin graph C++ suite - # GTEST_SKIPs on the missing fixture, which is how a dead - # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & - PID11=$! - # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph - # lower, which no other C++ fixture covers. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & - PID12=$! - # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both - # is_spin=true and dim_chg_spin>0, i.e. the only one on which the - # runtime charge_spin argument of DeepSpin::compute is not inert. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & - PID13=$! - # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear - # composition, including the source-freeze gate's cross-rank exchange - # through the nested with-comm artifact. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & - PID14=$! - # Minimal native-spin DPA4C for LAMMPS pair_style dpa4spin (no - # fparam/aparam/chg_spin). CPU-safe; used by the magnetic-force - # convention behavioral tests. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_spin_lmp.py & - PID14B=$! - # Native-spin DPA4C: the archives above all ship a with-comm artifact - # and are therefore served by DeepSpinPTExpt, which leaves - # NativeSpinPTExpt -- the class the dispatch selects for a native-spin - # archive without one -- uncovered. DPA4C keeps its messaging local and - # so reaches it. The compiled graph calls CUDA-only operators, so the - # fixture is generated only where the C++ suite has a GPU; the tests - # built on it skip when it is absent. - if [ "${DP_VARIANT}" = "cuda" ]; then - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & - PID15=$! - # Compressed DPA4C, an energy and a native-spin model, on the compact - # canonical lower: the only fixtures that reach the device-resident - # entry points LAMMPS + Kokkos calls, with the replay of repeated - # evaluations and a runtime charge state. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_canonical.py & - PID16=$! - fi - # DPA4C with a latent-charge long-range module: the uncompressed archive - # takes the all-pairs topology of the graph lower; for the CUDA variant the - # script also writes the compressed archive, whose compact canonical lower - # reads the node positions. - run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_long_range.py & - PID17=$! - wait $PID11 - wait $PID12 - wait $PID13 - wait $PID14 - wait $PID14B - if [ "${DP_VARIANT}" = "cuda" ]; then - wait $PID15 - wait $PID16 + # Native-spin DPA4 graph archives (baseline + model-level pair + # exclusion). Without this the whole native-spin graph C++ suite + # GTEST_SKIPs on the missing fixture, which is how a dead + # applyPairExclusion seam in DeepSpinPTExpt went unnoticed. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_spin.py & + PID11=$! + # DPA4 + analytical ZBL bridging: a linear COMPOSITION on the graph + # lower, which no other C++ fixture covers. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_zbl.py & + PID12=$! + # Native-spin DPA4 + charge-spin FiLM: the ONLY fixture with both + # is_spin=true and dim_chg_spin>0, i.e. the only one on which the + # runtime charge_spin argument of DeepSpin::compute is not inert. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_spin_chgspin.py & + PID13=$! + # Native-spin DPA4 + ZBL bridging COMBINED: spin reaching a linear + # composition, including the source-freeze gate's cross-rank exchange + # through the nested with-comm artifact. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4_spin_zbl.py & + PID14=$! + # Minimal native-spin DPA4C for LAMMPS pair_style dpa4spin (no + # fparam/aparam/chg_spin). CPU-safe; used by the magnetic-force + # convention behavioral tests. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_spin_lmp.py & + PID14B=$! + # Native-spin DPA4C: the archives above all ship a with-comm artifact + # and are therefore served by DeepSpinPTExpt, which leaves + # NativeSpinPTExpt -- the class the dispatch selects for a native-spin + # archive without one -- uncovered. DPA4C keeps its messaging local and + # so reaches it. The compiled graph calls CUDA-only operators, so the + # fixture is generated only where the C++ suite has a GPU; the tests + # built on it skip when it is absent. + if [ "${DP_VARIANT}" = "cuda" ]; then + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_spin.py & + PID15=$! + # Compressed DPA4C, an energy and a native-spin model, on the compact + # canonical lower: the only fixtures that reach the device-resident + # entry points LAMMPS + Kokkos calls, with the replay of repeated + # evaluations and a runtime charge state. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_canonical.py & + PID16=$! + fi + # DPA4C with a latent-charge long-range module: the uncompressed archive + # takes the all-pairs topology of the graph lower; for the CUDA variant the + # script also writes the compressed archive, whose compact canonical lower + # reads the node positions. + run_generator ${INFER_SCRIPT_PATH}/gen_dpa4c_long_range.py & + PID17=$! + wait $PID11 + wait $PID12 + wait $PID13 + wait $PID14 + wait $PID14B + if [ "${DP_VARIANT}" = "cuda" ]; then + wait $PID15 + wait $PID16 + fi + wait $PID17 + # Only publish the cache manifest after successful model generation. + for fixture in "${INFER_SCRIPT_PATH}"/*.pt2 "${INFER_SCRIPT_PATH}"/*.pte "${INFER_SCRIPT_PATH}"/*.expected; do + [ -f "${fixture}" ] && basename "${fixture}" + done | LC_ALL=C sort >"${_FIXTURE_MANIFEST}" + if [ ! -s "${_FIXTURE_MANIFEST}" ]; then + echo "ERROR: PT fixture generation produced no cacheable files" >&2 + exit 1 + fi fi - wait $PID17 fi if [ "${ENABLE_PADDLE:-TRUE}" == "TRUE" ]; then PADDLE_INFERENCE_DIR=${BUILD_TMP_DIR}/paddle_inference_install_dir diff --git a/source/tests/common/test_cc_fixture_cache_fingerprint.py b/source/tests/common/test_cc_fixture_cache_fingerprint.py new file mode 100644 index 0000000000..a870c25734 --- /dev/null +++ b/source/tests/common/test_cc_fixture_cache_fingerprint.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Fixture dependency changes invalidate the cache without hashing C++ tests.""" + +import hashlib +import shlex +import subprocess +import tempfile +import unittest +from pathlib import ( + Path, +) + + +class TestCCFixtureCacheFingerprint(unittest.TestCase): + def test_dependency_changes_invalidate_fingerprint(self): + workflow = ( + Path(__file__).resolve().parents[3] + / ".github" + / "workflows" + / "test_cc.yml" + ).read_text(encoding="utf-8") + command = workflow.split("source_fingerprint=$(git ls-files -s --", 1)[1].split( + "| sha256sum", 1 + )[0] + pathspecs = shlex.split(command.replace("\\\n", " ")) + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + generator = repo / "source" / "tests" / "infer" / "gen_dpa4_spin.py" + generator.parent.mkdir(parents=True) + generator.write_text("from dpa4_fixtures import jitter\n", encoding="utf-8") + dependencies = ( + "source/tests/dpa4_fixtures.py", + "source/lib/src/tabulate.cc", + "source/lib/include/tabulate.h", + "source/CMakeLists.txt", + ) + for path in dependencies: + dependency = repo / path + dependency.parent.mkdir(parents=True, exist_ok=True) + dependency.write_text("original\n", encoding="utf-8") + + def git(*args): + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + capture_output=True, + ).stdout + + def fingerprint(): + return hashlib.sha256( + git("ls-files", "-s", "--", *pathspecs) + ).hexdigest() + + git("init", "-q") + git("add", "--", "source") + before = fingerprint() + # Each dependency changes independently while the generator stays + # unchanged: native kernels/headers matter just as Python helpers do. + for path in dependencies: + with self.subTest(dependency=path): + (repo / path).write_text("changed\n", encoding="utf-8") + git("add", "--", path) + after = fingerprint() + self.assertNotEqual(before, after) + before = after + + # Editing a C++ test must not trigger expensive fixture generation. + unrelated = repo / "source/lib/tests/test_tabulate.cc" + unrelated.parent.mkdir(parents=True) + unrelated.write_text("new test\n", encoding="utf-8") + git("add", "--", "source/lib/tests/test_tabulate.cc") + self.assertEqual(before, fingerprint()) + + +if __name__ == "__main__": + unittest.main()