feat: add Qureg checkpointing via ADIOS2 (#747)

.github/workflows/compile.yml

@@ -47,6 +47,7 @@ jobs:
       ${{ matrix.cuda      == 'ON' && 'CUDA' || '' }}
       ${{ matrix.hip       == 'ON' && 'HIP'  || '' }}
       ${{ matrix.cuquantum == 'ON' && 'CUQ'  || '' }}
+      ${{ matrix.adios2    == 'ON' && 'CKPT' || '' }}
 
     runs-on: ${{ matrix.os }}
 
@@ -67,6 +68,7 @@ jobs:
         cuda:      [ON, OFF]
         hip:       [ON, OFF]
         cuquantum: [ON, OFF]
+        adios2:    [ON, OFF]
         mpilib:    ['', 'mpich', 'ompi', 'impi', 'msmpi']
 
         # disable deprecated API on MSVC, and assign unique compilers,
@@ -249,14 +251,16 @@ jobs:
           -DQUEST_ENABLE_CUDA=${{ matrix.cuda }}
           -DQUEST_ENABLE_HIP=${{ matrix.hip }}
           -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }}
+          -DQUEST_ENABLE_CHECKPOINTING=${{ matrix.adios2 }}
           -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }}
           -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }}
           -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
           -DCMAKE_CXX_FLAGS=${{ matrix.mpi == 'ON' && matrix.cuda == 'ON' && '-fno-lto' || '' }}
 
-      # force 'Release' build (needed by MSVC to enable optimisations)
+      # force 'Release' build (needed by MSVC to enable optimisations),
+      # temporarily forcing serial compilation to avoid ADIOS2 OOM error
       - name: Compile
-        run: cmake --build ${{ env.build_dir }} --config Release --parallel
+        run: cmake --build ${{ env.build_dir }} --config Release --parallel 1
 
       # run all compiled isolated examples to test for link-time errors,
       # continuing if any fail (since some deliberately fail)

.github/workflows/test_free.yml

@@ -68,16 +68,18 @@ jobs:
           -DQUEST_ENABLE_DEPRECATED_API=${{ matrix.version == 3 && 'ON' || 'OFF' }}
           -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }}
           -DQUEST_FLOAT_PRECISION=${{ matrix.precision }}
+          -DQUEST_ENABLE_CHECKPOINTING=ON
 
-      # force 'Release' build (needed by MSVC to enable optimisations)
+      # force 'Release' build (needed by MSVC to enable optimisations), and force serial (to avoid ADIOS2 OOM)
       - name: Compile
-        run: cmake --build ${{ env.build_dir }} --config Release --parallel
+        run: cmake --build ${{ env.build_dir }} --config Release --parallel 1
 
       # run v4 unit tests in random order, excluding the integration tests,
       # using the default environment variables (e.g. test all permutations)
       # TODO:
       # ctest currently doesn't know of our Catch2 tags, so we
       # are manually excluding each integration test by name
+
       - name: Run v4 tests
         if: ${{ matrix.version == 4 }}
         run: ctest -j2 --output-on-failure --schedule-random -C Release -E "density evolution"

CMakeLists.txt

@@ -543,6 +543,83 @@ endif()
 
 
 
+# Checkpointing (ADIOS2)
+option(QUEST_ENABLE_CHECKPOINTING "Enable Qureg checkpointing (saveQuregToFile / createQuregFromFile) via ADIOS2. Turned OFF by default." OFF)
+message(STATUS "Checkpointing is turned ${QUEST_ENABLE_CHECKPOINTING}. Set QUEST_ENABLE_CHECKPOINTING to modify.")
+if (QUEST_ENABLE_CHECKPOINTING)
+
+  find_package(adios2 QUIET)
+
+  # A distributed QuEST needs an MPI-enabled ADIOS2 (which provides the
+  # adios2::cxx_mpi target). A serial system install lacks it, so in that case we
+  # ignore the found package and fetch an MPI-enabled build instead of failing.
+  set(quest_use_found_adios2 ${adios2_FOUND})
+  if (adios2_FOUND AND QUEST_ENABLE_MPI AND NOT TARGET adios2::cxx_mpi)
+    message(STATUS "Found ADIOS2 lacks MPI support (no adios2::cxx_mpi target); fetching an MPI-enabled build instead")
+    set(quest_use_found_adios2 FALSE)
+  endif()
+
+  if(NOT quest_use_found_adios2)
+    message(STATUS "fetching ADIOS2 via FetchContent")
+
+    include(FetchContent)
+    FetchContent_Declare(
+      adios2
+      GIT_REPOSITORY https://github.com/ornladios/ADIOS2.git
+      GIT_TAG v2.12.1
+    )
+
+    # Match ADIOS2's MPI to QuEST's so distributed runs write per-rank slices
+    # into one shared file. ADIOS2's CUDA support is deliberately left OFF:
+    # checkpointing copies amps to host memory (syncQuregFromGpu/syncQuregToGpu)
+    # before any I/O, so ADIOS2 never touches device pointers. Building it with
+    # CUDA is unnecessary and stalls the Windows CUDA CI job.
+    set(ADIOS2_USE_MPI  ${QUEST_ENABLE_MPI} CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_CUDA OFF CACHE BOOL "" FORCE)
+
+    # Forego unused facilities
+    set(ADIOS2_BUILD_TESTING  OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_SODIUM     OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_Fortran    OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_HDF5       OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_ZeroMQ     OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_SST        OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_DataMan    OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_SSC        OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_MHS        OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_DAOS       OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_MGARD      OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_BZip2      OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_Blosc      OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_Blosc2     OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_SZ         OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_ZFP        OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_PNG        OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_Profiling  OFF CACHE BOOL "" FORCE)
+    set(ADIOS2_USE_Python     OFF CACHE BOOL "" FORCE)
+
+    FetchContent_MakeAvailable(adios2)
+
+  else()
+    # re-run non-QUIET so configuration fails with a clear error if the package
+    # somehow became unavailable between the two calls
+    find_package(adios2 REQUIRED)
+  endif()
+
+  # In distributed builds link ADIOS2's MPI-enabled C++ interface: it defines
+  # ADIOS2_USE_MPI, which exposes the adios2::ADIOS(MPI_Comm) constructor used in
+  # qureg.cpp for collective per-rank I/O. The serial target lacks it.
+  if (QUEST_ENABLE_MPI)
+    target_link_libraries(QuEST PRIVATE adios2::cxx_mpi)
+  else()
+    target_link_libraries(QuEST PRIVATE adios2::cxx)
+  endif()
+  set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON)
+endif()
+
+
+
 # ===============================
 # Set options to save in config.h
 # ===============================
@@ -553,6 +630,7 @@ set(QUEST_COMPILE_OMP ${QUEST_ENABLE_OMP})
 set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI})
 set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM})
 set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM})
+set(QUEST_COMPILE_CHECKPOINTING ${QUEST_ENABLE_CHECKPOINTING})
 set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API})
 
 

docs/compile.md

@@ -689,3 +689,33 @@ Note that distributed executables are launched in a distinct way to the other de
 > - UCX
 > - launch flags
 > - checking via reportenv
+
+
+
+
+------------------
+
+
+<!-- permit doxygen to reference section -->
+<a id="compile_checkpointing"></a>
+
+## Checkpointing
+
+QuEST can optionally _checkpoint_ a `Qureg` to disk; writing its state to a file with `saveQuregToFile()`, to later be restored into a new `Qureg` with `createQuregFromFile()`. This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. The file records only the `Qureg` dimension (the number of qubits, and whether it is a density matrix) and its amplitudes; never the incidental deployment configuration. A `Qureg` saved by one deployment (say, distributed over `8` nodes) can therefore be restored by any other (say, a single GPU-accelerated node).
+
+Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `QUEST_ENABLE_CHECKPOINTING` at configuration:
+```bash
+# configure
+cmake .. -D QUEST_ENABLE_CHECKPOINTING=ON
+
+# build
+cmake --build . --parallel
+```
+
+> [!IMPORTANT]
+> ADIOS2 must be discoverable by CMake. If it was installed to a non-standard location (such as `~/.local`), pass its prefix via `CMAKE_PREFIX_PATH`:
+> ```bash
+> cmake .. -D QUEST_ENABLE_CHECKPOINTING=ON -D CMAKE_PREFIX_PATH=$HOME/.local
+> ```
+
+Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled throws a validation error.

quest/include/config.h.in

@@ -41,6 +41,7 @@
     defined(QUEST_COMPILE_CUDA)                 || \
     defined(QUEST_COMPILE_HIP)                  || \
     defined(QUEST_COMPILE_CUQUANTUM)            || \
+    defined(QUEST_COMPILE_CHECKPOINTING)        || \
     defined(QUEST_ENABLE_NUMA)                  || \
     defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \
     defined(QUEST_DISABLE_DEPRECATION_WARNINGS)
@@ -84,6 +85,7 @@
 #cmakedefine01 QUEST_COMPILE_CUDA
 #cmakedefine01 QUEST_COMPILE_CUQUANTUM
 #cmakedefine01 QUEST_COMPILE_HIP
+#cmakedefine01 QUEST_COMPILE_CHECKPOINTING
 
 
 // crucial to QuEST source (informs optional NUMA usage)
@@ -125,6 +127,7 @@
     ! defined(QUEST_COMPILE_CUDA)                 || \
     ! defined(QUEST_COMPILE_HIP)                  || \
     ! defined(QUEST_COMPILE_CUQUANTUM)            || \
+    ! defined(QUEST_COMPILE_CHECKPOINTING)        || \
     ! defined(QUEST_ENABLE_NUMA)                  || \
     ! defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \
     ! defined(QUEST_DISABLE_DEPRECATION_WARNINGS)
@@ -152,6 +155,7 @@
     ! (QUEST_COMPILE_CUDA                 == 0 || QUEST_COMPILE_CUDA                 == 1) || \
     ! (QUEST_COMPILE_HIP                  == 0 || QUEST_COMPILE_HIP                  == 1) || \
     ! (QUEST_COMPILE_CUQUANTUM            == 0 || QUEST_COMPILE_CUQUANTUM            == 1) || \
+    ! (QUEST_COMPILE_CHECKPOINTING        == 0 || QUEST_COMPILE_CHECKPOINTING        == 1) || \
     ! (QUEST_ENABLE_NUMA                  == 0 || QUEST_ENABLE_NUMA                  == 1) || \
     ! (QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 0 || QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 1) || \
     ! (QUEST_DISABLE_DEPRECATION_WARNINGS == 0 || QUEST_DISABLE_DEPRECATION_WARNINGS == 1)

quest/include/qureg.h

@@ -488,6 +488,51 @@ void getDensityQuregAmps(qcomp** outAmps, Qureg qureg, qindex startRow, qindex s
 /** @} */
 
 
+
+/**
+ * @defgroup qureg_checkpoint Checkpointing
+ * @brief Functions for saving a Qureg to file and restoring it later.
+ * @details These functions are only available when QuEST is compiled with
+ *          checkpointing support (CMake variable @c QUEST_ENABLE_CHECKPOINTING=ON),
+ *          which additionally requires the ADIOS2 library. Calling them in a
+ *          build without checkpointing support throws a validation error.
+ * @{
+ */
+
+
+/** Writes the contents of @p qureg to the file @p fn, so that it may later be
+ * restored with createQuregFromFile(). The file records only the @p qureg
+ * dimension (number of qubits and whether it is a density matrix) and its full
+ * set of amplitudes; incidental deployment information (e.g. multithreading,
+ * GPU-acceleration, distribution) is not recorded.
+ *
+ * @param[in] qureg the Qureg to write to disk.
+ * @param[in] fn    the output file path.
+ * @notyetdoced
+ * @notyettested
+ * @see
+ * - createQuregFromFile() to restore a Qureg saved by this function.
+ */
+void saveQuregToFile(Qureg qureg, const char* fn);
+
+
+/** Creates a new Qureg from a file previously written by saveQuregToFile(),
+ * with automatically chosen deployments (independent of those used when the
+ * file was saved), and populates it with the stored amplitudes.
+ *
+ * @param[in] fn the input file path.
+ * @returns A new Qureg instance matching the saved dimension and amplitudes.
+ * @notyetdoced
+ * @notyettested
+ * @see
+ * - saveQuregToFile() to create a file readable by this function.
+ */
+Qureg createQuregFromFile(const char* fn);
+
+
+/** @} */
+
+
 // end de-mangler
 #ifdef __cplusplus
 }

quest/src/api/environment.cpp

@@ -5,6 +5,7 @@
  * @author Tyson Jones 
  */
 
+#include "quest/include/config.h"
 #include "quest/include/environment.h"
 #include "quest/include/precision.h"
 #include "quest/include/modes.h"
@@ -204,16 +205,23 @@ void printPrecisionInfo() {
 }
 
 
+// reports whether QuEST was compiled with Qureg checkpointing support (ADIOS2)
+static bool isCheckpointingCompiled() {
+    return (bool) QUEST_COMPILE_CHECKPOINTING;
+}
+
+
 void printCompilationInfo() {
 
     print_table(
         "compilation", {
-        {"isOmpCompiled",         cpu_isOpenmpCompiled()},
-        {"isMpiCompiled",         comm_isMpiCompiled()},
-        {"isMpiSubCommCompiled",  comm_isMpiSubCommCompiled()},
-        {"isGpuCompiled",         gpu_isGpuCompiled()},
-        {"isHipCompiled",         gpu_isHipCompiled()},
-        {"isCuQuantumCompiled",   gpu_isCuQuantumCompiled()},
+        {"isOmpCompiled",            cpu_isOpenmpCompiled()},
+        {"isMpiCompiled",            comm_isMpiCompiled()},
+        {"isMpiSubCommCompiled",     comm_isMpiSubCommCompiled()},
+        {"isGpuCompiled",            gpu_isGpuCompiled()},
+        {"isHipCompiled",            gpu_isHipCompiled()},
+        {"isCuQuantumCompiled",      gpu_isCuQuantumCompiled()},
+        {"isCheckpointingCompiled",  isCheckpointingCompiled()},
     });
 }