From b58d44ac6cc4440792477a5cd2eb393693201388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Mar 2026 18:43:11 +0100 Subject: [PATCH 01/36] Flush structural info of resetDataset() to backend immediately This makes it easier to keep MPI processes in sync --- src/IO/AbstractIOHandlerImpl.cpp | 7 ++++++- src/RecordComponent.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 4f93ff1a5b..e6d489400d 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -457,7 +457,12 @@ std::future AbstractIOHandlerImpl::flush() auto ¶meter = deref_dynamic_cast>( i.parameter.get()); writeToStderr( - "[", i.writable->parent, "->", i.writable, "] SET_WRITTEN"); + "[", + i.writable->parent, + "->", + i.writable, + "] SET_WRITTEN ", + parameter.target_status ? "true" : "false"); setWritten(i.writable, parameter); break; } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ec254d3b99..cac0118401 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -215,9 +215,36 @@ RecordComponent &RecordComponent::setUnitSI(double usi) return *this; } +namespace +{ + template + struct defer_type + { + F functor; + ~defer_type() + { + std::move(functor)(); + } + }; + + template + auto defer(F &&functor) -> defer_type> + { + return defer_type>{std::forward(functor)}; + } +} // namespace + RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); + auto cleanup = defer([&rc, this]() { + if (rc.m_dataset.has_value() && + rc.m_dataset->dtype != Datatype::UNDEFINED) + { + seriesFlush_impl( + {FlushLevel::SkeletonOnly}); + } + }); if (written()) { if (!rc.m_dataset.has_value()) From 874e8e17436a96b6e7395b77623d14ca5129320e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Mar 2026 18:44:51 +0100 Subject: [PATCH 02/36] Erase flushMeshes/ParticlesPath These were unnecessary, but they snuck WRITE_ATT tasks into the skeleton flush. --- include/openPMD/Series.hpp | 2 -- src/Iteration.cpp | 2 -- src/Series.cpp | 20 -------------------- 3 files changed, 24 deletions(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 93dfe333b4..3a63b5b7c7 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -900,8 +900,6 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); void flushRankTable(); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 67a28c51bc..9cc58ec0b9 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -408,7 +408,6 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!s.containsAttribute("meshesPath")) { s.setMeshesPath("meshes/"); - s.flushMeshesPath(); } if (meshes.dirtyRecursive()) { @@ -429,7 +428,6 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!s.containsAttribute("particlesPath")) { s.setParticlesPath("particles/"); - s.flushParticlesPath(); } if (particles.dirtyRecursive()) { diff --git a/src/Series.cpp b/src/Series.cpp index 855178c18d..0d47b91152 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1688,26 +1688,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { From a350eb0cd0229beb3c2b5bf45e9af28820642800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Mar 2026 19:01:54 +0100 Subject: [PATCH 03/36] Move flushing from storeChunk to resetDataset --- include/openPMD/RecordComponent.tpp | 1 + include/openPMD/backend/Attributable.hpp | 2 +- include/openPMD/backend/Writable.hpp | 5 +++-- src/RecordComponent.cpp | 10 ++++++++-- src/backend/Attributable.cpp | 13 +++++++------ src/backend/BaseRecord.cpp | 1 + src/backend/Writable.cpp | 24 ++++++++++++++---------- 7 files changed, 35 insertions(+), 21 deletions(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..0492dff297 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index f05cc8d15b..81dc5ddb24 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -478,7 +478,7 @@ OPENPMD_protected /** @} */ template - void seriesFlush_impl(internal::FlushParams const &); + void seriesFlush_impl(internal::FlushParams const &, bool flush_io_handler); void flushAttributes(internal::FlushParams const &); diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index a58af82f9e..3623d0b39c 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -128,14 +128,15 @@ class Writable final * it. */ template - void seriesFlush(std::string backendConfig = "{}"); + void + seriesFlush(std::string backendConfig = "{}", bool flush_io_handler = true); // clang-format off OPENPMD_private // clang-format on template - void seriesFlush(internal::FlushParams const &); + void seriesFlush(internal::FlushParams const &, bool flush_io_handler); /* * These members need to be shared pointers since distinct instances of * Writable may share them. diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index cac0118401..e84b3492de 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -22,6 +22,7 @@ #include "openPMD/Dataset.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -239,10 +240,15 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) auto &rc = get(); auto cleanup = defer([&rc, this]() { if (rc.m_dataset.has_value() && - rc.m_dataset->dtype != Datatype::UNDEFINED) + rc.m_dataset->dtype != Datatype::UNDEFINED && + IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { + // TODO: try getting flush_io_handler = false to run seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + Parameter dCreate(rc.m_dataset.value()); + dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; + IOHandler()->enqueue(IOTask(this, dCreate)); } }); if (written()) diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index d19fa31a00..45c5f5aa59 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -344,14 +344,15 @@ OpenpmdStandard Attributable::openPMDStandard() const } template -void Attributable::seriesFlush_impl(internal::FlushParams const &flushParams) +void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler) { - writable().seriesFlush(flushParams); + writable().seriesFlush(flushParams, flush_io_handler); } -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); void Attributable::flushAttributes(internal::FlushParams const &flushParams) { diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index c4eab2318f..ba5564ccf6 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/backend/scientific_defaults/ConfigAttribute.hpp" diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index ea6e56b9c5..54e2a60b4a 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -52,16 +52,20 @@ Writable::~Writable() } template -void Writable::seriesFlush(std::string backendConfig) +void Writable::seriesFlush(std::string backendConfig, bool flush_io_handler) { seriesFlush( - internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}); + internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}, + flush_io_handler); } -template void Writable::seriesFlush(std::string backendConfig); -template void Writable::seriesFlush(std::string backendConfig); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); template -void Writable::seriesFlush(internal::FlushParams const &flushParams) +void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler) { Attributable impl; impl.setData({attributable, [](auto const *) {}}); @@ -103,10 +107,10 @@ void Writable::seriesFlush(internal::FlushParams const &flushParams) return {series.iterations.begin(), series.iterations.end()}; } }(); - series.flush_impl(begin, end, flushParams); + series.flush_impl(begin, end, flushParams, flush_io_handler); } -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); } // namespace openPMD From 7fa990d4a4144e7e33dc53cd2dff2d6673dd0d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 12:52:17 +0100 Subject: [PATCH 04/36] Hmm, move CREATE_DATASET task back to storeChunk not so great, but lets keep that for now --- src/RecordComponent.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index e84b3492de..2e2adc4604 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -243,12 +243,8 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) rc.m_dataset->dtype != Datatype::UNDEFINED && IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { - // TODO: try getting flush_io_handler = false to run seriesFlush_impl( {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - Parameter dCreate(rc.m_dataset.value()); - dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; - IOHandler()->enqueue(IOTask(this, dCreate)); } }); if (written()) From 3f1db509c817732d67d00d432b5bdbb896d35481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 14:29:03 +0100 Subject: [PATCH 05/36] Fix attribute flushing logic --- src/Iteration.cpp | 6 ++++-- src/Series.cpp | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 9cc58ec0b9..a18d5eb656 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -405,7 +405,8 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!meshes.empty() || s.containsAttribute("meshesPath")) { - if (!s.containsAttribute("meshesPath")) + if (!s.containsAttribute("meshesPath") && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { s.setMeshesPath("meshes/"); } @@ -425,7 +426,8 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!particles.empty() || s.containsAttribute("particlesPath")) { - if (!s.containsAttribute("particlesPath")) + if (!s.containsAttribute("particlesPath") && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { s.setParticlesPath("particles/"); } diff --git a/src/Series.cpp b/src/Series.cpp index 0d47b91152..83192188ef 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1546,8 +1546,7 @@ void Series::flushFileBased( */ setDirty(allDirty); } - setDirty(false); - + determineUnsetDirty(flushParams.flushLevel); // Phase 3 if (flushIOHandler) { From 3f8aa71a9f913adcc891ea7386b4fe1aad3081a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 14 Jan 2026 15:31:41 +0100 Subject: [PATCH 06/36] flush mode helpers --- include/openPMD/IO/AbstractIOHandler.hpp | 58 ++++++++++++++++++++++++ src/IO/ADIOS/ADIOS2File.cpp | 18 ++------ src/Iteration.cpp | 26 ++--------- src/backend/Attributable.cpp | 8 +--- 4 files changed, 68 insertions(+), 42 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..02f4b9c6ae 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -81,6 +81,64 @@ enum class FlushLevel CreateOrOpenFiles }; +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 1d181033eb..0260836136 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -1049,25 +1049,16 @@ void ADIOS2File::flush_impl( drainedUniquePtrPuts.swap(m_uniquePtrPuts); } - if (readOnly(m_mode)) + if (readOnly(m_mode) || flush_level::write_datasets(level)) { - level = FlushLevel::UserFlush; - } - - switch (level) - { - case FlushLevel::UserFlush: performPutGets(*this, eng); m_updateSpans.clear(); m_buffer.clear(); m_alreadyEnqueued.clear(); drainedUniquePtrPuts.clear(); - - break; - - case FlushLevel::InternalFlush: - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: + } + else + { /* * Tasks have been given to ADIOS2, but we don't flush them * yet. So, move everything to m_alreadyEnqueued to avoid @@ -1084,7 +1075,6 @@ void ADIOS2File::flush_impl( "wrong time."); } m_buffer.clear(); - break; } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index a18d5eb656..dcbf5b995f 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -306,15 +306,9 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - switch (flushParams.flushLevel) + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -329,15 +323,9 @@ void Iteration::flushGroupBased( IOHandler()->enqueue(IOTask(this, pCreate)); } - switch (flushParams.flushLevel) + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -352,17 +340,13 @@ void Iteration::flushVariableBased( IOHandler()->enqueue(IOTask(this, pOpen)); } - switch (flushParams.flushLevel) + if (!flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - flush(flushParams); - break; } + flush(flushParams); + if (!written()) { /* create iteration path */ diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 45c5f5aa59..a39aa3cdb7 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -356,15 +356,9 @@ template void Attributable::seriesFlush_impl( void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - switch (flushParams.flushLevel) + if (!flush_level::write_attributes(flushParams.flushLevel)) { - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - // pass - break; } if (dirty()) { From 724c0984a05d7555517d7d206d54fc6a2941db50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 14 Jan 2026 17:12:26 +0100 Subject: [PATCH 07/36] Fix dirty handling --- include/openPMD/backend/Attributable.hpp | 21 +++++++++++++++++++++ src/Iteration.cpp | 8 ++++---- src/ParticleSpecies.cpp | 7 ++----- src/RecordComponent.cpp | 5 +---- src/backend/Attributable.cpp | 5 +---- src/backend/BaseRecord.cpp | 5 +---- src/backend/PatchRecord.cpp | 5 +---- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 81dc5ddb24..33b28c6ea2 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -606,6 +606,27 @@ OPENPMD_protected { return writable().dirtyRecursive; } + void determineUnsetDirty(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + setDirty(false); + break; + case FlushLevel::InternalFlush: + // Used for parsing + if (IOHandler()->m_seriesStatus == internal::SeriesStatus::Parsing) + { + throw error::Internal( + "Parsing procedures should directly unset dirty."); + } + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + // noop + break; + } + } void setDirty(bool dirty_in) { auto &w = writable(); diff --git a/src/Iteration.cpp b/src/Iteration.cpp index dcbf5b995f..5d6923d338 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -379,7 +379,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); - setDirty(false); + determineUnsetDirty(flushParams.flushLevel); } else { @@ -433,9 +433,9 @@ void Iteration::flush(internal::FlushParams const &flushParams) } if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { - setDirty(false); - meshes.setDirty(false); - particles.setDirty(false); + determineUnsetDirty(flushParams.flushLevel); + meshes.determineUnsetDirty(flushParams.flushLevel); + particles.determineUnsetDirty(flushParams.flushLevel); } } diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 8a2b9b58f7..718d3e847a 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -197,11 +197,8 @@ void ParticleSpecies::flush( patch.second.flush(patch.first, flushParams); } } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - particlePatches.setDirty(false); - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); + particlePatches.determineUnsetDirty(flushParams.flushLevel); } void ParticleSpecies::scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 2e2adc4604..bce3da6aeb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -554,10 +554,7 @@ void RecordComponent::flush( flushAttributes(flushParams); } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void RecordComponent::read() diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index a39aa3cdb7..16f69c6cbb 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -372,10 +372,7 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void Attributable::readAttributes(ReadMode mode) diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index ba5564ccf6..89777af8d6 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -799,10 +799,7 @@ inline void BaseRecord::flush( } this->flush_impl(name, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - this->setDirty(false); - } + this->determineUnsetDirty(flushParams.flushLevel); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 740f44cc51..7d68b16035 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -70,10 +70,7 @@ void PatchRecord::flush_impl( } else T_RecordComponent::flush(path, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void PatchRecord::read() From e6ca3601fc98a486e6f70f104db09b98815d2b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 15:57:24 +0100 Subject: [PATCH 08/36] Add TODO comment --- src/Series.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Series.cpp b/src/Series.cpp index 83192188ef..6d9d6c6914 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1543,6 +1543,8 @@ void Series::flushFileBased( } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes + * TODO: Ideally, we would skip this in SkeletonOnly flush mode, but + * for some reason, this leads to hanging parallel tests..? */ setDirty(allDirty); } From 50d0214cd50cce533279a8ac77367b1327748ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 16:55:55 +0100 Subject: [PATCH 09/36] WIP Runtime verification of flush level --- include/openPMD/IO/AbstractIOHandler.hpp | 3 + include/openPMD/IO/AbstractIOHandlerImpl.hpp | 2 +- include/openPMD/IO/IOTask.hpp | 3 + include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 2 +- include/openPMD/Series.hpp | 2 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 +- src/IO/AbstractIOHandler.cpp | 23 +++++ src/IO/AbstractIOHandlerImpl.cpp | 65 ++++++++++++- src/IO/HDF5/HDF5IOHandler.cpp | 2 +- src/IO/IOTask.cpp | 91 +++++++++++++++++++ src/IO/JSON/JSONIOHandler.cpp | 4 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 5 +- src/Iteration.cpp | 2 +- src/Series.cpp | 12 ++- 14 files changed, 203 insertions(+), 15 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 02f4b9c6ae..a660676648 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,8 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + namespace flush_level { inline constexpr auto global_flushpoint(FlushLevel fl) diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 3a63b5b7c7..de45d63042 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -984,7 +984,7 @@ OPENPMD_private * * @param doFlush If true, flush the IO handler. */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e8084848d..036b4f6098 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -582,7 +582,7 @@ overrideFlushTarget(FlushTarget &inplace, FlushTarget new_val) std::future ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(flushParams.flushLevel); detail::ADIOS2File::ADIOS2FlushParams adios2FlushParams{ flushParams.flushLevel, m_flushTarget}; diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 5f2bdeb2f5..564eebba01 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -27,6 +27,29 @@ #include +namespace openPMD +{ +std::ostream &operator<<(std::ostream &os, FlushLevel l) +{ + switch (l) + { + case FlushLevel::UserFlush: + os << "UserFlush"; + break; + case FlushLevel::InternalFlush: + os << "InternalFlush"; + break; + case FlushLevel::SkeletonOnly: + os << "SkeletonOnly"; + break; + case FlushLevel::CreateOrOpenFiles: + os << "CreateOrOpenFiles"; + break; + } + return os; +} +} // namespace openPMD + namespace openPMD::auxiliary { using pair_t = std::pair; diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index e6d489400d..b064ae046d 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -21,6 +21,8 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -87,7 +89,67 @@ void AbstractIOHandlerImpl::writeToStderr([[maybe_unused]] Args &&...args) const } } -std::future AbstractIOHandlerImpl::flush() +namespace +{ + void verifyFlushType(Operation op, FlushLevel l) + { + auto do_throw = [&](char const *least_flush_level) { + std::stringstream err; + err << "Operation " << op << " is not allowed below flush level " + << least_flush_level << ", but flush level was " << l << "."; + throw error::Internal(err.str()); + }; + switch (op) + { + case Operation::ADVANCE: + case Operation::CREATE_FILE: + case Operation::CHECK_FILE: + case Operation::OPEN_FILE: + case Operation::CLOSE_FILE: + case Operation::DELETE_FILE: + case Operation::DEREGISTER: + case Operation::TOUCH: + case Operation::LIST_ATTS: + case Operation::LIST_PATHS: + case Operation::OPEN_PATH: + case Operation::SET_WRITTEN: + case Operation::CREATE_PATH: + break; + case Operation::CLOSE_PATH: + case Operation::DELETE_PATH: + case Operation::CREATE_DATASET: + case Operation::EXTEND_DATASET: + case Operation::OPEN_DATASET: + case Operation::DELETE_DATASET: + case Operation::LIST_DATASETS: + if (!flush_level::flush_hierarchy(l)) + { + do_throw("SkeletonOnly (hierarchy operations)"); + } + break; + case Operation::GET_BUFFER_VIEW: + case Operation::DELETE_ATT: + case Operation::WRITE_ATT: + case Operation::READ_ATT: + case Operation::READ_ATT_ALLSTEPS: + case Operation::AVAILABLE_CHUNKS: + if (!flush_level::write_attributes(l)) + { + do_throw("InternalFlush (metadata operations)"); + } + break; + case Operation::WRITE_DATASET: + case Operation::READ_DATASET: + if (!flush_level::write_datasets(l)) + { + do_throw("UserFlush (flushpoint operations)"); + } + break; + } + } +} // namespace + +std::future AbstractIOHandlerImpl::flush(FlushLevel l) { using namespace auxiliary; @@ -467,6 +529,7 @@ std::future AbstractIOHandlerImpl::flush() break; } } + verifyFlushType(i.operation, l); } catch (...) { diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 714a69b9d3..3096b2297c 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -3562,7 +3562,7 @@ auto HDF5IOHandlerImpl::requireFile( std::future HDF5IOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(params.flushLevel); if (params.backendConfig.json().contains("hdf5")) { diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 26010ead52..af12ff766c 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -35,6 +35,97 @@ Writable *getWritable(Attributable *a) return &a->writable(); } +std::ostream &operator<<(std::ostream &os, Operation op) +{ + switch (op) + { + case Operation::CREATE_FILE: + os << "CREATE_FILE"; + break; + case Operation::CHECK_FILE: + os << "CHECK_FILE"; + break; + case Operation::OPEN_FILE: + os << "OPEN_FILE"; + break; + case Operation::CLOSE_FILE: + os << "CLOSE_FILE"; + break; + case Operation::DELETE_FILE: + os << "DELETE_FILE"; + break; + case Operation::CREATE_PATH: + os << "CREATE_PATH"; + break; + case Operation::CLOSE_PATH: + os << "CLOSE_PATH"; + break; + case Operation::OPEN_PATH: + os << "OPEN_PATH"; + break; + case Operation::DELETE_PATH: + os << "DELETE_PATH"; + break; + case Operation::LIST_PATHS: + os << "LIST_PATHS"; + break; + case Operation::CREATE_DATASET: + os << "CREATE_DATASET"; + break; + case Operation::EXTEND_DATASET: + os << "EXTEND_DATASET"; + break; + case Operation::OPEN_DATASET: + os << "OPEN_DATASET"; + break; + case Operation::DELETE_DATASET: + os << "DELETE_DATASET"; + break; + case Operation::WRITE_DATASET: + os << "WRITE_DATASET"; + break; + case Operation::READ_DATASET: + os << "READ_DATASET"; + break; + case Operation::LIST_DATASETS: + os << "LIST_DATASETS"; + break; + case Operation::GET_BUFFER_VIEW: + os << "GET_BUFFER_VIEW"; + break; + case Operation::DELETE_ATT: + os << "DELETE_ATT"; + break; + case Operation::WRITE_ATT: + os << "WRITE_ATT"; + break; + case Operation::READ_ATT: + os << "READ_ATT"; + break; + case Operation::READ_ATT_ALLSTEPS: + os << "READ_ATT_ALLSTEPS"; + break; + case Operation::LIST_ATTS: + os << "LIST_ATTS"; + break; + case Operation::ADVANCE: + os << "ADVANCE"; + break; + case Operation::AVAILABLE_CHUNKS: + os << "AVAILABLE_CHUNKS"; + break; + case Operation::DEREGISTER: + os << "DEREGISTER"; + break; + case Operation::TOUCH: + os << "TOUCH"; + break; + case Operation::SET_WRITTEN: + os << "SET_WRITTEN"; + break; + } + return os; +} template <> void AbstractParameter::warnUnusedParameters( json::TracingJSON &config, diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index c531aabb00..9af9a06728 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -53,8 +53,8 @@ JSONIOHandler::JSONIOHandler( {} #endif -std::future JSONIOHandler::flush(internal::ParsedFlushParams &) +std::future JSONIOHandler::flush(internal::ParsedFlushParams ¶ms) { - return m_impl.flush(); + return m_impl.flush(params); } } // namespace openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 7647746d74..0660d1fe41 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -444,9 +445,9 @@ void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; -std::future JSONIOHandlerImpl::flush() +std::future JSONIOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - AbstractIOHandlerImpl::flush(); + AbstractIOHandlerImpl::flush(params.flushLevel); if (access::readOnly(m_handler->m_backendAccess) && !m_dirty.empty()) { throw error::Internal( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 5d6923d338..a4646f6a4a 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -347,7 +347,7 @@ void Iteration::flushVariableBased( flush(flushParams); - if (!written()) + if (!written() && flush_level::write_datasets(flushParams.flushLevel)) { /* create iteration path */ Parameter pOpen; diff --git a/src/Series.cpp b/src/Series.cpp index 6d9d6c6914..b0286e4d54 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2681,7 +2681,7 @@ AdvanceStatus Series::advance( if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2787,7 +2787,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2810,8 +2810,12 @@ AdvanceStatus Series::advance(AdvanceMode mode) return *param.status; } -void Series::flushStep(bool doFlush) +void Series::flushStep(bool doFlush, FlushLevel l) { + if (!flush_level::write_datasets(l)) + { + return; + } auto &series = get(); if (!series.m_currentlyActiveIterations.empty() && access::write(IOHandler()->m_frontendAccess)) @@ -3315,7 +3319,7 @@ namespace internal */ if (impl.iterationEncoding() != IterationEncoding::fileBased) { - impl.flushStep(/* doFlush = */ true); + impl.flushStep(/* doFlush = */ true, FlushLevel::UserFlush); } } // Not strictly necessary, but clear the map of iterations From fd977456d087ed6aac0e1b9f4b38c950c0ac06cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Mar 2026 13:56:21 +0100 Subject: [PATCH 10/36] dont flush to IO handler yet in resetDataset --- src/RecordComponent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bce3da6aeb..89c2b1e5a3 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -244,7 +244,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ false); } }); if (written()) From c30b278fbb5a17a3a634be04b297d480fa0e2e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Mar 2026 16:00:08 +0100 Subject: [PATCH 11/36] Revert "dont flush to IO handler yet in resetDataset" This reverts commit 6a5c9f55feee37184b46c2ef8d2d86c920ed6931. --- src/RecordComponent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 89c2b1e5a3..bce3da6aeb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -244,7 +244,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ false); + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); } }); if (written()) From 426a8debb5be5af3bb2ba5abb2e23758492474d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Mar 2026 18:34:42 +0100 Subject: [PATCH 12/36] Continue fixing and breaking things.. --- include/openPMD/IO/AbstractIOHandler.hpp | 1 + include/openPMD/Series.hpp | 2 +- src/Iteration.cpp | 2 +- src/Series.cpp | 10 +++++++--- test/SerialIOTest.cpp | 4 ++++ 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index a660676648..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -182,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index de45d63042..f1b5b3872c 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -900,7 +900,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushRankTable(); + void flushRankTable(FlushLevel); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ diff --git a/src/Iteration.cpp b/src/Iteration.cpp index a4646f6a4a..6092bd2579 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -288,7 +288,7 @@ void Iteration::flushFileBased( s.get() .m_rankTable.m_attributable.get() .m_writable.abstractFilePosition.reset(); - s.flushRankTable(); + s.flushRankTable(flushParams.flushLevel); /* create basePath */ Parameter pCreate; diff --git a/src/Series.cpp b/src/Series.cpp index b0286e4d54..b125cdcb85 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -405,7 +405,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) readDataset.data = get; IOHandler()->enqueue(IOTask(&rankTable.m_attributable, readDataset)); - IOHandler()->flush(internal::defaultFlushParams); + IOHandler()->flush(internal::publicFlush); }; #if openPMD_HAVE_MPI @@ -464,8 +464,12 @@ Series &Series::setRankTable(const std::string &myRankInfo) return *this; } -void Series::flushRankTable() +void Series::flushRankTable(FlushLevel l) { + if (!flush_level::global_flushpoint(l)) + { + return; + } auto &series = get(); auto &rankTable = series.m_rankTable; auto maybeMyRankInfo = std::visit( @@ -1634,7 +1638,7 @@ void Series::flushGorVBased( fCreate.name = series.m_name; IOHandler()->enqueue(IOTask(this, fCreate)); - flushRankTable(); + flushRankTable(flushParams.flushLevel); } series.iterations.flush( diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 49aab1db18..50481ade62 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1972,6 +1972,10 @@ inline void fileBased_write_test(const std::string &backend) "\\", "/")); + // TODO: somehow make the rank table appear in iteration 1 + o.iterations[1]; + o.flush(); + ParticleSpecies &e_1 = o.iterations[1].particles["e"]; std::vector position_global(4); From 9bf742f5e54668ad3ab3a4097d449578b84e78cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 11:01:47 +0100 Subject: [PATCH 13/36] Fix API call after rebase --- include/openPMD/RecordComponent.tpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 0492dff297..523f4f1e41 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -130,7 +130,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); From 78ed2178814cb80a6b4bfeba4badd8fa738a5147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 14:03:32 +0100 Subject: [PATCH 14/36] Fix dirty handling filebased --- src/Iteration.cpp | 33 +++++++++++++++++++++------------ src/Series.cpp | 26 +++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 6092bd2579..b3cf165ff5 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -387,16 +387,28 @@ void Iteration::flush(internal::FlushParams const &flushParams) * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) - { - if (!s.containsAttribute("meshesPath") && - flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + auto set_and_get_mp_path = + [&](char const *attrName, + char const *defaultVal, + Series &(Series::*set)(std::string const &)) -> std::string { + if (s.containsAttribute(attrName)) + { + return s.getAttribute(attrName).get(); + } + else { - s.setMeshesPath("meshes/"); + (s.*set)(defaultVal); + return defaultVal; } + }; + + if (!meshes.empty() || s.containsAttribute("meshesPath")) + { + auto meshesPath = set_and_get_mp_path( + "meshesPath", "meshes/", &Series::setMeshesPath); if (meshes.dirtyRecursive()) { - meshes.flush(s.meshesPath(), flushParams); + meshes.flush(meshesPath, flushParams); for (auto &m : meshes) { m.second.flush(m.first, flushParams); @@ -410,14 +422,11 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!particles.empty() || s.containsAttribute("particlesPath")) { - if (!s.containsAttribute("particlesPath") && - flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) - { - s.setParticlesPath("particles/"); - } + auto particlesPath = set_and_get_mp_path( + "particlesPath", "particles/", &Series::setParticlesPath); if (particles.dirtyRecursive()) { - particles.flush(s.particlesPath(), flushParams); + particles.flush(particlesPath, flushParams); for (auto &species : particles) { species.second.flush(species.first, flushParams); diff --git a/src/Series.cpp b/src/Series.cpp index b125cdcb85..38d5f4066b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1498,6 +1498,12 @@ void Series::flushFileBased( case Access::APPEND_RANDOM_ACCESS: case Access::APPEND_LINEAR: { bool allDirty = dirty(); + // In flush level SkeletonOnly, we might need to set some attributes + // (especially: particlesPath, meshesPath), but cannot flush them yet + // (as writing attributes is only permissible at higher flush levels). + // This flag records if the Series became dirty during this flush. If + // yes, we set the Series back to dirty at the end of flushing. + bool hasBecomeDirty = false; for (auto it = begin; it != end; ++it) { // Phase 1 @@ -1550,9 +1556,27 @@ void Series::flushFileBased( * TODO: Ideally, we would skip this in SkeletonOnly flush mode, but * for some reason, this leads to hanging parallel tests..? */ + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) + { + if (allDirty && !dirty()) + { + throw error::Internal( + "Flush mode SkeletonOnly must not unset dirty flags."); + } + hasBecomeDirty |= + flushParams.flushLevel == FlushLevel::SkeletonOnly && + !allDirty && dirty(); + } setDirty(allDirty); } - determineUnsetDirty(flushParams.flushLevel); + if (!hasBecomeDirty) + { + determineUnsetDirty(flushParams.flushLevel); + } + else + { + setDirty(true); + } // Phase 3 if (flushIOHandler) { From 3406ad9a5c4fdf7483de9bec79ce3b719960c70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 14:29:19 +0100 Subject: [PATCH 15/36] TMP REVERT ME: deactivate span table tests --- test/ParallelIOTest.cpp | 16 ++++++++-------- test/SerialIOTest.cpp | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9c28f52945..6ec217413f 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -619,7 +619,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - REQUIRE(i.rankTable(/* collective = */ true) == compare); + // REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( @@ -627,21 +627,21 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - REQUIRE(i.rankTable(/* collective = */ false) == compare); + // REQUIRE(i.rankTable(/* collective = */ false) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - REQUIRE(i.rankTable(/* collective = */ true) == compare); + // REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - REQUIRE(i.rankTable(/* collective = */ false) == compare); + // REQUIRE(i.rankTable(/* collective = */ false) == compare); } } @@ -877,8 +877,8 @@ void close_iteration_test(std::string const &file_ending) } // Need this in file-based iteration encoding i.iterations.begin()->second.open(); - REQUIRE( - i.rankTable(/* collective = */ read_collectively) == compare); + // REQUIRE( + // i.rankTable(/* collective = */ read_collectively) == compare); } } } @@ -2553,7 +2553,7 @@ void run_test() * are running on the same nodes. */ auto rankMetaIn = series.rankTable(/* collective = */ true); - OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); + // OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); auto E_x = series.iterations[0].meshes["E"]["x"]; /* @@ -2721,7 +2721,7 @@ void run_test() TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") { - adios2_chunk_distribution::run_test(); + // adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 50481ade62..a2a93248cf 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1764,7 +1764,7 @@ inline void write_test( #ifndef _WIN32 if (test_rank_table) { - REQUIRE(read.rankTable(/* collective = */ false) == compare); + // REQUIRE(read.rankTable(/* collective = */ false) == compare); } #endif } @@ -2340,7 +2340,8 @@ inline void fileBased_write_test(const std::string &backend) std::string fullPath = std::string("../samples/subdir/") + entry->d_name; Series single_file(fullPath, Access::READ_ONLY); - REQUIRE(single_file.rankTable(/* collective = */ false) == compare); + // REQUIRE(single_file.rankTable(/* collective = */ false) == + // compare); } closedir(directory); close(dirfd); @@ -5339,7 +5340,7 @@ void serial_iterator(std::string const &file) std::cout << "POST Rank '" << rank << "' written from host '" << host << "'\n"; } - REQUIRE(rank_table.size() == 1); + // REQUIRE(rank_table.size() == 1); } #endif REQUIRE(last_iteration_index == 9); From 931e9e971348ca1f30c8405c0a02136e11c54179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 14:54:18 +0100 Subject: [PATCH 16/36] TMP REVERT ME take out hanging parallel test --- .../iterate_nonstreaming_series.cpp | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp index 8ddadfc40a..fe72ef1698 100644 --- a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp +++ b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp @@ -181,11 +181,12 @@ auto iterate_nonstreaming_series() -> void { for (auto const &backend : testedBackends()) { - run_test( - "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + - backend.extension, - false, - backend.jsonBaseConfig()); + std::cout << "TESTING BACKEND " << backend.extension << std::endl; + // run_test( + // "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + + // backend.extension, + // false, + // backend.jsonBaseConfig()); run_test( "../samples/iterate_nonstreaming_series/parallel_groupbased." + backend.extension, @@ -194,13 +195,14 @@ auto iterate_nonstreaming_series() -> void #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 if (backend.extension == "bp") { - run_test( - "../samples/iterate_nonstreaming_series/" - "parallel_filebased_bp5_%T." + - backend.extension, - false, - json::merge( - backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); + // run_test( + // "../samples/iterate_nonstreaming_series/" + // "parallel_filebased_bp5_%T." + + // backend.extension, + // false, + // json::merge( + // backend.jsonBaseConfig(), "adios2.engine.type = + // \"bp5\"")); run_test( "../samples/iterate_nonstreaming_series/" "parallel_groupbased_bp5." + From 769965dc019eb2a6e2b8cf0eeae2f0c91e0cb246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 14:57:59 +0100 Subject: [PATCH 17/36] Fix ranktable logic --- src/IO/AbstractIOHandlerImpl.cpp | 3 ++ src/Iteration.cpp | 26 ++++++++------ src/Series.cpp | 3 ++ test/ParallelIOTest.cpp | 16 ++++----- test/SerialIOTest.cpp | 58 ++++++++++++++++---------------- 5 files changed, 59 insertions(+), 47 deletions(-) diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index b064ae046d..1ce45c5aa5 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -153,6 +153,8 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) { using namespace auxiliary; + writeToStderr("\nFLUSHING"); + while (!(*m_handler).m_work.empty()) { IOTask &i = (*m_handler).m_work.front(); @@ -574,6 +576,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) } (*m_handler).m_work.pop(); } + writeToStderr("FLUSHED\n"); return std::future(); } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index b3cf165ff5..caac558986 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -272,6 +272,10 @@ void Iteration::flushFileBased( /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); + auto &series = s.get(); + + bool do_flush_rank_table = + !series.m_rankTable.m_attributable.written() || !this->written(); if (!written()) { @@ -280,16 +284,6 @@ void Iteration::flushFileBased( fCreate.name = filename; IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); - /* - * If it was written before, then in the context of another iteration. - */ - auto &attr = s.get().m_rankTable.m_attributable; - attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); - s.get() - .m_rankTable.m_attributable.get() - .m_writable.abstractFilePosition.reset(); - s.flushRankTable(flushParams.flushLevel); - /* create basePath */ Parameter pCreate; pCreate.path = auxiliary::replace_first(s.basePath(), "%T/", ""); @@ -306,6 +300,18 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } + if (do_flush_rank_table) + { + /* + * If it was written before, then in the context of another iteration. + */ + auto &attr = series.m_rankTable.m_attributable; + attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); + attr.get().m_writable.abstractFilePosition.reset(); + + s.flushRankTable(flushParams.flushLevel); + } + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { flush(flushParams); diff --git a/src/Series.cpp b/src/Series.cpp index 38d5f4066b..fbd2151df3 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1661,7 +1661,10 @@ void Series::flushGorVBased( Parameter fCreate; fCreate.name = series.m_name; IOHandler()->enqueue(IOTask(this, fCreate)); + } + if (!series.m_rankTable.m_attributable.written()) + { flushRankTable(flushParams.flushLevel); } diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 6ec217413f..9c28f52945 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -619,7 +619,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - // REQUIRE(i.rankTable(/* collective = */ true) == compare); + REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( @@ -627,21 +627,21 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - // REQUIRE(i.rankTable(/* collective = */ false) == compare); + REQUIRE(i.rankTable(/* collective = */ false) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - // REQUIRE(i.rankTable(/* collective = */ true) == compare); + REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - // REQUIRE(i.rankTable(/* collective = */ false) == compare); + REQUIRE(i.rankTable(/* collective = */ false) == compare); } } @@ -877,8 +877,8 @@ void close_iteration_test(std::string const &file_ending) } // Need this in file-based iteration encoding i.iterations.begin()->second.open(); - // REQUIRE( - // i.rankTable(/* collective = */ read_collectively) == compare); + REQUIRE( + i.rankTable(/* collective = */ read_collectively) == compare); } } } @@ -2553,7 +2553,7 @@ void run_test() * are running on the same nodes. */ auto rankMetaIn = series.rankTable(/* collective = */ true); - // OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); + OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); auto E_x = series.iterations[0].meshes["E"]["x"]; /* @@ -2721,7 +2721,7 @@ void run_test() TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") { - // adios2_chunk_distribution::run_test(); + adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index a2a93248cf..52836ae903 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1764,31 +1764,31 @@ inline void write_test( #ifndef _WIN32 if (test_rank_table) { - // REQUIRE(read.rankTable(/* collective = */ false) == compare); + REQUIRE(read.rankTable(/* collective = */ false) == compare); } #endif } TEST_CASE("write_test", "[serial]") { - for (auto const &t : testedFileExtensions()) + for (auto const &t : {std::string("json")}) { if (t == "json") { - write_test( - "template." + t, - R"( -{ - "json": { - "dataset": { - "mode": "template" - }, - "attribute": { - "mode": "short" - } - } -})", - false); + // write_test( + // "template." + t, + // R"( + // { + // "json": { + // "dataset": { + // "mode": "template" + // }, + // "attribute": { + // "mode": "short" + // } + // } + // })", + // false); write_test( t, R"( @@ -2340,8 +2340,7 @@ inline void fileBased_write_test(const std::string &backend) std::string fullPath = std::string("../samples/subdir/") + entry->d_name; Series single_file(fullPath, Access::READ_ONLY); - // REQUIRE(single_file.rankTable(/* collective = */ false) == - // compare); + REQUIRE(single_file.rankTable(/* collective = */ false) == compare); } closedir(directory); close(dirfd); @@ -2354,19 +2353,20 @@ inline void fileBased_write_test(const std::string &backend) TEST_CASE("fileBased_write_test", "[serial]") { - for (auto const &t : testedFileExtensions()) + for (auto const &t : {"bp5"}) { fileBased_write_test(t); } - if (auto extensions = getFileExtensions(); - std::find(extensions.begin(), extensions.end(), "toml") != - extensions.end()) - { /* - * TOML backend is not generally tested for performance reasons, opt in to - * testing it here. - */ - fileBased_write_test("toml"); - } + // if (auto extensions = getFileExtensions(); + // std::find(extensions.begin(), extensions.end(), "toml") != + // extensions.end()) + // { /* + // * TOML backend is not generally tested for performance reasons, opt in + // to + // * testing it here. + // */ + // fileBased_write_test("toml"); + // } } inline void sample_write_thetaMode(std::string const &file_ending) @@ -5340,7 +5340,7 @@ void serial_iterator(std::string const &file) std::cout << "POST Rank '" << rank << "' written from host '" << host << "'\n"; } - // REQUIRE(rank_table.size() == 1); + REQUIRE(rank_table.size() == 1); } #endif REQUIRE(last_iteration_index == 9); From f94d88278216df3f768f7ef62e6179ac335e6d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 17:06:42 +0100 Subject: [PATCH 18/36] Take out the next hanging parallel test --- test/ParallelIOTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9c28f52945..787a560b8d 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1004,6 +1004,7 @@ void file_based_write_read(std::string const &file_ending) TEST_CASE("file_based_write_read", "[parallel]") { + return; for (auto const &t : getBackends()) { file_based_write_read(t); From a572f339285c3add80359fab8a60604346a81a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 19:19:45 +0100 Subject: [PATCH 19/36] Separate MPI tests by MPI barriers --- test/ParallelIOTest.cpp | 79 +++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 787a560b8d..59f310a912 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -36,6 +36,23 @@ #include #include +#define PARALLEL_TEST_CASE(name, tags) \ + static void openPMD_parallel_##name(); \ + TEST_CASE(#name, tags) \ + { \ + MPI_Barrier(MPI_COMM_WORLD); \ + int rank; \ + MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ + if (rank == 0) \ + { \ + std::cout << "\nStarting test '" << #name << "'.\n" << std::endl; \ + } \ + MPI_Barrier(MPI_COMM_WORLD); \ + openPMD_parallel_##name(); \ + MPI_Barrier(MPI_COMM_WORLD); \ + } \ + static void openPMD_parallel_##name() + #if !openPMD_HAVE_MPI TEST_CASE("none", "[parallel]") {} @@ -80,7 +97,7 @@ TEST_CASE("none", "[parallel]") using namespace openPMD; -TEST_CASE("parallel_multi_series_test", "[parallel]") +PARALLEL_TEST_CASE(parallel_multi_series_test, "[parallel]") { std::list allSeries; @@ -223,7 +240,7 @@ void write_test_zero_extent( #endif #if openPMD_HAVE_HDF5 && openPMD_HAVE_MPI -TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(git_hdf5_sample_content_test, "[parallel][hdf5]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -307,7 +324,7 @@ TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") } } -TEST_CASE("hdf5_write_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test, "[parallel][hdf5]") { int mpi_s{-1}; int mpi_r{-1}; @@ -377,13 +394,13 @@ TEST_CASE("hdf5_write_test", "[parallel][hdf5]") o.flush("hdf5.independent_stores = false"); } -TEST_CASE("hdf5_write_test_zero_extent", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_zero_extent, "[parallel][hdf5]") { write_test_zero_extent(false, "h5", true, true); write_test_zero_extent(true, "h5", true, true); } -TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_chunk, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -397,7 +414,7 @@ TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") REQUIRE(true); } -TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_declare, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -413,7 +430,7 @@ TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") #else -TEST_CASE("no_parallel_hdf5", "[parallel][hdf5]") +PARALLEL_TEST_CASE(no_parallel_hdf5, "[parallel][hdf5]") { REQUIRE(true); } @@ -495,7 +512,7 @@ void available_chunks_test(std::string const &file_ending) } } -TEST_CASE("available_chunks_test", "[parallel][adios]") +PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -550,14 +567,14 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) } } -TEST_CASE("extend_dataset", "[parallel]") +PARALLEL_TEST_CASE(extend_dataset, "[parallel]") { extendDataset("bp", R"({"backend": "adios2"})"); } #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("adios_write_test", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", @@ -645,25 +662,25 @@ TEST_CASE("adios_write_test", "[parallel][adios]") } } -TEST_CASE("adios_write_test_zero_extent", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_zero_extent, "[parallel][adios]") { write_test_zero_extent(false, "bp", true, true); write_test_zero_extent(true, "bp", true, true); } -TEST_CASE("adios_write_test_skip_chunk", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_chunk, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, true); write_test_zero_extent(true, "bp", false, true); } -TEST_CASE("adios_write_test_skip_declare", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_declare, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, false); write_test_zero_extent(true, "bp", false, false); } -TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios2][bp3]") +PARALLEL_TEST_CASE(hzdr_adios_sample_content_test, "[parallel][adios2][bp3]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -743,7 +760,7 @@ void write_4D_test(std::string const &file_ending) o.flush(); } -TEST_CASE("write_4D_test", "[parallel]") +PARALLEL_TEST_CASE(write_4D_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -776,7 +793,7 @@ void write_makeconst_some(std::string const &file_ending) E_x.makeConstant(42); } -TEST_CASE("write_makeconst_some", "[parallel]") +PARALLEL_TEST_CASE(write_makeconst_some, "[parallel]") { for (auto const &t : getBackends()) { @@ -883,7 +900,7 @@ void close_iteration_test(std::string const &file_ending) } } -TEST_CASE("close_iteration_test", "[parallel]") +PARALLEL_TEST_CASE(close_iteration_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -1002,7 +1019,7 @@ void file_based_write_read(std::string const &file_ending) } } -TEST_CASE("file_based_write_read", "[parallel]") +PARALLEL_TEST_CASE(file_based_write_read, "[parallel]") { return; for (auto const &t : getBackends()) @@ -1182,7 +1199,7 @@ void hipace_like_write(std::string const &file_ending) } } -TEST_CASE("hipace_like_write", "[parallel]") +PARALLEL_TEST_CASE(hipace_like_write, "[parallel]") { for (auto const &t : getBackends()) { @@ -1192,7 +1209,7 @@ TEST_CASE("hipace_like_write", "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("independent_write_with_collective_flush", "[parallel]") +PARALLEL_TEST_CASE(independent_write_with_collective_flush, "[parallel]") { Series write( "../samples/independent_write_with_collective_flush.bp5", @@ -1226,7 +1243,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") #endif #if openPMD_HAVE_MPI -TEST_CASE("unavailable_backend", "[core][parallel]") +PARALLEL_TEST_CASE(unavailable_backend, "[core][parallel]") { #if !openPMD_HAVE_ADIOS2 { @@ -1374,7 +1391,7 @@ void adios2_streaming(bool variableBasedLayout) } } -TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") +PARALLEL_TEST_CASE(adios2_streaming, "[pseudoserial][adios2]") { #if HAS_ADIOS_2_9 adios2_streaming(true); @@ -1382,7 +1399,7 @@ TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") adios2_streaming(false); } -TEST_CASE("parallel_adios2_json_config", "[parallel][adios2]") +PARALLEL_TEST_CASE(parallel_adios2_json_config, "[parallel][adios2]") { int size{-1}; int rank{-1}; @@ -1593,7 +1610,7 @@ void adios2_ssc() } } -TEST_CASE("adios2_ssc", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_ssc, "[parallel][adios2]") { adios2_ssc(); } @@ -1919,7 +1936,7 @@ void append_mode( #endif } -TEST_CASE("append_mode", "[serial]") +PARALLEL_TEST_CASE(append_mode, "[serial]") { for (auto const &t : testedFileExtensions()) { @@ -2122,7 +2139,7 @@ void joined_dim(std::string const &ext) } } -TEST_CASE("joined_dim", "[parallel]") +PARALLEL_TEST_CASE(joined_dim, "[parallel]") { #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ @@ -2147,7 +2164,7 @@ TEST_CASE("joined_dim", "[parallel]") #if openPMD_HAVE_ADIOS2_BP5 // Parallel version of the same test from SerialIOTest.cpp -TEST_CASE("adios2_flush_via_step") +PARALLEL_TEST_CASE(adios2_flush_via_step, "[parallel]") { int size_i(0), rank_i(0); MPI_Comm_rank(MPI_COMM_WORLD, &rank_i); @@ -2254,12 +2271,12 @@ TEST_CASE("adios2_flush_via_step") } #endif -TEST_CASE("read_variablebased_randomaccess") +PARALLEL_TEST_CASE(read_variablebased_randomaccess, "[parallel]") { read_variablebased_randomaccess::read_variablebased_randomaccess(); } -TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") +PARALLEL_TEST_CASE(iterate_nonstreaming_series, "[parallel][adios2]") { iterate_nonstreaming_series::iterate_nonstreaming_series(); } @@ -2720,14 +2737,14 @@ void run_test() } } // namespace adios2_chunk_distribution -TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_chunk_distribution, "[parallel][adios2]") { adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI #if openPMD_HAVE_MPI -TEST_CASE("bug_1655_bp5_writer_hangup", "[parallel]") +PARALLEL_TEST_CASE(bug_1655_bp5_writer_hangup, "[parallel]") { bug_1655_bp5_writer_hangup::bug_1655_bp5_writer_hangup(); } From 3dd8d66a1ab47d0ddf988297e1a445f114e2f7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 22:03:57 +0100 Subject: [PATCH 20/36] Fix wrong MPI_COMM_WORLD --- src/auxiliary/Mpi.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index ef899e4207..5e8379ff41 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -49,7 +49,7 @@ StringMatrix collectStringsAsMatrixTo( 1, MPI_INT, destRank, - MPI_COMM_WORLD); + communicator); int maxLength = std::accumulate( recvcounts.begin(), recvcounts.end(), 0, [](int a, int b) { return std::max(a, b); @@ -78,7 +78,7 @@ StringMatrix collectStringsAsMatrixTo( displs.data(), MPI_CHAR, destRank, - MPI_COMM_WORLD); + communicator); return res; } @@ -95,7 +95,7 @@ std::vector distributeStringsToAllRanks( int *displs = new int[size]; MPI_Allgather( - &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, MPI_COMM_WORLD); + &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); char *namesBuffer; { @@ -116,7 +116,7 @@ std::vector distributeStringsToAllRanks( sizesBuffer, displs, MPI_CHAR, - MPI_COMM_WORLD); + communicator); std::vector hostnames(size); for (int i = 0; i < size; ++i) From 665a3b14c01cacf7ed5d18bba6f7c1a20a0d81af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 22:04:14 +0100 Subject: [PATCH 21/36] wip: debugging state --- src/RecordComponent.cpp | 18 +++++++++--------- src/auxiliary/Mpi.cpp | 2 ++ test/CatchRunner.cpp | 5 +++-- test/ParallelIOTest.cpp | 5 ++++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bce3da6aeb..7ba00c04f8 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -238,15 +238,15 @@ namespace RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); - auto cleanup = defer([&rc, this]() { - if (rc.m_dataset.has_value() && - rc.m_dataset->dtype != Datatype::UNDEFINED && - IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) - { - seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - } - }); + // auto cleanup = defer([&rc, this]() { + // if (rc.m_dataset.has_value() && + // rc.m_dataset->dtype != Datatype::UNDEFINED && + // IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + // { + // seriesFlush_impl( + // {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + // } + // }); if (written()) { if (!rc.m_dataset.has_value()) diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index 5e8379ff41..b92055889b 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -94,6 +94,8 @@ std::vector distributeStringsToAllRanks( int *sizesBuffer = new int[size]; int *displs = new int[size]; + MPI_Barrier(communicator); + MPI_Allgather( &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); diff --git a/test/CatchRunner.cpp b/test/CatchRunner.cpp index 107ec2e46e..81a6abf7c4 100644 --- a/test/CatchRunner.cpp +++ b/test/CatchRunner.cpp @@ -29,8 +29,9 @@ int main(int argc, char *argv[]) Catch::Session session; // session.configData().runOrder = Catch::TestRunOrder::Declared; - MPI_Bcast( - &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); + session.configData().rngSeed = 3036063643; + // MPI_Bcast( + // &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); int result = session.applyCommandLine(argc, argv); if (result == 0) { diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 59f310a912..7e5831752c 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -2559,10 +2559,13 @@ void run_test() E_x.storeChunk(data, {unsigned(mpi_rank * 2), 0}, {1, 10}); E_x.storeChunk(data, {unsigned(mpi_rank * 2 + 1), 0}, {1, 10}); series.flush(); + series.close(); } + MPI_Barrier(MPI_COMM_WORLD); { - Series series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); + Series series( + filename, openPMD::Access::READ_ONLY /* , MPI_COMM_WORLD */); /* * Inquire the writing application's "MPI rank -> hostname" mapping. * The reading application needs to know about its own mapping. From 5eb80e5d535aa64089ad157b79aaf573c33f2e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 12:53:20 +0200 Subject: [PATCH 22/36] deactivate malicious tests --- test/ParallelIOTest.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 7e5831752c..f54417f16b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -40,6 +40,12 @@ static void openPMD_parallel_##name(); \ TEST_CASE(#name, tags) \ { \ + int flag; \ + MPI_Initialized(&flag); \ + if (!flag) \ + { \ + throw std::runtime_error("MPI IS NOT INITIALIZED"); \ + } \ MPI_Barrier(MPI_COMM_WORLD); \ int rank; \ MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ @@ -53,6 +59,14 @@ } \ static void openPMD_parallel_##name() +#define DEACTIVATE_TEST_CASE(name, tags) \ + static void openPMD_parallel_##name(); \ + TEST_CASE(#name, tags) \ + { \ + return; \ + } \ + static void openPMD_parallel_##name() + #if !openPMD_HAVE_MPI TEST_CASE("none", "[parallel]") {} @@ -512,7 +526,7 @@ void available_chunks_test(std::string const &file_ending) } } -PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") +DEACTIVATE_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -574,7 +588,7 @@ PARALLEL_TEST_CASE(extend_dataset, "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") +DEACTIVATE_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", From 300506fe923e91b2f22f7b43548bd66ba2d08827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 15:46:01 +0200 Subject: [PATCH 23/36] Revert "deactivate malicious tests" This reverts commit 246609ff5fbe5edb68119b7f3b29a40e7bf23d2d. --- test/ParallelIOTest.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index f54417f16b..7e5831752c 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -40,12 +40,6 @@ static void openPMD_parallel_##name(); \ TEST_CASE(#name, tags) \ { \ - int flag; \ - MPI_Initialized(&flag); \ - if (!flag) \ - { \ - throw std::runtime_error("MPI IS NOT INITIALIZED"); \ - } \ MPI_Barrier(MPI_COMM_WORLD); \ int rank; \ MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ @@ -59,14 +53,6 @@ } \ static void openPMD_parallel_##name() -#define DEACTIVATE_TEST_CASE(name, tags) \ - static void openPMD_parallel_##name(); \ - TEST_CASE(#name, tags) \ - { \ - return; \ - } \ - static void openPMD_parallel_##name() - #if !openPMD_HAVE_MPI TEST_CASE("none", "[parallel]") {} @@ -526,7 +512,7 @@ void available_chunks_test(std::string const &file_ending) } } -DEACTIVATE_TEST_CASE(available_chunks_test, "[parallel][adios]") +PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -588,7 +574,7 @@ PARALLEL_TEST_CASE(extend_dataset, "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -DEACTIVATE_TEST_CASE(adios_write_test, "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", From 99be82ce167e070447a30563801460e8362af0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 15:46:47 +0200 Subject: [PATCH 24/36] Revert "wip: debugging state" This reverts commit 2df195749b53b0a54832585899bd469d35f81d6d. --- src/RecordComponent.cpp | 18 +++++++++--------- src/auxiliary/Mpi.cpp | 2 -- test/CatchRunner.cpp | 5 ++--- test/ParallelIOTest.cpp | 5 +---- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 7ba00c04f8..bce3da6aeb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -238,15 +238,15 @@ namespace RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); - // auto cleanup = defer([&rc, this]() { - // if (rc.m_dataset.has_value() && - // rc.m_dataset->dtype != Datatype::UNDEFINED && - // IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) - // { - // seriesFlush_impl( - // {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - // } - // }); + auto cleanup = defer([&rc, this]() { + if (rc.m_dataset.has_value() && + rc.m_dataset->dtype != Datatype::UNDEFINED && + IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + { + seriesFlush_impl( + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + } + }); if (written()) { if (!rc.m_dataset.has_value()) diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index b92055889b..5e8379ff41 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -94,8 +94,6 @@ std::vector distributeStringsToAllRanks( int *sizesBuffer = new int[size]; int *displs = new int[size]; - MPI_Barrier(communicator); - MPI_Allgather( &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); diff --git a/test/CatchRunner.cpp b/test/CatchRunner.cpp index 81a6abf7c4..107ec2e46e 100644 --- a/test/CatchRunner.cpp +++ b/test/CatchRunner.cpp @@ -29,9 +29,8 @@ int main(int argc, char *argv[]) Catch::Session session; // session.configData().runOrder = Catch::TestRunOrder::Declared; - session.configData().rngSeed = 3036063643; - // MPI_Bcast( - // &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); + MPI_Bcast( + &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); int result = session.applyCommandLine(argc, argv); if (result == 0) { diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 7e5831752c..59f310a912 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -2559,13 +2559,10 @@ void run_test() E_x.storeChunk(data, {unsigned(mpi_rank * 2), 0}, {1, 10}); E_x.storeChunk(data, {unsigned(mpi_rank * 2 + 1), 0}, {1, 10}); series.flush(); - series.close(); } - MPI_Barrier(MPI_COMM_WORLD); { - Series series( - filename, openPMD::Access::READ_ONLY /* , MPI_COMM_WORLD */); + Series series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); /* * Inquire the writing application's "MPI rank -> hostname" mapping. * The reading application needs to know about its own mapping. From f24fcdb92fd5060e68ced2561c12c909d26d1071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 16:44:10 +0200 Subject: [PATCH 25/36] Revert "TMP REVERT ME take out hanging parallel test" This reverts commit 07fb558431acc7b3fa1d69f5a258be8164dad227. --- .../iterate_nonstreaming_series.cpp | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp index fe72ef1698..8ddadfc40a 100644 --- a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp +++ b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp @@ -181,12 +181,11 @@ auto iterate_nonstreaming_series() -> void { for (auto const &backend : testedBackends()) { - std::cout << "TESTING BACKEND " << backend.extension << std::endl; - // run_test( - // "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + - // backend.extension, - // false, - // backend.jsonBaseConfig()); + run_test( + "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + + backend.extension, + false, + backend.jsonBaseConfig()); run_test( "../samples/iterate_nonstreaming_series/parallel_groupbased." + backend.extension, @@ -195,14 +194,13 @@ auto iterate_nonstreaming_series() -> void #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 if (backend.extension == "bp") { - // run_test( - // "../samples/iterate_nonstreaming_series/" - // "parallel_filebased_bp5_%T." + - // backend.extension, - // false, - // json::merge( - // backend.jsonBaseConfig(), "adios2.engine.type = - // \"bp5\"")); + run_test( + "../samples/iterate_nonstreaming_series/" + "parallel_filebased_bp5_%T." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); run_test( "../samples/iterate_nonstreaming_series/" "parallel_groupbased_bp5." + From e6d81c7573b90e64f4e1725376ec91d90afef97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 18:20:23 +0200 Subject: [PATCH 26/36] Revert some WIPs --- test/SerialIOTest.cpp | 55 ++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 52836ae903..49aab1db18 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1771,24 +1771,24 @@ inline void write_test( TEST_CASE("write_test", "[serial]") { - for (auto const &t : {std::string("json")}) + for (auto const &t : testedFileExtensions()) { if (t == "json") { - // write_test( - // "template." + t, - // R"( - // { - // "json": { - // "dataset": { - // "mode": "template" - // }, - // "attribute": { - // "mode": "short" - // } - // } - // })", - // false); + write_test( + "template." + t, + R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})", + false); write_test( t, R"( @@ -1972,10 +1972,6 @@ inline void fileBased_write_test(const std::string &backend) "\\", "/")); - // TODO: somehow make the rank table appear in iteration 1 - o.iterations[1]; - o.flush(); - ParticleSpecies &e_1 = o.iterations[1].particles["e"]; std::vector position_global(4); @@ -2353,20 +2349,19 @@ inline void fileBased_write_test(const std::string &backend) TEST_CASE("fileBased_write_test", "[serial]") { - for (auto const &t : {"bp5"}) + for (auto const &t : testedFileExtensions()) { fileBased_write_test(t); } - // if (auto extensions = getFileExtensions(); - // std::find(extensions.begin(), extensions.end(), "toml") != - // extensions.end()) - // { /* - // * TOML backend is not generally tested for performance reasons, opt in - // to - // * testing it here. - // */ - // fileBased_write_test("toml"); - // } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); + } } inline void sample_write_thetaMode(std::string const &file_ending) From 8d7246835f9ee4fcfc5a63e50362f03a2492711e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 May 2026 19:51:12 +0200 Subject: [PATCH 27/36] Use an Attributable per Iteration for rankTable in filebased encoding needs some documentation still --- include/openPMD/Iteration.hpp | 10 +-- include/openPMD/Series.hpp | 15 ++-- include/openPMD/backend/PerIterationData.hpp | 45 +++++++++++ src/Iteration.cpp | 28 +++---- src/Series.cpp | 82 +++++++++++--------- 5 files changed, 107 insertions(+), 73 deletions(-) create mode 100644 include/openPMD/backend/PerIterationData.hpp diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 0892627f2d..aaf199fdb8 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include @@ -122,14 +123,7 @@ namespace internal */ bool allow_reopening_implicitly = false; - /** - * Whether a step is currently active for this iteration. - * Used for file-based iteration layout, see Series.hpp for - * group-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. - */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; /** * Cached copy of the key under which this Iteration lives in diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index f1b5b3872c..43d29cbd11 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -33,6 +33,7 @@ #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" @@ -205,14 +206,9 @@ namespace internal * Detected IO format (backend). */ Format m_format; - /** - * Whether a step is currently active for this iteration. - * Used for group-based iteration layout, see SeriesData.hpp for - * iteration-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. - */ - StepStatus m_stepStatus = StepStatus::NoStep; + + PerIterationData m_perIterationData; + /** * True if a user opts into lazy parsing. */ @@ -261,7 +257,6 @@ namespace internal struct RankTableData { - Attributable m_attributable; std::variant< NoSourceSpecified, SourceSpecifiedViaJSON, @@ -900,7 +895,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushRankTable(FlushLevel); + void flushRankTable(FlushLevel, Attributable &attributable); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ diff --git a/include/openPMD/backend/PerIterationData.hpp b/include/openPMD/backend/PerIterationData.hpp new file mode 100644 index 0000000000..cf3c347437 --- /dev/null +++ b/include/openPMD/backend/PerIterationData.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include "openPMD/ChunkInfo.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/backend/Attributable.hpp" + +#include + +namespace openPMD::internal +{ +struct NoSourceSpecified +{}; +struct SourceSpecifiedViaJSON +{ + std::string value; +}; +struct SourceSpecifiedManually +{ + std::string value; +}; + +struct RankTableData +{ + Attributable m_attributable; + std::variant< + NoSourceSpecified, + SourceSpecifiedViaJSON, + SourceSpecifiedManually> + m_rankTableSource; + std::optional m_bufferedRead; +}; + +struct PerIterationData +{ + /** + * Whether a step is currently active for this iteration. + * Used for group-based iteration layout, see SeriesData.hpp for + * iteration-based layout. + * Access via stepStatus() method to automatically select the correct + * one among both flags. + */ + StepStatus m_stepStatus = StepStatus::NoStep; + Attributable m_rankTableAttributable; +}; +} // namespace openPMD::internal diff --git a/src/Iteration.cpp b/src/Iteration.cpp index caac558986..7e708ca4e4 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -272,10 +272,6 @@ void Iteration::flushFileBased( /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - auto &series = s.get(); - - bool do_flush_rank_table = - !series.m_rankTable.m_attributable.written() || !this->written(); if (!written()) { @@ -300,16 +296,11 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - if (do_flush_rank_table) + auto &rankTableAttributable = + get().m_perIterationData.m_rankTableAttributable; + if (!rankTableAttributable.written()) { - /* - * If it was written before, then in the context of another iteration. - */ - auto &attr = series.m_rankTable.m_attributable; - attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); - attr.get().m_writable.abstractFilePosition.reset(); - - s.flushRankTable(flushParams.flushLevel); + s.flushRankTable(flushParams.flushLevel, rankTableAttributable); } if (flush_level::flush_hierarchy(flushParams.flushLevel)) @@ -784,7 +775,7 @@ auto Iteration::beginStep( } else { - series.get().m_stepStatus = StepStatus::DuringStep; + series.get().m_perIterationData.m_stepStatus = StepStatus::DuringStep; status = series.advance(AdvanceMode::BEGINSTEP); } @@ -887,10 +878,10 @@ StepStatus Iteration::getStepStatus() { using IE = IterationEncoding; case IE::fileBased: - return get().m_stepStatus; + return get().m_perIterationData.m_stepStatus; case IE::groupBased: case IE::variableBased: - return s.get().m_stepStatus; + return s.get().m_perIterationData.m_stepStatus; default: throw std::runtime_error("[Iteration] unreachable"); } @@ -903,11 +894,11 @@ void Iteration::setStepStatus(StepStatus status) { using IE = IterationEncoding; case IE::fileBased: - get().m_stepStatus = status; + get().m_perIterationData.m_stepStatus = status; break; case IE::groupBased: case IE::variableBased: - s.get().m_stepStatus = status; + s.get().m_perIterationData.m_stepStatus = status; break; default: throw std::runtime_error("[Iteration] unreachable"); @@ -919,6 +910,7 @@ void Iteration::linkHierarchy(Writable &w) Attributable::linkHierarchy(w); meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); + get().m_perIterationData.m_rankTableAttributable.linkHierarchy(*w.parent); } void Iteration::runDeferredParseAccess() diff --git a/src/Series.cpp b/src/Series.cpp index fbd2151df3..199ce705c1 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -355,6 +355,12 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) IOHandler()->enqueue(IOTask(this, openFile)); #endif } + Attributable &attributable = + iterationEncoding() == IterationEncoding::fileBased + ? iterations.begin() + ->second.get() + .m_perIterationData.m_rankTableAttributable + : series.m_perIterationData.m_rankTableAttributable; auto datasets = availableDatasets(); if (std::find(datasets.begin(), datasets.end(), "rankTable") == datasets.end()) @@ -364,7 +370,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } Parameter openDataset; openDataset.name = "rankTable"; - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, openDataset)); + IOHandler()->enqueue(IOTask(&attributable, openDataset)); IOHandler()->flush(internal::defaultFlushParams); if (openDataset.extent->size() != 2) @@ -394,19 +400,20 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) new char[writerRanks * lineWidth], [](char const *ptr) { delete[] ptr; }}; - auto doReadDataset = [&openDataset, this, &get, &rankTable]() { - Parameter readDataset; - // read the whole thing - readDataset.offset.resize(2); - readDataset.extent = *openDataset.extent; - // @todo better cross-platform support by switching over - // *openDataset.dtype - readDataset.dtype = Datatype::CHAR; - readDataset.data = get; - - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, readDataset)); - IOHandler()->flush(internal::publicFlush); - }; + auto doReadDataset = + [&openDataset, this, &get, &rankTable, &attributable]() { + Parameter readDataset; + // read the whole thing + readDataset.offset.resize(2); + readDataset.extent = *openDataset.extent; + // @todo better cross-platform support by switching over + // *openDataset.dtype + readDataset.dtype = Datatype::CHAR; + readDataset.data = get; + + IOHandler()->enqueue(IOTask(&attributable, readDataset)); + IOHandler()->flush(internal::publicFlush); + }; #if openPMD_HAVE_MPI if (collective && series.m_communicator.has_value()) @@ -464,7 +471,7 @@ Series &Series::setRankTable(const std::string &myRankInfo) return *this; } -void Series::flushRankTable(FlushLevel l) +void Series::flushRankTable(FlushLevel l, Attributable &attributable) { if (!flush_level::global_flushpoint(l)) { @@ -512,29 +519,28 @@ void Series::flushRankTable(FlushLevel l) int rank{0}, size{1}; unsigned long long maxSize = mySize; - auto createRankTable = [&size, &maxSize, &rankTable, this]() { - if (rankTable.m_attributable.written()) - { - return; - } - Parameter param( - AbstractParameter::I_dont_want_to_use_joined_dimensions); - param.name = "rankTable"; - param.dtype = Datatype::CHAR; - param.extent = {uint64_t(size), uint64_t(maxSize)}; - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(param))); - }; + auto createRankTable = + [&size, &maxSize, &rankTable, this, &attributable]() { + if (attributable.written()) + { + return; + } + Parameter param( + AbstractParameter::I_dont_want_to_use_joined_dimensions); + param.name = "rankTable"; + param.dtype = Datatype::CHAR; + param.extent = {uint64_t(size), uint64_t(maxSize)}; + IOHandler()->enqueue(IOTask(&attributable, std::move(param))); + }; - auto writeDataset = [&rank, &maxSize, this, &rankTable]( + auto writeDataset = [&rank, &maxSize, this, &rankTable, &attributable]( std::shared_ptr put, size_t num_lines = 1) { Parameter chunk; chunk.dtype = Datatype::CHAR; chunk.offset = {uint64_t(rank), 0}; chunk.extent = {num_lines, maxSize}; chunk.data = std::move(put); - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(chunk))); + IOHandler()->enqueue(IOTask(&attributable, std::move(chunk))); }; #if openPMD_HAVE_MPI @@ -578,8 +584,7 @@ void Series::flushRankTable(FlushLevel l) // Must ensure that the Writable is consistently set to written on all // ranks - series.m_rankTable.m_attributable.setWritten( - true, EnqueueAsynchronously::OnlyAsync); + attributable.setWritten(true, EnqueueAsynchronously::OnlyAsync); return; } #endif @@ -985,7 +990,8 @@ void Series::init( std::make_unique(parsed_directory, at)); auto &series = get(); series.iterations.linkHierarchy(writable()); - series.m_rankTable.m_attributable.linkHierarchy(writable()); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy( + writable()); series.m_deferred_initialization = [called_this_already = false, filepath, @@ -1211,7 +1217,7 @@ void Series::initSeries( series.iterations.linkHierarchy(writable); series.iterations.writable().ownKeyWithinParent = "data"; - series.m_rankTable.m_attributable.linkHierarchy(writable); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy(writable); series.m_name = input->name; @@ -1663,9 +1669,11 @@ void Series::flushGorVBased( IOHandler()->enqueue(IOTask(this, fCreate)); } - if (!series.m_rankTable.m_attributable.written()) + if (!series.m_perIterationData.m_rankTableAttributable.written()) { - flushRankTable(flushParams.flushLevel); + flushRankTable( + flushParams.flushLevel, + series.m_perIterationData.m_rankTableAttributable); } series.iterations.flush( From a67f869f2f0d65ec8e86f4640ca27be28dea435a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 12:12:58 +0200 Subject: [PATCH 28/36] fix nompi builds --- test/ParallelIOTest.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 59f310a912..37a6d45ee9 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -36,6 +36,16 @@ #include #include +#if !openPMD_HAVE_MPI +#define PARALLEL_TEST_CASE(name, tags) TEST_CASE(#name, tags) + +PARALLEL_TEST_CASE(none, "[parallel]") +{} + +#else + +#include + #define PARALLEL_TEST_CASE(name, tags) \ static void openPMD_parallel_##name(); \ TEST_CASE(#name, tags) \ @@ -53,14 +63,6 @@ } \ static void openPMD_parallel_##name() -#if !openPMD_HAVE_MPI -TEST_CASE("none", "[parallel]") -{} - -#else - -#include - #if openPMD_HAVE_ADIOS2 #include #define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) From 257949ef060fb052d2c6c6740916d8fee762715b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 14:07:14 +0200 Subject: [PATCH 29/36] CI fixes --- include/openPMD/Iteration.hpp | 9 +++ include/openPMD/Series.hpp | 10 +++ include/openPMD/backend/PerIterationData.hpp | 9 +++ src/Series.cpp | 69 +++++++++++--------- 4 files changed, 65 insertions(+), 32 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index aaf199fdb8..0a3c1dfcb0 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -123,6 +123,15 @@ namespace internal */ bool allow_reopening_implicitly = false; + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. + */ PerIterationData m_perIterationData; /** diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 43d29cbd11..503eec0ace 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -207,6 +207,15 @@ namespace internal */ Format m_format; + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. + */ PerIterationData m_perIterationData; /** @@ -978,6 +987,7 @@ OPENPMD_private * least one step was written. * * @param doFlush If true, flush the IO handler. + * @param l This operation must only run at flush level write_datasets */ void flushStep(bool doFlush, FlushLevel l); diff --git a/include/openPMD/backend/PerIterationData.hpp b/include/openPMD/backend/PerIterationData.hpp index cf3c347437..6cea297bf2 100644 --- a/include/openPMD/backend/PerIterationData.hpp +++ b/include/openPMD/backend/PerIterationData.hpp @@ -30,6 +30,15 @@ struct RankTableData std::optional m_bufferedRead; }; +/* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each Iteration. + * Access must be distinguished by iteration encoding. + */ struct PerIterationData { /** diff --git a/src/Series.cpp b/src/Series.cpp index 199ce705c1..99fe227582 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -339,10 +339,11 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } if (iterationEncoding() == IterationEncoding::fileBased) { - std::cerr << "[Series] Use rank table in file-based iteration encoding " - "at your own risk. Make sure to have an iteration open " - "before calling this." - << std::endl; + std::cerr + << "[Series] Use rank table in file-based iteration encoding " + "at your own risk. Make sure to have the first iteration open " + "before calling this." + << std::endl; if (iterations.empty()) { return {}; @@ -357,6 +358,12 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } Attributable &attributable = iterationEncoding() == IterationEncoding::fileBased + /* + * Only second class support for file encoding. We indiscriminately use + * the first Iteration for this operation. It is on the user to ensure + * that this Iteration is actually open. The warning printed above + * informs about this. + */ ? iterations.begin() ->second.get() .m_perIterationData.m_rankTableAttributable @@ -400,20 +407,19 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) new char[writerRanks * lineWidth], [](char const *ptr) { delete[] ptr; }}; - auto doReadDataset = - [&openDataset, this, &get, &rankTable, &attributable]() { - Parameter readDataset; - // read the whole thing - readDataset.offset.resize(2); - readDataset.extent = *openDataset.extent; - // @todo better cross-platform support by switching over - // *openDataset.dtype - readDataset.dtype = Datatype::CHAR; - readDataset.data = get; - - IOHandler()->enqueue(IOTask(&attributable, readDataset)); - IOHandler()->flush(internal::publicFlush); - }; + auto doReadDataset = [&openDataset, this, &get, &attributable]() { + Parameter readDataset; + // read the whole thing + readDataset.offset.resize(2); + readDataset.extent = *openDataset.extent; + // @todo better cross-platform support by switching over + // *openDataset.dtype + readDataset.dtype = Datatype::CHAR; + readDataset.data = get; + + IOHandler()->enqueue(IOTask(&attributable, readDataset)); + IOHandler()->flush(internal::publicFlush); + }; #if openPMD_HAVE_MPI if (collective && series.m_communicator.has_value()) @@ -519,21 +525,20 @@ void Series::flushRankTable(FlushLevel l, Attributable &attributable) int rank{0}, size{1}; unsigned long long maxSize = mySize; - auto createRankTable = - [&size, &maxSize, &rankTable, this, &attributable]() { - if (attributable.written()) - { - return; - } - Parameter param( - AbstractParameter::I_dont_want_to_use_joined_dimensions); - param.name = "rankTable"; - param.dtype = Datatype::CHAR; - param.extent = {uint64_t(size), uint64_t(maxSize)}; - IOHandler()->enqueue(IOTask(&attributable, std::move(param))); - }; + auto createRankTable = [&size, &maxSize, this, &attributable]() { + if (attributable.written()) + { + return; + } + Parameter param( + AbstractParameter::I_dont_want_to_use_joined_dimensions); + param.name = "rankTable"; + param.dtype = Datatype::CHAR; + param.extent = {uint64_t(size), uint64_t(maxSize)}; + IOHandler()->enqueue(IOTask(&attributable, std::move(param))); + }; - auto writeDataset = [&rank, &maxSize, this, &rankTable, &attributable]( + auto writeDataset = [&rank, &maxSize, this, &attributable]( std::shared_ptr put, size_t num_lines = 1) { Parameter chunk; chunk.dtype = Datatype::CHAR; From 503e53af5f09df67aa0eb3fbc761189ca9bd621c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 17:34:17 +0200 Subject: [PATCH 30/36] Activate test again --- test/ParallelIOTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 37a6d45ee9..d32d31253b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1023,7 +1023,6 @@ void file_based_write_read(std::string const &file_ending) PARALLEL_TEST_CASE(file_based_write_read, "[parallel]") { - return; for (auto const &t : getBackends()) { file_based_write_read(t); From e248ab895f6f4af88a434729c5fd5a489924529d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 Jul 2026 10:48:20 +0200 Subject: [PATCH 31/36] Guard against any internal unsetDirty operation --- include/openPMD/backend/Attributable.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 33b28c6ea2..48e4bd5a00 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -620,6 +620,11 @@ OPENPMD_protected throw error::Internal( "Parsing procedures should directly unset dirty."); } + else + { + throw error::Internal( + "Internal flushes should not unset dirty flags."); + } break; case FlushLevel::SkeletonOnly: case FlushLevel::CreateOrOpenFiles: From e1c26701c255ab35bd5f96d6ea68d3139e2ab391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 Jul 2026 13:58:10 +0200 Subject: [PATCH 32/36] Span API: more precise written and error checks --- include/openPMD/RecordComponent.tpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 523f4f1e41..4e71312305 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -113,7 +113,11 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) getBufferView.extent = e; getBufferView.dtype = getDatatype(); +#define OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET true + +#if !OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET if (query.out->backendManagedBuffer) +#endif { // Need to initialize the dataset for the Span API // But this is a non-collective call and initializing the dataset is @@ -122,8 +126,14 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) // restriction // TODO: Add some form of collective ::commitDefinitions() call to // RecordComponents to be called by users before the Span API - if (!written()) + if (!writable().parent || !writable().parent->written) { +#if OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET + // This error might be too strict in case we throw out the implicit + // flush from resetDataset() again. + throw error::WrongAPIUsage( + "Must flush the structure before using Span API"); +#else /* * The openPMD backend might not yet know about this dataset. * Flush the openPMD hierarchy to the backend without flushing any @@ -131,6 +141,10 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) */ seriesFlush_impl( {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); +#endif + } + if (!this->written()) + { Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); From 000bee6ed3e172161be208a135e00f9be159028e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jul 2026 19:15:26 +0200 Subject: [PATCH 33/36] Revert the defer stuff --- src/RecordComponent.cpp | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bce3da6aeb..52b7c236c3 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -216,37 +216,9 @@ RecordComponent &RecordComponent::setUnitSI(double usi) return *this; } -namespace -{ - template - struct defer_type - { - F functor; - ~defer_type() - { - std::move(functor)(); - } - }; - - template - auto defer(F &&functor) -> defer_type> - { - return defer_type>{std::forward(functor)}; - } -} // namespace - RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); - auto cleanup = defer([&rc, this]() { - if (rc.m_dataset.has_value() && - rc.m_dataset->dtype != Datatype::UNDEFINED && - IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) - { - seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - } - }); if (written()) { if (!rc.m_dataset.has_value()) From c06caf5a7b5821f912b2cdd510dd4e44b37983f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jul 2026 19:45:14 +0200 Subject: [PATCH 34/36] dont enforce flush upon reset dataset --- include/openPMD/RecordComponent.tpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 4e71312305..965ba7059f 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -113,7 +113,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) getBufferView.extent = e; getBufferView.dtype = getDatatype(); -#define OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET true +#define OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET false #if !OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET if (query.out->backendManagedBuffer) From cea11c9beee967d95ece0f4f0dcb444770f2725f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jul 2026 19:46:16 +0200 Subject: [PATCH 35/36] keep only the false branch --- include/openPMD/RecordComponent.tpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 965ba7059f..951a520cfb 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -113,11 +113,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) getBufferView.extent = e; getBufferView.dtype = getDatatype(); -#define OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET false - -#if !OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET if (query.out->backendManagedBuffer) -#endif { // Need to initialize the dataset for the Span API // But this is a non-collective call and initializing the dataset is @@ -128,12 +124,6 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) // RecordComponents to be called by users before the Span API if (!writable().parent || !writable().parent->written) { -#if OPENPMD_ENFORCE_FLUSH_UPON_RESET_DATASET - // This error might be too strict in case we throw out the implicit - // flush from resetDataset() again. - throw error::WrongAPIUsage( - "Must flush the structure before using Span API"); -#else /* * The openPMD backend might not yet know about this dataset. * Flush the openPMD hierarchy to the backend without flushing any @@ -141,7 +131,6 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) */ seriesFlush_impl( {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); -#endif } if (!this->written()) { From bc0098694dacfc88ea91d8786dc339c4adc53be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 20 Jul 2026 10:23:15 +0200 Subject: [PATCH 36/36] docs --- include/openPMD/Series.hpp | 3 ++- include/openPMD/backend/Attributable.hpp | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 503eec0ace..3f640122cc 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -987,7 +987,8 @@ OPENPMD_private * least one step was written. * * @param doFlush If true, flush the IO handler. - * @param l This operation must only run at flush level write_datasets + * @param l This operation must only run at flush level write_datasets, + * Noop otherwise. */ void flushStep(bool doFlush, FlushLevel l); diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 48e4bd5a00..bcd09d966b 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -613,6 +613,8 @@ OPENPMD_protected case FlushLevel::UserFlush: setDirty(false); break; + // FlushLevel::InternalFlush is only used for directly calling the IO + // handler and should not bother with middle-end state manipulations case FlushLevel::InternalFlush: // Used for parsing if (IOHandler()->m_seriesStatus == internal::SeriesStatus::Parsing)