From 3d9900b770a37e9a6540c28c9cd68e1288a4e16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 28 Jun 2023 14:47:30 +0200 Subject: [PATCH 01/27] Introduce SharedAttributableData --- include/openPMD/backend/Attributable.hpp | 79 +++++++++++++++++------- include/openPMD/backend/Writable.hpp | 7 +++ src/Iteration.cpp | 6 +- src/Series.cpp | 10 +-- src/backend/Attributable.cpp | 11 +++- src/backend/Writable.cpp | 5 +- 6 files changed, 86 insertions(+), 32 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 850d3bf35c..97dad22460 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -56,18 +56,19 @@ namespace internal class IterationData; class SeriesData; - class AttributableData + class SharedAttributableData { friend class openPMD::Attributable; public: - AttributableData(); - AttributableData(AttributableData const &) = delete; - AttributableData(AttributableData &&) = delete; - virtual ~AttributableData() = default; + SharedAttributableData(AttributableData *); + SharedAttributableData(SharedAttributableData const &) = delete; + SharedAttributableData(SharedAttributableData &&) = delete; + virtual ~SharedAttributableData() = default; - AttributableData &operator=(AttributableData const &) = delete; - AttributableData &operator=(AttributableData &&) = delete; + SharedAttributableData & + operator=(SharedAttributableData const &) = delete; + SharedAttributableData &operator=(SharedAttributableData &&) = delete; using A_MAP = std::map; /** @@ -77,6 +78,46 @@ namespace internal */ Writable m_writable; + private: + /** + * The attributes defined by this Attributable. + */ + A_MAP m_attributes; + }; + + /* + * This is essentially a two-level pointer. + * + * 1. level: Our public API hands out handles to users that are (shared) + * pointers to an internal object (PIMPL). + * 2. level: Multiple internal objects might refer to the same item in an + * openPMD file, e.g. to the same backend object. + * So, the internal object for an Attributable is a shared pointer to the + * unique object identifying this item. + * + * Such sharing occurs in the CustomHierarchy class where multiple + * containers refer to the same group in the openPMD hierarchy + * (container of groups, of meshes, of particle species, of datasets). + * This might also become relevant for links as in HDF5 if we choose to + * implement them. + */ + + class AttributableData : public std::shared_ptr + { + friend class openPMD::Attributable; + + using SharedData_t = std::shared_ptr; + + public: + AttributableData(); + AttributableData(SharedAttributableData *); + AttributableData(AttributableData const &) = delete; + AttributableData(AttributableData &&) = delete; + virtual ~AttributableData() = default; + + AttributableData &operator=(AttributableData const &) = delete; + AttributableData &operator=(AttributableData &&) = delete; + template T asInternalCopyOf() { @@ -112,12 +153,6 @@ namespace internal std::shared_ptr(self, [](auto const *) {})); return res; } - - private: - /** - * The attributes defined by this Attributable. - */ - A_MAP m_attributes; }; template @@ -429,7 +464,7 @@ OPENPMD_protected } AbstractIOHandler const *IOHandler() const { - auto &opt = m_attri->m_writable.IOHandler; + auto &opt = writable().IOHandler; if (!opt || !opt->has_value()) { return nullptr; @@ -438,19 +473,19 @@ OPENPMD_protected } Writable *&parent() { - return m_attri->m_writable.parent; + return writable().parent; } Writable const *parent() const { - return m_attri->m_writable.parent; + return writable().parent; } Writable &writable() { - return m_attri->m_writable; + return (*m_attri)->m_writable; } Writable const &writable() const { - return m_attri->m_writable; + return (*m_attri)->m_writable; } inline void setData(std::shared_ptr attri) @@ -458,13 +493,13 @@ OPENPMD_protected m_attri = std::move(attri); } - inline internal::AttributableData &get() + inline internal::SharedAttributableData &get() { - return *m_attri; + return **m_attri; } - inline internal::AttributableData const &get() const + inline internal::SharedAttributableData const &get() const { - return *m_attri; + return **m_attri; } bool dirty() const diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 25a2154665..be36f47758 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -48,6 +48,7 @@ class Series; namespace internal { + class SharedAttributableData; class AttributableData; class SeriesData; } // namespace internal @@ -73,6 +74,7 @@ namespace debug */ class Writable final { + friend class internal::SharedAttributableData; friend class internal::AttributableData; friend class internal::SeriesData; friend class Attributable; @@ -142,6 +144,11 @@ OPENPMD_private */ std::shared_ptr>> IOHandler = nullptr; + /* + * Link to the containing Attributable. + * If multiple Attributables share the same Writable, then the creating one. + * (See SharedAttributableData) + */ internal::AttributableData *attributable = nullptr; Writable *parent = nullptr; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 366fea0de1..93318fdeb0 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -27,6 +27,7 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -708,7 +709,8 @@ auto Iteration::beginStep( case IE::fileBased: if (thisObject.has_value()) { - file = &static_cast(*thisObject).get(); + file = static_cast( + thisObject.value().m_attri.get()); } else { @@ -790,7 +792,7 @@ void Iteration::endStep() switch (series.iterationEncoding()) { case IE::fileBased: - file = &Attributable::get(); + file = m_attri.get(); break; case IE::groupBased: case IE::variableBased: diff --git a/src/Series.cpp b/src/Series.cpp index a3749c5e73..4f37a7cf78 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1066,7 +1066,7 @@ void Series::initSeries( std::unique_ptr input) { auto &series = get(); - auto &writable = series.m_writable; + auto &writable = series->m_writable; /* * In Access modes READ_LINEAR and APPEND, the Series constructor might have @@ -2427,7 +2427,7 @@ AdvanceStatus Series::advance( // If the backend does not support steps, we cannot continue here param.isThisStepMandatory = true; } - IOTask task(&file.m_writable, param); + IOTask task(&file->m_writable, param); IOHandler()->enqueue(task); } @@ -2524,7 +2524,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) // If the backend does not support steps, we cannot continue here param.isThisStepMandatory = true; } - IOTask task(&series.m_writable, param); + IOTask task(&series->m_writable, param); IOHandler()->enqueue(task); // We cannot call Series::flush now, since the IO handler is still filled @@ -2888,9 +2888,9 @@ namespace internal // This releases the openPMD hierarchy iterations.container().clear(); // Release the IO Handler - if (m_writable.IOHandler) + if (operator*().m_writable.IOHandler) { - *m_writable.IOHandler = std::nullopt; + *operator*().m_writable.IOHandler = std::nullopt; } } } // namespace internal diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 9ffbe25ee3..ef83f1e85c 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -38,7 +38,16 @@ namespace openPMD { namespace internal { - AttributableData::AttributableData() : m_writable{this} + SharedAttributableData::SharedAttributableData(AttributableData *attr) + : m_writable{attr} + {} + + AttributableData::AttributableData() + : SharedData_t(std::make_shared(this)) + {} + + AttributableData::AttributableData(SharedAttributableData *raw_ptr) + : SharedData_t({raw_ptr, [](auto const *) {}}) {} } // namespace internal diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index ee15f2a1a0..95bd6bccea 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -19,10 +19,11 @@ * If not, see . */ #include "openPMD/backend/Writable.hpp" -#include "openPMD/Error.hpp" + +#include "openPMD/backend/Attributable.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Series.hpp" -#include "openPMD/auxiliary/DerefDynamicCast.hpp" + #include namespace openPMD From 8b1837375a1d00946caec950acd0345178ca3a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 28 Jun 2023 14:47:30 +0200 Subject: [PATCH 02/27] Introduce SharedAttributableData --- src/backend/Writable.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index 95bd6bccea..948ff71580 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -20,9 +20,9 @@ */ #include "openPMD/backend/Writable.hpp" -#include "openPMD/backend/Attributable.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Series.hpp" +#include "openPMD/backend/Attributable.hpp" #include From 309d5f125a45380d4c17c94c79dfd19e2a1a1005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 May 2023 14:37:48 +0200 Subject: [PATCH 03/27] JSON backend: Fail when trying to open non-existing groups --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 3 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 54 +++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index b67ac9138a..aec068f571 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -323,7 +323,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // make sure that the given path exists in proper form in // the passed json value - static void ensurePath(nlohmann::json *json, std::string const &path); + static void + ensurePath(nlohmann::json *json, std::string const &path, Access); // In order not to insert the same file name into the data structures // with a new pointer (e.g. when reopening), search for a possibly diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 38de34f072..ae510bd6cd 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -25,6 +25,8 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -256,13 +258,13 @@ void JSONIOHandlerImpl::createPath( auto filepos = setAndGetFilePosition(writable, false); jsonVal = &(*jsonVal)[filepos->id]; - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); path = filepos->id.to_string() + "/" + path; } else { - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); } m_dirty.emplace(file); @@ -672,7 +674,10 @@ void JSONIOHandlerImpl::openPath( std::make_shared(json::json_pointer(path)); } - ensurePath(j, removeSlashes(parameters.path)); + ensurePath( + j, + removeSlashes(parameters.path), + /* Must not modify j */ Access::READ_ONLY); writable->written = true; } @@ -1226,18 +1231,45 @@ bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) } void JSONIOHandlerImpl::ensurePath( - nlohmann::json *jsonp, std::string const &path) + nlohmann::json *jsonp, std::string const &path, Access access) { auto groups = auxiliary::split(path, "/"); - for (std::string &group : groups) + if (access::readOnly(access)) { - // Enforce a JSON object - // the library will automatically create a list if the first - // key added to it is parseable as an int - jsonp = &(*jsonp)[group]; - if (jsonp->is_null()) + for (std::string const &group : groups) { - *jsonp = nlohmann::json::object(); + if (!jsonp->contains(group)) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::NotFound, + "JSON", + "Required group '" + path + "' not present."); + } + jsonp = &(*jsonp).at(group); + if (!jsonp->is_object()) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + "JSON", + "Required group '" + path + + "' is present, but not a JSON object."); + } + } + } + else + { + for (std::string const &group : groups) + { + // Enforce a JSON object + // the library will automatically create a list if the first + // key added to it is parseable as an int + jsonp = &(*jsonp)[group]; + if (jsonp->is_null()) + { + *jsonp = nlohmann::json::object(); + } } } } From 72399c4184a7217aaf667b1ce9fd5b17d9ab4df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Apr 2023 15:15:02 +0200 Subject: [PATCH 04/27] Insert CustomHierarchy class to Iteration --- CMakeLists.txt | 1 + include/openPMD/CustomHierarchy.hpp | 62 +++++++++++++++++++++++++++ include/openPMD/Iteration.hpp | 14 ++++-- include/openPMD/backend/Container.hpp | 16 ++++++- src/CustomHierarchy.cpp | 30 +++++++++++++ src/Iteration.cpp | 11 ++--- src/Series.cpp | 4 +- 7 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 include/openPMD/CustomHierarchy.hpp create mode 100644 src/CustomHierarchy.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fc5d6d5de5..3993f7205d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -384,6 +384,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..5c57e814f8 --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,62 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/backend/Container.hpp" + +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + using CustomHierarchyData = ContainerData; +} + +class CustomHierarchy : public Container +{ + friend class Iteration; + +private: + using Container_t = Container; + using Data_t = typename Container_t::ContainerData; + static_assert(std::is_base_of_v); + +protected: + CustomHierarchy(); + CustomHierarchy(NoInit); + + inline void setData(std::shared_ptr data) + { + Container_t::setData(std::move(data)); + } + +public: + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; +}; +} // namespace openPMD diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index a35ceccd74..6a4cd98a13 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" @@ -79,7 +80,7 @@ namespace internal bool beginStep = false; }; - class IterationData : public AttributableData + class IterationData : public CustomHierarchyData { public: /* @@ -123,7 +124,7 @@ namespace internal * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration : public CustomHierarchy { template friend class Container; @@ -269,14 +270,19 @@ class Iteration : public Attributable inline void setData(std::shared_ptr data) { m_iterationData = std::move(data); - Attributable::setData(m_iterationData); + CustomHierarchy::setData(m_iterationData); } void flushFileBased( std::string const &, IterationIndex_t, internal::FlushParams const &); void flushGroupBased(IterationIndex_t, internal::FlushParams const &); void flushVariableBased(IterationIndex_t, internal::FlushParams const &); - void flush(internal::FlushParams const &); + /* + * Named flushIteration instead of flush to avoid naming + * conflicts with overridden virtual flush from CustomHierarchy + * class. + */ + void flushIteration(internal::FlushParams const &); void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for runDeferredParseAccess(), readFileBased(), diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index cee50f9baf..da2e2c06d6 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -57,8 +57,22 @@ namespace traits }; } // namespace traits +class CustomHierarchy; + namespace internal { + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; + + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; + class SeriesData; template class EraseStaleEntries; @@ -103,7 +117,7 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, + internal::isDerivedFromAttributable, "Type of container element must be derived from Writable"); friend class Iteration; diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..9dbfa79f7c --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,30 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" + +#include "openPMD/backend/Attributable.hpp" + +namespace openPMD +{ +CustomHierarchy::CustomHierarchy() = default; +CustomHierarchy::CustomHierarchy(NoInit): Container_t(NoInit()) {} +} \ No newline at end of file diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 93318fdeb0..db71c92b5d 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" @@ -39,7 +40,7 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; -Iteration::Iteration() : Attributable(NoInit()) +Iteration::Iteration() : CustomHierarchy(NoInit()) { setData(std::make_shared()); setTime(static_cast(0)); @@ -244,7 +245,7 @@ void Iteration::flushFileBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -267,7 +268,7 @@ void Iteration::flushGroupBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -290,7 +291,7 @@ void Iteration::flushVariableBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } @@ -316,7 +317,7 @@ void Iteration::flushVariableBased( } } -void Iteration::flush(internal::FlushParams const &flushParams) +void Iteration::flushIteration(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); diff --git a/src/Series.cpp b/src/Series.cpp index 4f37a7cf78..a3e78539b4 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1320,7 +1320,7 @@ void Series::flushFileBased( break; case IO::HasBeenOpened: // continue below - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1433,7 +1433,7 @@ void Series::flushGorVBased( break; case IO::HasBeenOpened: // continue below - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } From 5d4c5746ccb6fbc72b00540143925b12371cc88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 25 May 2023 13:36:33 +0200 Subject: [PATCH 05/27] Help older compilers deal with this --- include/openPMD/Iteration.hpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 6a4cd98a13..bb4327ccaf 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -126,8 +126,20 @@ namespace internal */ class Iteration : public CustomHierarchy { - template - friend class Container; +public: + using IterationIndex_t = uint64_t; + + /* + * Some old compilers have trouble with befriending the entire Container + * template here, so we restrict it + * to Container, more is not needed anyway. + * + * E.g. on gcc-7: + * > error: specialization of 'openPMD::Container' + * > after instantiation + * > friend class Container; + */ + friend class Container; friend class Series; friend class WriteIterations; friend class SeriesIterator; @@ -136,12 +148,9 @@ class Iteration : public CustomHierarchy friend T &internal::makeOwning(T &self, Series); friend class Writable; -public: Iteration(Iteration const &) = default; Iteration &operator=(Iteration const &) = default; - using IterationIndex_t = uint64_t; - /** * @tparam T Floating point type of user-selected precision (e.g. float, * double). From a407e9d2d1cd4bddd082be11b6969a027efe7339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 28 Jul 2023 18:02:48 +0200 Subject: [PATCH 06/27] Add vector variants of meshes/particlesPath --- include/openPMD/Series.hpp | 6 +- include/openPMD/backend/Attribute.hpp | 19 +++++ src/Iteration.cpp | 2 - src/Series.cpp | 104 ++++++++++++++++++++------ 4 files changed, 103 insertions(+), 28 deletions(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index dbad461dc1..9af4271b74 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -410,6 +410,7 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + std::vector meshesPaths() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -420,6 +421,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setMeshesPath(std::string const &meshesPath); + Series &setMeshesPath(std::vector const &meshesPath); /** * @throw no_such_attribute_error If optional attribute is not present. @@ -453,6 +455,7 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + std::vector particlesPaths() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -463,6 +466,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setParticlesPath(std::string const &particlesPath); + Series &setParticlesPath(std::vector const &particlesPath); /** * @throw no_such_attribute_error If optional attribute is not present. @@ -813,8 +817,6 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); void flushRankTable(); void readFileBased(); void readOneIterationFileBased(std::string const &filePath); diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index a183b7818a..aaab8e565d 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -292,6 +292,25 @@ namespace detail } } } + // conversion cast: turn a 1-element vector into a single value + else if constexpr (auxiliary::IsVector_v) + { + if constexpr (std::is_convertible_v) + { + if (pv->size() != 1) + { + return {std::runtime_error( + "getCast: vector to scalar conversion requires " + "single-element vectors")}; + } + return {U(*pv->begin())}; + } + else + { + return {std::runtime_error( + "getCast: no vector to scalar conversion possible.")}; + } + } else { return {std::runtime_error("getCast: no cast possible.")}; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index db71c92b5d..322e07cb6e 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -339,7 +339,6 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) if (!s.containsAttribute("meshesPath")) { s.setMeshesPath("meshes/"); - s.flushMeshesPath(); } meshes.flush(s.meshesPath(), flushParams); for (auto &m : meshes) @@ -355,7 +354,6 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) if (!s.containsAttribute("particlesPath")) { s.setParticlesPath("particles/"); - s.flushParticlesPath(); } particles.flush(s.particlesPath(), flushParams); for (auto &species : particles) diff --git a/src/Series.cpp b/src/Series.cpp index a3e78539b4..6ddfc57fab 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -178,7 +178,27 @@ Series &Series::setBasePath(std::string const &bp) std::string Series::meshesPath() const { - return getAttribute("meshesPath").get(); + auto res = meshesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("meshesPath"); + } + /* + * @todo: Verify that meshesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::meshesPaths() const +{ + if (containsAttribute("meshesPath")) + { + return getAttribute("meshesPath").get>(); + } + else + { + return {}; + } } Series &Series::setMeshesPath(std::string const &mp) @@ -201,6 +221,23 @@ Series &Series::setMeshesPath(std::string const &mp) setDirty(true); return *this; } +Series &Series::setMeshesPath(std::vector const &mp) +{ + // @todo if already written, then append + switch (mp.size()) + { + case 0: + return *this; + case 1: + setAttribute("meshesPath", *mp.begin()); + break; + default: + setAttribute("meshesPath", mp); + break; + } + dirty() = true; + return *this; +} #if openPMD_HAVE_MPI chunk_assignment::RankMeta Series::rankTable(bool collective) @@ -468,7 +505,27 @@ void Series::flushRankTable() std::string Series::particlesPath() const { - return getAttribute("particlesPath").get(); + auto res = particlesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("particlesPath"); + } + /* + * @todo: Verify that particlesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::particlesPaths() const +{ + if (containsAttribute("particlesPath")) + { + return getAttribute("particlesPath").get>(); + } + else + { + return {}; + } } Series &Series::setParticlesPath(std::string const &pp) @@ -491,6 +548,23 @@ Series &Series::setParticlesPath(std::string const &pp) setDirty(true); return *this; } +Series &Series::setParticlesPath(std::vector const &pp) +{ + // @todo if already written, then append + switch (pp.size()) + { + case 0: + return *this; + case 1: + setAttribute("particlesPath", *pp.begin()); + break; + default: + setAttribute("particlesPath", pp); + break; + } + dirty() = true; + return *this; +} std::string Series::author() const { @@ -1536,26 +1610,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.resource = a.getResource(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.resource = a.getResource(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased() { auto &series = get(); @@ -2234,7 +2288,8 @@ void Series::readBase() aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(*aRead.resource) + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ @@ -2266,7 +2321,8 @@ void Series::readBase() aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(*aRead.resource) + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ From 79cb2d30c11aad6fa61e3f7400ef5f7bc9a48540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 28 Jul 2023 18:38:32 +0200 Subject: [PATCH 07/27] Move meshes and particles over to CustomHierarchies class --- include/openPMD/CustomHierarchy.hpp | 12 ++++++++++++ include/openPMD/Iteration.hpp | 10 ---------- include/openPMD/Mesh.hpp | 1 + include/openPMD/ParticleSpecies.hpp | 1 + include/openPMD/backend/Attributable.hpp | 4 ++++ include/openPMD/backend/Container.hpp | 1 + include/openPMD/backend/Writable.hpp | 1 + src/CustomHierarchy.cpp | 21 ++++++++++++++++++--- src/Iteration.cpp | 7 ------- 9 files changed, 38 insertions(+), 20 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 5c57e814f8..bfa8b9501a 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -20,6 +20,8 @@ */ #pragma once +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" #include "openPMD/backend/Container.hpp" #include @@ -52,11 +54,21 @@ class CustomHierarchy : public Container Container_t::setData(std::move(data)); } + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + public: CustomHierarchy(CustomHierarchy const &other) = default; CustomHierarchy(CustomHierarchy &&other) = default; CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; + + Container meshes{}; + Container particles{}; }; } // namespace openPMD diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index bb4327ccaf..a2968f8966 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -250,9 +250,6 @@ class Iteration : public CustomHierarchy [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - Container meshes{}; - Container particles{}; // particleSpecies? - virtual ~Iteration() = default; private: @@ -404,13 +401,6 @@ class Iteration : public CustomHierarchy */ void setStepStatus(StepStatus); - /** - * @brief Link with parent. - * - * @param w The Writable representing the parent. - */ - virtual void linkHierarchy(Writable &w); - /** * @brief Access an iteration in read mode that has potentially not been * parsed yet. diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index 53274ac7d4..1c255e3910 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -41,6 +41,7 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class CustomHierarchy; public: Mesh(Mesh const &) = default; diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index af7aa50375..9f454a0ed5 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -37,6 +37,7 @@ class ParticleSpecies : public Container friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: ParticlePatches particlePatches; diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 97dad22460..e2eae0f1a6 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -50,6 +50,7 @@ class AbstractFilePosition; class Attributable; class Iteration; class Series; +class CustomHierarchy; namespace internal { @@ -59,6 +60,7 @@ namespace internal class SharedAttributableData { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; public: SharedAttributableData(AttributableData *); @@ -105,6 +107,7 @@ namespace internal class AttributableData : public std::shared_ptr { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; using SharedData_t = std::shared_ptr; @@ -209,6 +212,7 @@ class Attributable friend void debug::printDirty(Series const &); template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; protected: // tag for internal constructor diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index da2e2c06d6..96c7c0c099 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -128,6 +128,7 @@ class Container : virtual public Attributable template friend class internal::EraseStaleEntries; friend class SeriesIterator; + friend class CustomHierarchy; protected: using ContainerData = internal::ContainerData; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index be36f47758..7bac7ef929 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -103,6 +103,7 @@ class Writable final template friend class Span; friend void debug::printDirty(Series const &); + friend class CustomHierarchy; private: Writable(internal::AttributableData *); diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 9dbfa79f7c..c3c3eab77e 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -22,9 +22,24 @@ #include "openPMD/CustomHierarchy.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Writable.hpp" + +#include namespace openPMD { -CustomHierarchy::CustomHierarchy() = default; -CustomHierarchy::CustomHierarchy(NoInit): Container_t(NoInit()) {} -} \ No newline at end of file +CustomHierarchy::CustomHierarchy() +{ + meshes.writable().ownKeyWithinParent = "meshes"; + particles.writable().ownKeyWithinParent = "particles"; +} +CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit()) +{} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); + meshes.linkHierarchy(this->writable()); + particles.linkHierarchy(this->writable()); +} +} // namespace openPMD diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 322e07cb6e..6877fbe333 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -837,13 +837,6 @@ void Iteration::setStepStatus(StepStatus status) } } -void Iteration::linkHierarchy(Writable &w) -{ - Attributable::linkHierarchy(w); - meshes.linkHierarchy(this->writable()); - particles.linkHierarchy(this->writable()); -} - void Iteration::runDeferredParseAccess() { if (access::read(IOHandler()->m_frontendAccess)) From 996acc60a37468de128d4de971c54673ba0c9c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 May 2023 15:08:51 +0200 Subject: [PATCH 08/27] Move dirtyRecursive to CustomHierarchy --- include/openPMD/CustomHierarchy.hpp | 10 +++++++ include/openPMD/backend/BaseRecord.hpp | 1 + src/CustomHierarchy.cpp | 37 ++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index bfa8b9501a..ee2706f0ce 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -61,6 +61,16 @@ class CustomHierarchy : public Container */ void linkHierarchy(Writable &w) override; + /* + * @brief Check recursively whether this object is dirty. + * It is dirty if any attribute or dataset is read from or written to + * the backend. + * + * @return true If dirty. + * @return false Otherwise. + */ + bool dirtyRecursive() const; + public: CustomHierarchy(CustomHierarchy const &other) = default; CustomHierarchy(CustomHierarchy &&other) = default; diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index ba137b10db..914c541ccf 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -228,6 +228,7 @@ class BaseRecord private: using T_Self = BaseRecord; + friend class CustomHierarchy; friend class Iteration; friend class ParticleSpecies; friend class PatchRecord; diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index c3c3eab77e..19a577fb23 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -21,9 +21,12 @@ #include "openPMD/CustomHierarchy.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" +#include #include namespace openPMD @@ -42,4 +45,38 @@ void CustomHierarchy::linkHierarchy(Writable &w) meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); } + +bool CustomHierarchy::dirtyRecursive() const +{ + if (dirty()) + { + return true; + } + if (particles.dirty() || meshes.dirty()) + { + return true; + } + for (auto const &pair : particles) + { + if (pair.second.dirtyRecursive()) + { + return true; + } + } + for (auto const &pair : meshes) + { + if (pair.second.dirtyRecursive()) + { + return true; + } + } + for (auto const &pair : *this) + { + if (pair.second.dirtyRecursive()) + { + return true; + } + } + return false; +} } // namespace openPMD From 2ee8e165a53def857b8b27ccd4a56df705bb5a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 28 Jul 2023 19:39:59 +0200 Subject: [PATCH 09/27] Move Iteration reading logic to CustomHierarchy --- include/openPMD/CustomHierarchy.hpp | 38 +++- include/openPMD/Iteration.hpp | 2 - include/openPMD/RecordComponent.hpp | 1 + src/CustomHierarchy.cpp | 271 ++++++++++++++++++++++++++++ src/Iteration.cpp | 184 +------------------ 5 files changed, 319 insertions(+), 177 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index ee2706f0ce..802a0d7998 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -33,18 +33,49 @@ namespace openPMD class CustomHierarchy; namespace internal { + enum class ContainedType + { + Group, + Mesh, + Particle + }; + struct MeshesParticlesPath + { + std::optional meshesPath; + std::optional particlesPath; + + explicit MeshesParticlesPath() = default; + MeshesParticlesPath( + std::optional meshesPath, + std::optional particlesPath); + + [[nodiscard]] ContainedType determineType( + std::vector const &path, + std::string const &name) const; + [[nodiscard]] bool isParticle( + std::vector const &path, + std::string const &name) const; + [[nodiscard]] bool isMesh( + std::vector const &path, + std::string const &name) const; + }; + using CustomHierarchyData = ContainerData; -} +} // namespace internal class CustomHierarchy : public Container { friend class Iteration; + friend class Container; private: using Container_t = Container; using Data_t = typename Container_t::ContainerData; static_assert(std::is_base_of_v); + void readMeshes(std::string const &meshesPath); + void readParticles(std::string const &particlesPath); + protected: CustomHierarchy(); CustomHierarchy(NoInit); @@ -54,6 +85,11 @@ class CustomHierarchy : public Container Container_t::setData(std::move(data)); } + void read(internal::MeshesParticlesPath const &); + void read( + internal::MeshesParticlesPath const &, + std::vector ¤tPath); + /** * @brief Link with parent. * diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index a2968f8966..99f2021e06 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -316,8 +316,6 @@ class Iteration : public CustomHierarchy bool beginStep); void readGorVBased(std::string const &groupPath, bool beginStep); void read_impl(std::string const &groupPath); - void readMeshes(std::string const &meshesPath); - void readParticles(std::string const &particlesPath); /** * Status after beginning an IO step. Currently includes: diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 9072b93a32..4da9547579 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -135,6 +135,7 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: enum class Allocation diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 19a577fb23..0c1f320e28 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -20,17 +20,75 @@ */ #include "openPMD/CustomHierarchy.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/IOTask.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/Writable.hpp" +#include +#include #include #include namespace openPMD { +namespace internal +{ + MeshesParticlesPath::MeshesParticlesPath( + std::optional meshesPath_in, + std::optional particlesPath_in) + : meshesPath{std::move(meshesPath_in)} + , particlesPath{std::move(particlesPath_in)} + { + if (meshesPath.has_value()) + { + meshesPath = auxiliary::removeSlashes(*meshesPath); + } + if (particlesPath.has_value()) + { + particlesPath = auxiliary::removeSlashes(*particlesPath); + } + } + + ContainedType MeshesParticlesPath::determineType( + std::vector const &path, std::string const &name) const + { + if (isMesh(path, name)) + { + return ContainedType::Mesh; + } + else if (isParticle(path, name)) + { + return ContainedType::Particle; + } + else + { + return ContainedType::Group; + } + } + + bool MeshesParticlesPath::isParticle( + std::vector const &path, std::string const &name) const + { + (void)path; + return particlesPath.has_value() ? name == *particlesPath : false; + } + bool MeshesParticlesPath::isMesh( + std::vector const &path, std::string const &name) const + { + (void)path; + return meshesPath.has_value() ? name == *meshesPath : false; + } +} // namespace internal + CustomHierarchy::CustomHierarchy() { meshes.writable().ownKeyWithinParent = "meshes"; @@ -39,6 +97,219 @@ CustomHierarchy::CustomHierarchy() CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit()) {} +void CustomHierarchy::readMeshes(std::string const &meshesPath) +{ + Parameter pOpen; + Parameter pList; + + pOpen.path = meshesPath; + IOHandler()->enqueue(IOTask(&meshes, pOpen)); + + meshes.readAttributes(ReadMode::FullyReread); + + internal::EraseStaleEntries map{meshes}; + + /* obtain all non-scalar meshes */ + IOHandler()->enqueue(IOTask(&meshes, pList)); + IOHandler()->flush(internal::defaultFlushParams); + + Parameter aList; + for (auto const &mesh_name : *pList.paths) + { + Mesh &m = map[mesh_name]; + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); + IOHandler()->flush(internal::defaultFlushParams); + + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) + { + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.get().m_isConstant = true; + } + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } + } + + /* obtain all scalar meshes */ + Parameter dList; + IOHandler()->enqueue(IOTask(&meshes, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + Parameter dOpen; + for (auto const &mesh_name : *dList.datasets) + { + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + IOHandler()->enqueue(IOTask(&m, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } + } +} + +void CustomHierarchy::readParticles(std::string const &particlesPath) +{ + Parameter pOpen; + Parameter pList; + + pOpen.path = particlesPath; + IOHandler()->enqueue(IOTask(&particles, pOpen)); + + particles.readAttributes(ReadMode::FullyReread); + + /* obtain all particle species */ + pList.paths->clear(); + IOHandler()->enqueue(IOTask(&particles, pList)); + IOHandler()->flush(internal::defaultFlushParams); + + internal::EraseStaleEntries map{particles}; + for (auto const &species_name : *pList.paths) + { + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try + { + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" + << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } + } +} + +void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp) +{ + std::vector currentPath; + read(mpp, currentPath); +} + +void CustomHierarchy::read( + internal::MeshesParticlesPath const &mpp, + std::vector ¤tPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + for (auto const &path : *pList.paths) + { + switch (mpp.determineType(currentPath, path)) + { + case internal::ContainedType::Group: { + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + currentPath.emplace_back(path); + try + { + subpath.read(mpp, currentPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read subgroup '" << path << "' at path '" + << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + container().erase(path); + } + currentPath.pop_back(); + break; + } + case internal::ContainedType::Mesh: { + try + { + readMeshes(*mpp.meshesPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read meshes at path '" + << myPath().openPMDPath() + << "' and will skip them due to read error:\n" + << err.what() << std::endl; + meshes = {}; + meshes.dirty() = false; + } + break; + } + case internal::ContainedType::Particle: { + try + { + readParticles(*mpp.particlesPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particles at path '" + << myPath().openPMDPath() + << "' and will skip them due to read error:\n" + << err.what() << std::endl; + particles = {}; + particles.dirty() = false; + } + break; + } + } + } + + if (!mpp.meshesPath.has_value()) + { + meshes.dirty() = false; + } + if (!mpp.particlesPath.has_value()) + { + particles.dirty() = false; + } +} + void CustomHierarchy::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 6877fbe333..8ec68d98af 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -488,68 +488,23 @@ void Iteration::read_impl(std::string const &groupPath) "found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); std::string version = s.openPMD(); - bool hasMeshes = false; - bool hasParticles = false; - if (version == "1.0.0" || version == "1.0.1") - { - IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(internal::defaultFlushParams); - hasMeshes = std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.meshesPath(), "/", "")) == 1; - hasParticles = - std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.particlesPath(), "/", "")) == 1; - pList.paths->clear(); - } - else - { - hasMeshes = s.containsAttribute("meshesPath"); - hasParticles = s.containsAttribute("particlesPath"); - } - if (hasMeshes) - { - try - { - readMeshes(s.meshesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read meshes in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - meshes = {}; - } - } - meshes.setDirty(false); + // @todo restore compatibility with openPMD 1.0.*: + // hasMeshes <-> meshesPath is defined - if (hasParticles) - { - try - { - readParticles(s.particlesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particles in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - particles = {}; - } - } - particles.setDirty(false); + internal::MeshesParticlesPath mpp( + s.containsAttribute("meshesPath") ? std::make_optional(s.meshesPath()) + : std::nullopt, + s.containsAttribute("particlesPath") + ? std::make_optional(s.particlesPath()) + : std::nullopt); + CustomHierarchy::read(std::move(mpp)); - readAttributes(ReadMode::FullyReread); #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) { @@ -566,125 +521,6 @@ void Iteration::read_impl(std::string const &groupPath) #endif } -void Iteration::readMeshes(std::string const &meshesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = meshesPath; - IOHandler()->enqueue(IOTask(&meshes, pOpen)); - - meshes.readAttributes(ReadMode::FullyReread); - - internal::EraseStaleEntries map{meshes}; - - /* obtain all non-scalar meshes */ - IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter aList; - for (auto const &mesh_name : *pList.paths) - { - Mesh &m = map[mesh_name]; - pOpen.path = mesh_name; - aList.attributes->clear(); - IOHandler()->enqueue(IOTask(&m, pOpen)); - IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(internal::defaultFlushParams); - - auto att_begin = aList.attributes->begin(); - auto att_end = aList.attributes->end(); - auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) - { - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; - } - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } - - /* obtain all scalar meshes */ - Parameter dList; - IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter dOpen; - for (auto const &mesh_name : *dList.datasets) - { - Mesh &m = map[mesh_name]; - dOpen.name = mesh_name; - IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); - mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } -} - -void Iteration::readParticles(std::string const &particlesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = particlesPath; - IOHandler()->enqueue(IOTask(&particles, pOpen)); - - particles.readAttributes(ReadMode::FullyReread); - - /* obtain all particle species */ - pList.paths->clear(); - IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - internal::EraseStaleEntries map{particles}; - for (auto const &species_name : *pList.paths) - { - ParticleSpecies &p = map[species_name]; - pOpen.path = species_name; - IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - try - { - p.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particle species with name '" - << species_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(species_name); - } - } -} - auto Iteration::beginStep(bool reread) -> BeginStepStatus { BeginStepStatus res; From 41c8c4876cee06f2408527791cd6f07144dd4ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 31 Jul 2023 11:07:50 +0200 Subject: [PATCH 10/27] Move Iteration flushing logic to CustomHierarchy class --- include/openPMD/CustomHierarchy.hpp | 6 ++ src/CustomHierarchy.cpp | 94 +++++++++++++++++++++++++++++ src/Iteration.cpp | 44 ++------------ 3 files changed, 104 insertions(+), 40 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 802a0d7998..ad6c24e47b 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -90,6 +90,12 @@ class CustomHierarchy : public Container internal::MeshesParticlesPath const &, std::vector ¤tPath); + void flush_internal( + internal::FlushParams const &, + internal::MeshesParticlesPath &, + std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + /** * @brief Link with parent. * diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 0c1f320e28..55c4d8a658 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -20,6 +20,8 @@ */ #include "openPMD/CustomHierarchy.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/Series.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/Dataset.hpp" @@ -310,6 +312,98 @@ void CustomHierarchy::read( } } +void CustomHierarchy::flush_internal( + internal::FlushParams const &flushParams, + internal::MeshesParticlesPath &mpp, + std::vector currentPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : *this) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + currentPath.emplace_back(name); + subpath.flush_internal(flushParams, mpp, currentPath); + currentPath.pop_back(); + } + if (!meshes.empty() || mpp.meshesPath.has_value()) + { + if (!access::readOnly(IOHandler()->m_frontendAccess)) + { + if (!mpp.meshesPath.has_value()) + { + mpp.meshesPath = "meshes"; + } + meshes.flush(mpp.meshesPath.value(), flushParams); + } + for (auto &m : meshes) + m.second.flush(m.first, flushParams); + } + else + { + meshes.dirty() = false; + } + if (!particles.empty() || mpp.particlesPath.has_value()) + { + if (!access::readOnly(IOHandler()->m_frontendAccess)) + { + if (!mpp.particlesPath.has_value()) + { + mpp.particlesPath = "particles"; + } + particles.flush(mpp.particlesPath.value(), flushParams); + } + for (auto &m : particles) + m.second.flush(m.first, flushParams); + } + else + { + particles.dirty() = false; + } +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &flushParams) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Series s = this->retrieveSeries(); + internal::MeshesParticlesPath mpp( + s.containsAttribute("meshesPath") ? std::make_optional(s.meshesPath()) + : std::nullopt, + s.containsAttribute("particlesPath") + ? std::make_optional(s.particlesPath()) + : std::nullopt); + std::vector currentPath; + flush_internal(flushParams, mpp, currentPath); + if (mpp.meshesPath.has_value() && !s.containsAttribute("meshesPath")) + { + s.setMeshesPath(*mpp.meshesPath); + } + if (mpp.particlesPath.has_value() && !s.containsAttribute("particlesPath")) + { + s.setParticlesPath(*mpp.particlesPath); + } +} + void CustomHierarchy::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 8ec68d98af..75377573d2 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -321,49 +321,13 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); - if (access::readOnly(IOHandler()->m_frontendAccess)) + if (flushParams.flushLevel == FlushLevel::CreateOrOpenFiles) { - for (auto &m : meshes) - m.second.flush(m.first, flushParams); - for (auto &species : particles) - species.second.flush(species.first, flushParams); + return; } - else + CustomHierarchy::flush("", flushParams); + if (access::write(IOHandler()->m_frontendAccess)) { - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ - Series s = retrieveSeries(); - - if (!meshes.empty() || s.containsAttribute("meshesPath")) - { - if (!s.containsAttribute("meshesPath")) - { - s.setMeshesPath("meshes/"); - } - meshes.flush(s.meshesPath(), flushParams); - for (auto &m : meshes) - m.second.flush(m.first, flushParams); - } - else - { - meshes.setDirty(false); - } - - if (!particles.empty() || s.containsAttribute("particlesPath")) - { - if (!s.containsAttribute("particlesPath")) - { - s.setParticlesPath("particles/"); - } - particles.flush(s.particlesPath(), flushParams); - for (auto &species : particles) - species.second.flush(species.first, flushParams); - } - else - { - particles.setDirty(false); - } - flushAttributes(flushParams); } if (flushParams.flushLevel != FlushLevel::SkeletonOnly) From 15345d780c3f2d884d655aa579cc2f687fe6c100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 31 Jul 2023 14:54:11 +0200 Subject: [PATCH 11/27] Support for custom datasets --- include/openPMD/CustomHierarchy.hpp | 26 +++++- include/openPMD/backend/Attributable.hpp | 13 +++ include/openPMD/backend/Container.hpp | 2 + src/CustomHierarchy.cpp | 102 +++++++++++++++++++++++ test/CoreTest.cpp | 98 ++++++++++++++++++++++ 5 files changed, 239 insertions(+), 2 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index ad6c24e47b..30f181b04f 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -60,7 +60,14 @@ namespace internal std::string const &name) const; }; - using CustomHierarchyData = ContainerData; + struct CustomHierarchyData : ContainerData + { + explicit CustomHierarchyData(); + + void syncAttributables(); + + Container m_embeddedDatasets; + }; } // namespace internal class CustomHierarchy : public Container @@ -70,9 +77,20 @@ class CustomHierarchy : public Container private: using Container_t = Container; - using Data_t = typename Container_t::ContainerData; + using Data_t = internal::CustomHierarchyData; static_assert(std::is_base_of_v); + std::shared_ptr m_customHierarchyData; + + [[nodiscard]] Data_t &get() + { + return *m_customHierarchyData; + } + [[nodiscard]] Data_t const &get() const + { + return *m_customHierarchyData; + } + void readMeshes(std::string const &meshesPath); void readParticles(std::string const &particlesPath); @@ -82,6 +100,7 @@ class CustomHierarchy : public Container inline void setData(std::shared_ptr data) { + m_customHierarchyData = data; Container_t::setData(std::move(data)); } @@ -120,6 +139,9 @@ class CustomHierarchy : public Container CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; + template + auto asContainerOf() -> Container &; + Container meshes{}; Container particles{}; }; diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index e2eae0f1a6..08dfe0e1c5 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -118,6 +118,17 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline std::shared_ptr & + asSharedPtrOfAttributable() + { + return *this; + } + inline std::shared_ptr const & + asSharedPtrOfAttributable() const + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; @@ -162,6 +173,7 @@ namespace internal class BaseRecordData; class RecordComponentData; + struct CustomHierarchyData; /* * Internal function to turn a handle into an owning handle that will keep @@ -213,6 +225,7 @@ class Attributable template friend T &internal::makeOwning(T &self, Series); friend class CustomHierarchy; + friend struct internal::CustomHierarchyData; protected: // tag for internal constructor diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 96c7c0c099..b62f584b74 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -76,6 +76,7 @@ namespace internal class SeriesData; template class EraseStaleEntries; + struct CustomHierarchyData; template < typename T, @@ -128,6 +129,7 @@ class Container : virtual public Attributable template friend class internal::EraseStaleEntries; friend class SeriesIterator; + friend struct internal::CustomHierarchyData; friend class CustomHierarchy; protected: diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 55c4d8a658..53f9e65ba5 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -89,10 +89,31 @@ namespace internal (void)path; return meshesPath.has_value() ? name == *meshesPath : false; } + + CustomHierarchyData::CustomHierarchyData() + { + syncAttributables(); + } + + void CustomHierarchyData::syncAttributables() + { + /* + * m_embeddeddatasets and its friends should point to the same instance + * of Attributable. + * (next commits will add members for meshes and particles) + */ + for (auto p : + std::initializer_list{&m_embeddedDatasets}) + { + p->m_attri->asSharedPtrOfAttributable() = + this->asSharedPtrOfAttributable(); + } + } } // namespace internal CustomHierarchy::CustomHierarchy() { + setData(std::make_shared()); meshes.writable().ownKeyWithinParent = "meshes"; particles.writable().ownKeyWithinParent = "particles"; } @@ -242,6 +263,8 @@ void CustomHierarchy::read( IOHandler()->enqueue(IOTask(this, dList)); IOHandler()->flush(internal::defaultFlushParams); + std::deque constantComponentsPushback; + auto &data = get(); for (auto const &path : *pList.paths) { switch (mpp.determineType(currentPath, path)) @@ -265,6 +288,15 @@ void CustomHierarchy::read( container().erase(path); } currentPath.pop_back(); + if (subpath.size() == 0 && subpath.containsAttribute("shape") && + subpath.containsAttribute("value")) + { + // This is not a group, but a constant record component + // Writable::~Writable() will deal with removing this from the + // backend again. + constantComponentsPushback.push_back(path); + container().erase(path); + } break; } case internal::ContainedType::Mesh: { @@ -310,6 +342,40 @@ void CustomHierarchy::read( { particles.dirty() = false; } + + for (auto const &path : *dList.datasets) + { + auto &rc = data.m_embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.written() = false; + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.written() = true; + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().printGroup() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + data.m_embeddedDatasets.container().erase(path); + } + } + + for (auto const &path : constantComponentsPushback) + { + auto &rc = data.m_embeddedDatasets[path]; + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(); + } } void CustomHierarchy::flush_internal( @@ -374,6 +440,10 @@ void CustomHierarchy::flush_internal( { particles.dirty() = false; } + for (auto &[name, dataset] : get().m_embeddedDatasets) + { + dataset.flush(name, flushParams); + } } void CustomHierarchy::flush( @@ -435,6 +505,13 @@ bool CustomHierarchy::dirtyRecursive() const return true; } } + for (auto const &pair : get().m_embeddedDatasets) + { + if (pair.second.dirtyRecursive()) + { + return true; + } + } for (auto const &pair : *this) { if (pair.second.dirtyRecursive()) @@ -444,4 +521,29 @@ bool CustomHierarchy::dirtyRecursive() const } return false; } + +template +auto CustomHierarchy::asContainerOf() -> Container & +{ + if constexpr (std::is_same_v) + { + return *static_cast *>(this); + } + else if constexpr (std::is_same_v) + { + return get().m_embeddedDatasets; + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent."); + } +} + +template auto CustomHierarchy::asContainerOf() + -> Container &; +template auto CustomHierarchy::asContainerOf() + -> Container &; } // namespace openPMD diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index d27d68a8c5..236db48f2b 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -169,6 +169,104 @@ TEST_CASE("attribute_dtype_test", "[core]") } } +TEST_CASE("custom_hierarchies", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies.json"; + Series write(filePath, Access::CREATE); + write.iterations[0]; + write.close(); + + Series read(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 0); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + write.close(); + + read = Series(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 2); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE( + read.iterations[0]["custom"] + .getAttribute("string") + .get() == "attribute"); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .getAttribute("number") + .get() == 3); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + REQUIRE(read.iterations[0].size() == 2); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + + REQUIRE( + read.iterations[0].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .asContainerOf() + .size() == 0); + REQUIRE( + read.iterations[0]["no_attributes"] + .asContainerOf() + .size() == 0); + + auto iteration_level_ds = + read.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + REQUIRE(iteration_level_ds.getDatatype() == Datatype::INT); + REQUIRE(iteration_level_ds.getExtent() == Extent{10}); + auto loaded_chunk = iteration_level_ds.loadChunk(); + iteration_level_ds.seriesFlush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(loaded_chunk.get()[i] == 5); + } + + auto constant_dataset = + read.iterations[0]["custom"] + .asContainerOf()["emptyDataset"]; + REQUIRE(constant_dataset.getDatatype() == Datatype::FLOAT); + REQUIRE(constant_dataset.getExtent() == Extent{0, 0, 0}); + } +} + TEST_CASE("myPath", "[core]") { #if openPMD_USE_INVASIVE_TESTS From 35a24329f7e999d7212154a96ba5fe574264ad91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 31 Jul 2023 17:22:26 +0200 Subject: [PATCH 12/27] Treat "meshes"/"particles" as normal subgroups Introduction of iteration["meshes"].asContainerOf() as a more explicit variant for iteration.meshes. --- include/openPMD/CustomHierarchy.hpp | 28 +- src/CustomHierarchy.cpp | 507 +++++++++++++++++----------- test/CoreTest.cpp | 44 ++- 3 files changed, 386 insertions(+), 193 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 30f181b04f..c47671adc6 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -44,6 +44,17 @@ namespace internal std::optional meshesPath; std::optional particlesPath; + inline std::string requestMeshesPath() + { + meshesPath = meshesPath.value_or("meshes"); + return *meshesPath; + } + inline std::string requestParticlesPath() + { + particlesPath = particlesPath.value_or("particles"); + return *particlesPath; + } + explicit MeshesParticlesPath() = default; MeshesParticlesPath( std::optional meshesPath, @@ -67,6 +78,8 @@ namespace internal void syncAttributables(); Container m_embeddedDatasets; + Container m_embeddedMeshes; + Container m_embeddedParticles; }; } // namespace internal @@ -91,8 +104,12 @@ class CustomHierarchy : public Container return *m_customHierarchyData; } - void readMeshes(std::string const &meshesPath); - void readParticles(std::string const &particlesPath); + using EraseStaleMeshes = internal::EraseStaleEntries>; + using EraseStaleParticles = + internal::EraseStaleEntries>; + void readNonscalarMesh(EraseStaleMeshes &map, std::string const &name); + void readScalarMesh(EraseStaleMeshes &map, std::string const &name); + void readParticleSpecies(EraseStaleParticles &map, std::string const &name); protected: CustomHierarchy(); @@ -139,10 +156,17 @@ class CustomHierarchy : public Container CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; + mapped_type &operator[](key_type &&key); + mapped_type &operator[](key_type const &key); + template auto asContainerOf() -> Container &; Container meshes{}; Container particles{}; + +private: + template + mapped_type &bracketOperatorImpl(KeyType &&); }; } // namespace openPMD diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 53f9e65ba5..f4e06f0497 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -20,17 +20,16 @@ */ #include "openPMD/CustomHierarchy.hpp" -#include "openPMD/IO/Access.hpp" -#include "openPMD/Series.hpp" -#include "openPMD/auxiliary/StringManip.hpp" - #include "openPMD/Dataset.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/RecordComponent.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/Writable.hpp" @@ -38,8 +37,35 @@ #include #include #include +#include #include +namespace +{ +template +std::string +concatWithSep(Iterator &&begin, Iterator const &end, std::string const &sep) +{ + if (begin == end) + { + return ""; + } + std::stringstream res; + res << *(begin++); + for (; begin != end; ++begin) + { + res << sep << *begin; + } + return res.str(); +} + +std::string +concatWithSep(std::vector const &v, std::string const &sep) +{ + return concatWithSep(v.begin(), v.end(), sep); +} +} // namespace + namespace openPMD { namespace internal @@ -80,14 +106,18 @@ namespace internal bool MeshesParticlesPath::isParticle( std::vector const &path, std::string const &name) const { - (void)path; - return particlesPath.has_value() ? name == *particlesPath : false; + (void)name; + return particlesPath.has_value() && !path.empty() + ? *path.rbegin() == *particlesPath + : false; } bool MeshesParticlesPath::isMesh( std::vector const &path, std::string const &name) const { - (void)path; - return meshesPath.has_value() ? name == *meshesPath : false; + (void)name; + return meshesPath.has_value() && !path.empty() + ? *path.rbegin() == *meshesPath + : false; } CustomHierarchyData::CustomHierarchyData() @@ -100,10 +130,9 @@ namespace internal /* * m_embeddeddatasets and its friends should point to the same instance * of Attributable. - * (next commits will add members for meshes and particles) */ - for (auto p : - std::initializer_list{&m_embeddedDatasets}) + for (auto p : std::initializer_list{ + &m_embeddedDatasets, &m_embeddedMeshes, &m_embeddedParticles}) { p->m_attri->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); @@ -120,122 +149,92 @@ CustomHierarchy::CustomHierarchy() CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit()) {} -void CustomHierarchy::readMeshes(std::string const &meshesPath) +void CustomHierarchy::readNonscalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) { Parameter pOpen; - Parameter pList; - - pOpen.path = meshesPath; - IOHandler()->enqueue(IOTask(&meshes, pOpen)); - - meshes.readAttributes(ReadMode::FullyReread); + Parameter aList; - internal::EraseStaleEntries map{meshes}; + Mesh &m = map[mesh_name]; - /* obtain all non-scalar meshes */ - IOHandler()->enqueue(IOTask(&meshes, pList)); + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); IOHandler()->flush(internal::defaultFlushParams); - Parameter aList; - for (auto const &mesh_name : *pList.paths) + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) { - Mesh &m = map[mesh_name]; - pOpen.path = mesh_name; - aList.attributes->clear(); - IOHandler()->enqueue(IOTask(&m, pOpen)); - IOHandler()->enqueue(IOTask(&m, aList)); + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - - auto att_begin = aList.attributes->begin(); - auto att_end = aList.attributes->end(); - auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) - { - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; - } - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } + mrc.get().m_isConstant = true; + } + try + { + m.read(); } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} - /* obtain all scalar meshes */ - Parameter dList; - IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(internal::defaultFlushParams); +void CustomHierarchy::readScalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter pList; Parameter dOpen; - for (auto const &mesh_name : *dList.datasets) + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try { - Mesh &m = map[mesh_name]; - dOpen.name = mesh_name; - IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); - mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); } } -void CustomHierarchy::readParticles(std::string const &particlesPath) +void CustomHierarchy::readParticleSpecies( + EraseStaleParticles &map, std::string const &species_name) { Parameter pOpen; Parameter pList; - pOpen.path = particlesPath; - IOHandler()->enqueue(IOTask(&particles, pOpen)); - - particles.readAttributes(ReadMode::FullyReread); - - /* obtain all particle species */ - pList.paths->clear(); - IOHandler()->enqueue(IOTask(&particles, pList)); + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - - internal::EraseStaleEntries map{particles}; - for (auto const &species_name : *pList.paths) + try { - ParticleSpecies &p = map[species_name]; - pOpen.path = species_name; - IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - try - { - p.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particle species with name '" - << species_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(species_name); - } + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); } } @@ -265,6 +264,8 @@ void CustomHierarchy::read( std::deque constantComponentsPushback; auto &data = get(); + EraseStaleMeshes meshesMap(data.m_embeddedMeshes); + EraseStaleParticles particlesMap(data.m_embeddedParticles); for (auto const &path : *pList.paths) { switch (mpp.determineType(currentPath, path)) @@ -302,32 +303,30 @@ void CustomHierarchy::read( case internal::ContainedType::Mesh: { try { - readMeshes(*mpp.meshesPath); + readNonscalarMesh(meshesMap, path); } catch (error::ReadError const &err) { - std::cerr << "Cannot read meshes at path '" - << myPath().openPMDPath() + std::cerr << "Cannot read mesh at location '" + << myPath().openPMDPath() << "/" << path << "' and will skip them due to read error:\n" << err.what() << std::endl; - meshes = {}; - meshes.dirty() = false; + meshesMap.forget(path); } break; } case internal::ContainedType::Particle: { try { - readParticles(*mpp.particlesPath); + readParticleSpecies(particlesMap, path); } catch (error::ReadError const &err) { - std::cerr << "Cannot read particles at path '" - << myPath().openPMDPath() + std::cerr << "Cannot read particle species at location '" + << myPath().openPMDPath() << "/" << path << "' and will skip them due to read error:\n" << err.what() << std::endl; - particles = {}; - particles.dirty() = false; + particlesMap.forget(path); } break; } @@ -345,25 +344,44 @@ void CustomHierarchy::read( for (auto const &path : *dList.datasets) { - auto &rc = data.m_embeddedDatasets[path]; - Parameter dOpen; - dOpen.name = path; - IOHandler()->enqueue(IOTask(&rc, dOpen)); - try + switch (mpp.determineType(currentPath, path)) { - IOHandler()->flush(internal::defaultFlushParams); - rc.written() = false; - rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - rc.written() = true; - rc.read(); + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path + case internal::ContainedType::Group: { + auto &rc = data.m_embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + data.m_embeddedDatasets.container().erase(path); + } + break; } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read contained custom dataset '" << path - << "' at path '" << myPath().printGroup() - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - data.m_embeddedDatasets.container().erase(path); + case internal::ContainedType::Mesh: + readScalarMesh(meshesMap, path); + break; + case internal::ContainedType::Particle: + std::cerr + << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << " that matches one of the given particle paths. A particle " + "species is always a group, never a dataset. Will skip." + << std::endl; + break; } } @@ -389,8 +407,21 @@ void CustomHierarchy::flush_internal( * to create/open path for contained subpaths. */ + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); if (access::write(IOHandler()->m_frontendAccess)) { + if (!meshes.empty()) + { + (*this)[mpp.requestMeshesPath()]; + } + + if (!particles.empty()) + { + (*this)[mpp.requestParticlesPath()]; + } + flushAttributes(flushParams); } @@ -406,39 +437,23 @@ void CustomHierarchy::flush_internal( subpath.flush_internal(flushParams, mpp, currentPath); currentPath.pop_back(); } - if (!meshes.empty() || mpp.meshesPath.has_value()) + for (auto &[name, mesh] : data.m_embeddedMeshes) { - if (!access::readOnly(IOHandler()->m_frontendAccess)) + if (!mpp.isMesh(currentPath, name)) { - if (!mpp.meshesPath.has_value()) - { - mpp.meshesPath = "meshes"; - } - meshes.flush(mpp.meshesPath.value(), flushParams); + throw std::runtime_error( + "Unimplemented: Extended meshesPath functionality."); } - for (auto &m : meshes) - m.second.flush(m.first, flushParams); + mesh.flush(name, flushParams); } - else + for (auto &[name, particleSpecies] : data.m_embeddedParticles) { - meshes.dirty() = false; - } - if (!particles.empty() || mpp.particlesPath.has_value()) - { - if (!access::readOnly(IOHandler()->m_frontendAccess)) + if (!mpp.isParticle(currentPath, name)) { - if (!mpp.particlesPath.has_value()) - { - mpp.particlesPath = "particles"; - } - particles.flush(mpp.particlesPath.value(), flushParams); + throw std::runtime_error( + "Unimplemented: Extended particlesPath functionality."); } - for (auto &m : particles) - m.second.flush(m.first, flushParams); - } - else - { - particles.dirty() = false; + particleSpecies.flush(name, flushParams); } for (auto &[name, dataset] : get().m_embeddedDatasets) { @@ -487,39 +502,38 @@ bool CustomHierarchy::dirtyRecursive() const { return true; } - if (particles.dirty() || meshes.dirty()) - { - return true; - } - for (auto const &pair : particles) - { - if (pair.second.dirtyRecursive()) - { - return true; - } - } - for (auto const &pair : meshes) - { - if (pair.second.dirtyRecursive()) + auto check = [](auto const &container) { + for (auto const &pair : container) { - return true; - } - } - for (auto const &pair : get().m_embeddedDatasets) - { - if (pair.second.dirtyRecursive()) - { - return true; - } - } - for (auto const &pair : *this) - { - if (pair.second.dirtyRecursive()) - { - return true; + if (pair.second.dirtyRecursive()) + { + return true; + } } - } - return false; + return false; + }; + auto &data = get(); + return check(data.m_embeddedMeshes) || check(data.m_embeddedParticles) || + + /* + * Need to check this, too. It might be that the `meshes` alias has not + * been synced yet with the "meshes" subgroup. + * The CustomHierarchy object needs to be flushed in order for that to + * happen (or the "meshes" group needs to be accessed explicitly via + * operator[]()). + */ + check(meshes) || check(particles) || check(data.m_embeddedDatasets) || + check(*this); +} + +auto CustomHierarchy::operator[](key_type &&key) -> mapped_type & +{ + return bracketOperatorImpl(std::move(key)); +} + +auto CustomHierarchy::operator[](key_type const &key) -> mapped_type & +{ + return bracketOperatorImpl(key); } template @@ -529,6 +543,14 @@ auto CustomHierarchy::asContainerOf() -> Container & { return *static_cast *>(this); } + else if constexpr (std::is_same_v) + { + return get().m_embeddedMeshes; + } + else if constexpr (std::is_same_v) + { + return get().m_embeddedParticles; + } else if constexpr (std::is_same_v) { return get().m_embeddedDatasets; @@ -538,7 +560,8 @@ auto CustomHierarchy::asContainerOf() -> Container & static_assert( auxiliary::dependent_false_v, "[CustomHierarchy::asContainerOf] Type parameter must be " - "one of: CustomHierarchy, RecordComponent."); + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); } } @@ -546,4 +569,108 @@ template auto CustomHierarchy::asContainerOf() -> Container &; template auto CustomHierarchy::asContainerOf() -> Container &; +template auto CustomHierarchy::asContainerOf() -> Container &; +template auto CustomHierarchy::asContainerOf() + -> Container &; + +/* + * This method implements the usual job of ::operator[](), but additionally + * ensures that returned entries are properly linked with ::particles and + * ::meshes. + */ +template +auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) + -> mapped_type & +{ + auto &data = get(); + auto &cont = data.m_container; + auto find_special_key = + [&cont, &provided_key, this]( + std::string const &special_key, + auto &alias, + auto &&embeddedAccessor) -> std::optional { + if (provided_key != special_key) + { + return std::nullopt; + } + if (auto it = cont.find(provided_key); it != cont.end()) + { + if (it->second.m_attri->get() != alias.m_attri->get() || + embeddedAccessor(it->second)->m_containerData.get() != + alias.m_containerData.get()) + { + /* + * This might happen if a user first creates a custom group + * "fields" and sets the default meshes path as "fields" + * only later. + * If the CustomHierarchy::meshes alias carries no data yet, + * we can just redirect it to that group now. + * Otherwise, we need to fail. + */ + if (alias.empty() && alias.attributes().empty()) + { + alias.m_containerData = + embeddedAccessor(it->second)->m_containerData; + alias.m_attri->asSharedPtrOfAttributable() = + it->second.m_attri->asSharedPtrOfAttributable(); + return &it->second; + } + throw error::WrongAPIUsage( + "Found a group '" + provided_key + "' at path '" + + myPath().openPMDPath() + + "' which is not synchronous with mesh/particles alias " + "despite '" + + special_key + + "' being the default meshes/particles path. This can " + "have happened because setting default " + "meshes/particles path too late (after first flush). " + "If that's not the case, this is likely an internal " + "bug."); + } + return &it->second; + } + else + { + auto *res = + &Container::operator[](std::forward(provided_key)); + embeddedAccessor(*res)->m_containerData = alias.m_containerData; + res->m_attri->asSharedPtrOfAttributable() = + alias.m_attri->asSharedPtrOfAttributable(); + res->m_customHierarchyData->syncAttributables(); + return res; + } + }; + auto series = retrieveSeries(); + std::string meshesPath = series.containsAttribute("meshesPath") + ? auxiliary::removeSlashes(series.meshesPath()) + : "meshes"; + if (auto res = find_special_key( + meshesPath, + meshes, + [](auto &group) { + return &group.m_customHierarchyData->m_embeddedMeshes; + }); + res.has_value()) + { + return **res; + } + std::string particlesPath = series.containsAttribute("particlesPath") + ? auxiliary::removeSlashes(series.particlesPath()) + : "particles"; + if (auto res = find_special_key( + particlesPath, + particles, + [](auto &group) { + return &group.m_customHierarchyData->m_embeddedParticles; + }); + res.has_value()) + { + return **res; + } + else + { + return (*this).Container::operator[]( + std::forward(provided_key)); + } +} } // namespace openPMD diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 236db48f2b..8176424ebf 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -219,18 +219,60 @@ TEST_CASE("custom_hierarchies", "[core]") iteration_level_ds.resetDataset({Datatype::INT, {10}}); std::vector data(10, 5); iteration_level_ds.storeChunk(data); + + auto meshesViaAlias = write.iterations[0].meshes; + meshesViaAlias["E"]["x"].makeEmpty(2); + write.setMeshesPath("fields"); + auto meshesManually = + write.iterations[0]["fields"].asContainerOf(); + REQUIRE(meshesManually.contains("E")); + REQUIRE(meshesManually.size() == 1); + meshesManually["B"]["x"].makeEmpty(2); + REQUIRE(meshesViaAlias.contains("B")); + REQUIRE(meshesViaAlias.size() == 2); + + write.setParticlesPath("species"); + auto particlesManually = + write.iterations[0]["species"].asContainerOf(); + particlesManually["e"]["position"]["x"].makeEmpty(1); + auto particlesViaAlias = write.iterations[0].particles; + particlesViaAlias["i"]["position"]["x"].makeEmpty(1); + write.close(); } read = Series(filePath, Access::READ_ONLY); { - REQUIRE(read.iterations[0].size() == 2); + REQUIRE(read.iterations[0].size() == 4); REQUIRE(read.iterations[0].count("custom") == 1); REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0].count("fields") == 1); + REQUIRE(read.iterations[0].count("species") == 1); REQUIRE(read.iterations[0]["custom"].size() == 1); REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE(read.iterations[0]["fields"].asContainerOf().size() == 2); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("E")); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("B")); + REQUIRE(read.iterations[0].meshes.size() == 2); + REQUIRE(read.iterations[0].meshes.contains("E")); + REQUIRE(read.iterations[0].meshes.contains("B")); + REQUIRE( + read.iterations[0]["species"] + .asContainerOf() + .size() == 2); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("e")); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("i")); + REQUIRE(read.iterations[0].particles.size() == 2); + REQUIRE(read.iterations[0].particles.contains("e")); + REQUIRE(read.iterations[0].particles.contains("i")); REQUIRE( read.iterations[0].asContainerOf().size() == 1); From 3bba668b1551951f9378a8a79bc160d2b3e6d34b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 31 Jul 2023 18:08:51 +0200 Subject: [PATCH 13/27] Regex-based list of meshes/particlesPaths --- include/openPMD/CustomHierarchy.hpp | 47 +++-- include/openPMD/Series.hpp | 1 + src/CustomHierarchy.cpp | 272 ++++++++++++++++++++++------ src/Iteration.cpp | 7 +- 4 files changed, 247 insertions(+), 80 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index c47671adc6..b238ff0efe 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -20,13 +20,18 @@ */ #pragma once +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/backend/Container.hpp" #include +#include +#include +#include #include #include +#include namespace openPMD { @@ -41,24 +46,24 @@ namespace internal }; struct MeshesParticlesPath { - std::optional meshesPath; - std::optional particlesPath; - - inline std::string requestMeshesPath() - { - meshesPath = meshesPath.value_or("meshes"); - return *meshesPath; - } - inline std::string requestParticlesPath() - { - particlesPath = particlesPath.value_or("particles"); - return *particlesPath; - } + std::regex meshRegex; + std::set collectNewMeshesPaths; + std::regex particleRegex; + std::set collectNewParticlesPaths; + + /* + * These values decide which path will be returned upon use of the + * shorthand notation s.iterations[0].meshes or .particles. + * + */ + std::string m_defaultMeshesPath = "meshes"; + std::string m_defaultParticlesPath = "particles"; explicit MeshesParticlesPath() = default; MeshesParticlesPath( - std::optional meshesPath, - std::optional particlesPath); + std::vector const &meshes, + std::vector const &particles); + MeshesParticlesPath(Series const &); [[nodiscard]] ContainedType determineType( std::vector const &path, @@ -80,6 +85,16 @@ namespace internal Container m_embeddedDatasets; Container m_embeddedMeshes; Container m_embeddedParticles; + + /* + * Each call to operator[]() needs to check the Series if the meshes/ + * particlesPath has changed, so the Series gets buffered. + * + * Alternative: Require that the meshesPath/particlesPath is fixed as + * soon as operator[]() has been called for the first time, check + * at flush time. + */ + std::unique_ptr m_bufferedSeries; }; } // namespace internal @@ -168,5 +183,7 @@ class CustomHierarchy : public Container private: template mapped_type &bracketOperatorImpl(KeyType &&); + + Series &getBufferedSeries(); }; } // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 9af4271b74..a67f17c106 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -257,6 +257,7 @@ class Series : public Attributable friend class internal::SeriesData; friend class internal::AttributableData; friend class WriteIterations; + friend class CustomHierarchy; public: explicit Series(); diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index f4e06f0497..ed26efdd3c 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/CustomHierarchy.hpp" + #include "openPMD/Dataset.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" @@ -31,14 +32,21 @@ #include "openPMD/Series.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/BaseRecord.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/Writable.hpp" #include +#include +#include #include +#include #include -#include +#include +#include #include +#include +#include namespace { @@ -64,6 +72,87 @@ concatWithSep(std::vector const &v, std::string const &sep) { return concatWithSep(v.begin(), v.end(), sep); } + +// Not specifying std::regex_constants::optimize here, only using it where +// it makes sense to. +constexpr std::regex_constants::syntax_option_type regex_flags = + std::regex_constants::egrep; + +template +void setDefaultMeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles, + OutParam &writeTarget) +{ + std::regex is_default_path_specification("[[:alnum:]_]+/", regex_flags); + constexpr char const *default_default_mesh = "meshes"; + constexpr char const *default_default_particle = "particles"; + for (auto [vec, defaultPath, default_default] : + {std::make_tuple( + &meshes, &writeTarget.m_defaultMeshesPath, default_default_mesh), + std::make_tuple( + &particles, + &writeTarget.m_defaultParticlesPath, + default_default_particle)}) + { + bool set_default = true; + /* + * The first eligible path in meshesPath/particlesPath is used as + * the default, "meshes"/"particles" otherwise. + */ + for (auto const &path : *vec) + { + if (std::regex_match(path, is_default_path_specification)) + { + *defaultPath = openPMD::auxiliary::replace_last(path, "/", ""); + set_default = false; + break; + } + } + if (set_default) + { + *defaultPath = default_default; + } + } +} + +bool anyPathRegexMatches( + std::regex regex, + std::vector const &path, + std::string const &name) +{ + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) '/group/meshes/E' (absolute path to mesh itself) + * 3) 'meshes/' (relative path to mesh container) + * + * The potential fourth option 'E' (relative path to mesh itself) + * is not supported. ("Anything that is named 'E' is a mesh" is not + * really a semantic that we want to explicitly support.) + * '/' is never a valid meshes path. + * + * All this analogously for particles path. + */ + std::vector pathsToMatch = { + /* option 2) from above */ + "/" + (path.empty() ? "" : concatWithSep(path, "/") + "/") + name}; + if (!path.empty()) + { + // option 1) from above + pathsToMatch.emplace_back("/" + concatWithSep(path, "/") + "/"); + + // option 3 from above + pathsToMatch.emplace_back(*path.rbegin() + "/"); + } + return std::any_of( + pathsToMatch.begin(), + pathsToMatch.end(), + [®ex](std::string const &candidate_path) { + return std::regex_match(candidate_path, regex); + }); +} } // namespace namespace openPMD @@ -71,19 +160,34 @@ namespace openPMD namespace internal { MeshesParticlesPath::MeshesParticlesPath( - std::optional meshesPath_in, - std::optional particlesPath_in) - : meshesPath{std::move(meshesPath_in)} - , particlesPath{std::move(particlesPath_in)} + std::vector const &meshes, + std::vector const &particles) { - if (meshesPath.has_value()) + std::regex is_default_path_specification("[[:alnum:]_]+/", regex_flags); + for (auto [target_regex, vec] : + {std::make_tuple(&this->meshRegex, &meshes), + std::make_tuple(&this->particleRegex, &particles)}) { - meshesPath = auxiliary::removeSlashes(*meshesPath); - } - if (particlesPath.has_value()) - { - particlesPath = auxiliary::removeSlashes(*particlesPath); + if (vec->empty()) + { + *target_regex = std::regex( + /* does not match anything */ "a^", + regex_flags | std::regex_constants::optimize); + continue; + } + auto begin = vec->begin(); + std::stringstream build_regex; + build_regex << '(' << *begin++ << ')'; + for (; begin != vec->end(); ++begin) + { + build_regex << "|(" << *begin << ')'; + } + auto regex_string = build_regex.str(); + // std::cout << "Using regex string: " << regex_string << std::endl; + *target_regex = std::regex( + regex_string, regex_flags | std::regex_constants::optimize); } + setDefaultMeshesParticlesPath(meshes, particles, *this); } ContainedType MeshesParticlesPath::determineType( @@ -106,18 +210,12 @@ namespace internal bool MeshesParticlesPath::isParticle( std::vector const &path, std::string const &name) const { - (void)name; - return particlesPath.has_value() && !path.empty() - ? *path.rbegin() == *particlesPath - : false; + return anyPathRegexMatches(particleRegex, path, name); } bool MeshesParticlesPath::isMesh( std::vector const &path, std::string const &name) const { - (void)name; - return meshesPath.has_value() && !path.empty() - ? *path.rbegin() == *meshesPath - : false; + return anyPathRegexMatches(meshRegex, path, name); } CustomHierarchyData::CustomHierarchyData() @@ -332,16 +430,6 @@ void CustomHierarchy::read( } } } - - if (!mpp.meshesPath.has_value()) - { - meshes.dirty() = false; - } - if (!mpp.particlesPath.has_value()) - { - particles.dirty() = false; - } - for (auto const &path : *dList.datasets) { switch (mpp.determineType(currentPath, path)) @@ -414,12 +502,12 @@ void CustomHierarchy::flush_internal( { if (!meshes.empty()) { - (*this)[mpp.requestMeshesPath()]; + (*this)[mpp.m_defaultMeshesPath]; } if (!particles.empty()) { - (*this)[mpp.requestParticlesPath()]; + (*this)[mpp.m_defaultParticlesPath]; } flushAttributes(flushParams); @@ -441,8 +529,24 @@ void CustomHierarchy::flush_internal( { if (!mpp.isMesh(currentPath, name)) { - throw std::runtime_error( - "Unimplemented: Extended meshesPath functionality."); + std::string extend_meshes_path; + // Check if this can be covered by shorthand notation + // (e.g. meshesPath == "meshes/") + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultMeshesPath) + { + extend_meshes_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_meshes_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/") + + name; + } + mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); } mesh.flush(name, flushParams); } @@ -450,8 +554,25 @@ void CustomHierarchy::flush_internal( { if (!mpp.isParticle(currentPath, name)) { - throw std::runtime_error( - "Unimplemented: Extended particlesPath functionality."); + std::string extend_particles_path; + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultParticlesPath) + { + // Check if this can be covered by shorthand notation + // (e.g. particlesPath == "particles/") + extend_particles_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_particles_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/") + + name; + } + mpp.collectNewParticlesPaths.emplace( + std::move(extend_particles_path)); } particleSpecies.flush(name, flushParams); } @@ -470,22 +591,27 @@ void CustomHierarchy::flush( * to create/open path for contained subpaths. */ - Series s = this->retrieveSeries(); - internal::MeshesParticlesPath mpp( - s.containsAttribute("meshesPath") ? std::make_optional(s.meshesPath()) - : std::nullopt, - s.containsAttribute("particlesPath") - ? std::make_optional(s.particlesPath()) - : std::nullopt); + Series s = this->getBufferedSeries(); + std::vector meshesPaths = s.meshesPaths(), + particlesPaths = s.particlesPaths(); + internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); std::vector currentPath; flush_internal(flushParams, mpp, currentPath); - if (mpp.meshesPath.has_value() && !s.containsAttribute("meshesPath")) - { - s.setMeshesPath(*mpp.meshesPath); - } - if (mpp.particlesPath.has_value() && !s.containsAttribute("particlesPath")) + if (!mpp.collectNewMeshesPaths.empty() || + !mpp.collectNewParticlesPaths.empty()) { - s.setParticlesPath(*mpp.particlesPath); + for (auto [newly_added_paths, vec] : + {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), + std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) + { + std::transform( + newly_added_paths->begin(), + newly_added_paths->end(), + std::back_inserter(*vec), + [](auto const &pair) { return pair; }); + } + s.setMeshesPath(meshesPaths); + s.setParticlesPath(particlesPaths); } } @@ -582,8 +708,7 @@ template auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) -> mapped_type & { - auto &data = get(); - auto &cont = data.m_container; + auto &cont = container(); auto find_special_key = [&cont, &provided_key, this]( std::string const &special_key, @@ -640,12 +765,27 @@ auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) return res; } }; - auto series = retrieveSeries(); - std::string meshesPath = series.containsAttribute("meshesPath") - ? auxiliary::removeSlashes(series.meshesPath()) - : "meshes"; + + /* + * @todo Buffer this somehow while still ensuring that changed meshesPath + * or particlesPath will be recorded. + */ + struct + { + std::string m_defaultMeshesPath; + std::string m_defaultParticlesPath; + } defaultPaths; + + { + auto const &series = getBufferedSeries(); + auto meshes_paths = series.meshesPaths(); + auto particles_paths = series.particlesPaths(); + setDefaultMeshesParticlesPath( + meshes_paths, particles_paths, defaultPaths); + } + if (auto res = find_special_key( - meshesPath, + defaultPaths.m_defaultMeshesPath, meshes, [](auto &group) { return &group.m_customHierarchyData->m_embeddedMeshes; @@ -654,11 +794,8 @@ auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) { return **res; } - std::string particlesPath = series.containsAttribute("particlesPath") - ? auxiliary::removeSlashes(series.particlesPath()) - : "particles"; if (auto res = find_special_key( - particlesPath, + defaultPaths.m_defaultParticlesPath, particles, [](auto &group) { return &group.m_customHierarchyData->m_embeddedParticles; @@ -673,4 +810,21 @@ auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) std::forward(provided_key)); } } + +Series &CustomHierarchy::getBufferedSeries() +{ + auto &data = get(); + if (!data.m_bufferedSeries) + { + /* + * retrieveSeries() returns a non-owning Series handle anyway, but let's + * be explicit here that we need a non-owning Series to avoid creating + * a memory cycle. + */ + data.m_bufferedSeries = std::make_unique(); + data.m_bufferedSeries->setData(std::shared_ptr( + &retrieveSeries().get(), [](auto const *) {})); + } + return *data.m_bufferedSeries; +} } // namespace openPMD diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 75377573d2..9fd4279795 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -461,12 +461,7 @@ void Iteration::read_impl(std::string const &groupPath) // @todo restore compatibility with openPMD 1.0.*: // hasMeshes <-> meshesPath is defined - internal::MeshesParticlesPath mpp( - s.containsAttribute("meshesPath") ? std::make_optional(s.meshesPath()) - : std::nullopt, - s.containsAttribute("particlesPath") - ? std::make_optional(s.particlesPath()) - : std::nullopt); + internal::MeshesParticlesPath mpp(s.meshesPaths(), s.particlesPaths()); CustomHierarchy::read(std::move(mpp)); #ifdef openPMD_USE_INVASIVE_TESTS From 5d191df51342ed3330abace9e55620cde35eb898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 31 Jul 2023 18:22:57 +0200 Subject: [PATCH 14/27] More extended testing --- test/CoreTest.cpp | 85 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 8176424ebf..2e3f28fda3 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -222,7 +222,7 @@ TEST_CASE("custom_hierarchies", "[core]") auto meshesViaAlias = write.iterations[0].meshes; meshesViaAlias["E"]["x"].makeEmpty(2); - write.setMeshesPath("fields"); + write.setMeshesPath(std::vector{"fields/", ".*/meshes/"}); auto meshesManually = write.iterations[0]["fields"].asContainerOf(); REQUIRE(meshesManually.contains("E")); @@ -307,6 +307,89 @@ TEST_CASE("custom_hierarchies", "[core]") REQUIRE(constant_dataset.getDatatype() == Datatype::FLOAT); REQUIRE(constant_dataset.getExtent() == Extent{0, 0, 0}); } + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"].meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = write.iterations[0]["custom_particles"] + .particles["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + auto it0 = read.iterations[0]; + auto custom_meshes = it0["custom_meshes"]; + REQUIRE(custom_meshes.meshes.size() == 1); + REQUIRE(read.iterations[0]["custom_meshes"].meshes.count("E") == 1); + auto E_x_loaded = read.iterations[0]["custom_meshes"] + .meshes["E"]["x"] + .loadChunk(); + REQUIRE(read.iterations[0]["custom_particles"].particles.size() == 1); + REQUIRE( + read.iterations[0]["custom_particles"].particles.count("e") == 1); + auto e_pos_x_loaded = read.iterations[0]["custom_particles"] + .particles["e"]["position"]["x"] + .loadChunk(); + read.flush(); + + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(E_x_loaded.get()[i] == 3); + REQUIRE(e_pos_x_loaded.get()[i] == 3); + } + } +} + +TEST_CASE("custom_hierarchies_no_rw", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies_no_rw.json"; + Series write(filePath, Access::CREATE); + write.setMeshesPath(std::vector{".*/meshes/"}); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + write.flush(); + } + + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"].meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = write.iterations[0]["custom_particles"] + .particles["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + write.close(); + } + + Series read(filePath, Access::READ_ONLY); } TEST_CASE("myPath", "[core]") From 700043aa8f55f4444474091ec83015693ab13cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Aug 2023 15:10:41 +0200 Subject: [PATCH 15/27] Fix Python bindings without adding new functionality yet Overload resolution --- src/binding/python/Series.cpp | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 59068c2728..36a878f03b 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -283,11 +283,25 @@ this method. &Series::setOpenPMDextension) .def_property("base_path", &Series::basePath, &Series::setBasePath) .def_property( - "meshes_path", &Series::meshesPath, &Series::setMeshesPath) + "meshes_path", + &Series::meshesPath, + py::overload_cast(&Series::setMeshesPath)) .def("get_rank_table", &Series::rankTable, py::arg("collective")) .def("set_rank_table", &Series::setRankTable, py::arg("my_rank_info")) .def_property( - "particles_path", &Series::particlesPath, &Series::setParticlesPath) + "particles_path", + &Series::particlesPath, + py::overload_cast(&Series::setParticlesPath)) + .def_property( + "meshes_paths", + &Series::meshesPath, + py::overload_cast const &>( + &Series::setMeshesPath)) + .def_property( + "particles_paths", + &Series::particlesPath, + py::overload_cast const &>( + &Series::setParticlesPath)) .def_property("author", &Series::author, &Series::setAuthor) .def_property( "machine", @@ -330,8 +344,20 @@ this method. .def("set_openPMD", &Series::setOpenPMD) .def("set_openPMD_extension", &Series::setOpenPMDextension) .def("set_base_path", &Series::setBasePath) - .def("set_meshes_path", &Series::setMeshesPath) - .def("set_particles_path", &Series::setParticlesPath) + .def( + "set_meshes_path", + py::overload_cast(&Series::setMeshesPath)) + .def( + "set_meshes_path", + py::overload_cast const &>( + &Series::setMeshesPath)) + .def( + "set_particles_path", + py::overload_cast const &>( + &Series::setParticlesPath)) + .def( + "set_particles_path", + py::overload_cast(&Series::setParticlesPath)) .def("set_author", &Series::setAuthor) .def("set_date", &Series::setDate) .def("set_iteration_encoding", &Series::setIterationEncoding) From 46ef87cf16ae1b95b9a050294ef3fc0ee8e5dca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Aug 2023 15:25:25 +0200 Subject: [PATCH 16/27] Add simple Python bindings and an example --- CMakeLists.txt | 2 + examples/14_custom_hierarchy.py | 48 +++++++++++++++ include/openPMD/binding/python/Common.hpp | 3 + src/binding/python/Container.cpp | 71 +++++++++++++++++++++++ src/binding/python/CustomHierarchy.cpp | 42 ++++++++++++++ src/binding/python/Iteration.cpp | 8 ++- src/binding/python/openPMD.cpp | 2 + 7 files changed, 174 insertions(+), 2 deletions(-) create mode 100755 examples/14_custom_hierarchy.py create mode 100644 src/binding/python/Container.cpp create mode 100644 src/binding/python/CustomHierarchy.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3993f7205d..4dc6cce472 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -543,6 +543,7 @@ if(openPMD_HAVE_PYTHON) src/binding/python/Attributable.cpp src/binding/python/BaseRecordComponent.cpp src/binding/python/ChunkInfo.cpp + src/binding/python/CustomHierarchy.cpp src/binding/python/Dataset.cpp src/binding/python/Datatype.cpp src/binding/python/Error.cpp @@ -708,6 +709,7 @@ set(openPMD_PYTHON_EXAMPLE_NAMES 11_particle_dataframe 12_span_write 13_write_dynamic_configuration + 14_custom_hierarchy ) if(openPMD_USE_INVASIVE_TESTS) diff --git a/examples/14_custom_hierarchy.py b/examples/14_custom_hierarchy.py new file mode 100755 index 0000000000..b3eff208a9 --- /dev/null +++ b/examples/14_custom_hierarchy.py @@ -0,0 +1,48 @@ +import numpy as np +import openpmd_api as io + + +def main(): + if "bp" in io.file_extensions: + filename = "../samples/custom_hierarchy.bp" + else: + filename = "../samples/custom_hierarchy.json" + s = io.Series(filename, io.Access.create) + it = s.write_iterations()[100] + + # write openPMD part + temp = it.meshes["temperature"] + temp.axis_labels = ["x", "y"] + temp.unit_dimension = {io.Unit_Dimension.T: 1} + temp.position = [0.5, 0.5] + temp.grid_spacing = [1, 1] + temp.grid_global_offset = [0, 0] + temp.reset_dataset(io.Dataset(np.dtype("double"), [5, 5])) + temp[()] = np.zeros((5, 5)) + + # write NeXus part + nxentry = it["Scan"] + nxentry.set_attribute("NX_class", "NXentry") + nxentry.set_attribute("default", "data") + + data = nxentry["data"] + data.set_attribute("NX_class", "NXdata") + data.set_attribute("signal", "counts") + data.set_attribute("axes", ["two_theta"]) + data.set_attribute("two_theta_indices", [0]) + + counts = data.as_container_of_datasets()["counts"] + counts.set_attribute("units", "counts") + counts.set_attribute("long_name", "photodiode counts") + counts.reset_dataset(io.Dataset(np.dtype("int"), [15])) + counts[()] = np.zeros(15, dtype=np.dtype("int")) + + two_theta = data.as_container_of_datasets()["two_theta"] + two_theta.set_attribute("units", "degrees") + two_theta.set_attribute("long_name", "two_theta (degrees)") + two_theta.reset_dataset(io.Dataset(np.dtype("double"), [15])) + two_theta[()] = np.zeros(15) + + +if __name__ == "__main__": + main() diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp index 7b42b919d8..a27358837b 100644 --- a/include/openPMD/binding/python/Common.hpp +++ b/include/openPMD/binding/python/Common.hpp @@ -8,6 +8,7 @@ */ #pragma once +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -46,6 +47,7 @@ using PyPatchRecordComponentContainer = Container; using PyBaseRecordRecordComponent = BaseRecord; using PyBaseRecordMeshRecordComponent = BaseRecord; using PyBaseRecordPatchRecordComponent = BaseRecord; +using PyCustomHierarchyContainer = Container; PYBIND11_MAKE_OPAQUE(PyIterationContainer) PYBIND11_MAKE_OPAQUE(PyMeshContainer) PYBIND11_MAKE_OPAQUE(PyPartContainer) @@ -57,3 +59,4 @@ PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyBaseRecordRecordComponent) PYBIND11_MAKE_OPAQUE(PyBaseRecordPatchRecordComponent) +PYBIND11_MAKE_OPAQUE(PyCustomHierarchyContainer) diff --git a/src/binding/python/Container.cpp b/src/binding/python/Container.cpp new file mode 100644 index 0000000000..d49185e57d --- /dev/null +++ b/src/binding/python/Container.cpp @@ -0,0 +1,71 @@ +/* Copyright 2018-2021 Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + * + * The function `bind_container` is based on std_bind.h in pybind11 + * Copyright (c) 2016 Sergey Lyskov and Wenzel Jakob + * + * BSD-style license, see pybind11 LICENSE file. + */ + +#include + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" +#include "openPMD/binding/python/Container.hpp" + +#include "openPMD/binding/python/Common.hpp" + +void init_Container(py::module &m) +{ + ::detail::create_and_bind_container( + m, "Iteration_Container"); + ::detail::create_and_bind_container( + m, "Mesh_Container"); + ::detail::create_and_bind_container( + m, "Particle_Container"); + ::detail::create_and_bind_container( + m, "Particle_Patches_Container"); + ::detail::create_and_bind_container( + m, "Record_Container"); + ::detail::create_and_bind_container( + m, "Patch_Record_Container"); + ::detail:: + create_and_bind_container( + m, "Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Mesh_Record_Component_Container"); + ::detail::create_and_bind_container< + PyPatchRecordComponentContainer, + Attributable>(m, "Patch_Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Custom_Hierarchy_Container"); +} diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp new file mode 100644 index 0000000000..3d7ac651b8 --- /dev/null +++ b/src/binding/python/CustomHierarchy.cpp @@ -0,0 +1,42 @@ + + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Container.H" +#include + +namespace py = pybind11; +using namespace openPMD; + +void init_CustomHierarchy(py::module &m) +{ + auto py_ch_cont = declare_container(m, "Container_CustomHierarchy"); + + py::class_, Attributable>( + m, "CustomHierarchy") + .def( + "as_container_of_datasets", + &CustomHierarchy::asContainerOf) + .def("as_container_of_meshes", &CustomHierarchy::asContainerOf) + .def( + "as_container_of_particles", + &CustomHierarchy::asContainerOf) + + .def_readwrite( + "meshes", + &CustomHierarchy::meshes, + py::return_value_policy::copy, + // garbage collection: return value must be freed before Iteration + py::keep_alive<1, 0>()) + .def_readwrite( + "particles", + &CustomHierarchy::particles, + py::return_value_policy::copy, + // garbage collection: return value must be freed before Iteration + py::keep_alive<1, 0>()); + + finalize_container(py_ch_cont); +} diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index cd5fecacb0..4c92062e61 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/binding/python/Common.hpp" @@ -40,8 +41,11 @@ void init_Iteration(py::module &m) #define OPENPMD_AVOID_CLANG_FORMAT auto cl = OPENPMD_AVOID_CLANG_FORMAT #undef OPENPMD_AVOID_CLANG_FORMAT - - py::class_(m, "Iteration") + py::class_< + Iteration, + CustomHierarchy, + PyCustomHierarchyContainer, + Attributable>(m, "Iteration") .def(py::init()) .def( diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index fe26bfced8..4e2b74b892 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -37,6 +37,7 @@ void init_Dataset(py::module &); void init_Datatype(py::module &); void init_Error(py::module &); void init_Helper(py::module &); +void init_CustomHierarchy(py::module &); void init_Iteration(py::module &); void init_IterationEncoding(py::module &); void init_Mesh(py::module &); @@ -90,6 +91,7 @@ PYBIND11_MODULE(openpmd_api_cxx, m) init_Datatype(m); init_Dataset(m); + init_CustomHierarchy(m); init_BaseRecordComponent(m); init_RecordComponent(m); init_MeshRecordComponent(m); From acabbe3564b85f4d8e5ad9274773d4b49be5afda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Oct 2023 14:40:40 +0200 Subject: [PATCH 17/27] Replace Regexes with Globbing TODO: Since meshes/particles can no longer be directly addressed with this, maybe adapt the class hierarchy to disallow mixed groups that contain meshes, particles, groups and datasets at the same time. Only maybe though.. --- include/openPMD/CustomHierarchy.hpp | 15 +-- src/CustomHierarchy.cpp | 164 +++++++++++++++++----------- test/CoreTest.cpp | 4 +- 3 files changed, 108 insertions(+), 75 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index b238ff0efe..10ad959b13 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -65,15 +65,12 @@ namespace internal std::vector const &particles); MeshesParticlesPath(Series const &); - [[nodiscard]] ContainedType determineType( - std::vector const &path, - std::string const &name) const; - [[nodiscard]] bool isParticle( - std::vector const &path, - std::string const &name) const; - [[nodiscard]] bool isMesh( - std::vector const &path, - std::string const &name) const; + [[nodiscard]] ContainedType + determineType(std::vector const &path) const; + [[nodiscard]] bool + isParticleContainer(std::vector const &path) const; + [[nodiscard]] bool + isMeshContainer(std::vector const &path) const; }; struct CustomHierarchyData : ContainerData diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index ed26efdd3c..cf388a23ab 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -43,11 +43,18 @@ #include #include #include +#include #include #include #include #include +// @todo add handselected choice of [:punct:] characters to this +// using a macro here to make string interpolation simpler +#define OPENPMD_LEGAL_IDENTIFIER_CHARS "[:alnum:]_" +#define OPENPMD_SINGLE_GLOBBING_CHAR "%" +#define OPENPMD_DOUBLE_GLOBBING_CHAR "%%" + namespace { template @@ -84,7 +91,9 @@ void setDefaultMeshesParticlesPath( std::vector const &particles, OutParam &writeTarget) { - std::regex is_default_path_specification("[[:alnum:]_]+/", regex_flags); + std::regex is_default_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); constexpr char const *default_default_mesh = "meshes"; constexpr char const *default_default_particle = "particles"; for (auto [vec, defaultPath, default_default] : @@ -117,41 +126,10 @@ void setDefaultMeshesParticlesPath( } bool anyPathRegexMatches( - std::regex regex, - std::vector const &path, - std::string const &name) + std::regex const ®ex, std::vector const &path) { - /* - * /group/meshes/E is a mesh if the meshes path contains: - * - * 1) '/group/meshes/' (absolute path to mesh container) - * 2) '/group/meshes/E' (absolute path to mesh itself) - * 3) 'meshes/' (relative path to mesh container) - * - * The potential fourth option 'E' (relative path to mesh itself) - * is not supported. ("Anything that is named 'E' is a mesh" is not - * really a semantic that we want to explicitly support.) - * '/' is never a valid meshes path. - * - * All this analogously for particles path. - */ - std::vector pathsToMatch = { - /* option 2) from above */ - "/" + (path.empty() ? "" : concatWithSep(path, "/") + "/") + name}; - if (!path.empty()) - { - // option 1) from above - pathsToMatch.emplace_back("/" + concatWithSep(path, "/") + "/"); - - // option 3 from above - pathsToMatch.emplace_back(*path.rbegin() + "/"); - } - return std::any_of( - pathsToMatch.begin(), - pathsToMatch.end(), - [®ex](std::string const &candidate_path) { - return std::regex_match(candidate_path, regex); - }); + std::string pathToMatch = '/' + concatWithSep(path, "/") + '/'; + return std::regex_match(pathToMatch, regex); } } // namespace @@ -159,28 +137,83 @@ namespace openPMD { namespace internal { + namespace + { + std::string globToRegexLongForm(std::string const &glob) + { + return auxiliary::replace_all( + auxiliary::replace_all( + glob, + OPENPMD_DOUBLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*)"), + OPENPMD_SINGLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "]*)"); + } + + std::string globToRegexShortForm(std::string const &glob) + { + return "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*/" + glob; + } + } // namespace + MeshesParticlesPath::MeshesParticlesPath( std::vector const &meshes, std::vector const &particles) { - std::regex is_default_path_specification("[[:alnum:]_]+/", regex_flags); + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) 'meshes/' (relative path to mesh container) + * + * All this analogously for particles path. + */ + + // regex for detecting option 1) + // e.g. '/path/to/meshes/': The path to the meshes. Mandatory slashes at + // beginning and end, possibly slashes in + // between. Mandatory slash at beginning might + // be replaced with '%%' to enable paths like + // '%%/path/to/meshes'. + // resolves to: `(/|%%)[[:alnum:]_%/]+/` + std::regex is_legal_long_path_specification( + "(/|" OPENPMD_DOUBLE_GLOBBING_CHAR + ")[" OPENPMD_LEGAL_IDENTIFIER_CHARS OPENPMD_SINGLE_GLOBBING_CHAR + "/]+/", + regex_flags | std::regex_constants::optimize); + + // Regex for detecting option 2) + // e.g. 'meshes/': The name without path. One single mandatory slash + // at the end, no slashes otherwise. + // resolves to `[[:alnum:]_]+/` + std::regex is_legal_short_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + for (auto [target_regex, vec] : {std::make_tuple(&this->meshRegex, &meshes), std::make_tuple(&this->particleRegex, &particles)}) { - if (vec->empty()) - { - *target_regex = std::regex( - /* does not match anything */ "a^", - regex_flags | std::regex_constants::optimize); - continue; - } - auto begin = vec->begin(); std::stringstream build_regex; - build_regex << '(' << *begin++ << ')'; - for (; begin != vec->end(); ++begin) + // neutral element: empty language, regex doesn't match anything + build_regex << "(a^)"; + for (auto const &entry : *vec) { - build_regex << "|(" << *begin << ')'; + if (std::regex_match(entry, is_legal_short_path_specification)) + { + build_regex << "|(" << globToRegexShortForm(entry) << ')'; + } + else if (std::regex_match( + entry, is_legal_long_path_specification)) + { + build_regex << "|(" << globToRegexLongForm(entry) << ')'; + } + else + { + std::cerr + << "[WARNING] Not a legal meshes-/particles-path: '" + << entry << "'. Will skip." << std::endl; + } } auto regex_string = build_regex.str(); // std::cout << "Using regex string: " << regex_string << std::endl; @@ -191,13 +224,13 @@ namespace internal } ContainedType MeshesParticlesPath::determineType( - std::vector const &path, std::string const &name) const + std::vector const &path) const { - if (isMesh(path, name)) + if (isMeshContainer(path)) { return ContainedType::Mesh; } - else if (isParticle(path, name)) + else if (isParticleContainer(path)) { return ContainedType::Particle; } @@ -207,15 +240,15 @@ namespace internal } } - bool MeshesParticlesPath::isParticle( - std::vector const &path, std::string const &name) const + bool MeshesParticlesPath::isParticleContainer( + std::vector const &path) const { - return anyPathRegexMatches(particleRegex, path, name); + return anyPathRegexMatches(particleRegex, path); } - bool MeshesParticlesPath::isMesh( - std::vector const &path, std::string const &name) const + bool MeshesParticlesPath::isMeshContainer( + std::vector const &path) const { - return anyPathRegexMatches(meshRegex, path, name); + return anyPathRegexMatches(meshRegex, path); } CustomHierarchyData::CustomHierarchyData() @@ -366,7 +399,7 @@ void CustomHierarchy::read( EraseStaleParticles particlesMap(data.m_embeddedParticles); for (auto const &path : *pList.paths) { - switch (mpp.determineType(currentPath, path)) + switch (mpp.determineType(currentPath)) { case internal::ContainedType::Group: { Parameter pOpen; @@ -432,7 +465,7 @@ void CustomHierarchy::read( } for (auto const &path : *dList.datasets) { - switch (mpp.determineType(currentPath, path)) + switch (mpp.determineType(currentPath)) { // Group is a bit of an internal misnomer here, it just means that // it matches neither meshes nor particles path @@ -527,7 +560,7 @@ void CustomHierarchy::flush_internal( } for (auto &[name, mesh] : data.m_embeddedMeshes) { - if (!mpp.isMesh(currentPath, name)) + if (!mpp.isMeshContainer(currentPath)) { std::string extend_meshes_path; // Check if this can be covered by shorthand notation @@ -543,8 +576,7 @@ void CustomHierarchy::flush_internal( extend_meshes_path = "/" + (currentPath.empty() ? "" - : concatWithSep(currentPath, "/") + "/") + - name; + : concatWithSep(currentPath, "/") + "/"); } mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); } @@ -552,7 +584,7 @@ void CustomHierarchy::flush_internal( } for (auto &[name, particleSpecies] : data.m_embeddedParticles) { - if (!mpp.isParticle(currentPath, name)) + if (!mpp.isParticleContainer(currentPath)) { std::string extend_particles_path; if (!currentPath.empty() && @@ -568,8 +600,8 @@ void CustomHierarchy::flush_internal( extend_particles_path = "/" + (currentPath.empty() ? "" - : concatWithSep(currentPath, "/") + "/") + - name; + : concatWithSep(currentPath, "/") + "/"); + ; } mpp.collectNewParticlesPaths.emplace( std::move(extend_particles_path)); @@ -828,3 +860,7 @@ Series &CustomHierarchy::getBufferedSeries() return *data.m_bufferedSeries; } } // namespace openPMD + +#undef OPENPMD_LEGAL_IDENTIFIER_CHARS +#undef OPENPMD_SINGLE_GLOBBING_CHAR +#undef OPENPMD_DOUBLE_GLOBBING_CHAR diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 2e3f28fda3..10075451e7 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -222,7 +222,7 @@ TEST_CASE("custom_hierarchies", "[core]") auto meshesViaAlias = write.iterations[0].meshes; meshesViaAlias["E"]["x"].makeEmpty(2); - write.setMeshesPath(std::vector{"fields/", ".*/meshes/"}); + write.setMeshesPath(std::vector{"fields/", "%%/meshes/"}); auto meshesManually = write.iterations[0]["fields"].asContainerOf(); REQUIRE(meshesManually.contains("E")); @@ -353,7 +353,7 @@ TEST_CASE("custom_hierarchies_no_rw", "[core]") { std::string filePath = "../samples/custom_hierarchies_no_rw.json"; Series write(filePath, Access::CREATE); - write.setMeshesPath(std::vector{".*/meshes/"}); + write.setMeshesPath(std::vector{"%%/meshes/"}); write.iterations[0]["custom"]["hierarchy"]; write.iterations[0]["custom"].setAttribute("string", "attribute"); write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); From 8a084ca31969eb31aa70ef1732452ca7a782e286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 24 Oct 2023 17:40:00 +0200 Subject: [PATCH 18/27] Move .meshes and .particles back to Iteration class The have their own meaning now and are no longer just carefully maintained for backwards compatibility. Instead, they are supposed to serve as a shortcut to all openPMD data found further down the hierarchy. --- include/openPMD/CustomHierarchy.hpp | 23 +-- include/openPMD/Iteration.hpp | 32 ++++ src/CustomHierarchy.cpp | 198 +------------------------ src/Iteration.cpp | 143 +++++++++++++++++- src/binding/python/CustomHierarchy.cpp | 19 +-- test/CoreTest.cpp | 58 +++++--- test/SerialIOTest.cpp | 24 ++- 7 files changed, 243 insertions(+), 254 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 10ad959b13..fb3402cf23 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -82,16 +83,6 @@ namespace internal Container m_embeddedDatasets; Container m_embeddedMeshes; Container m_embeddedParticles; - - /* - * Each call to operator[]() needs to check the Series if the meshes/ - * particlesPath has changed, so the Series gets buffered. - * - * Alternative: Require that the meshesPath/particlesPath is fixed as - * soon as operator[]() has been called for the first time, check - * at flush time. - */ - std::unique_ptr m_bufferedSeries; }; } // namespace internal @@ -168,19 +159,7 @@ class CustomHierarchy : public Container CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; - mapped_type &operator[](key_type &&key); - mapped_type &operator[](key_type const &key); - template auto asContainerOf() -> Container &; - - Container meshes{}; - Container particles{}; - -private: - template - mapped_type &bracketOperatorImpl(KeyType &&); - - Series &getBufferedSeries(); }; } // namespace openPMD diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 99f2021e06..e50adf9cd8 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -149,7 +149,16 @@ class Iteration : public CustomHierarchy friend class Writable; Iteration(Iteration const &) = default; + Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; + Iteration &operator=(Iteration &&) = default; + + // These use the openPMD Container class mainly for consistency. + // But they are in fact only aliases that don't actually exist + // in the backend. + // Hence meshes.written() and particles.written() will always be false. + Container meshes{}; + Container particles{}; /** * @tparam T Floating point type of user-selected precision (e.g. float, @@ -289,6 +298,12 @@ class Iteration : public CustomHierarchy * class. */ void flushIteration(internal::FlushParams const &); + + void sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &); + void sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &); + void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for runDeferredParseAccess(), readFileBased(), @@ -399,6 +414,23 @@ class Iteration : public CustomHierarchy */ void setStepStatus(StepStatus); + /* + * @brief Check recursively whether this Iteration is dirty. + * It is dirty if any attribute or dataset is read from or written to + * the backend. + * + * @return true If dirty. + * @return false Otherwise. + */ + bool dirtyRecursive() const; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w); + /** * @brief Access an iteration in read mode that has potentially not been * parsed yet. diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index cf388a23ab..c46e9c3639 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -274,8 +275,6 @@ namespace internal CustomHierarchy::CustomHierarchy() { setData(std::make_shared()); - meshes.writable().ownKeyWithinParent = "meshes"; - particles.writable().ownKeyWithinParent = "particles"; } CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit()) {} @@ -533,16 +532,6 @@ void CustomHierarchy::flush_internal( auto &data = get(); if (access::write(IOHandler()->m_frontendAccess)) { - if (!meshes.empty()) - { - (*this)[mpp.m_defaultMeshesPath]; - } - - if (!particles.empty()) - { - (*this)[mpp.m_defaultParticlesPath]; - } - flushAttributes(flushParams); } @@ -615,43 +604,16 @@ void CustomHierarchy::flush_internal( } void CustomHierarchy::flush( - std::string const & /* path */, internal::FlushParams const &flushParams) + std::string const & /* path */, internal::FlushParams const &) { - /* - * Convention for CustomHierarchy::flush and CustomHierarchy::read: - * Path is created/opened already at entry point of method, method needs - * to create/open path for contained subpaths. - */ - - Series s = this->getBufferedSeries(); - std::vector meshesPaths = s.meshesPaths(), - particlesPaths = s.particlesPaths(); - internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); - std::vector currentPath; - flush_internal(flushParams, mpp, currentPath); - if (!mpp.collectNewMeshesPaths.empty() || - !mpp.collectNewParticlesPaths.empty()) - { - for (auto [newly_added_paths, vec] : - {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), - std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) - { - std::transform( - newly_added_paths->begin(), - newly_added_paths->end(), - std::back_inserter(*vec), - [](auto const &pair) { return pair; }); - } - s.setMeshesPath(meshesPaths); - s.setParticlesPath(particlesPaths); - } + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); } void CustomHierarchy::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); - meshes.linkHierarchy(this->writable()); - particles.linkHierarchy(this->writable()); } bool CustomHierarchy::dirtyRecursive() const @@ -672,26 +634,7 @@ bool CustomHierarchy::dirtyRecursive() const }; auto &data = get(); return check(data.m_embeddedMeshes) || check(data.m_embeddedParticles) || - - /* - * Need to check this, too. It might be that the `meshes` alias has not - * been synced yet with the "meshes" subgroup. - * The CustomHierarchy object needs to be flushed in order for that to - * happen (or the "meshes" group needs to be accessed explicitly via - * operator[]()). - */ - check(meshes) || check(particles) || check(data.m_embeddedDatasets) || - check(*this); -} - -auto CustomHierarchy::operator[](key_type &&key) -> mapped_type & -{ - return bracketOperatorImpl(std::move(key)); -} - -auto CustomHierarchy::operator[](key_type const &key) -> mapped_type & -{ - return bracketOperatorImpl(key); + check(data.m_embeddedDatasets) || check(*this); } template @@ -730,135 +673,6 @@ template auto CustomHierarchy::asContainerOf() template auto CustomHierarchy::asContainerOf() -> Container &; template auto CustomHierarchy::asContainerOf() -> Container &; - -/* - * This method implements the usual job of ::operator[](), but additionally - * ensures that returned entries are properly linked with ::particles and - * ::meshes. - */ -template -auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) - -> mapped_type & -{ - auto &cont = container(); - auto find_special_key = - [&cont, &provided_key, this]( - std::string const &special_key, - auto &alias, - auto &&embeddedAccessor) -> std::optional { - if (provided_key != special_key) - { - return std::nullopt; - } - if (auto it = cont.find(provided_key); it != cont.end()) - { - if (it->second.m_attri->get() != alias.m_attri->get() || - embeddedAccessor(it->second)->m_containerData.get() != - alias.m_containerData.get()) - { - /* - * This might happen if a user first creates a custom group - * "fields" and sets the default meshes path as "fields" - * only later. - * If the CustomHierarchy::meshes alias carries no data yet, - * we can just redirect it to that group now. - * Otherwise, we need to fail. - */ - if (alias.empty() && alias.attributes().empty()) - { - alias.m_containerData = - embeddedAccessor(it->second)->m_containerData; - alias.m_attri->asSharedPtrOfAttributable() = - it->second.m_attri->asSharedPtrOfAttributable(); - return &it->second; - } - throw error::WrongAPIUsage( - "Found a group '" + provided_key + "' at path '" + - myPath().openPMDPath() + - "' which is not synchronous with mesh/particles alias " - "despite '" + - special_key + - "' being the default meshes/particles path. This can " - "have happened because setting default " - "meshes/particles path too late (after first flush). " - "If that's not the case, this is likely an internal " - "bug."); - } - return &it->second; - } - else - { - auto *res = - &Container::operator[](std::forward(provided_key)); - embeddedAccessor(*res)->m_containerData = alias.m_containerData; - res->m_attri->asSharedPtrOfAttributable() = - alias.m_attri->asSharedPtrOfAttributable(); - res->m_customHierarchyData->syncAttributables(); - return res; - } - }; - - /* - * @todo Buffer this somehow while still ensuring that changed meshesPath - * or particlesPath will be recorded. - */ - struct - { - std::string m_defaultMeshesPath; - std::string m_defaultParticlesPath; - } defaultPaths; - - { - auto const &series = getBufferedSeries(); - auto meshes_paths = series.meshesPaths(); - auto particles_paths = series.particlesPaths(); - setDefaultMeshesParticlesPath( - meshes_paths, particles_paths, defaultPaths); - } - - if (auto res = find_special_key( - defaultPaths.m_defaultMeshesPath, - meshes, - [](auto &group) { - return &group.m_customHierarchyData->m_embeddedMeshes; - }); - res.has_value()) - { - return **res; - } - if (auto res = find_special_key( - defaultPaths.m_defaultParticlesPath, - particles, - [](auto &group) { - return &group.m_customHierarchyData->m_embeddedParticles; - }); - res.has_value()) - { - return **res; - } - else - { - return (*this).Container::operator[]( - std::forward(provided_key)); - } -} - -Series &CustomHierarchy::getBufferedSeries() -{ - auto &data = get(); - if (!data.m_bufferedSeries) - { - /* - * retrieveSeries() returns a non-owning Series handle anyway, but let's - * be explicit here that we need a non-owning Series to avoid creating - * a memory cycle. - */ - data.m_bufferedSeries = std::make_unique(); - data.m_bufferedSeries->setData(std::shared_ptr( - &retrieveSeries().get(), [](auto const *) {})); - } - return *data.m_bufferedSeries; -} } // namespace openPMD #undef OPENPMD_LEGAL_IDENTIFIER_CHARS diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 9fd4279795..211213c622 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -33,7 +33,10 @@ #include #include +#include #include +#include +#include namespace openPMD { @@ -325,7 +328,42 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) { return; } - CustomHierarchy::flush("", flushParams); + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Series s = retrieveSeries(); + std::vector meshesPaths = s.meshesPaths(), + particlesPaths = s.particlesPaths(); + internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); + + sync_meshes_and_particles_from_alias_to_subgroups(mpp); + + std::vector currentPath; + CustomHierarchy::flush_internal(flushParams, mpp, currentPath); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); + + if (!mpp.collectNewMeshesPaths.empty() || + !mpp.collectNewParticlesPaths.empty()) + { + for (auto [newly_added_paths, vec] : + {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), + std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) + { + std::transform( + newly_added_paths->begin(), + newly_added_paths->end(), + std::back_inserter(*vec), + [](auto const &pair) { return pair; }); + } + s.setMeshesPath(meshesPaths); + s.setParticlesPath(particlesPaths); + } + if (access::write(IOHandler()->m_frontendAccess)) { flushAttributes(flushParams); @@ -338,6 +376,75 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) } } +void Iteration::sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + + if (m_or_p.empty()) + { + return; + } + auto &container = (*this)[defaultPath].asContainerOf(); + + for (auto &[name, entry] : m_or_p) + { + if (auxiliary::contains(name, '/')) + { + throw std::runtime_error( + "Unimplemented: Multi-level paths in " + "Iteration::meshes/Iteration::particles"); + } + if (auto it = container.find(name); it != container.end()) + { + if (it->second.m_attri->asSharedPtrOfAttributable() == + entry.m_attri->asSharedPtrOfAttributable()) + { + continue; // has been emplaced previously + } + else + { + throw std::runtime_error("asdfasdfasdfasd"); + } + } + else + { + container.emplace(name, entry); + entry.linkHierarchy(container.writable()); + } + } + }; + + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + +void Iteration::sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + auto it = this->find(defaultPath); + if (it == this->end()) + { + return; + } + auto &container = it->second.asContainerOf(); + for (auto &[name, entry] : container) + { + m_or_p.emplace(name, entry); + } + }; + + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + void Iteration::deferParseAccess(DeferredParseAccess dr) { get().m_deferredParseAccess = @@ -462,7 +569,9 @@ void Iteration::read_impl(std::string const &groupPath) // hasMeshes <-> meshesPath is defined internal::MeshesParticlesPath mpp(s.meshesPaths(), s.particlesPaths()); - CustomHierarchy::read(std::move(mpp)); + CustomHierarchy::read(mpp); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) @@ -632,6 +741,36 @@ void Iteration::setStepStatus(StepStatus status) } } +bool Iteration::dirtyRecursive() const +{ + if (dirty() || CustomHierarchy::dirtyRecursive()) + { + return true; + } + for (auto const &pair : particles) + { + if (!pair.second.written()) + { + return true; + } + } + for (auto const &pair : meshes) + { + if (!pair.second.written()) + { + return true; + } + } + return false; +} + +void Iteration::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); + meshes.linkHierarchy(this->writable()); + particles.linkHierarchy(this->writable()); +} + void Iteration::runDeferredParseAccess() { if (access::read(IOHandler()->m_frontendAccess)) diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp index 3d7ac651b8..d618226d15 100644 --- a/src/binding/python/CustomHierarchy.cpp +++ b/src/binding/python/CustomHierarchy.cpp @@ -13,7 +13,9 @@ using namespace openPMD; void init_CustomHierarchy(py::module &m) { - auto py_ch_cont = declare_container(m, "Container_CustomHierarchy"); + auto py_ch_cont = + declare_container( + m, "Container_CustomHierarchy"); py::class_, Attributable>( m, "CustomHierarchy") @@ -23,20 +25,7 @@ void init_CustomHierarchy(py::module &m) .def("as_container_of_meshes", &CustomHierarchy::asContainerOf) .def( "as_container_of_particles", - &CustomHierarchy::asContainerOf) - - .def_readwrite( - "meshes", - &CustomHierarchy::meshes, - py::return_value_policy::copy, - // garbage collection: return value must be freed before Iteration - py::keep_alive<1, 0>()) - .def_readwrite( - "particles", - &CustomHierarchy::particles, - py::return_value_policy::copy, - // garbage collection: return value must be freed before Iteration - py::keep_alive<1, 0>()); + &CustomHierarchy::asContainerOf); finalize_container(py_ch_cont); } diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 10075451e7..76d1ff3998 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -225,9 +225,13 @@ TEST_CASE("custom_hierarchies", "[core]") write.setMeshesPath(std::vector{"fields/", "%%/meshes/"}); auto meshesManually = write.iterations[0]["fields"].asContainerOf(); + REQUIRE(meshesManually.size() == 0); + write.flush(); // Synchronized upon flushing REQUIRE(meshesManually.contains("E")); REQUIRE(meshesManually.size() == 1); meshesManually["B"]["x"].makeEmpty(2); + REQUIRE(meshesViaAlias.size() == 1); + write.flush(); REQUIRE(meshesViaAlias.contains("B")); REQUIRE(meshesViaAlias.size() == 2); @@ -313,12 +317,14 @@ TEST_CASE("custom_hierarchies", "[core]") { std::vector data(10, 3); - auto E_x = write.iterations[0]["custom_meshes"].meshes["E"]["x"]; + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; E_x.resetDataset({Datatype::INT, {10}}); E_x.storeChunk(data, {0}, {10}); - auto e_pos_x = write.iterations[0]["custom_particles"] - .particles["e"]["position"]["x"]; + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; e_pos_x.resetDataset({Datatype::INT, {10}}); e_pos_x.storeChunk(data, {0}, {10}); write.close(); @@ -328,17 +334,26 @@ TEST_CASE("custom_hierarchies", "[core]") { auto it0 = read.iterations[0]; auto custom_meshes = it0["custom_meshes"]; - REQUIRE(custom_meshes.meshes.size() == 1); - REQUIRE(read.iterations[0]["custom_meshes"].meshes.count("E") == 1); - auto E_x_loaded = read.iterations[0]["custom_meshes"] - .meshes["E"]["x"] + REQUIRE(custom_meshes["meshes"].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf() + .count("E") == 1); + auto E_x_loaded = read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"] .loadChunk(); - REQUIRE(read.iterations[0]["custom_particles"].particles.size() == 1); REQUIRE( - read.iterations[0]["custom_particles"].particles.count("e") == 1); - auto e_pos_x_loaded = read.iterations[0]["custom_particles"] - .particles["e"]["position"]["x"] - .loadChunk(); + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .count("e") == 1); + auto e_pos_x_loaded = + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"] + .loadChunk(); read.flush(); for (size_t i = 0; i < 10; ++i) @@ -378,12 +393,14 @@ TEST_CASE("custom_hierarchies_no_rw", "[core]") { std::vector data(10, 3); - auto E_x = write.iterations[0]["custom_meshes"].meshes["E"]["x"]; + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; E_x.resetDataset({Datatype::INT, {10}}); E_x.storeChunk(data, {0}, {10}); - auto e_pos_x = write.iterations[0]["custom_particles"] - .particles["e"]["position"]["x"]; + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; e_pos_x.resetDataset({Datatype::INT, {10}}); e_pos_x.storeChunk(data, {0}, {10}); write.close(); @@ -420,7 +437,9 @@ TEST_CASE("myPath", "[core]") recordComponent.template makeConstant(5678); }; - REQUIRE(pathOf(iteration.meshes) == vec_t{"iterations", "1234", "meshes"}); + // iteration.meshes is only an alias without a path of its own + // REQUIRE(pathOf(iteration.meshes) == vec_t{"iterations", "1234", + // "meshes"}); auto scalarMesh = iteration.meshes["e_chargeDensity"]; REQUIRE( @@ -439,9 +458,10 @@ TEST_CASE("myPath", "[core]") pathOf(vectorMeshComponent) == vec_t{"iterations", "1234", "meshes", "E", "x"}); - REQUIRE( - pathOf(iteration.particles) == - vec_t{"iterations", "1234", "particles"}); + // iteration.particles is only an alias without a path of its own + // REQUIRE( + // pathOf(iteration.particles) == + // vec_t{"iterations", "1234", "particles"}); auto speciesE = iteration.particles["e"]; REQUIRE(pathOf(speciesE) == vec_t{"iterations", "1234", "particles", "e"}); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 3ff3f7572c..5a0e4b6d3a 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2729,9 +2729,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes.parent() == getWritable(&o.iterations[100])); + REQUIRE( + getWritable( + &o.iterations[100]["fields"].asContainerOf()["E"]) == + getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + o.iterations[100]["fields"].asContainerOf()["E"].parent() == + getWritable(&o.iterations[100]["fields"].asContainerOf())); REQUIRE( o.iterations[100].meshes["E"].parent() == - getWritable(&o.iterations[100].meshes)); + // Iteration::meshes is only an alias, this is the actual parent + getWritable(&o.iterations[100]["fields"].asContainerOf())); REQUIRE( o.iterations[100].meshes["E"]["x"].parent() == getWritable(&o.iterations[100].meshes["E"])); @@ -2741,13 +2749,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes["E"]["z"].parent() == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + getWritable(&o.iterations[100].meshes["rho"]) == + getWritable( + &o.iterations[100]["fields"].asContainerOf()["rho"])); REQUIRE( o.iterations[100].meshes["rho"].parent() == - getWritable(&o.iterations[100].meshes)); + getWritable(&o.iterations[100]["fields"])); REQUIRE( o.iterations[100] .meshes["rho"][MeshRecordComponent::SCALAR] - .parent() == getWritable(&o.iterations[100].meshes)); + .parent() == getWritable(&o.iterations[100]["fields"])); REQUIRE_THROWS_AS( o.iterations[100].meshes["cherries"], std::out_of_range); REQUIRE( @@ -2755,7 +2767,11 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") getWritable(&o.iterations[100])); REQUIRE( o.iterations[100].particles["electrons"].parent() == - getWritable(&o.iterations[100].particles)); + getWritable(&o.iterations[100]["particles"])); + REQUIRE( + getWritable(&o.iterations[100].particles["electrons"]) == + getWritable(&o.iterations[100]["particles"] + .asContainerOf()["electrons"])); REQUIRE( o.iterations[100].particles["electrons"]["charge"].parent() == getWritable(&o.iterations[100].particles["electrons"])); From 3582a1184858f3dddde1f0df0b3b882698a40853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 26 Oct 2023 13:45:25 +0200 Subject: [PATCH 19/27] Some fixes in read error handling --- src/CustomHierarchy.cpp | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index c46e9c3639..bba12419a2 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -439,7 +439,7 @@ void CustomHierarchy::read( { std::cerr << "Cannot read mesh at location '" << myPath().openPMDPath() << "/" << path - << "' and will skip them due to read error:\n" + << "' and will skip it due to read error:\n" << err.what() << std::endl; meshesMap.forget(path); } @@ -454,7 +454,7 @@ void CustomHierarchy::read( { std::cerr << "Cannot read particle species at location '" << myPath().openPMDPath() << "/" << path - << "' and will skip them due to read error:\n" + << "' and will skip it due to read error:\n" << err.what() << std::endl; particlesMap.forget(path); } @@ -492,7 +492,18 @@ void CustomHierarchy::read( break; } case internal::ContainedType::Mesh: - readScalarMesh(meshesMap, path); + try + { + readScalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } break; case internal::ContainedType::Particle: std::cerr @@ -508,11 +519,22 @@ void CustomHierarchy::read( for (auto const &path : constantComponentsPushback) { auto &rc = data.m_embeddedDatasets[path]; - Parameter pOpen; - pOpen.path = path; - IOHandler()->enqueue(IOTask(&rc, pOpen)); - rc.get().m_isConstant = true; - rc.read(); + try + { + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read dataset at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + data.m_embeddedDatasets.container().erase(path); + } } } From 4b9f95c0c751c0fbb45df9c25e160783e879ea1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 26 Oct 2023 15:34:15 +0200 Subject: [PATCH 20/27] More symmetric design for container types --- include/openPMD/CustomHierarchy.hpp | 111 ++++++++++++++++++++----- src/CustomHierarchy.cpp | 81 ++++++------------ src/Iteration.cpp | 4 +- src/binding/python/CustomHierarchy.cpp | 40 +++++++-- src/binding/python/openPMD.cpp | 2 +- test/SerialIOTest.cpp | 4 +- 6 files changed, 154 insertions(+), 88 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index fb3402cf23..ee585ed6b3 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -23,6 +23,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" #include "openPMD/backend/Container.hpp" #include @@ -74,27 +75,67 @@ namespace internal isMeshContainer(std::vector const &path) const; }; - struct CustomHierarchyData : ContainerData + struct CustomHierarchyData + : ContainerData + , ContainerData + , ContainerData + , ContainerData { explicit CustomHierarchyData(); void syncAttributables(); - Container m_embeddedDatasets; - Container m_embeddedMeshes; - Container m_embeddedParticles; + Container customHierarchies() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + Container embeddedDatasets() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + Container embeddedMeshes() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + + Container embeddedParticles() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } }; } // namespace internal -class CustomHierarchy : public Container +template +class ConversibleContainer : public Container { - friend class Iteration; - friend class Container; + template + friend class ConversibleContainer; -private: - using Container_t = Container; +protected: + using Container_t = Container; using Data_t = internal::CustomHierarchyData; - static_assert(std::is_base_of_v); + static_assert( + std::is_base_of_v); + + ConversibleContainer(Attributable::NoInit) + : Container_t(Attributable::NoInit{}) + {} std::shared_ptr m_customHierarchyData; @@ -107,6 +148,47 @@ class CustomHierarchy : public Container return *m_customHierarchyData; } + inline void setData(std::shared_ptr data) + { + m_customHierarchyData = data; + Container_t::setData(std::move(data)); + } + +public: + template + auto asContainerOf() -> ConversibleContainer + { + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + ConversibleContainer res(Attributable::NoInit{}); + res.setData(m_customHierarchyData); + return res; + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); + } + } +}; + +class CustomHierarchy : public ConversibleContainer +{ + friend class Iteration; + friend class Container; + +private: + using Container_t = Container; + using Parent_t = ConversibleContainer; + using Data_t = typename Parent_t::Data_t; + using EraseStaleMeshes = internal::EraseStaleEntries>; using EraseStaleParticles = internal::EraseStaleEntries>; @@ -118,12 +200,6 @@ class CustomHierarchy : public Container CustomHierarchy(); CustomHierarchy(NoInit); - inline void setData(std::shared_ptr data) - { - m_customHierarchyData = data; - Container_t::setData(std::move(data)); - } - void read(internal::MeshesParticlesPath const &); void read( internal::MeshesParticlesPath const &, @@ -158,8 +234,5 @@ class CustomHierarchy : public Container CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; - - template - auto asContainerOf() -> Container &; }; } // namespace openPMD diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index bba12419a2..daf46f4e1c 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -262,21 +262,29 @@ namespace internal /* * m_embeddeddatasets and its friends should point to the same instance * of Attributable. + * Not strictly necessary to do this explicitly due to virtual + * inheritance (all Attributable instances are the same anyway), + * but let's be explicit about this. */ - for (auto p : std::initializer_list{ - &m_embeddedDatasets, &m_embeddedMeshes, &m_embeddedParticles}) + for (auto p : std::initializer_list{ + static_cast *>(this), + static_cast *>(this), + static_cast *>(this), + static_cast *>(this)}) { - p->m_attri->asSharedPtrOfAttributable() = - this->asSharedPtrOfAttributable(); + p->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); } } } // namespace internal -CustomHierarchy::CustomHierarchy() +// template +// class ConversibleContainer; + +CustomHierarchy::CustomHierarchy() : ConversibleContainer(NoInit{}) { setData(std::make_shared()); } -CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit()) +CustomHierarchy::CustomHierarchy(NoInit) : ConversibleContainer(NoInit{}) {} void CustomHierarchy::readNonscalarMesh( @@ -394,8 +402,8 @@ void CustomHierarchy::read( std::deque constantComponentsPushback; auto &data = get(); - EraseStaleMeshes meshesMap(data.m_embeddedMeshes); - EraseStaleParticles particlesMap(data.m_embeddedParticles); + EraseStaleMeshes meshesMap(data.embeddedMeshes()); + EraseStaleParticles particlesMap(data.embeddedParticles()); for (auto const &path : *pList.paths) { switch (mpp.determineType(currentPath)) @@ -469,7 +477,7 @@ void CustomHierarchy::read( // Group is a bit of an internal misnomer here, it just means that // it matches neither meshes nor particles path case internal::ContainedType::Group: { - auto &rc = data.m_embeddedDatasets[path]; + auto &rc = data.embeddedDatasets()[path]; Parameter dOpen; dOpen.name = path; IOHandler()->enqueue(IOTask(&rc, dOpen)); @@ -487,7 +495,7 @@ void CustomHierarchy::read( << "' at path '" << myPath().openPMDPath() << "' and will skip it due to read error:\n" << err.what() << std::endl; - data.m_embeddedDatasets.container().erase(path); + data.embeddedDatasets().container().erase(path); } break; } @@ -518,7 +526,7 @@ void CustomHierarchy::read( for (auto const &path : constantComponentsPushback) { - auto &rc = data.m_embeddedDatasets[path]; + auto &rc = data.embeddedDatasets()[path]; try { Parameter pOpen; @@ -533,7 +541,7 @@ void CustomHierarchy::read( << myPath().openPMDPath() << "/" << path << "' and will skip it due to read error:\n" << err.what() << std::endl; - data.m_embeddedDatasets.container().erase(path); + data.embeddedDatasets().container().erase(path); } } } @@ -569,7 +577,7 @@ void CustomHierarchy::flush_internal( subpath.flush_internal(flushParams, mpp, currentPath); currentPath.pop_back(); } - for (auto &[name, mesh] : data.m_embeddedMeshes) + for (auto &[name, mesh] : data.embeddedMeshes()) { if (!mpp.isMeshContainer(currentPath)) { @@ -593,7 +601,7 @@ void CustomHierarchy::flush_internal( } mesh.flush(name, flushParams); } - for (auto &[name, particleSpecies] : data.m_embeddedParticles) + for (auto &[name, particleSpecies] : data.embeddedParticles()) { if (!mpp.isParticleContainer(currentPath)) { @@ -619,7 +627,7 @@ void CustomHierarchy::flush_internal( } particleSpecies.flush(name, flushParams); } - for (auto &[name, dataset] : get().m_embeddedDatasets) + for (auto &[name, dataset] : get().embeddedDatasets()) { dataset.flush(name, flushParams); } @@ -654,47 +662,10 @@ bool CustomHierarchy::dirtyRecursive() const } return false; }; - auto &data = get(); - return check(data.m_embeddedMeshes) || check(data.m_embeddedParticles) || - check(data.m_embeddedDatasets) || check(*this); -} - -template -auto CustomHierarchy::asContainerOf() -> Container & -{ - if constexpr (std::is_same_v) - { - return *static_cast *>(this); - } - else if constexpr (std::is_same_v) - { - return get().m_embeddedMeshes; - } - else if constexpr (std::is_same_v) - { - return get().m_embeddedParticles; - } - else if constexpr (std::is_same_v) - { - return get().m_embeddedDatasets; - } - else - { - static_assert( - auxiliary::dependent_false_v, - "[CustomHierarchy::asContainerOf] Type parameter must be " - "one of: CustomHierarchy, RecordComponent, Mesh, " - "ParticleSpecies."); - } + auto &data = const_cast(get()); // @todo do this better + return check(data.embeddedMeshes()) || check(data.embeddedParticles()) || + check(data.embeddedDatasets()) || check(data.customHierarchies()); } - -template auto CustomHierarchy::asContainerOf() - -> Container &; -template auto CustomHierarchy::asContainerOf() - -> Container &; -template auto CustomHierarchy::asContainerOf() -> Container &; -template auto CustomHierarchy::asContainerOf() - -> Container &; } // namespace openPMD #undef OPENPMD_LEGAL_IDENTIFIER_CHARS diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 211213c622..11ef556f40 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -388,7 +388,7 @@ void Iteration::sync_meshes_and_particles_from_alias_to_subgroups( { return; } - auto &container = (*this)[defaultPath].asContainerOf(); + auto container = (*this)[defaultPath].asContainerOf(); for (auto &[name, entry] : m_or_p) { @@ -434,7 +434,7 @@ void Iteration::sync_meshes_and_particles_from_subgroups_to_alias( { return; } - auto &container = it->second.asContainerOf(); + auto container = it->second.asContainerOf(); for (auto &[name, entry] : container) { m_or_p.emplace(name, entry); diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp index d618226d15..cd4b893909 100644 --- a/src/binding/python/CustomHierarchy.cpp +++ b/src/binding/python/CustomHierarchy.cpp @@ -11,21 +11,43 @@ namespace py = pybind11; using namespace openPMD; +template +void define_conversible_container(py::module &m, std::string const &name) +{ + using CC = ConversibleContainer; + py::class_, Attributable>(m, name.c_str()) + .def( + "as_container_of_datasets", + &CC::template asContainerOf) + .def("as_container_of_meshes", &CC::template asContainerOf) + .def( + "as_container_of_particles", + &CC::template asContainerOf) + .def( + "as_container_of_custom_hierarchy", + &CC::template asContainerOf); +} + void init_CustomHierarchy(py::module &m) { auto py_ch_cont = declare_container( m, "Container_CustomHierarchy"); - py::class_, Attributable>( - m, "CustomHierarchy") - .def( - "as_container_of_datasets", - &CustomHierarchy::asContainerOf) - .def("as_container_of_meshes", &CustomHierarchy::asContainerOf) - .def( - "as_container_of_particles", - &CustomHierarchy::asContainerOf); + define_conversible_container( + m, "ConversibleContainer_CustomHierarchy"); + define_conversible_container( + m, "ConversibleContainer_ParticleSpecies"); + define_conversible_container( + m, "ConversibleContainer_RecordComponent"); + define_conversible_container(m, "ConversibleContainer_Mesh"); + + [[maybe_unused]] py::class_< + CustomHierarchy, + ConversibleContainer, + Container, + Attributable> + custom_hierarchy(m, "CustomHierarchy"); finalize_container(py_ch_cont); } diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index 4e2b74b892..996208bd55 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -91,7 +91,6 @@ PYBIND11_MODULE(openpmd_api_cxx, m) init_Datatype(m); init_Dataset(m); - init_CustomHierarchy(m); init_BaseRecordComponent(m); init_RecordComponent(m); init_MeshRecordComponent(m); @@ -103,6 +102,7 @@ PYBIND11_MODULE(openpmd_api_cxx, m) init_ParticleSpecies(m); init_Mesh(m); + init_CustomHierarchy(m); init_Iteration(m); init_IterationEncoding(m); init_Series(m); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5a0e4b6d3a..41cad658c1 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2735,11 +2735,11 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") getWritable(&o.iterations[100].meshes["E"])); REQUIRE( o.iterations[100]["fields"].asContainerOf()["E"].parent() == - getWritable(&o.iterations[100]["fields"].asContainerOf())); + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"].parent() == // Iteration::meshes is only an alias, this is the actual parent - getWritable(&o.iterations[100]["fields"].asContainerOf())); + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"]["x"].parent() == getWritable(&o.iterations[100].meshes["E"])); From 1113280b12439850628bd37967ad222de988cfc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Nov 2023 17:25:36 +0100 Subject: [PATCH 21/27] Don't write unitSI in custom datasets --- include/openPMD/RecordComponent.hpp | 7 ++++--- src/CustomHierarchy.cpp | 6 +++--- src/ParticlePatches.cpp | 2 +- src/Record.cpp | 23 ++++++++++++++--------- src/RecordComponent.cpp | 14 ++++++++------ src/backend/MeshRecordComponent.cpp | 4 ++-- src/backend/PatchRecord.cpp | 7 ++++--- 7 files changed, 36 insertions(+), 27 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 4da9547579..2ce7786154 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -488,8 +488,9 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void flush( + std::string const &, internal::FlushParams const &, bool set_defaults); + void read(bool read_defaults); private: /** @@ -537,7 +538,7 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(bool read_defaults); template void verifyChunk(Offset const &, Extent const &) const; diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index daf46f4e1c..f031aa52e2 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -487,7 +487,7 @@ void CustomHierarchy::read( rc.setWritten(false, Attributable::EnqueueAsynchronously::No); rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.setWritten(true, Attributable::EnqueueAsynchronously::No); - rc.read(); + rc.read(/* read_defaults = */ false); } catch (error::ReadError const &err) { @@ -533,7 +533,7 @@ void CustomHierarchy::read( pOpen.path = path; IOHandler()->enqueue(IOTask(&rc, pOpen)); rc.get().m_isConstant = true; - rc.read(); + rc.read(/* read_defaults = */ false); } catch (error::ReadError const &err) { @@ -629,7 +629,7 @@ void CustomHierarchy::flush_internal( } for (auto &[name, dataset] : get().embeddedDatasets()) { - dataset.flush(name, flushParams); + dataset.flush(name, flushParams, /* set_defaults = */ false); } } diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 491add8be7..c7fac4b654 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -100,7 +100,7 @@ void ParticlePatches::read() pr.setDirty(false); try { - prc.PatchRecordComponent::read(/* require_unit_si = */ false); + prc.PatchRecordComponent::read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/Record.cpp b/src/Record.cpp index 3bcac4d7e1..5178135dc4 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -50,12 +50,14 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } else @@ -65,7 +67,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams); + rc.flush(name, flushParams, /* set_defaults = */ true); } else { @@ -75,7 +77,8 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } } @@ -84,12 +87,14 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } @@ -104,7 +109,7 @@ void Record::read() /* using operator[] will incorrectly update parent */ try { - T_RecordComponent::read(/* require_unit_si = */ true); + T_RecordComponent::read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -128,7 +133,7 @@ void Record::read() rc.get().m_isConstant = true; try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -155,7 +160,7 @@ void Record::read() rc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 0387268514..3e51599882 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -232,7 +232,9 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, internal::FlushParams const &flushParams) + std::string const &name, + internal::FlushParams const &flushParams, + bool set_defaults) { auto &rc = get(); if (flushParams.flushLevel == FlushLevel::SkeletonOnly) @@ -271,7 +273,7 @@ void RecordComponent::flush( "before flushing (see RecordComponent::resetDataset())."); } } - if (!containsAttribute("unitSI")) + if (set_defaults && !containsAttribute("unitSI")) { setUnitSI(1); } @@ -358,9 +360,9 @@ void RecordComponent::flush( } } -void RecordComponent::read(bool require_unit_si) +void RecordComponent::read(bool read_defaults) { - readBase(require_unit_si); + readBase(read_defaults); } namespace @@ -385,7 +387,7 @@ namespace }; } // namespace -void RecordComponent::readBase(bool require_unit_si) +void RecordComponent::readBase(bool read_defaults) { using DT = Datatype; // auto & rc = get(); @@ -433,7 +435,7 @@ void RecordComponent::readBase(bool require_unit_si) readAttributes(ReadMode::FullyReread); - if (require_unit_si) + if (read_defaults) { if (!containsAttribute("unitSI")) { diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index ed50080757..10bee1cf45 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -64,7 +64,7 @@ void MeshRecordComponent::read() "of any floating point type, found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); - readBase(/* require_unit_si = */ true); + readBase(/* read_defaults = */ true); } void MeshRecordComponent::flush( @@ -75,7 +75,7 @@ void MeshRecordComponent::flush( { setPosition(std::vector{0}); } - RecordComponent::flush(name, params); + RecordComponent::flush(name, params, /* set_defaults = */ true); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 5d2b38d50f..47874daae7 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -48,10 +48,11 @@ void PatchRecord::flush_impl( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } else - T_RecordComponent::flush(path, flushParams); + T_RecordComponent::flush(path, flushParams, /* set_defaults = */ true); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); @@ -95,7 +96,7 @@ void PatchRecord::read() prc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - prc.read(/* require_unit_si = */ false); + prc.read(/* read_defaults = */ false); } catch (error::ReadError const &err) { From 6174cf8d942b6bf1061eaf8a575b13184218833e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 21 Dec 2023 18:53:51 +0100 Subject: [PATCH 22/27] Discouraged support for custom datasets inside the particlesPath --- src/CustomHierarchy.cpp | 22 ++++++++++++---------- test/CoreTest.cpp | 6 ++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index f031aa52e2..51d5368857 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -474,8 +474,18 @@ void CustomHierarchy::read( { switch (mpp.determineType(currentPath)) { - // Group is a bit of an internal misnomer here, it just means that - // it matches neither meshes nor particles path + + case internal::ContainedType::Particle: + std::cerr << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << "' inside the particles path. A particle species is " + "always a group, never a dataset. Will parse as a " + "custom dataset. Storing custom datasets inside the " + "particles path is discouraged." + << std::endl; + [[fallthrough]]; + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path case internal::ContainedType::Group: { auto &rc = data.embeddedDatasets()[path]; Parameter dOpen; @@ -513,14 +523,6 @@ void CustomHierarchy::read( meshesMap.forget(path); } break; - case internal::ContainedType::Particle: - std::cerr - << "[Warning] Dataset found at '" - << (concatWithSep(currentPath, "/") + "/" + path) - << " that matches one of the given particle paths. A particle " - "species is always a group, never a dataset. Will skip." - << std::endl; - break; } } diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 76d1ff3998..bdad64e117 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -403,6 +403,12 @@ TEST_CASE("custom_hierarchies_no_rw", "[core]") .asContainerOf()["e"]["position"]["x"]; e_pos_x.resetDataset({Datatype::INT, {10}}); e_pos_x.storeChunk(data, {0}, {10}); + + auto gnihihi = write.iterations[0]["custom_particles"]["particles"] + .asContainerOf(); + auto dataset = gnihihi["custom_dataset"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(std::unique_ptr(new int[10]{}), {0}, {10}); write.close(); } From b74748b2edb3f9754e1a6c2abd39b4c0a86f6c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 26 Mar 2024 14:30:27 +0100 Subject: [PATCH 23/27] Fix after rebase: dirtyRecursive --- src/CustomHierarchy.cpp | 2 ++ src/Series.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 51d5368857..c2b3466c30 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -546,6 +546,7 @@ void CustomHierarchy::read( data.embeddedDatasets().container().erase(path); } } + setDirty(false); } void CustomHierarchy::flush_internal( @@ -633,6 +634,7 @@ void CustomHierarchy::flush_internal( { dataset.flush(name, flushParams, /* set_defaults = */ false); } + setDirty(false); } void CustomHierarchy::flush( diff --git a/src/Series.cpp b/src/Series.cpp index 6ddfc57fab..f7d0ca2d1d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -235,7 +235,7 @@ Series &Series::setMeshesPath(std::vector const &mp) setAttribute("meshesPath", mp); break; } - dirty() = true; + setDirty(true); return *this; } @@ -562,7 +562,7 @@ Series &Series::setParticlesPath(std::vector const &pp) setAttribute("particlesPath", pp); break; } - dirty() = true; + setDirty(true); return *this; } From 5376ba678f947842174506337650ee0154ab1f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 May 2024 16:33:37 +0200 Subject: [PATCH 24/27] Fixes to the dirty/dirtyRecursive logic --- src/CustomHierarchy.cpp | 6 +++++- src/Iteration.cpp | 11 ++++++----- src/Series.cpp | 11 +++++------ src/backend/Attributable.cpp | 9 +++++---- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index c2b3466c30..4398121a64 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -634,7 +634,11 @@ void CustomHierarchy::flush_internal( { dataset.flush(name, flushParams, /* set_defaults = */ false); } - setDirty(false); + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } } void CustomHierarchy::flush( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 11ef556f40..6b3b45dcb1 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -364,12 +364,13 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) s.setParticlesPath(particlesPaths); } - if (access::write(IOHandler()->m_frontendAccess)) - { - flushAttributes(flushParams); - } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } setDirty(false); meshes.setDirty(false); particles.setDirty(false); diff --git a/src/Series.cpp b/src/Series.cpp index f7d0ca2d1d..1cec4a9be7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1418,9 +1418,13 @@ void Series::flushFileBased( case Access::READ_WRITE: case Access::CREATE: case Access::APPEND: { - bool allDirty = dirty(); + bool const allDirty = dirty(); for (auto it = begin; it != end; ++it) { + /* reset the dirty bit for every iteration (i.e. file) + * otherwise only the first iteration will have updates attributes + */ + setDirty(allDirty); // Phase 1 switch (openIterationIfDirty(it->first, it->second)) { @@ -1467,12 +1471,7 @@ void Series::flushFileBased( it->second.get().m_closed = internal::CloseStatus::ClosedInBackend; } - /* reset the dirty bit for every iteration (i.e. file) - * otherwise only the first iteration will have updates attributes - */ - setDirty(allDirty); } - setDirty(false); // Phase 3 if (flushIOHandler) diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index ef83f1e85c..583c334b73 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/backend/Attributable.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/RecordComponent.hpp" @@ -293,10 +294,10 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + assert( + flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles); + setDirty(false); } void Attributable::readAttributes(ReadMode mode) From a8784043f6a29b62b17e41526c535e285c61f33e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:58:18 +0000 Subject: [PATCH 25/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/binding/python/CustomHierarchy.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp index cd4b893909..b893d6a174 100644 --- a/src/binding/python/CustomHierarchy.cpp +++ b/src/binding/python/CustomHierarchy.cpp @@ -46,8 +46,7 @@ void init_CustomHierarchy(py::module &m) CustomHierarchy, ConversibleContainer, Container, - Attributable> - custom_hierarchy(m, "CustomHierarchy"); + Attributable> custom_hierarchy(m, "CustomHierarchy"); finalize_container(py_ch_cont); } From 6b47a35a8196d3503b442c688571caf11e25bce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 14 Aug 2024 13:12:54 +0200 Subject: [PATCH 26/27] Some cleanup in CustomHierarchies class --- include/openPMD/CustomHierarchy.hpp | 46 ++++++++++++++++++++--------- src/CustomHierarchy.cpp | 41 +++++++------------------ 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index ee585ed6b3..dc23a97e64 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -85,7 +85,8 @@ namespace internal void syncAttributables(); - Container customHierarchies() +#if 0 + inline Container customHierarchiesWrapped() { Container res; res.setData( @@ -93,7 +94,8 @@ namespace internal [](auto const *) {}}); return res; } - Container embeddedDatasets() +#endif + inline Container embeddedDatasetsWrapped() { Container res; res.setData( @@ -101,7 +103,7 @@ namespace internal [](auto const *) {}}); return res; } - Container embeddedMeshes() + inline Container embeddedMeshesWrapped() { Container res; res.setData( @@ -110,7 +112,7 @@ namespace internal return res; } - Container embeddedParticles() + inline Container embeddedParticlesWrapped() { Container res; res.setData( @@ -118,6 +120,32 @@ namespace internal [](auto const *) {}}); return res; } + +#if 0 + inline Container::InternalContainer & + customHierarchiesInternal() + { + return static_cast *>(this) + ->m_container; + } +#endif + inline Container::InternalContainer & + embeddedDatasetsInternal() + { + return static_cast *>(this) + ->m_container; + } + inline Container::InternalContainer &embeddedMeshesInternal() + { + return static_cast *>(this)->m_container; + } + + inline Container::InternalContainer & + embeddedParticlesInternal() + { + return static_cast *>(this) + ->m_container; + } }; } // namespace internal @@ -218,16 +246,6 @@ class CustomHierarchy : public ConversibleContainer */ void linkHierarchy(Writable &w) override; - /* - * @brief Check recursively whether this object is dirty. - * It is dirty if any attribute or dataset is read from or written to - * the backend. - * - * @return true If dirty. - * @return false Otherwise. - */ - bool dirtyRecursive() const; - public: CustomHierarchy(CustomHierarchy const &other) = default; CustomHierarchy(CustomHierarchy &&other) = default; diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 4398121a64..1dc0fb1788 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -402,8 +402,8 @@ void CustomHierarchy::read( std::deque constantComponentsPushback; auto &data = get(); - EraseStaleMeshes meshesMap(data.embeddedMeshes()); - EraseStaleParticles particlesMap(data.embeddedParticles()); + EraseStaleMeshes meshesMap(data.embeddedMeshesWrapped()); + EraseStaleParticles particlesMap(data.embeddedParticlesWrapped()); for (auto const &path : *pList.paths) { switch (mpp.determineType(currentPath)) @@ -487,7 +487,8 @@ void CustomHierarchy::read( // Group is a bit of an internal misnomer here, it just means that // it matches neither meshes nor particles path case internal::ContainedType::Group: { - auto &rc = data.embeddedDatasets()[path]; + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; Parameter dOpen; dOpen.name = path; IOHandler()->enqueue(IOTask(&rc, dOpen)); @@ -505,7 +506,7 @@ void CustomHierarchy::read( << "' at path '" << myPath().openPMDPath() << "' and will skip it due to read error:\n" << err.what() << std::endl; - data.embeddedDatasets().container().erase(path); + embeddedDatasets.erase(path); } break; } @@ -528,7 +529,8 @@ void CustomHierarchy::read( for (auto const &path : constantComponentsPushback) { - auto &rc = data.embeddedDatasets()[path]; + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; try { Parameter pOpen; @@ -543,7 +545,7 @@ void CustomHierarchy::read( << myPath().openPMDPath() << "/" << path << "' and will skip it due to read error:\n" << err.what() << std::endl; - data.embeddedDatasets().container().erase(path); + embeddedDatasets.erase(path); } } setDirty(false); @@ -580,7 +582,7 @@ void CustomHierarchy::flush_internal( subpath.flush_internal(flushParams, mpp, currentPath); currentPath.pop_back(); } - for (auto &[name, mesh] : data.embeddedMeshes()) + for (auto &[name, mesh] : data.embeddedMeshesInternal()) { if (!mpp.isMeshContainer(currentPath)) { @@ -604,7 +606,7 @@ void CustomHierarchy::flush_internal( } mesh.flush(name, flushParams); } - for (auto &[name, particleSpecies] : data.embeddedParticles()) + for (auto &[name, particleSpecies] : data.embeddedParticlesInternal()) { if (!mpp.isParticleContainer(currentPath)) { @@ -630,7 +632,7 @@ void CustomHierarchy::flush_internal( } particleSpecies.flush(name, flushParams); } - for (auto &[name, dataset] : get().embeddedDatasets()) + for (auto &[name, dataset] : get().embeddedDatasetsInternal()) { dataset.flush(name, flushParams, /* set_defaults = */ false); } @@ -653,27 +655,6 @@ void CustomHierarchy::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); } - -bool CustomHierarchy::dirtyRecursive() const -{ - if (dirty()) - { - return true; - } - auto check = [](auto const &container) { - for (auto const &pair : container) - { - if (pair.second.dirtyRecursive()) - { - return true; - } - } - return false; - }; - auto &data = const_cast(get()); // @todo do this better - return check(data.embeddedMeshes()) || check(data.embeddedParticles()) || - check(data.embeddedDatasets()) || check(data.customHierarchies()); -} } // namespace openPMD #undef OPENPMD_LEGAL_IDENTIFIER_CHARS From 20f2cd5bca0c60e2302803d4c634d4b30f5d06f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 Nov 2024 22:13:04 +0100 Subject: [PATCH 27/27] Use polymorphism for meshes/particlesPath in Python --- src/binding/python/Series.cpp | 68 ++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 36a878f03b..2ceaf4b4b3 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -22,10 +22,12 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/config.hpp" #include "openPMD/binding/python/Common.hpp" +#include #if openPMD_HAVE_MPI // re-implemented signatures: @@ -282,26 +284,60 @@ this method. &Series::openPMDextension, &Series::setOpenPMDextension) .def_property("base_path", &Series::basePath, &Series::setBasePath) - .def_property( - "meshes_path", - &Series::meshesPath, - py::overload_cast(&Series::setMeshesPath)) .def("get_rank_table", &Series::rankTable, py::arg("collective")) .def("set_rank_table", &Series::setRankTable, py::arg("my_rank_info")) .def_property( - "particles_path", - &Series::particlesPath, - py::overload_cast(&Series::setParticlesPath)) - .def_property( - "meshes_paths", - &Series::meshesPath, - py::overload_cast const &>( - &Series::setMeshesPath)) + "meshes_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.meshesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setMeshesPath(arg_resolved); + }, + arg); + return self; + }) .def_property( - "particles_paths", - &Series::particlesPath, - py::overload_cast const &>( - &Series::setParticlesPath)) + "particles_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.particlesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setParticlesPath(arg_resolved); + }, + arg); + return self; + }) .def_property("author", &Series::author, &Series::setAuthor) .def_property( "machine",